diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md
index d0f53936eb56a..bd85d67772cd9 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md
@@ -25,6 +25,7 @@ export interface SearchSourceFields
| [highlightAll](./kibana-plugin-plugins-data-public.searchsourcefields.highlightall.md) | boolean | |
| [index](./kibana-plugin-plugins-data-public.searchsourcefields.index.md) | IndexPattern | |
| [parent](./kibana-plugin-plugins-data-public.searchsourcefields.parent.md) | SearchSourceFields | |
+| [pit](./kibana-plugin-plugins-data-public.searchsourcefields.pit.md) | {
id: string;
keep_alive?: string;
} | |
| [query](./kibana-plugin-plugins-data-public.searchsourcefields.query.md) | Query | [Query](./kibana-plugin-plugins-data-public.query.md) |
| [searchAfter](./kibana-plugin-plugins-data-public.searchsourcefields.searchafter.md) | EsQuerySearchAfter | |
| [size](./kibana-plugin-plugins-data-public.searchsourcefields.size.md) | number | |
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.pit.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.pit.md
new file mode 100644
index 0000000000000..c7e426d3f47ca
--- /dev/null
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.pit.md
@@ -0,0 +1,14 @@
+
+
+[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [SearchSourceFields](./kibana-plugin-plugins-data-public.searchsourcefields.md) > [pit](./kibana-plugin-plugins-data-public.searchsourcefields.pit.md)
+
+## SearchSourceFields.pit property
+
+Signature:
+
+```typescript
+pit?: {
+ id: string;
+ keep_alive?: string;
+ };
+```
diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts
index f11e7f06b6ab9..4f0b88f840d60 100644
--- a/src/plugins/data/common/search/search_source/search_source.ts
+++ b/src/plugins/data/common/search/search_source/search_source.ts
@@ -439,6 +439,8 @@ export class SearchSource {
getConfig(UI_SETTINGS.SORT_OPTIONS)
);
return addToBody(key, sort);
+ case 'pit':
+ return addToRoot(key, val);
default:
return addToBody(key, val);
}
diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts
index a178b38693d92..0fbbaec5c4a6a 100644
--- a/src/plugins/data/common/search/search_source/types.ts
+++ b/src/plugins/data/common/search/search_source/types.ts
@@ -100,7 +100,10 @@ export interface SearchSourceFields {
searchAfter?: EsQuerySearchAfter;
timeout?: string;
terminate_after?: number;
-
+ pit?: {
+ id: string;
+ keep_alive?: string;
+ };
parent?: SearchSourceFields;
}
diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md
index 9720395881ea3..9f6c7e424cdbf 100644
--- a/src/plugins/data/public/public.api.md
+++ b/src/plugins/data/public/public.api.md
@@ -2438,6 +2438,11 @@ export interface SearchSourceFields {
// (undocumented)
parent?: SearchSourceFields;
// (undocumented)
+ pit?: {
+ id: string;
+ keep_alive?: string;
+ };
+ // (undocumented)
query?: Query;
// Warning: (ae-forgotten-export) The symbol "EsQuerySearchAfter" needs to be exported by the entry point index.d.ts
//
diff --git a/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts
index a1215836f9c5f..65fef2e4d030f 100644
--- a/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts
+++ b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts
@@ -97,8 +97,7 @@ export const getTopNavLinks = ({
const sharingData = await getSharingData(
searchSource,
state.appStateContainer.getState(),
- services.uiSettings,
- getFieldCounts
+ services.uiSettings
);
services.share.toggleShareContextMenu({
anchorElement,
diff --git a/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts b/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts
index 5e0e48e619a27..d3a23fa6a939a 100644
--- a/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts
+++ b/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts
@@ -16,54 +16,26 @@ import { SORT_DEFAULT_ORDER_SETTING } from '../../../common';
describe('getSharingData', () => {
test('returns valid data for sharing', async () => {
const searchSourceMock = createSearchSourceMock({ index: indexPatternMock });
- const result = await getSharingData(
- searchSourceMock,
- { columns: [] },
- ({
- get: (key: string) => {
- if (key === SORT_DEFAULT_ORDER_SETTING) {
- return 'desc';
- }
- return false;
- },
- } as unknown) as IUiSettingsClient,
- () => Promise.resolve({})
- );
+ const result = await getSharingData(searchSourceMock, { columns: [] }, ({
+ get: (key: string) => {
+ if (key === SORT_DEFAULT_ORDER_SETTING) {
+ return 'desc';
+ }
+ return false;
+ },
+ } as unknown) as IUiSettingsClient);
expect(result).toMatchInlineSnapshot(`
Object {
- "conflictedTypesFields": Array [],
- "fields": Array [],
- "indexPatternId": "the-index-pattern-id",
- "metaFields": Array [
- "_index",
- "_score",
- ],
- "searchRequest": Object {
- "body": Object {
- "_source": Object {},
- "fields": Array [],
- "query": Object {
- "bool": Object {
- "filter": Array [],
- "must": Array [],
- "must_not": Array [],
- "should": Array [],
- },
+ "searchSource": Object {
+ "fields": Array [
+ "*",
+ ],
+ "index": "the-index-pattern-id",
+ "sort": Array [
+ Object {
+ "_score": "desc",
},
- "runtime_mappings": Object {},
- "script_fields": Object {},
- "sort": Array [
- Object {
- "_score": Object {
- "order": "desc",
- },
- },
- ],
- "stored_fields": Array [
- "*",
- ],
- },
- "index": "the-index-pattern-title",
+ ],
},
}
`);
diff --git a/src/plugins/discover/public/application/helpers/get_sharing_data.ts b/src/plugins/discover/public/application/helpers/get_sharing_data.ts
index 2455589cf69fc..11b2309df305e 100644
--- a/src/plugins/discover/public/application/helpers/get_sharing_data.ts
+++ b/src/plugins/discover/public/application/helpers/get_sharing_data.ts
@@ -6,57 +6,24 @@
* Side Public License, v 1.
*/
-import { Capabilities, IUiSettingsClient } from 'kibana/public';
+import type { Capabilities, IUiSettingsClient } from 'kibana/public';
import { DOC_HIDE_TIME_COLUMN_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common';
import { getSortForSearchSource } from '../angular/doc_table';
import { ISearchSource } from '../../../../data/common';
import { AppState } from '../angular/discover_state';
-import { SortOrder } from '../../saved_searches/types';
-
-const getSharingDataFields = async (
- getFieldCounts: () => Promise>,
- selectedFields: string[],
- timeFieldName: string,
- hideTimeColumn: boolean
-) => {
- if (
- selectedFields.length === 0 ||
- (selectedFields.length === 1 && selectedFields[0] === '_source')
- ) {
- const fieldCounts = await getFieldCounts();
- return {
- searchFields: undefined,
- selectFields: Object.keys(fieldCounts).sort(),
- };
- }
-
- const fields =
- timeFieldName && !hideTimeColumn ? [timeFieldName, ...selectedFields] : selectedFields;
- return {
- searchFields: fields,
- selectFields: fields,
- };
-};
+import { SavedSearch, SortOrder } from '../../saved_searches/types';
/**
* Preparing data to share the current state as link or CSV/Report
*/
export async function getSharingData(
currentSearchSource: ISearchSource,
- state: AppState,
- config: IUiSettingsClient,
- getFieldCounts: () => Promise>
+ state: AppState | SavedSearch,
+ config: IUiSettingsClient
) {
const searchSource = currentSearchSource.createCopy();
const index = searchSource.getField('index')!;
- const { searchFields, selectFields } = await getSharingDataFields(
- getFieldCounts,
- state.columns || [],
- index.timeFieldName || '',
- config.get(DOC_HIDE_TIME_COLUMN_SETTING)
- );
- searchSource.setField('fieldsFromSource', searchFields);
searchSource.setField(
'sort',
getSortForSearchSource(state.sort as SortOrder[], index, config.get(SORT_DEFAULT_ORDER_SETTING))
@@ -66,20 +33,45 @@ export async function getSharingData(
searchSource.removeField('aggs');
searchSource.removeField('size');
- const body = await searchSource.getSearchRequestBody();
+ // Set the fields of the search source to match the saved search columns
+ searchSource.removeField('fields');
+ searchSource.removeField('fieldsFromSource');
+
+ let columns = state.columns || [];
+
+ // NOTE: A newly saved search with no columns selected has a bug(?) where the
+ // column array is a single '_source' value which is invalid for CSV export
+ if (columns && columns.length === 1 && /^_source$/.test(columns.join())) {
+ columns = [];
+ }
+
+ // conditionally add the time field column
+ let timeFieldName: string | undefined;
+ const hideTimeColumn = config.get(DOC_HIDE_TIME_COLUMN_SETTING);
+ if (!hideTimeColumn && index && index.timeFieldName) {
+ timeFieldName = index.timeFieldName;
+ }
+
+ if (columns && columns.length > 0 && timeFieldName) {
+ columns = [timeFieldName, ...columns];
+ }
+
+ if (columns.length === 0) {
+ searchSource.setField('fields', ['*']);
+ } else {
+ searchSource.setField('fields', columns);
+ }
return {
- searchRequest: {
- index: index.title,
- body,
- },
- fields: selectFields,
- metaFields: index.metaFields,
- conflictedTypesFields: index.fields.filter((f) => f.type === 'conflict').map((f) => f.name),
- indexPatternId: index.id,
+ searchSource: searchSource.getSerializedFields(true),
};
}
+/**
+ * makes getSharingData lazy loadable
+ */
+export function getSharingDataModule() {}
+
export interface DiscoverCapabilities {
createShortUrl?: boolean;
save?: boolean;
diff --git a/src/plugins/discover/public/index.ts b/src/plugins/discover/public/index.ts
index de76c65ccdc98..fbe853ec6deb5 100644
--- a/src/plugins/discover/public/index.ts
+++ b/src/plugins/discover/public/index.ts
@@ -16,4 +16,5 @@ export function plugin(initializerContext: PluginInitializerContext) {
export { SavedSearch, SavedSearchLoader, createSavedSearchesLoader } from './saved_searches';
export { ISearchEmbeddable, SEARCH_EMBEDDABLE_TYPE, SearchInput } from './application/embeddable';
+export { loadSharingDataHelpers } from './shared';
export { DISCOVER_APP_URL_GENERATOR, DiscoverUrlGeneratorState } from './url_generator';
diff --git a/src/plugins/discover/public/shared/index.ts b/src/plugins/discover/public/shared/index.ts
new file mode 100644
index 0000000000000..b1e4d9d87000e
--- /dev/null
+++ b/src/plugins/discover/public/shared/index.ts
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/*
+ * Allows the getSharingData function to be lazy loadable
+ */
+export async function loadSharingDataHelpers() {
+ return await import('../application/helpers/get_sharing_data');
+}
diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts
index e3f58dd20cecb..2a95557473fc0 100644
--- a/x-pack/plugins/reporting/common/constants.ts
+++ b/x-pack/plugins/reporting/common/constants.ts
@@ -50,6 +50,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];
export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';
+export const UI_SETTINGS_DATEFORMAT_TZ = 'dateFormat:tz';
export const LAYOUT_TYPES = {
PRESERVE_LAYOUT: 'preserve_layout',
@@ -57,13 +58,16 @@ export const LAYOUT_TYPES = {
};
// Export Type Definitions
+export const CSV_REPORT_TYPE = 'CSV';
+export const CSV_JOB_TYPE = 'csv_searchsource';
+
export const PDF_REPORT_TYPE = 'printablePdf';
export const PDF_JOB_TYPE = 'printable_pdf';
export const PNG_REPORT_TYPE = 'PNG';
export const PNG_JOB_TYPE = 'PNG';
-export const CSV_FROM_SAVEDOBJECT_JOB_TYPE = 'csv_from_savedobject';
+export const CSV_SEARCHSOURCE_IMMEDIATE_TYPE = 'csv_searchsource_immediate';
// This is deprecated because it lacks support for runtime fields
// but the extension points are still needed for pre-existing scripted automation, until 8.0
@@ -86,9 +90,9 @@ export const API_BASE_GENERATE = `${API_BASE_URL}/generate`;
export const API_LIST_URL = `${API_BASE_URL}/jobs`;
export const API_DIAGNOSE_URL = `${API_BASE_URL}/diagnose`;
-// hacky endpoint
+// hacky endpoint: download CSV without queueing a report
export const API_BASE_URL_V1 = '/api/reporting/v1'; //
-export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv/saved-object`;
+export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv_searchsource`;
// Management UI route
export const REPORTING_MANAGEMENT_HOME = '/app/management/insightsAndAlerting/reporting';
diff --git a/x-pack/plugins/reporting/common/types.ts b/x-pack/plugins/reporting/common/types.ts
index 3af329cbf0303..5e20381e35898 100644
--- a/x-pack/plugins/reporting/common/types.ts
+++ b/x-pack/plugins/reporting/common/types.ts
@@ -47,9 +47,10 @@ export interface ReportDocumentHead {
export interface TaskRunResult {
content_type: string | null;
content: string | null;
- csv_contains_formulas?: boolean;
size: number;
+ csv_contains_formulas?: boolean;
max_size_reached?: boolean;
+ needs_sorting?: boolean;
warnings?: string[];
}
diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx b/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx
index 6673aded2ecbe..db9ddec90b605 100644
--- a/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx
+++ b/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx
@@ -11,11 +11,7 @@ import React, { Component, ReactElement } from 'react';
import { ToastsSetup } from 'src/core/public';
import url from 'url';
import { toMountPoint } from '../../../../../src/plugins/kibana_react/public';
-import {
- CSV_REPORT_TYPE_DEPRECATED,
- PDF_REPORT_TYPE,
- PNG_REPORT_TYPE,
-} from '../../common/constants';
+import { CSV_REPORT_TYPE, PDF_REPORT_TYPE, PNG_REPORT_TYPE } from '../../common/constants';
import { BaseParams } from '../../common/types';
import { ReportingAPIClient } from '../lib/reporting_api_client';
@@ -177,8 +173,8 @@ class ReportingPanelContentUi extends Component {
switch (this.props.reportType) {
case PDF_REPORT_TYPE:
return 'PDF';
- case 'csv':
- return CSV_REPORT_TYPE_DEPRECATED;
+ case 'csv_searchsource':
+ return CSV_REPORT_TYPE;
case 'png':
return PNG_REPORT_TYPE;
default:
diff --git a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts
index f452719e91713..4e1b9ccd2642f 100644
--- a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts
+++ b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts
@@ -52,7 +52,20 @@ describe('GetCsvReportPanelAction', () => {
context = {
embeddable: {
type: 'search',
- getSavedSearch: () => ({ id: 'lebowski' }),
+ getSavedSearch: () => {
+ const searchSource = {
+ createCopy: () => searchSource,
+ removeField: jest.fn(),
+ setField: jest.fn(),
+ getField: jest.fn().mockImplementation((key: string) => {
+ if (key === 'index') {
+ return 'my-test-index-*';
+ }
+ }),
+ getSerializedFields: jest.fn().mockImplementation(() => ({})),
+ };
+ return { searchSource };
+ },
getTitle: () => `The Dude`,
getInspectorAdapters: () => null,
getInput: () => ({
diff --git a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx
index cc1da146eff32..d440edc3f3fe9 100644
--- a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx
+++ b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx
@@ -5,13 +5,13 @@
* 2.0.
*/
-import dateMath from '@elastic/datemath';
import { i18n } from '@kbn/i18n';
-import _ from 'lodash';
import moment from 'moment-timezone';
import { CoreSetup } from 'src/core/public';
import {
+ loadSharingDataHelpers,
ISearchEmbeddable,
+ SavedSearch,
SEARCH_EMBEDDABLE_TYPE,
} from '../../../../../src/plugins/discover/public';
import { IEmbeddable, ViewMode } from '../../../../../src/plugins/embeddable/public';
@@ -21,6 +21,7 @@ import {
} from '../../../../../src/plugins/ui_actions/public';
import { LicensingPluginSetup } from '../../../licensing/public';
import { API_GENERATE_IMMEDIATE, CSV_REPORTING_ACTION } from '../../common/constants';
+import { JobParamsDownloadCSV } from '../../server/export_types/csv_searchsource_immediate/types';
import { checkLicense } from '../lib/license_check';
function isSavedSearchEmbeddable(
@@ -61,17 +62,16 @@ export class GetCsvReportPanelAction implements ActionDefinition
});
}
- public getSearchRequestBody({ searchEmbeddable }: { searchEmbeddable: any }) {
- const adapters = searchEmbeddable.getInspectorAdapters();
- if (!adapters) {
- return {};
- }
-
- if (adapters.requests.requests.length === 0) {
- return {};
- }
+ public async getSearchSource(savedSearch: SavedSearch, embeddable: ISearchEmbeddable) {
+ const { getSharingData } = await loadSharingDataHelpers();
+ const searchSource = savedSearch.searchSource.createCopy();
+ const { searchSource: serializedSearchSource } = await getSharingData(
+ searchSource,
+ savedSearch, // TODO: get unsaved state (using embeddale.searchScope): https://github.com/elastic/kibana/issues/43977
+ this.core.uiSettings
+ );
- return searchEmbeddable.getSavedSearch().searchSource.getSearchRequestBody();
+ return serializedSearchSource;
}
public isCompatible = async (context: ActionContext) => {
@@ -95,34 +95,18 @@ export class GetCsvReportPanelAction implements ActionDefinition
return;
}
- const {
- timeRange: { to, from },
- } = embeddable.getInput();
+ const savedSearch = embeddable.getSavedSearch();
+ const searchSource = await this.getSearchSource(savedSearch, embeddable);
- const searchEmbeddable = embeddable;
- const searchRequestBody = await this.getSearchRequestBody({ searchEmbeddable });
- const state = _.pick(searchRequestBody, ['sort', 'docvalue_fields', 'query']);
const kibanaTimezone = this.core.uiSettings.get('dateFormat:tz');
+ const browserTimezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone;
+ const immediateJobParams: JobParamsDownloadCSV = {
+ searchSource,
+ browserTimezone,
+ title: savedSearch.title,
+ };
- const id = `search:${embeddable.getSavedSearch().id}`;
- const timezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone;
- const fromTime = dateMath.parse(from);
- const toTime = dateMath.parse(to, { roundUp: true });
-
- if (!fromTime || !toTime) {
- return this.onGenerationFail(
- new Error(`Invalid time range: From: ${fromTime}, To: ${toTime}`)
- );
- }
-
- const body = JSON.stringify({
- timerange: {
- min: fromTime.format(),
- max: toTime.format(),
- timezone,
- },
- state,
- });
+ const body = JSON.stringify(immediateJobParams);
this.isDownloading = true;
@@ -137,11 +121,11 @@ export class GetCsvReportPanelAction implements ActionDefinition
});
await this.core.http
- .post(`${API_GENERATE_IMMEDIATE}/${id}`, { body })
+ .post(`${API_GENERATE_IMMEDIATE}`, { body })
.then((rawResponse: string) => {
this.isDownloading = false;
- const download = `${embeddable.getSavedSearch().title}.csv`;
+ const download = `${savedSearch.title}.csv`;
const blob = new Blob([rawResponse], { type: 'text/csv;charset=utf-8;' });
// Hack for IE11 Support
diff --git a/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx b/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx
index 31c86ae4c5669..97433f7a4f0c1 100644
--- a/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx
+++ b/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx
@@ -11,10 +11,8 @@ import React from 'react';
import { IUiSettingsClient, ToastsSetup } from 'src/core/public';
import { ShareContext } from '../../../../../src/plugins/share/public';
import { LicensingPluginSetup } from '../../../licensing/public';
-import {
- JobParamsDeprecatedCSV,
- SearchRequestDeprecatedCSV,
-} from '../../server/export_types/csv/types';
+import { CSV_JOB_TYPE } from '../../common/constants';
+import { JobParamsCSV } from '../../server/export_types/csv_searchsource/types';
import { ReportingPanelContent } from '../components/reporting_panel_content_lazy';
import { checkLicense } from '../lib/license_check';
import { ReportingAPIClient } from '../lib/reporting_api_client';
@@ -56,22 +54,18 @@ export const csvReportingProvider = ({
objectType,
objectId,
sharingData,
- isDirty,
onClose,
+ isDirty,
}: ShareContext) => {
if ('search' !== objectType) {
return [];
}
- const jobParams: JobParamsDeprecatedCSV = {
+ const jobParams: JobParamsCSV = {
browserTimezone,
- objectType,
title: sharingData.title as string,
- indexPatternId: sharingData.indexPatternId as string,
- searchRequest: sharingData.searchRequest as SearchRequestDeprecatedCSV,
- fields: sharingData.fields as string[],
- metaFields: sharingData.metaFields as string[],
- conflictedTypesFields: sharingData.conflictedTypesFields as string[],
+ objectType,
+ searchSource: sharingData.searchSource,
};
const getJobParams = () => jobParams;
@@ -99,7 +93,7 @@ export const csvReportingProvider = ({
;
@@ -45,6 +47,8 @@ export interface ReportingInternalStart {
store: ReportingStore;
savedObjects: SavedObjectsServiceStart;
uiSettings: UiSettingsServiceStart;
+ esClient: IClusterClient;
+ data: DataPluginStart;
}
export class ReportingCore {
@@ -239,4 +243,14 @@ export class ReportingCore {
const savedObjectsClient = await this.getSavedObjectsClient(request);
return await this.getUiSettingsServiceFactory(savedObjectsClient);
}
+
+ public async getSearchService() {
+ const startDeps = await this.getPluginStartDeps();
+ return startDeps.data.search;
+ }
+
+ public async getEsClient() {
+ const startDeps = await this.getPluginStartDeps();
+ return startDeps.esClient;
+ }
}
diff --git a/x-pack/plugins/reporting/server/export_types/common/index.ts b/x-pack/plugins/reporting/server/export_types/common/index.ts
index 1003ecf83601c..8832577281bb2 100644
--- a/x-pack/plugins/reporting/server/export_types/common/index.ts
+++ b/x-pack/plugins/reporting/server/export_types/common/index.ts
@@ -12,7 +12,6 @@ export { omitBlockedHeaders } from './omit_blocked_headers';
export { validateUrls } from './validate_urls';
export interface TimeRangeParams {
- timezone: string;
min?: Date | string | number | null;
max?: Date | string | number | null;
}
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/check_cells_for_formulas.ts b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/check_cells_for_formulas.ts
index 942739f0d9945..c2114bc6bcd1e 100644
--- a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/check_cells_for_formulas.ts
+++ b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/check_cells_for_formulas.ts
@@ -5,8 +5,8 @@
* 2.0.
*/
-import { pick, keys, values, some } from 'lodash';
-import { cellHasFormulas } from './cell_has_formula';
+import { keys, pick, some, values } from 'lodash';
+import { cellHasFormulas } from '../../csv_searchsource/generate_csv/cell_has_formula';
interface IFlattened {
[header: string]: string;
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts
index ed05180501e32..629a81df350be 100644
--- a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts
+++ b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts
@@ -13,15 +13,18 @@ import { CSV_BOM_CHARS } from '../../../../common/constants';
import { byteSizeValueToNumber } from '../../../../common/schema_utils';
import { LevelLogger } from '../../../lib';
import { getFieldFormats } from '../../../services';
-import { IndexPatternSavedObjectDeprecatedCSV, SavedSearchGeneratorResult } from '../types';
+import { createEscapeValue } from '../../csv_searchsource/generate_csv/escape_value';
+import { MaxSizeStringBuilder } from '../../csv_searchsource/generate_csv/max_size_string_builder';
+import {
+ IndexPatternSavedObjectDeprecatedCSV,
+ SavedSearchGeneratorResultDeprecatedCSV,
+} from '../types';
import { checkIfRowsHaveFormulas } from './check_cells_for_formulas';
-import { createEscapeValue } from './escape_value';
import { fieldFormatMapFactory } from './field_format_map';
import { createFlattenHit } from './flatten_hit';
import { createFormatCsvValues } from './format_csv_values';
import { getUiSettings } from './get_ui_settings';
import { createHitIterator, EndpointCaller } from './hit_iterator';
-import { MaxSizeStringBuilder } from './max_size_string_builder';
interface SearchRequest {
index: string;
@@ -55,7 +58,7 @@ export function createGenerateCsv(logger: LevelLogger) {
uiSettingsClient: IUiSettingsClient,
callEndpoint: EndpointCaller,
cancellationToken: CancellationToken
- ): Promise {
+ ): Promise {
const settings = await getUiSettings(job.browserTimezone, uiSettingsClient, config, logger);
const escapeValue = createEscapeValue(settings.quoteValues, settings.escapeFormulaValues);
const bom = config.get('csv', 'useByteOrderMarkEncoding') ? CSV_BOM_CHARS : '';
diff --git a/x-pack/plugins/reporting/server/export_types/csv/types.d.ts b/x-pack/plugins/reporting/server/export_types/csv/types.d.ts
index 4c4f33d0ee9f7..604d451d822b6 100644
--- a/x-pack/plugins/reporting/server/export_types/csv/types.d.ts
+++ b/x-pack/plugins/reporting/server/export_types/csv/types.d.ts
@@ -77,15 +77,10 @@ type FormatsMapDeprecatedCSV = Map<
}
>;
-export interface SavedSearchGeneratorResult {
+export interface SavedSearchGeneratorResultDeprecatedCSV {
content: string;
size: number;
maxSizeReached: boolean;
csvContainsFormulas?: boolean;
warnings: string[];
}
-
-export interface CsvResultFromSearch {
- type: string;
- result: SavedSearchGeneratorResult;
-}
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts
deleted file mode 100644
index b27c244aa11ae..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { notFound, notImplemented } from '@hapi/boom';
-import { get } from 'lodash';
-import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants';
-import { CsvFromSavedObjectRequest } from '../../routes/generate_from_savedobject_immediate';
-import { CreateJobFnFactory } from '../../types';
-import {
- JobParamsPanelCsv,
- JobPayloadPanelCsv,
- SavedObject,
- SavedObjectReference,
- SavedObjectServiceError,
- VisObjectAttributesJSON,
-} from './types';
-import type { ReportingRequestHandlerContext } from '../../types';
-
-export type ImmediateCreateJobFn = (
- jobParams: JobParamsPanelCsv,
- context: ReportingRequestHandlerContext,
- req: CsvFromSavedObjectRequest
-) => Promise;
-
-export const createJobFnFactory: CreateJobFnFactory = function createJobFactoryFn(
- reporting,
- parentLogger
-) {
- const logger = parentLogger.clone([CSV_FROM_SAVEDOBJECT_JOB_TYPE, 'create-job']);
-
- return async function createJob(jobParams, context, req) {
- const { savedObjectType, savedObjectId } = jobParams;
-
- const panel = await Promise.resolve()
- .then(() => context.core.savedObjects.client.get(savedObjectType, savedObjectId))
- .then(async (savedObject: SavedObject) => {
- const { attributes, references } = savedObject;
- const { kibanaSavedObjectMeta: kibanaSavedObjectMetaJSON } = attributes;
- const { timerange } = req.body;
-
- if (!kibanaSavedObjectMetaJSON) {
- throw new Error('Could not parse saved object data!');
- }
-
- const kibanaSavedObjectMeta = {
- ...kibanaSavedObjectMetaJSON,
- searchSource: JSON.parse(kibanaSavedObjectMetaJSON.searchSourceJSON),
- };
-
- const { visState: visStateJSON } = attributes as VisObjectAttributesJSON;
- if (visStateJSON) {
- throw notImplemented('Visualization types are not yet implemented');
- }
-
- // saved search type
- const { searchSource } = kibanaSavedObjectMeta;
- if (!searchSource || !references) {
- throw new Error('The saved search object is missing configuration fields!');
- }
-
- const indexPatternMeta = references.find(
- (ref: SavedObjectReference) => ref.type === 'index-pattern'
- );
- if (!indexPatternMeta) {
- throw new Error('Could not find index pattern for the saved search!');
- }
-
- return {
- attributes: {
- ...attributes,
- kibanaSavedObjectMeta: { searchSource },
- },
- indexPatternSavedObjectId: indexPatternMeta.id,
- timerange,
- };
- })
- .catch((err: Error) => {
- const boomErr = (err as unknown) as { isBoom: boolean };
- if (boomErr.isBoom) {
- throw err;
- }
- const errPayload: SavedObjectServiceError = get(err, 'output.payload', { statusCode: 0 });
- if (errPayload.statusCode === 404) {
- throw notFound(errPayload.message);
- }
- logger.error(err);
- throw new Error(`Unable to create a job from saved object data! Error: ${err}`);
- });
-
- return { ...jobParams, panel };
- };
-};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts
deleted file mode 100644
index b037e72699dd6..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { KibanaRequest } from 'src/core/server';
-import { CancellationToken } from '../../../common';
-import { CONTENT_TYPE_CSV, CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants';
-import { TaskRunResult } from '../../lib/tasks';
-import { RunTaskFnFactory } from '../../types';
-import { createGenerateCsv } from '../csv/generate_csv';
-import { getGenerateCsvParams } from './lib/get_csv_job';
-import { JobPayloadPanelCsv } from './types';
-import type { ReportingRequestHandlerContext } from '../../types';
-
-/*
- * ImmediateExecuteFn receives the job doc payload because the payload was
- * generated in the ScheduleFn
- */
-export type ImmediateExecuteFn = (
- jobId: null,
- job: JobPayloadPanelCsv,
- context: ReportingRequestHandlerContext,
- req: KibanaRequest
-) => Promise;
-
-export const runTaskFnFactory: RunTaskFnFactory = function executeJobFactoryFn(
- reporting,
- parentLogger
-) {
- const config = reporting.getConfig();
- const logger = parentLogger.clone([CSV_FROM_SAVEDOBJECT_JOB_TYPE, 'execute-job']);
-
- return async function runTask(jobId, jobPayload, context, req) {
- const generateCsv = createGenerateCsv(logger);
- const { panel } = jobPayload;
-
- logger.debug(`Execute job generating saved search CSV`);
-
- const savedObjectsClient = context.core.savedObjects.client;
- const uiSettingsClient = await reporting.getUiSettingsServiceFactory(savedObjectsClient);
- const job = await getGenerateCsvParams(
- jobPayload,
- panel,
- savedObjectsClient,
- uiSettingsClient,
- logger
- );
-
- const elasticsearch = reporting.getElasticsearchService();
- const { callAsCurrentUser } = elasticsearch.legacy.client.asScoped(req);
-
- const { content, maxSizeReached, size, csvContainsFormulas, warnings } = await generateCsv(
- job,
- config,
- uiSettingsClient,
- callAsCurrentUser,
- new CancellationToken() // can not be cancelled
- );
-
- if (csvContainsFormulas) {
- logger.warn(`CSV may contain formulas whose values have been escaped`);
- }
-
- if (maxSizeReached) {
- logger.warn(`Max size reached: CSV output truncated to ${size} bytes`);
- }
-
- return {
- content_type: CONTENT_TYPE_CSV,
- content,
- max_size_reached: maxSizeReached,
- size,
- csv_contains_formulas: csvContainsFormulas,
- warnings,
- };
- };
-};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.test.ts
deleted file mode 100644
index fc6e092962d3b..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.test.ts
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { createMockLevelLogger } from '../../../test_helpers';
-import { JobParamsPanelCsv, SearchPanel } from '../types';
-import { getGenerateCsvParams } from './get_csv_job';
-
-const logger = createMockLevelLogger();
-
-describe('Get CSV Job', () => {
- let mockJobParams: JobParamsPanelCsv;
- let mockSearchPanel: SearchPanel;
- let mockSavedObjectsClient: any;
- let mockUiSettingsClient: any;
- beforeEach(() => {
- mockJobParams = { savedObjectType: 'search', savedObjectId: '234-ididid' };
- mockSearchPanel = {
- indexPatternSavedObjectId: '123-indexId',
- attributes: {
- title: 'my search',
- sort: [],
- kibanaSavedObjectMeta: {
- searchSource: { query: { isSearchSourceQuery: true }, filter: [] },
- },
- uiState: 56,
- },
- timerange: { timezone: 'PST', min: 0, max: 100 },
- };
- mockSavedObjectsClient = {
- get: () => ({
- attributes: { fields: null, title: null, timeFieldName: null },
- }),
- };
- mockUiSettingsClient = {
- get: () => ({}),
- };
- });
-
- it('creates a data structure needed by generateCsv', async () => {
- const result = await getGenerateCsvParams(
- mockJobParams,
- mockSearchPanel,
- mockSavedObjectsClient,
- mockUiSettingsClient,
- logger
- );
- expect(result).toMatchInlineSnapshot(`
- Object {
- "browserTimezone": "PST",
- "conflictedTypesFields": Array [],
- "fields": Array [],
- "indexPatternSavedObject": Object {
- "attributes": Object {
- "fields": null,
- "timeFieldName": null,
- "title": null,
- },
- "fields": Array [],
- "timeFieldName": null,
- "title": null,
- },
- "metaFields": Array [],
- "searchRequest": Object {
- "body": Object {
- "_source": Object {
- "includes": Array [],
- },
- "docvalue_fields": undefined,
- "query": Object {
- "bool": Object {
- "filter": Array [],
- "must": Array [],
- "must_not": Array [],
- "should": Array [],
- },
- },
- "script_fields": Object {},
- "sort": Array [],
- },
- "index": null,
- },
- }
- `);
- });
-
- it('uses query and sort from the payload', async () => {
- mockJobParams.post = {
- state: {
- query: ['this is the query'],
- sort: ['this is the sort'],
- },
- };
- const result = await getGenerateCsvParams(
- mockJobParams,
- mockSearchPanel,
- mockSavedObjectsClient,
- mockUiSettingsClient,
- logger
- );
- expect(result).toMatchInlineSnapshot(`
- Object {
- "browserTimezone": "PST",
- "conflictedTypesFields": Array [],
- "fields": Array [],
- "indexPatternSavedObject": Object {
- "attributes": Object {
- "fields": null,
- "timeFieldName": null,
- "title": null,
- },
- "fields": Array [],
- "timeFieldName": null,
- "title": null,
- },
- "metaFields": Array [],
- "searchRequest": Object {
- "body": Object {
- "_source": Object {
- "includes": Array [],
- },
- "docvalue_fields": undefined,
- "query": Object {
- "bool": Object {
- "filter": Array [
- Object {
- "0": "this is the query",
- },
- ],
- "must": Array [],
- "must_not": Array [],
- "should": Array [],
- },
- },
- "script_fields": Object {},
- "sort": Array [
- "this is the sort",
- ],
- },
- "index": null,
- },
- }
- `);
- });
-
- it('uses timerange timezone from the payload', async () => {
- mockJobParams.post = {
- timerange: { timezone: 'Africa/Timbuktu', min: 0, max: 9000 },
- };
- const result = await getGenerateCsvParams(
- mockJobParams,
- mockSearchPanel,
- mockSavedObjectsClient,
- mockUiSettingsClient,
- logger
- );
- expect(result).toMatchInlineSnapshot(`
- Object {
- "browserTimezone": "Africa/Timbuktu",
- "conflictedTypesFields": Array [],
- "fields": Array [],
- "indexPatternSavedObject": Object {
- "attributes": Object {
- "fields": null,
- "timeFieldName": null,
- "title": null,
- },
- "fields": Array [],
- "timeFieldName": null,
- "title": null,
- },
- "metaFields": Array [],
- "searchRequest": Object {
- "body": Object {
- "_source": Object {
- "includes": Array [],
- },
- "docvalue_fields": undefined,
- "query": Object {
- "bool": Object {
- "filter": Array [],
- "must": Array [],
- "must_not": Array [],
- "should": Array [],
- },
- },
- "script_fields": Object {},
- "sort": Array [],
- },
- "index": null,
- },
- }
- `);
- });
-
- it('uses timerange min and max (numeric) when index pattern has timefieldName', async () => {
- mockJobParams.post = {
- timerange: { timezone: 'Africa/Timbuktu', min: 0, max: 900000000 },
- };
- mockSavedObjectsClient = {
- get: () => ({
- attributes: { fields: null, title: 'test search', timeFieldName: '@test_time' },
- }),
- };
- const result = await getGenerateCsvParams(
- mockJobParams,
- mockSearchPanel,
- mockSavedObjectsClient,
- mockUiSettingsClient,
- logger
- );
- expect(result).toMatchInlineSnapshot(`
- Object {
- "browserTimezone": "Africa/Timbuktu",
- "conflictedTypesFields": Array [],
- "fields": Array [
- "@test_time",
- ],
- "indexPatternSavedObject": Object {
- "attributes": Object {
- "fields": null,
- "timeFieldName": "@test_time",
- "title": "test search",
- },
- "fields": Array [],
- "timeFieldName": "@test_time",
- "title": "test search",
- },
- "metaFields": Array [],
- "searchRequest": Object {
- "body": Object {
- "_source": Object {
- "includes": Array [
- "@test_time",
- ],
- },
- "docvalue_fields": undefined,
- "query": Object {
- "bool": Object {
- "filter": Array [
- Object {
- "range": Object {
- "@test_time": Object {
- "format": "strict_date_time",
- "gte": "1970-01-01T00:00:00Z",
- "lte": "1970-01-11T10:00:00Z",
- },
- },
- },
- ],
- "must": Array [],
- "must_not": Array [],
- "should": Array [],
- },
- },
- "script_fields": Object {},
- "sort": Array [],
- },
- "index": "test search",
- },
- }
- `);
- });
-
- it('uses timerange min and max (string) when index pattern has timefieldName', async () => {
- mockJobParams.post = {
- timerange: {
- timezone: 'Africa/Timbuktu',
- min: '1980-01-01T00:00:00Z',
- max: '1990-01-01T00:00:00Z',
- },
- };
- mockSavedObjectsClient = {
- get: () => ({
- attributes: { fields: null, title: 'test search', timeFieldName: '@test_time' },
- }),
- };
- const result = await getGenerateCsvParams(
- mockJobParams,
- mockSearchPanel,
- mockSavedObjectsClient,
- mockUiSettingsClient,
- logger
- );
- expect(result).toMatchInlineSnapshot(`
- Object {
- "browserTimezone": "Africa/Timbuktu",
- "conflictedTypesFields": Array [],
- "fields": Array [
- "@test_time",
- ],
- "indexPatternSavedObject": Object {
- "attributes": Object {
- "fields": null,
- "timeFieldName": "@test_time",
- "title": "test search",
- },
- "fields": Array [],
- "timeFieldName": "@test_time",
- "title": "test search",
- },
- "metaFields": Array [],
- "searchRequest": Object {
- "body": Object {
- "_source": Object {
- "includes": Array [
- "@test_time",
- ],
- },
- "docvalue_fields": undefined,
- "query": Object {
- "bool": Object {
- "filter": Array [
- Object {
- "range": Object {
- "@test_time": Object {
- "format": "strict_date_time",
- "gte": "1980-01-01T00:00:00Z",
- "lte": "1990-01-01T00:00:00Z",
- },
- },
- },
- ],
- "must": Array [],
- "must_not": Array [],
- "should": Array [],
- },
- },
- "script_fields": Object {},
- "sort": Array [],
- },
- "index": "test search",
- },
- }
- `);
- });
-});
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.ts
deleted file mode 100644
index e4570816e26ff..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.ts
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { IUiSettingsClient, SavedObjectsClientContract } from 'kibana/server';
-import { EsQueryConfig } from 'src/plugins/data/server';
-import { esQuery, Filter, Query } from '../../../../../../../src/plugins/data/server';
-import { LevelLogger } from '../../../lib';
-import { TimeRangeParams } from '../../common';
-import { GenerateCsvParams } from '../../csv/generate_csv';
-import {
- DocValueFields,
- IndexPatternField,
- JobParamsPanelCsv,
- QueryFilter,
- SavedSearchObjectAttributes,
- SearchPanel,
- SearchSource,
-} from '../types';
-import { getDataSource } from './get_data_source';
-import { getFilters } from './get_filters';
-
-export const getEsQueryConfig = async (config: IUiSettingsClient) => {
- const configs = await Promise.all([
- config.get('query:allowLeadingWildcards'),
- config.get('query:queryString:options'),
- config.get('courier:ignoreFilterIfFieldNotInIndex'),
- ]);
- const [allowLeadingWildcards, queryStringOptions, ignoreFilterIfFieldNotInIndex] = configs;
- return {
- allowLeadingWildcards,
- queryStringOptions,
- ignoreFilterIfFieldNotInIndex,
- } as EsQueryConfig;
-};
-
-/*
- * Create a CSV Job object for CSV From SavedObject to use as a job parameter
- * for generateCsv
- */
-export const getGenerateCsvParams = async (
- jobParams: JobParamsPanelCsv,
- panel: SearchPanel,
- savedObjectsClient: SavedObjectsClientContract,
- uiConfig: IUiSettingsClient,
- logger: LevelLogger
-): Promise => {
- let timerange: TimeRangeParams | null;
- if (jobParams.post?.timerange) {
- timerange = jobParams.post?.timerange;
- } else {
- timerange = panel.timerange || null;
- }
- const { indexPatternSavedObjectId } = panel;
- const savedSearchObjectAttr = panel.attributes as SavedSearchObjectAttributes;
- const { indexPatternSavedObject } = await getDataSource(
- savedObjectsClient,
- indexPatternSavedObjectId
- );
- const esQueryConfig = await getEsQueryConfig(uiConfig);
-
- const {
- kibanaSavedObjectMeta: {
- searchSource: {
- filter: [searchSourceFilter],
- query: searchSourceQuery,
- },
- },
- } = savedSearchObjectAttr as { kibanaSavedObjectMeta: { searchSource: SearchSource } };
-
- const {
- timeFieldName: indexPatternTimeField,
- title: esIndex,
- fields: indexPatternFields,
- } = indexPatternSavedObject;
-
- if (!indexPatternFields || indexPatternFields.length === 0) {
- logger.error(
- new Error(
- `No fields are selected in the saved search! Please select fields as columns in the saved search and try again.`
- )
- );
- }
-
- let payloadQuery: QueryFilter | undefined;
- let payloadSort: any[] = [];
- let docValueFields: DocValueFields[] | undefined;
- if (jobParams.post && jobParams.post.state) {
- ({
- post: {
- state: { query: payloadQuery, sort: payloadSort = [], docvalue_fields: docValueFields },
- },
- } = jobParams);
- }
- const { includes, combinedFilter } = getFilters(
- indexPatternSavedObjectId,
- indexPatternTimeField,
- timerange,
- savedSearchObjectAttr,
- searchSourceFilter,
- payloadQuery
- );
-
- const savedSortConfigs = savedSearchObjectAttr.sort;
- const sortConfig = [...payloadSort];
- savedSortConfigs.forEach(([savedSortField, savedSortOrder]) => {
- sortConfig.push({ [savedSortField]: { order: savedSortOrder } });
- });
-
- const scriptFieldsConfig =
- indexPatternFields &&
- indexPatternFields
- .filter((f: IndexPatternField) => f.scripted)
- .reduce((accum: any, curr: IndexPatternField) => {
- return {
- ...accum,
- [curr.name]: {
- script: {
- source: curr.script,
- lang: curr.lang,
- },
- },
- };
- }, {});
-
- const searchRequest = {
- index: esIndex,
- body: {
- _source: { includes },
- docvalue_fields: docValueFields,
- query: esQuery.buildEsQuery(
- // compromise made while factoring out IIndexPattern type
- // @ts-expect-error
- indexPatternSavedObject,
- (searchSourceQuery as unknown) as Query,
- (combinedFilter as unknown) as Filter,
- esQueryConfig
- ),
- script_fields: scriptFieldsConfig,
- sort: sortConfig,
- },
- };
-
- return {
- browserTimezone: timerange?.timezone,
- indexPatternSavedObject,
- searchRequest,
- fields: includes,
- metaFields: [],
- conflictedTypesFields: [],
- };
-};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_data_source.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_data_source.ts
deleted file mode 100644
index d903a1d8ba9e8..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_data_source.ts
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { IndexPatternSavedObjectDeprecatedCSV } from '../../csv/types';
-import { SavedObjectReference, SavedSearchObjectAttributesJSON, SearchSource } from '../types';
-
-export async function getDataSource(
- savedObjectsClient: any,
- indexPatternId?: string,
- savedSearchObjectId?: string
-): Promise<{
- indexPatternSavedObject: IndexPatternSavedObjectDeprecatedCSV;
- searchSource: SearchSource | null;
-}> {
- let indexPatternSavedObject: IndexPatternSavedObjectDeprecatedCSV;
- let searchSource: SearchSource | null = null;
-
- if (savedSearchObjectId) {
- try {
- const { attributes, references } = (await savedObjectsClient.get(
- 'search',
- savedSearchObjectId
- )) as { attributes: SavedSearchObjectAttributesJSON; references: SavedObjectReference[] };
- searchSource = JSON.parse(attributes.kibanaSavedObjectMeta.searchSourceJSON);
- const { id: indexPatternFromSearchId } = references.find(
- ({ type }) => type === 'index-pattern'
- ) as { id: string };
- ({ indexPatternSavedObject } = await getDataSource(
- savedObjectsClient,
- indexPatternFromSearchId
- ));
- return { searchSource, indexPatternSavedObject };
- } catch (err) {
- throw new Error(`Could not get saved search info! ${err}`);
- }
- }
- try {
- const { attributes } = await savedObjectsClient.get('index-pattern', indexPatternId);
- const { fields, title, timeFieldName } = attributes;
- const parsedFields = fields ? JSON.parse(fields) : [];
-
- indexPatternSavedObject = {
- fields: parsedFields,
- title,
- timeFieldName,
- attributes,
- };
- } catch (err) {
- throw new Error(`Could not get index pattern saved object! ${err}`);
- }
- return { indexPatternSavedObject, searchSource };
-}
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.test.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.test.ts
deleted file mode 100644
index ca5bf12e1d510..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.test.ts
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { TimeRangeParams } from '../../common';
-import { QueryFilter, SavedSearchObjectAttributes, SearchSourceFilter } from '../types';
-import { getFilters } from './get_filters';
-
-interface Args {
- indexPatternId: string;
- indexPatternTimeField: string | null;
- timerange: TimeRangeParams | null;
- savedSearchObjectAttr: SavedSearchObjectAttributes;
- searchSourceFilter: SearchSourceFilter;
- queryFilter: QueryFilter;
-}
-
-describe('CSV from Saved Object: get_filters', () => {
- let args: Args;
- beforeEach(() => {
- args = {
- indexPatternId: 'logs-test-*',
- indexPatternTimeField: 'testtimestamp',
- timerange: {
- timezone: 'UTC',
- min: '1901-01-01T00:00:00.000Z',
- max: '1902-01-01T00:00:00.000Z',
- },
- savedSearchObjectAttr: {
- title: 'test',
- sort: [{ sortField: { order: 'asc' } }],
- kibanaSavedObjectMeta: {
- searchSource: {
- query: { isSearchSourceQuery: true },
- filter: ['hello searchSource filter 1'],
- },
- },
- columns: ['larry'],
- uiState: null,
- },
- searchSourceFilter: { isSearchSourceFilter: true, isFilter: true },
- queryFilter: { isQueryFilter: true, isFilter: true },
- };
- });
-
- describe('search', () => {
- it('for timebased search', () => {
- const filters = getFilters(
- args.indexPatternId,
- args.indexPatternTimeField,
- args.timerange,
- args.savedSearchObjectAttr,
- args.searchSourceFilter,
- args.queryFilter
- );
-
- expect(filters).toEqual({
- combinedFilter: [
- {
- range: {
- testtimestamp: {
- format: 'strict_date_time',
- gte: '1901-01-01T00:00:00Z',
- lte: '1902-01-01T00:00:00Z',
- },
- },
- },
- { isFilter: true, isSearchSourceFilter: true },
- { isFilter: true, isQueryFilter: true },
- ],
- includes: ['testtimestamp', 'larry'],
- timezone: 'UTC',
- });
- });
-
- it('for non-timebased search', () => {
- args.indexPatternTimeField = null;
- args.timerange = null;
-
- const filters = getFilters(
- args.indexPatternId,
- args.indexPatternTimeField,
- args.timerange,
- args.savedSearchObjectAttr,
- args.searchSourceFilter,
- args.queryFilter
- );
-
- expect(filters).toEqual({
- combinedFilter: [
- { isFilter: true, isSearchSourceFilter: true },
- { isFilter: true, isQueryFilter: true },
- ],
- includes: ['larry'],
- timezone: null,
- });
- });
- });
-
- describe('errors', () => {
- it('throw if timebased and timerange is missing', () => {
- args.timerange = null;
-
- const throwFn = () =>
- getFilters(
- args.indexPatternId,
- args.indexPatternTimeField,
- args.timerange,
- args.savedSearchObjectAttr,
- args.searchSourceFilter,
- args.queryFilter
- );
-
- expect(throwFn).toThrow(
- 'Time range params are required for index pattern [logs-test-*], using time field [testtimestamp]'
- );
- });
- });
-
- it('composes the defined filters', () => {
- expect(
- getFilters(
- args.indexPatternId,
- args.indexPatternTimeField,
- args.timerange,
- args.savedSearchObjectAttr,
- undefined,
- undefined
- )
- ).toEqual({
- combinedFilter: [
- {
- range: {
- testtimestamp: {
- format: 'strict_date_time',
- gte: '1901-01-01T00:00:00Z',
- lte: '1902-01-01T00:00:00Z',
- },
- },
- },
- ],
- includes: ['testtimestamp', 'larry'],
- timezone: 'UTC',
- });
-
- expect(
- getFilters(
- args.indexPatternId,
- args.indexPatternTimeField,
- args.timerange,
- args.savedSearchObjectAttr,
- undefined,
- args.queryFilter
- )
- ).toEqual({
- combinedFilter: [
- {
- range: {
- testtimestamp: {
- format: 'strict_date_time',
- gte: '1901-01-01T00:00:00Z',
- lte: '1902-01-01T00:00:00Z',
- },
- },
- },
- { isFilter: true, isQueryFilter: true },
- ],
- includes: ['testtimestamp', 'larry'],
- timezone: 'UTC',
- });
- });
-
- describe('timefilter', () => {
- it('formats the datetime to the provided timezone', () => {
- args.timerange = {
- timezone: 'MST',
- min: '1901-01-01T00:00:00Z',
- max: '1902-01-01T00:00:00Z',
- };
-
- expect(
- getFilters(
- args.indexPatternId,
- args.indexPatternTimeField,
- args.timerange,
- args.savedSearchObjectAttr
- )
- ).toEqual({
- combinedFilter: [
- {
- range: {
- testtimestamp: {
- format: 'strict_date_time',
- gte: '1900-12-31T17:00:00-07:00',
- lte: '1901-12-31T17:00:00-07:00',
- },
- },
- },
- ],
- includes: ['testtimestamp', 'larry'],
- timezone: 'MST',
- });
- });
- });
-});
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.ts
deleted file mode 100644
index c252b66952360..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { badRequest } from '@hapi/boom';
-import moment from 'moment-timezone';
-import { TimeRangeParams } from '../../common';
-import { Filter, QueryFilter, SavedSearchObjectAttributes, SearchSourceFilter } from '../types';
-
-export function getFilters(
- indexPatternId: string,
- indexPatternTimeField: string | null,
- timerange: TimeRangeParams | null,
- savedSearchObjectAttr: SavedSearchObjectAttributes,
- searchSourceFilter?: SearchSourceFilter,
- queryFilter?: QueryFilter
-) {
- let includes: string[];
- let timeFilter: any | null;
- let timezone: string | null;
-
- if (indexPatternTimeField) {
- if (!timerange || timerange.min == null || timerange.max == null) {
- throw badRequest(
- `Time range params are required for index pattern [${indexPatternId}], using time field [${indexPatternTimeField}]`
- );
- }
-
- timezone = timerange.timezone;
- const { min: gte, max: lte } = timerange;
- timeFilter = {
- range: {
- [indexPatternTimeField]: {
- format: 'strict_date_time',
- gte: moment.tz(moment(gte), timezone).format(),
- lte: moment.tz(moment(lte), timezone).format(),
- },
- },
- };
-
- const savedSearchCols = savedSearchObjectAttr.columns || [];
- includes = [indexPatternTimeField, ...savedSearchCols];
- } else {
- includes = savedSearchObjectAttr.columns || [];
- timeFilter = null;
- timezone = null;
- }
-
- const combinedFilter: Filter[] = [timeFilter, searchSourceFilter, queryFilter].filter(Boolean); // builds an array of defined filters
-
- return { timezone, combinedFilter, includes };
-}
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/types.d.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/types.d.ts
deleted file mode 100644
index a4fbdb69bbbba..0000000000000
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/types.d.ts
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { TimeRangeParams } from '../common';
-
-export interface FakeRequest {
- headers: Record;
-}
-
-export interface JobParamsPanelCsvPost {
- timerange?: TimeRangeParams;
- state?: any;
-}
-
-export interface SearchPanel {
- indexPatternSavedObjectId: string;
- attributes: SavedSearchObjectAttributes;
- timerange?: TimeRangeParams;
-}
-
-export interface JobPayloadPanelCsv extends JobParamsPanelCsv {
- panel: SearchPanel;
-}
-
-export interface JobParamsPanelCsv {
- savedObjectType: string;
- savedObjectId: string;
- post?: JobParamsPanelCsvPost;
- visType?: string;
-}
-
-export interface SavedObjectServiceError {
- statusCode: number;
- error?: string;
- message?: string;
-}
-
-export interface SavedObjectMetaJSON {
- searchSourceJSON: string;
-}
-
-export interface SavedObjectMeta {
- searchSource: SearchSource;
-}
-
-export interface SavedSearchObjectAttributesJSON {
- title: string;
- sort: any[];
- columns: string[];
- kibanaSavedObjectMeta: SavedObjectMetaJSON;
- uiState: any;
-}
-
-export interface SavedSearchObjectAttributes {
- title: string;
- sort: any[];
- columns?: string[];
- kibanaSavedObjectMeta: SavedObjectMeta;
- uiState: any;
-}
-
-export interface VisObjectAttributesJSON {
- title: string;
- visState: string; // JSON string
- type: string;
- params: any;
- uiStateJSON: string; // also JSON string
- aggs: any[];
- sort: any[];
- kibanaSavedObjectMeta: SavedObjectMeta;
-}
-
-export interface VisObjectAttributes {
- title: string;
- visState: string; // JSON string
- type: string;
- params: any;
- uiState: {
- vis: {
- params: {
- sort: {
- columnIndex: string;
- direction: string;
- };
- };
- };
- };
- aggs: any[];
- sort: any[];
- kibanaSavedObjectMeta: SavedObjectMeta;
-}
-
-export interface SavedObjectReference {
- name: string; // should be kibanaSavedObjectMeta.searchSourceJSON.index
- type: string; // should be index-pattern
- id: string;
-}
-
-export interface SavedObject {
- attributes: any;
- references: SavedObjectReference[];
-}
-
-export interface VisPanel {
- indexPatternSavedObjectId?: string;
- savedSearchObjectId?: string;
- attributes: VisObjectAttributes;
- timerange: TimeRangeParams;
-}
-
-export interface DocValueFields {
- field: string;
- format: string;
-}
-
-export interface SearchSourceQuery {
- isSearchSourceQuery: boolean;
-}
-
-export interface SearchSource {
- query: SearchSourceQuery;
- filter: any[];
-}
-
-/*
- * These filter types are stub types to help ensure things get passed to
- * non-Typescript functions in the right order. An actual structure is not
- * needed because the code doesn't look into the properties; just combines them
- * and passes them through to other non-TS modules.
- */
-export interface Filter {
- isFilter: boolean;
-}
-export interface TimeFilter extends Filter {
- isTimeFilter: boolean;
-}
-export interface QueryFilter extends Filter {
- isQueryFilter: boolean;
-}
-export interface SearchSourceFilter extends Filter {
- isSearchSourceFilter: boolean;
-}
-
-export interface IndexPatternField {
- scripted: boolean;
- lang?: string;
- script?: string;
- name: string;
-}
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/create_job.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/create_job.ts
new file mode 100644
index 0000000000000..a389f2a3252ca
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/create_job.ts
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CSV_JOB_TYPE } from '../../../common/constants';
+import { cryptoFactory } from '../../lib';
+import { CreateJobFn, CreateJobFnFactory } from '../../types';
+import { JobParamsCSV, TaskPayloadCSV } from './types';
+
+export const createJobFnFactory: CreateJobFnFactory<
+ CreateJobFn
+> = function createJobFactoryFn(reporting, parentLogger) {
+ const logger = parentLogger.clone([CSV_JOB_TYPE, 'create-job']);
+
+ const config = reporting.getConfig();
+ const crypto = cryptoFactory(config.get('encryptionKey'));
+
+ return async function createJob(jobParams, context, request) {
+ const serializedEncryptedHeaders = await crypto.encrypt(request.headers);
+
+ return {
+ headers: serializedEncryptedHeaders,
+ spaceId: reporting.getSpaceId(request, logger),
+ ...jobParams,
+ };
+ };
+};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.test.ts
new file mode 100644
index 0000000000000..1c2e15ebc5d9b
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.test.ts
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+jest.mock('./generate_csv/generate_csv', () => ({
+ CsvGenerator: class CsvGeneratorMock {
+ generateData() {
+ return {
+ content: 'test\n123',
+ };
+ }
+ },
+}));
+
+import nodeCrypto from '@elastic/node-crypto';
+import { ReportingCore } from '../../';
+import { CancellationToken } from '../../../common';
+import {
+ createMockConfig,
+ createMockConfigSchema,
+ createMockLevelLogger,
+ createMockReportingCore,
+} from '../../test_helpers';
+import { runTaskFnFactory } from './execute_job';
+
+const logger = createMockLevelLogger();
+const encryptionKey = 'tetkey';
+const headers = { sid: 'cooltestheaders' };
+let encryptedHeaders: string;
+let reportingCore: ReportingCore;
+
+beforeAll(async () => {
+ const crypto = nodeCrypto({ encryptionKey });
+ const config = createMockConfig(
+ createMockConfigSchema({
+ encryptionKey,
+ csv: {
+ checkForFormulas: true,
+ escapeFormulaValues: true,
+ maxSizeBytes: 180000,
+ scroll: { size: 500, duration: '30s' },
+ },
+ })
+ );
+
+ encryptedHeaders = await crypto.encrypt(headers);
+
+ reportingCore = await createMockReportingCore(config);
+});
+
+test('gets the csv content from job parameters', async () => {
+ const runTask = runTaskFnFactory(reportingCore, logger);
+
+ const payload = await runTask(
+ 'cool-job-id',
+ {
+ headers: encryptedHeaders,
+ browserTimezone: 'US/Alaska',
+ searchSource: {},
+ objectType: 'search',
+ title: 'Test Search',
+ },
+ new CancellationToken()
+ );
+
+ expect(payload).toMatchInlineSnapshot(`
+ Object {
+ "content": "test
+ 123",
+ }
+ `);
+});
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.ts
new file mode 100644
index 0000000000000..0453af4ef23e3
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.ts
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CSV_JOB_TYPE } from '../../../common/constants';
+import { RunTaskFn, RunTaskFnFactory } from '../../types';
+import { decryptJobHeaders } from '../common';
+import { CsvGenerator } from './generate_csv/generate_csv';
+import { TaskPayloadCSV } from './types';
+import { getFieldFormats } from '../../services';
+
+export const runTaskFnFactory: RunTaskFnFactory> = (
+ reporting,
+ parentLogger
+) => {
+ const config = reporting.getConfig();
+
+ return async function runTask(jobId, job, cancellationToken) {
+ const logger = parentLogger.clone([CSV_JOB_TYPE, 'execute-job', jobId]);
+
+ const encryptionKey = config.get('encryptionKey');
+ const headers = await decryptJobHeaders(encryptionKey, job.headers, logger);
+ const fakeRequest = reporting.getFakeRequest({ headers }, job.spaceId, logger);
+ const uiSettingsClient = await reporting.getUiSettingsClient(fakeRequest, logger);
+
+ const searchService = await reporting.getSearchService();
+ const searchSourceService = await searchService.searchSource.asScoped(fakeRequest);
+ const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(
+ uiSettingsClient
+ );
+
+ const esClient = (await reporting.getEsClient()).asScoped(fakeRequest);
+
+ const csv = new CsvGenerator(
+ job,
+ config,
+ esClient,
+ uiSettingsClient,
+ searchSourceService,
+ fieldFormatsRegistry,
+ cancellationToken,
+ logger
+ );
+
+ return await csv.generateData();
+ };
+};
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/cell_has_formula.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/cell_has_formula.ts
similarity index 100%
rename from x-pack/plugins/reporting/server/export_types/csv/generate_csv/cell_has_formula.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/cell_has_formula.ts
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/escape_value.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/escape_value.test.ts
similarity index 100%
rename from x-pack/plugins/reporting/server/export_types/csv/generate_csv/escape_value.test.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/escape_value.test.ts
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/escape_value.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/escape_value.ts
similarity index 100%
rename from x-pack/plugins/reporting/server/export_types/csv/generate_csv/escape_value.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/escape_value.ts
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts
new file mode 100644
index 0000000000000..430f8cadbe90d
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts
@@ -0,0 +1,551 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { identity, range } from 'lodash';
+import { IScopedClusterClient, IUiSettingsClient } from 'src/core/server';
+import {
+ elasticsearchServiceMock,
+ savedObjectsClientMock,
+ uiSettingsServiceMock,
+} from 'src/core/server/mocks';
+import {
+ EsQuerySearchAfter,
+ FieldFormatsRegistry,
+ ISearchStartSearchSource,
+} from 'src/plugins/data/common';
+import { searchSourceInstanceMock } from 'src/plugins/data/common/search/search_source/mocks';
+import { ReportingConfig } from '../../../';
+import { CancellationToken } from '../../../../common';
+import {
+ UI_SETTINGS_CSV_QUOTE_VALUES,
+ UI_SETTINGS_CSV_SEPARATOR,
+ UI_SETTINGS_DATEFORMAT_TZ,
+} from '../../../../common/constants';
+import {
+ createMockConfig,
+ createMockConfigSchema,
+ createMockLevelLogger,
+} from '../../../test_helpers';
+import { JobParamsCSV } from '../types';
+import { CsvGenerator } from './generate_csv';
+
+const createMockJob = (baseObj: any = {}): JobParamsCSV => ({
+ ...baseObj,
+});
+
+let mockEsClient: IScopedClusterClient;
+let mockConfig: ReportingConfig;
+let uiSettingsClient: IUiSettingsClient;
+
+const searchSourceMock = { ...searchSourceInstanceMock };
+const mockSearchSourceService: jest.Mocked = {
+ create: jest.fn().mockReturnValue(searchSourceMock),
+ createEmpty: jest.fn().mockReturnValue(searchSourceMock),
+};
+const mockSearchSourceFetchDefault = jest.fn().mockResolvedValue({
+ hits: {
+ hits: [],
+ total: 0,
+ },
+});
+const mockSearchSourceGetFieldDefault = jest.fn().mockImplementation((key: string) => {
+ switch (key) {
+ case 'fields':
+ return ['date', 'ip', 'message'];
+ case 'index':
+ return {
+ fields: {
+ getByName: jest.fn().mockImplementation(() => []),
+ getByType: jest.fn().mockImplementation(() => []),
+ },
+ getFormatterForField: jest.fn(),
+ };
+ }
+});
+
+const mockFieldFormatsRegistry = ({
+ deserialize: jest
+ .fn()
+ .mockImplementation(() => ({ id: 'string', convert: jest.fn().mockImplementation(identity) })),
+} as unknown) as FieldFormatsRegistry;
+
+beforeEach(async () => {
+ mockEsClient = elasticsearchServiceMock.createScopedClusterClient();
+
+ uiSettingsClient = uiSettingsServiceMock
+ .createStartContract()
+ .asScopedToClient(savedObjectsClientMock.create());
+ uiSettingsClient.get = jest.fn().mockImplementation((key): any => {
+ switch (key) {
+ case UI_SETTINGS_CSV_QUOTE_VALUES:
+ return true;
+ case UI_SETTINGS_CSV_SEPARATOR:
+ return ',';
+ case UI_SETTINGS_DATEFORMAT_TZ:
+ return 'Browser';
+ }
+ });
+
+ mockConfig = createMockConfig(
+ createMockConfigSchema({
+ csv: {
+ checkForFormulas: true,
+ escapeFormulaValues: true,
+ maxSizeBytes: 180000,
+ scroll: { size: 500, duration: '30s' },
+ },
+ })
+ );
+
+ searchSourceMock.getField = mockSearchSourceGetFieldDefault;
+ searchSourceMock.fetch = mockSearchSourceFetchDefault;
+});
+
+const logger = createMockLevelLogger();
+
+it('formats an empty search result to CSV content', async () => {
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "date,ip,message
+ "
+ `);
+ expect(csvResult.csv_contains_formulas).toBe(false);
+ expect(csvResult.needs_sorting).toBe(false);
+});
+
+it('formats a search result to CSV content', async () => {
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: [
+ {
+ fields: {
+ date: `["2020-12-31T00:14:28.000Z"]`,
+ ip: `["110.135.176.89"]`,
+ message: `["This is a great message!"]`,
+ },
+ sort: [1, 'a'] as EsQuerySearchAfter,
+ },
+ ],
+ total: 1,
+ },
+ });
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "date,ip,message
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"This is a great message!\\"
+ "
+ `);
+ expect(csvResult.csv_contains_formulas).toBe(false);
+ expect(csvResult.needs_sorting).toBe(false);
+});
+
+const TEST_NUM_TOTAL = 100;
+
+it('calculates the bytes of the content', async () => {
+ searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
+ if (key === 'fields') {
+ return ['message'];
+ }
+ return mockSearchSourceGetFieldDefault(key);
+ });
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: range(0, TEST_NUM_TOTAL).map((hit, i) => ({
+ fields: {
+ message: ['this is a great message'],
+ },
+ sort: [i, 1] as EsQuerySearchAfter,
+ })),
+ total: TEST_NUM_TOTAL,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+ expect(csvResult.size).toBe(2608);
+ expect(csvResult.max_size_reached).toBe(false);
+ expect(csvResult.needs_sorting).toBe(false);
+ expect(csvResult.warnings).toEqual([]);
+});
+
+it('warns if max size was reached', async () => {
+ const TEST_MAX_SIZE = 500;
+
+ mockConfig = createMockConfig(
+ createMockConfigSchema({
+ csv: {
+ checkForFormulas: true,
+ escapeFormulaValues: true,
+ maxSizeBytes: TEST_MAX_SIZE,
+ scroll: { size: 500, duration: '30s' },
+ },
+ })
+ );
+
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: range(0, TEST_NUM_TOTAL).map((hit, i) => ({
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ ip: ['110.135.176.89'],
+ message: ['super cali fragile istic XPLA docious'],
+ },
+ sort: [i, ''] as EsQuerySearchAfter,
+ })),
+ total: TEST_NUM_TOTAL,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+ expect(csvResult.max_size_reached).toBe(true);
+ expect(csvResult.needs_sorting).toBe(false);
+ expect(csvResult.warnings).toEqual([]);
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "date,ip,message
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
+ "
+ `);
+});
+
+it('warns if it detects paging through unsorted search results', async () => {
+ searchSourceMock.fetch = jest.fn().mockResolvedValue({
+ hits: {
+ hits: range(0, 15).map(() => ({
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ ip: ['110.135.176.89'],
+ message: ['super cali fragile istic XPLA docious'],
+ },
+ })),
+ total: 50,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+ expect(csvResult.needs_sorting).toBe(true);
+});
+
+it('warns if it detects paging through poorly sorted data', async () => {
+ searchSourceMock.fetch = jest.fn().mockResolvedValue({
+ hits: {
+ hits: range(0, 15).map(() => ({
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ ip: ['110.135.176.89'],
+ message: ['super cali fragile istic XPLA docious'],
+ sort: [0, 0] as EsQuerySearchAfter,
+ },
+ })),
+ total: 50,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+ expect(csvResult.needs_sorting).toBe(true);
+});
+
+describe('fields', () => {
+ it('cells can be multi-value', async () => {
+ searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
+ if (key === 'fields') {
+ return ['_id', 'sku'];
+ }
+ return mockSearchSourceGetFieldDefault(key);
+ });
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: [
+ {
+ _id: 'my-cool-id',
+ _index: 'my-cool-index',
+ _version: 4,
+ fields: {
+ sku: [`This is a cool SKU.`, `This is also a cool SKU.`],
+ },
+ sort: [1, 'a'] as EsQuerySearchAfter,
+ },
+ ],
+ total: 1,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({ searchSource: {} }),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+ const csvResult = await generateCsv.generateData();
+
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "\\"_id\\",sku
+ \\"my-cool-id\\",\\"This is a cool SKU., This is also a cool SKU.\\"
+ "
+ `);
+ expect(csvResult.needs_sorting).toBe(false);
+ });
+
+ it('provides top-level underscored fields as columns', async () => {
+ searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
+ if (key === 'fields') {
+ return ['_id', '_index', 'date', 'message'];
+ }
+ return mockSearchSourceGetFieldDefault(key);
+ });
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: [
+ {
+ _id: 'my-cool-id',
+ _index: 'my-cool-index',
+ _version: 4,
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ message: [`it's nice to see you`],
+ },
+ sort: [1, 'a'] as EsQuerySearchAfter,
+ },
+ ],
+ total: 1,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({
+ searchSource: {
+ query: { query: '', language: 'kuery' },
+ sort: [{ '@date': 'desc' }],
+ index: '93f4bc50-6662-11eb-98bc-f550e2308366',
+ fields: ['_id', '_index', '@date', 'message'],
+ filter: [],
+ },
+ }),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+
+ const csvResult = await generateCsv.generateData();
+
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "\\"_id\\",\\"_index\\",date,message
+ \\"my-cool-id\\",\\"my-cool-index\\",\\"2020-12-31T00:14:28.000Z\\",\\"it's nice to see you\\"
+ "
+ `);
+ expect(csvResult.csv_contains_formulas).toBe(false);
+ expect(csvResult.needs_sorting).toBe(false);
+ });
+});
+
+describe('formulas', () => {
+ const TEST_FORMULA = '=SUM(A1:A2)';
+
+ it(`escapes formula values in a cell, doesn't warn the csv contains formulas`, async () => {
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: [
+ {
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ ip: ['110.135.176.89'],
+ message: [TEST_FORMULA],
+ },
+ sort: [1, 'a'] as EsQuerySearchAfter,
+ },
+ ],
+ total: 1,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+
+ const csvResult = await generateCsv.generateData();
+
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "date,ip,message
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"'=SUM(A1:A2)\\"
+ "
+ `);
+ expect(csvResult.csv_contains_formulas).toBe(false);
+ expect(csvResult.needs_sorting).toBe(false);
+ });
+
+ it(`escapes formula values in a header, doesn't warn the csv contains formulas`, async () => {
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: [
+ {
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ ip: ['110.135.176.89'],
+ [TEST_FORMULA]: 'This is great data',
+ },
+ sort: [1, 'a'] as EsQuerySearchAfter,
+ },
+ ],
+ total: 1,
+ },
+ });
+
+ searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
+ if (key === 'fields') {
+ return ['date', 'ip', TEST_FORMULA];
+ }
+ return mockSearchSourceGetFieldDefault(key);
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+
+ const csvResult = await generateCsv.generateData();
+
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "date,ip,\\"'=SUM(A1:A2)\\"
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"This is great data\\"
+ "
+ `);
+ expect(csvResult.csv_contains_formulas).toBe(false);
+ expect(csvResult.needs_sorting).toBe(false);
+ });
+
+ it('can check for formulas, without escaping them', async () => {
+ mockConfig = createMockConfig(
+ createMockConfigSchema({
+ csv: {
+ checkForFormulas: true,
+ escapeFormulaValues: false,
+ maxSizeBytes: 180000,
+ scroll: { size: 500, duration: '30s' },
+ },
+ })
+ );
+ searchSourceMock.fetch = jest.fn().mockResolvedValueOnce({
+ hits: {
+ hits: [
+ {
+ fields: {
+ date: ['2020-12-31T00:14:28.000Z'],
+ ip: ['110.135.176.89'],
+ message: [TEST_FORMULA],
+ },
+ sort: [1, 'a'] as EsQuerySearchAfter,
+ },
+ ],
+ total: 1,
+ },
+ });
+
+ const generateCsv = new CsvGenerator(
+ createMockJob({}),
+ mockConfig,
+ mockEsClient,
+ uiSettingsClient,
+ mockSearchSourceService,
+ mockFieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+
+ const csvResult = await generateCsv.generateData();
+
+ expect(csvResult.content).toMatchInlineSnapshot(`
+ "date,ip,message
+ \\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"=SUM(A1:A2)\\"
+ "
+ `);
+ expect(csvResult.csv_contains_formulas).toBe(true);
+ expect(csvResult.needs_sorting).toBe(false);
+ });
+});
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts
new file mode 100644
index 0000000000000..9c0486a40fee9
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts
@@ -0,0 +1,394 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { SearchResponse } from 'elasticsearch';
+import { IScopedClusterClient, IUiSettingsClient } from 'src/core/server';
+import { Datatable } from 'src/plugins/expressions/server';
+import { ReportingConfig } from '../../..';
+import {
+ EsQuerySearchAfter,
+ FieldFormat,
+ FieldFormatConfig,
+ IFieldFormatsRegistry,
+ ISearchStartSearchSource,
+ SearchFieldValue,
+ tabifyDocs,
+} from '../../../../../../../src/plugins/data/common';
+import { CancellationToken } from '../../../../common';
+import { CONTENT_TYPE_CSV } from '../../../../common/constants';
+import { byteSizeValueToNumber } from '../../../../common/schema_utils';
+import { LevelLogger } from '../../../lib';
+import { TaskRunResult } from '../../../lib/tasks';
+import { JobParamsCSV } from '../types';
+import { cellHasFormulas } from './cell_has_formula';
+import { CsvExportSettings, getExportSettings } from './get_export_settings';
+import { MaxSizeStringBuilder } from './max_size_string_builder';
+
+export class CsvGenerator {
+ private _columnMap: number[] | null = null;
+ private _formatters: Record | null = null;
+ private csvContainsFormulas = false;
+ private maxSizeReached = false;
+ private needsSorting = false;
+ private csvRowCount = 0;
+
+ constructor(
+ private job: JobParamsCSV,
+ private config: ReportingConfig,
+ private esClient: IScopedClusterClient,
+ private uiSettingsClient: IUiSettingsClient,
+ private searchSourceService: ISearchStartSearchSource,
+ private fieldFormatsRegistry: IFieldFormatsRegistry,
+ private cancellationToken: CancellationToken,
+ private logger: LevelLogger
+ ) {}
+
+ /*
+ * Build a map for ordering the fields of search results into CSV columns
+ */
+ private getColumnMap(fields: SearchFieldValue[] | undefined, table: Datatable) {
+ if (this._columnMap) {
+ return this._columnMap;
+ }
+
+ // if there are selected fields, re-initialize columnMap with field order is set in searchSource fields
+ if (fields && fields[0] !== '*') {
+ this._columnMap = fields.map((field) =>
+ table.columns.findIndex((column) => column.id === field)
+ );
+ }
+
+ // initialize default columnMap, works if fields are asterisk and order doesn't matter
+ if (!this._columnMap) {
+ this._columnMap = table.columns.map((c, columnIndex) => columnIndex);
+ }
+
+ return this._columnMap;
+ }
+
+ /*
+ * Load field formats for each field in the list
+ */
+ private getFormatters(table: Datatable) {
+ if (this._formatters) {
+ return this._formatters;
+ }
+
+ // initialize field formats
+ const formatters: Record = {};
+ table.columns.forEach((c) => {
+ const fieldFormat = this.fieldFormatsRegistry.deserialize(c.meta.params);
+ formatters[c.id] = fieldFormat;
+ });
+
+ this._formatters = formatters;
+ return this._formatters;
+ }
+
+ /*
+ * Use the list of fields to generate the header row
+ */
+ private generateHeader(
+ fields: SearchFieldValue[] | undefined,
+ table: Datatable,
+ builder: MaxSizeStringBuilder,
+ settings: CsvExportSettings
+ ) {
+ const { checkForFormulas, escapeValue, separator } = settings;
+ const columnMap = this.getColumnMap(fields, table);
+
+ this.logger.debug(`Building CSV header row...`);
+ const header =
+ columnMap
+ .map((columnIndex, position) => {
+ let value: string;
+ if (columnIndex > -1) {
+ value = table.columns[columnIndex].name;
+ } else {
+ value = fields && fields[position] ? (fields[position] as string) : 'unknown';
+ }
+
+ if (checkForFormulas && cellHasFormulas(value)) {
+ this.csvContainsFormulas = true; // set warning if heading value has a formula
+ }
+
+ return escapeValue(value);
+ })
+ .join(separator) + `\n`;
+
+ if (!builder.tryAppend(header)) {
+ return {
+ size: 0,
+ content: '',
+ maxSizeReached: true,
+ warnings: [],
+ };
+ }
+ }
+
+ /*
+ * Format a Datatable into rows of CSV content
+ */
+ private generateRows(
+ fields: SearchFieldValue[] | undefined,
+ table: Datatable,
+ builder: MaxSizeStringBuilder,
+ formatters: Record,
+ settings: CsvExportSettings
+ ) {
+ // write the rows
+ this.logger.debug(`Building ${table.rows.length} CSV data rows...`);
+ const { checkForFormulas, escapeValue, separator } = settings;
+
+ for (const dataTableRow of table.rows) {
+ if (this.cancellationToken.isCancelled()) {
+ break;
+ }
+
+ const columnMap = this.getColumnMap(fields, table);
+ const row =
+ columnMap
+ .map((columnIndex, position) => {
+ const tableColumn = table.columns[columnIndex];
+ let cell: string[] | string = '-';
+
+ if (tableColumn != null) {
+ cell = formatters[tableColumn.id].convert(dataTableRow[tableColumn.id]);
+
+ try {
+ // expected values are a string of JSON where the value(s) is in an array
+ cell = JSON.parse(cell);
+ } catch (e) {
+ // ignore
+ }
+
+ // We have to strip singular array values out of their array wrapper,
+ // So that the value appears the visually the same as seen in Discover
+ if (Array.isArray(cell)) {
+ cell = cell.join(', '); // mimic Discover behavior
+ }
+ } else {
+ this.logger.warn(`Unrecognized field: ${(fields && fields[position]) || 'unknown'}`);
+ }
+
+ return cell;
+ })
+ .map((value) => {
+ if (checkForFormulas && cellHasFormulas(value)) {
+ this.csvContainsFormulas = true; // set warning if cell value has a formula
+ }
+ // Escape the values in Data
+ return escapeValue(value);
+ })
+ .join(separator) + '\n';
+
+ if (!builder.tryAppend(row)) {
+ this.logger.warn('max Size Reached');
+ this.maxSizeReached = true;
+ if (this.cancellationToken) {
+ this.cancellationToken.cancel();
+ }
+ break;
+ }
+
+ this.csvRowCount++;
+ }
+ }
+
+ /*
+ * Open a Point In Time on the Elasticsearch index and collect all the data from the index
+ */
+ public async generateData(): Promise {
+ const [settings, searchSource] = await Promise.all([
+ getExportSettings(this.uiSettingsClient, this.config, this.job.browserTimezone, this.logger),
+ this.searchSourceService.create(this.job.searchSource),
+ ]);
+
+ const index = searchSource.getField('index');
+ const fields = searchSource.getField('fields');
+
+ const { maxSizeBytes, bom, escapeFormulaValues } = settings;
+ searchSource.setField('size', settings.scroll.size);
+
+ const builder = new MaxSizeStringBuilder(byteSizeValueToNumber(maxSizeBytes), bom);
+
+ let currentRecord = -1;
+ let totalRecords = 0;
+ let first = true;
+ let lastSortId: EsQuerySearchAfter | undefined;
+ let pitId: string | undefined;
+ const warnings: string[] = [];
+
+ // open PIT and set field format config
+ if (index) {
+ // use _pit API
+ const { title: indexPatternTitle } = index;
+ const { duration } = settings.scroll;
+ this.logger.debug(`Open PIT: index: '${indexPatternTitle}' keep_alive: '${duration}'`);
+ const { body, statusCode } = await this.esClient.asCurrentUser.openPointInTime({
+ index: indexPatternTitle,
+ keep_alive: duration,
+ });
+ if (statusCode === 404) {
+ this.logger.error('Unable to open point in time!');
+ this.logger.error(new Error(JSON.stringify(body)));
+ }
+ pitId = body.id as string;
+
+ // apply timezone from the job to all date field formatters
+ try {
+ index.fields.getByType('date').forEach(({ name }) => {
+ this.logger.debug(`setting timezone on ${name}`);
+ const format: FieldFormatConfig = {
+ ...index.fieldFormatMap[name],
+ id: index.fieldFormatMap[name]?.id || 'date', // allow id: date_nanos
+ params: {
+ ...index.fieldFormatMap[name]?.params,
+ timezone: settings.timezone,
+ },
+ };
+ index.setFieldFormat(name, format);
+ });
+ } catch (err) {
+ this.logger.error(err);
+ }
+ }
+
+ this.logger.debug(`Received PIT ID: ${pitId?.substring(0, 9)}`);
+ if (pitId) {
+ searchSource.setField('pit', { id: pitId, keep_alive: '2m' });
+ } else {
+ this.logger.error(`Unable to open a point in time with ${searchSource.getId()}!`);
+ }
+
+ do {
+ if (this.cancellationToken.isCancelled()) {
+ break;
+ }
+
+ if (lastSortId) {
+ searchSource.setField('searchAfter', lastSortId);
+ }
+ let results: SearchResponse | undefined;
+ try {
+ results = await searchSource.fetch();
+ } catch (err) {
+ this.logger.error(err);
+ }
+
+ if (!results) {
+ break;
+ }
+
+ if (results.hits?.total == null) {
+ this.logger.error('Expected total number of records in the search response');
+ totalRecords = -1;
+ } else {
+ totalRecords = results.hits.total;
+ }
+ this.logger.debug(`Search results: ${totalRecords}`);
+
+ let table: Datatable | undefined;
+ try {
+ table = tabifyDocs(results, index, { shallow: true, meta: true });
+ } catch (err) {
+ this.logger.error(err);
+ }
+
+ if (!table) {
+ break;
+ }
+
+ // write the header and initialize formatters / column orderings
+ // depends on the table to know what order to place the columns
+ if (first) {
+ first = false;
+ this.generateHeader(fields, table, builder, settings);
+ }
+
+ if (table.rows.length < 1) {
+ break; // empty report with just the header
+ }
+
+ const formatters = this.getFormatters(table);
+ this.generateRows(fields, table, builder, formatters, settings);
+
+ const hits = results.hits?.hits || [];
+
+ let tempSortId: EsQuerySearchAfter | undefined;
+ try {
+ tempSortId = hits[hits.length - 1]?.sort as EsQuerySearchAfter;
+ if (lastSortId && tempSortId?.join() === lastSortId?.join()) {
+ // repeated sort ID: results are not thoroughly sorted
+ this.needsSorting = true;
+ }
+
+ const uniqueSortIds = new Set();
+ hits.forEach((hit) => {
+ uniqueSortIds.add(hit.sort?.join());
+ });
+ if (lastSortId && (!uniqueSortIds.size || uniqueSortIds.size < hits.length)) {
+ // not enough unique sort IDs for the number of hits: results are not thoroughly sorted
+ this.needsSorting = true;
+ }
+ } catch (err) {
+ this.logger.error(err);
+ }
+
+ // update last sort ID for next query
+ lastSortId = tempSortId;
+
+ // update iterator
+ currentRecord += table.rows.length;
+ } while (currentRecord < totalRecords - 1);
+
+ const size = builder.getSizeInBytes();
+ this.logger.debug(
+ `Finished generating. Total size in bytes: ${size}. Row count: ${this.csvRowCount}.`
+ );
+
+ // the row count is wrong and it's not because of max size limit
+ // results are not thoroughly sorted
+ if (this.csvRowCount !== totalRecords && !this.maxSizeReached) {
+ this.needsSorting = true;
+ }
+
+ // Add warnings to be logged
+ if (this.csvContainsFormulas && escapeFormulaValues) {
+ warnings.push(
+ i18n.translate('xpack.reporting.exportTypes.csv.generateCsv.escapedFormulaValues', {
+ defaultMessage: 'CSV may contain formulas whose values have been escaped',
+ })
+ );
+ }
+
+ if (this.needsSorting) {
+ warnings.push(
+ i18n.translate('xpack.reporting.exportTypes.csv.generateCsv.unsortedResults', {
+ defaultMessage: `The export may have duplicated or missing data: sort values in the search results must be unique`,
+ })
+ );
+ }
+
+ // clean PIT
+ if (pitId) {
+ this.logger.debug(`Closing PIT`);
+ await this.esClient.asCurrentUser.closePointInTime({ body: { id: pitId } });
+ }
+
+ return {
+ content: builder.getString(),
+ content_type: CONTENT_TYPE_CSV,
+ csv_contains_formulas: this.csvContainsFormulas && !escapeFormulaValues,
+ max_size_reached: this.maxSizeReached,
+ needs_sorting: this.needsSorting,
+ size,
+ warnings,
+ };
+ }
+}
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts
new file mode 100644
index 0000000000000..efdb583a89dc8
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+ UI_SETTINGS_DATEFORMAT_TZ,
+ UI_SETTINGS_CSV_QUOTE_VALUES,
+ UI_SETTINGS_CSV_SEPARATOR,
+} from '../../../../common/constants';
+import { IUiSettingsClient } from 'kibana/server';
+import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks';
+import {
+ createMockConfig,
+ createMockConfigSchema,
+ createMockLevelLogger,
+} from '../../../test_helpers';
+import { getExportSettings } from './get_export_settings';
+
+describe('getExportSettings', () => {
+ let uiSettingsClient: IUiSettingsClient;
+ const config = createMockConfig(createMockConfigSchema({}));
+ const logger = createMockLevelLogger();
+
+ beforeEach(() => {
+ uiSettingsClient = uiSettingsServiceMock
+ .createStartContract()
+ .asScopedToClient(savedObjectsClientMock.create());
+ uiSettingsClient.get = jest.fn().mockImplementation((key: string) => {
+ switch (key) {
+ case UI_SETTINGS_CSV_QUOTE_VALUES:
+ return true;
+ case UI_SETTINGS_CSV_SEPARATOR:
+ return ',';
+ case UI_SETTINGS_DATEFORMAT_TZ:
+ return 'Browser';
+ }
+
+ return 'helo world';
+ });
+ });
+
+ test('getExportSettings: returns the expected result', async () => {
+ expect(await getExportSettings(uiSettingsClient, config, '', logger)).toMatchInlineSnapshot(`
+ Object {
+ "bom": "",
+ "checkForFormulas": undefined,
+ "escapeFormulaValues": undefined,
+ "escapeValue": [Function],
+ "maxSizeBytes": undefined,
+ "scroll": Object {
+ "duration": undefined,
+ "size": undefined,
+ },
+ "separator": ",",
+ "timezone": "UTC",
+ }
+ `);
+ });
+
+ test('escapeValue function', async () => {
+ const { escapeValue } = await getExportSettings(uiSettingsClient, config, '', logger);
+ expect(escapeValue(`test`)).toBe(`test`);
+ expect(escapeValue(`this is, a test`)).toBe(`"this is, a test"`);
+ expect(escapeValue(`"tet"`)).toBe(`"""tet"""`);
+ expect(escapeValue(`@foo`)).toBe(`"@foo"`);
+ });
+
+ test('non-default timezone', async () => {
+ uiSettingsClient.get = jest.fn().mockImplementation((key: string) => {
+ switch (key) {
+ case UI_SETTINGS_DATEFORMAT_TZ:
+ return `America/Aruba`;
+ }
+ });
+
+ expect(
+ await getExportSettings(uiSettingsClient, config, '', logger).then(({ timezone }) => timezone)
+ ).toBe(`America/Aruba`);
+ });
+});
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts
new file mode 100644
index 0000000000000..17a10f3034242
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ByteSizeValue } from '@kbn/config-schema';
+import { i18n } from '@kbn/i18n';
+import { IUiSettingsClient } from 'kibana/server';
+import { ReportingConfig } from '../../../';
+import {
+ CSV_BOM_CHARS,
+ UI_SETTINGS_DATEFORMAT_TZ,
+ UI_SETTINGS_CSV_QUOTE_VALUES,
+ UI_SETTINGS_CSV_SEPARATOR,
+} from '../../../../common/constants';
+import { LevelLogger } from '../../../lib';
+import { createEscapeValue } from './escape_value';
+
+export interface CsvExportSettings {
+ timezone: string;
+ scroll: {
+ size: number;
+ duration: string;
+ };
+ bom: string;
+ separator: string;
+ maxSizeBytes: number | ByteSizeValue;
+ checkForFormulas: boolean;
+ escapeFormulaValues: boolean;
+ escapeValue: (value: string) => string;
+}
+
+export const getExportSettings = async (
+ client: IUiSettingsClient,
+ config: ReportingConfig,
+ timezone: string | undefined,
+ logger: LevelLogger
+): Promise => {
+ // Timezone
+ let setTimezone: string;
+ // timezone in job params?
+ if (timezone) {
+ setTimezone = timezone;
+ } else {
+ // timezone in settings?
+ setTimezone = await client.get(UI_SETTINGS_DATEFORMAT_TZ);
+ if (setTimezone === 'Browser') {
+ // if `Browser`, hardcode it to 'UTC' so the export has data that makes sense
+ logger.warn(
+ i18n.translate('xpack.reporting.exportTypes.csv.executeJob.dateFormateSetting', {
+ defaultMessage:
+ 'Kibana Advanced Setting "{dateFormatTimezone}" is set to "Browser". Dates will be formatted as UTC to avoid ambiguity.',
+ values: { dateFormatTimezone: 'dateFormat:tz' },
+ })
+ );
+ setTimezone = 'UTC';
+ }
+ }
+
+ // Separator, QuoteValues
+ const [separator, quoteValues] = await Promise.all([
+ client.get(UI_SETTINGS_CSV_SEPARATOR),
+ client.get(UI_SETTINGS_CSV_QUOTE_VALUES),
+ ]);
+
+ const escapeFormulaValues = config.get('csv', 'escapeFormulaValues');
+ const escapeValue = createEscapeValue(quoteValues, escapeFormulaValues);
+ const bom = config.get('csv', 'useByteOrderMarkEncoding') ? CSV_BOM_CHARS : '';
+
+ return {
+ timezone: setTimezone,
+ scroll: {
+ size: config.get('csv', 'scroll', 'size'),
+ duration: config.get('csv', 'scroll', 'duration'),
+ },
+ bom,
+ separator,
+ maxSizeBytes: config.get('csv', 'maxSizeBytes'),
+ checkForFormulas: config.get('csv', 'checkForFormulas'),
+ escapeFormulaValues,
+ escapeValue,
+ };
+};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/index.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/index.ts
new file mode 100644
index 0000000000000..4e08ff2a222dc
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/index.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { CsvGenerator } from './generate_csv';
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/max_size_string_builder.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/max_size_string_builder.test.ts
similarity index 100%
rename from x-pack/plugins/reporting/server/export_types/csv/generate_csv/max_size_string_builder.test.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/max_size_string_builder.test.ts
diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/max_size_string_builder.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/max_size_string_builder.ts
similarity index 100%
rename from x-pack/plugins/reporting/server/export_types/csv/generate_csv/max_size_string_builder.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/max_size_string_builder.ts
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/index.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/index.ts
new file mode 100644
index 0000000000000..65126a0a62cb8
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/index.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+ CSV_JOB_TYPE as jobType,
+ LICENSE_TYPE_BASIC,
+ LICENSE_TYPE_ENTERPRISE,
+ LICENSE_TYPE_GOLD,
+ LICENSE_TYPE_PLATINUM,
+ LICENSE_TYPE_STANDARD,
+ LICENSE_TYPE_TRIAL,
+} from '../../../common/constants';
+import { CreateJobFn, ExportTypeDefinition, RunTaskFn } from '../../types';
+import { createJobFnFactory } from './create_job';
+import { runTaskFnFactory } from './execute_job';
+import { metadata } from './metadata';
+import { JobParamsCSV, TaskPayloadCSV } from './types';
+
+export const getExportType = (): ExportTypeDefinition<
+ CreateJobFn,
+ RunTaskFn
+> => ({
+ ...metadata,
+ jobType,
+ jobContentExtension: 'csv',
+ createJobFnFactory,
+ runTaskFnFactory,
+ validLicenses: [
+ LICENSE_TYPE_TRIAL,
+ LICENSE_TYPE_BASIC,
+ LICENSE_TYPE_STANDARD,
+ LICENSE_TYPE_GOLD,
+ LICENSE_TYPE_PLATINUM,
+ LICENSE_TYPE_ENTERPRISE,
+ ],
+});
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/metadata.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/metadata.ts
similarity index 65%
rename from x-pack/plugins/reporting/server/export_types/csv_from_savedobject/metadata.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource/metadata.ts
index 76bf106acb5de..187d64d872a9d 100644
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/metadata.ts
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/metadata.ts
@@ -5,9 +5,9 @@
* 2.0.
*/
-import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants';
+import { CSV_JOB_TYPE } from '../../../common/constants';
export const metadata = {
- id: CSV_FROM_SAVEDOBJECT_JOB_TYPE,
- name: CSV_FROM_SAVEDOBJECT_JOB_TYPE,
+ id: 'csv_searchsource',
+ name: CSV_JOB_TYPE,
};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/types.d.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/types.d.ts
new file mode 100644
index 0000000000000..f0ad4e00ebd5c
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/types.d.ts
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { BaseParams, BasePayload } from '../../types';
+
+export type RawValue = string | object | null | undefined;
+
+interface BaseParamsCSV {
+ browserTimezone: string;
+ searchSource: any;
+}
+
+export type JobParamsCSV = BaseParamsCSV & BaseParams;
+export type TaskPayloadCSV = BaseParamsCSV & BasePayload;
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/execute_job.ts
new file mode 100644
index 0000000000000..eab8a06b10121
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/execute_job.ts
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaRequest } from 'src/core/server';
+import { CancellationToken } from '../../../common';
+import { CSV_SEARCHSOURCE_IMMEDIATE_TYPE } from '../../../common/constants';
+import { TaskRunResult } from '../../lib/tasks';
+import { getFieldFormats } from '../../services';
+import { ReportingRequestHandlerContext, RunTaskFnFactory } from '../../types';
+import { CsvGenerator } from '../csv_searchsource/generate_csv/generate_csv';
+import { JobParamsDownloadCSV } from './types';
+
+/*
+ * ImmediateExecuteFn receives the job doc payload because the payload was
+ * generated in the ScheduleFn
+ */
+export type ImmediateExecuteFn = (
+ jobId: null,
+ job: JobParamsDownloadCSV,
+ context: ReportingRequestHandlerContext,
+ req: KibanaRequest
+) => Promise;
+
+export const runTaskFnFactory: RunTaskFnFactory = function executeJobFactoryFn(
+ reporting,
+ parentLogger
+) {
+ const config = reporting.getConfig();
+ const logger = parentLogger.clone([CSV_SEARCHSOURCE_IMMEDIATE_TYPE, 'execute-job']);
+
+ return async function runTask(jobId, immediateJobParams, context, req) {
+ const esClient = (await reporting.getEsClient()).asScoped(req);
+ const savedObjectsClient = context.core.savedObjects.client;
+ const uiSettingsClient = await reporting.getUiSettingsServiceFactory(savedObjectsClient);
+ const searchService = await reporting.getSearchService();
+ const searchSourceService = await searchService.searchSource.asScoped(req);
+
+ const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(
+ uiSettingsClient
+ );
+
+ const job = {
+ objectType: 'immediate-search',
+ ...immediateJobParams,
+ };
+
+ const csv = new CsvGenerator(
+ job,
+ config,
+ esClient,
+ uiSettingsClient,
+ searchSourceService,
+ fieldFormatsRegistry,
+ new CancellationToken(),
+ logger
+ );
+
+ const result = await csv.generateData();
+
+ if (result.csv_contains_formulas) {
+ logger.warn(`CSV may contain formulas whose values have been escaped`);
+ }
+
+ if (result.max_size_reached) {
+ logger.warn(`Max size reached: CSV output truncated to ${result.size} bytes`);
+ }
+
+ const { warnings } = result;
+ if (warnings) {
+ warnings.forEach((warning) => {
+ logger.warning(warning);
+ });
+ }
+
+ return result;
+ };
+};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/index.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/index.ts
similarity index 75%
rename from x-pack/plugins/reporting/server/export_types/csv_from_savedobject/index.ts
rename to x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/index.ts
index c3a0df9529a4d..9d915db4797b3 100644
--- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/index.ts
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/index.ts
@@ -6,7 +6,7 @@
*/
import {
- CSV_FROM_SAVEDOBJECT_JOB_TYPE,
+ CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
LICENSE_TYPE_BASIC,
LICENSE_TYPE_ENTERPRISE,
LICENSE_TYPE_GOLD,
@@ -15,7 +15,6 @@ import {
LICENSE_TYPE_TRIAL,
} from '../../../common/constants';
import { ExportTypeDefinition } from '../../types';
-import { createJobFnFactory, ImmediateCreateJobFn } from './create_job';
import { ImmediateExecuteFn, runTaskFnFactory } from './execute_job';
import { metadata } from './metadata';
@@ -23,17 +22,13 @@ import { metadata } from './metadata';
* These functions are exported to share with the API route handler that
* generates csv from saved object immediately on request.
*/
-export { createJobFnFactory } from './create_job';
export { runTaskFnFactory } from './execute_job';
-export const getExportType = (): ExportTypeDefinition<
- ImmediateCreateJobFn,
- ImmediateExecuteFn
-> => ({
+export const getExportType = (): ExportTypeDefinition => ({
...metadata,
- jobType: CSV_FROM_SAVEDOBJECT_JOB_TYPE,
+ jobType: CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
jobContentExtension: 'csv',
- createJobFnFactory,
+ createJobFnFactory: null,
runTaskFnFactory,
validLicenses: [
LICENSE_TYPE_TRIAL,
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/metadata.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/metadata.ts
new file mode 100644
index 0000000000000..c27b8484697dd
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/metadata.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CSV_SEARCHSOURCE_IMMEDIATE_TYPE } from '../../../common/constants';
+
+export const metadata = {
+ id: CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
+ name: CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
+};
diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/types.d.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/types.d.ts
new file mode 100644
index 0000000000000..276016dd61233
--- /dev/null
+++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/types.d.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { TimeRangeParams } from '../common';
+
+export interface FakeRequest {
+ headers: Record;
+}
+
+export interface JobParamsDownloadCSV {
+ browserTimezone: string;
+ title: string;
+ searchSource: any;
+}
+
+export interface SavedObjectServiceError {
+ statusCode: number;
+ error?: string;
+ message?: string;
+}
diff --git a/x-pack/plugins/reporting/server/lib/enqueue_job.ts b/x-pack/plugins/reporting/server/lib/enqueue_job.ts
index 1c0eb8f4f5b77..5eb6bcb0b42a9 100644
--- a/x-pack/plugins/reporting/server/lib/enqueue_job.ts
+++ b/x-pack/plugins/reporting/server/lib/enqueue_job.ts
@@ -47,12 +47,16 @@ export function enqueueJobFactory(
throw new Error(`Export type ${exportTypeId} does not exist in the registry!`);
}
+ if (!exportType.createJobFnFactory) {
+ throw new Error(`Export type ${exportTypeId} is not an async job type!`);
+ }
+
const [createJob, { store }] = await Promise.all([
exportType.createJobFnFactory(reporting, logger),
reporting.getPluginStartDeps(),
]);
- const job = await createJob(jobParams, context, request);
+ const job = await createJob!(jobParams, context, request);
const pendingReport = new Report({
jobtype: exportType.jobType,
created_by: user ? user.username : false,
diff --git a/x-pack/plugins/reporting/server/lib/export_types_registry.ts b/x-pack/plugins/reporting/server/lib/export_types_registry.ts
index 5502692306319..890af43297751 100644
--- a/x-pack/plugins/reporting/server/lib/export_types_registry.ts
+++ b/x-pack/plugins/reporting/server/lib/export_types_registry.ts
@@ -6,8 +6,9 @@
*/
import { isString } from 'lodash';
-import { getExportType as getTypeCsv } from '../export_types/csv';
-import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_from_savedobject';
+import { getExportType as getTypeCsvDeprecated } from '../export_types/csv';
+import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_searchsource_immediate';
+import { getExportType as getTypeCsv } from '../export_types/csv_searchsource';
import { getExportType as getTypePng } from '../export_types/png';
import { getExportType as getTypePrintablePdf } from '../export_types/printable_pdf';
import { CreateJobFn, ExportTypeDefinition } from '../types';
@@ -82,8 +83,9 @@ export function getExportTypesRegistry(): ExportTypesRegistry {
const registry = new ExportTypesRegistry();
type CreateFnType = CreateJobFn; // can not specify params types because different type of params are not assignable to each other
type RunFnType = any; // can not specify because ImmediateExecuteFn is not assignable to RunTaskFn
- const getTypeFns: Array<() => ExportTypeDefinition> = [
+ const getTypeFns: Array<() => ExportTypeDefinition> = [
getTypeCsv,
+ getTypeCsvDeprecated,
getTypeCsvFromSavedObject,
getTypePng,
getTypePrintablePdf,
diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts
index 7d30ae78e5c84..6605ec37e57f8 100644
--- a/x-pack/plugins/reporting/server/plugin.ts
+++ b/x-pack/plugins/reporting/server/plugin.ts
@@ -118,11 +118,13 @@ export class ReportingPlugin
const esqueue = await createQueueFactory(reportingCore, store, logger); // starts polling for pending jobs
reportingCore.pluginStart({
+ data: plugins.data,
browserDriverFactory,
savedObjects: core.savedObjects,
uiSettings: core.uiSettings,
esqueue,
store,
+ esClient: core.elasticsearch.client,
});
this.logger.debug('Start complete');
diff --git a/x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts b/x-pack/plugins/reporting/server/routes/csv_searchsource_immediate.ts
similarity index 55%
rename from x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts
rename to x-pack/plugins/reporting/server/routes/csv_searchsource_immediate.ts
index 6d000cffb9195..55092b5236ce6 100644
--- a/x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts
+++ b/x-pack/plugins/reporting/server/routes/csv_searchsource_immediate.ts
@@ -8,26 +8,17 @@
import { schema } from '@kbn/config-schema';
import { KibanaRequest } from 'src/core/server';
import { ReportingCore } from '../';
-import { createJobFnFactory } from '../export_types/csv_from_savedobject/create_job';
-import { runTaskFnFactory } from '../export_types/csv_from_savedobject/execute_job';
-import {
- JobParamsPanelCsv,
- JobParamsPanelCsvPost,
-} from '../export_types/csv_from_savedobject/types';
+import { runTaskFnFactory } from '../export_types/csv_searchsource_immediate/execute_job';
+import { JobParamsDownloadCSV } from '../export_types/csv_searchsource_immediate/types';
import { LevelLogger as Logger } from '../lib';
import { TaskRunResult } from '../lib/tasks';
import { authorizedUserPreRoutingFactory } from './lib/authorized_user_pre_routing';
-import { getJobParamsFromRequest } from './lib/get_job_params_from_request';
import { HandlerErrorFunction } from './types';
const API_BASE_URL_V1 = '/api/reporting/v1';
const API_BASE_GENERATE_V1 = `${API_BASE_URL_V1}/generate`;
-export type CsvFromSavedObjectRequest = KibanaRequest<
- JobParamsPanelCsv,
- unknown,
- JobParamsPanelCsvPost
->;
+export type CsvFromSavedObjectRequest = KibanaRequest;
/*
* This function registers API Endpoints for immediate Reporting jobs. The API inputs are:
@@ -47,43 +38,28 @@ export function registerGenerateCsvFromSavedObjectImmediate(
const userHandler = authorizedUserPreRoutingFactory(reporting);
const { router } = setupDeps;
- /*
- * CSV export with the `immediate` option does not queue a job with Reporting's ESQueue to run the job async. Instead, this does:
- * - re-use the createJob function to build up es query config
- * - re-use the runTask function to run the scan and scroll queries and capture the entire CSV in a result object.
- */
+ // This API calls run the SearchSourceImmediate export type's runTaskFn directly
router.post(
{
- path: `${API_BASE_GENERATE_V1}/immediate/csv/saved-object/{savedObjectType}:{savedObjectId}`,
+ path: `${API_BASE_GENERATE_V1}/immediate/csv_searchsource`,
validate: {
- params: schema.object({
- savedObjectType: schema.string({ minLength: 5 }),
- savedObjectId: schema.string({ minLength: 5 }),
- }),
body: schema.object({
- state: schema.object({}, { unknowns: 'allow' }),
- timerange: schema.object({
- timezone: schema.string({ defaultValue: 'UTC' }),
- min: schema.nullable(schema.oneOf([schema.number(), schema.string({ minLength: 5 })])),
- max: schema.nullable(schema.oneOf([schema.number(), schema.string({ minLength: 5 })])),
- }),
+ searchSource: schema.object({}, { unknowns: 'allow' }),
+ browserTimezone: schema.string({ defaultValue: 'UTC' }),
+ title: schema.string(),
}),
},
},
userHandler(async (user, context, req: CsvFromSavedObjectRequest, res) => {
- const logger = parentLogger.clone(['savedobject-csv']);
- const jobParams = getJobParamsFromRequest(req);
- const createJob = createJobFnFactory(reporting, logger);
+ const logger = parentLogger.clone(['csv_searchsource_immediate']);
const runTaskFn = runTaskFnFactory(reporting, logger);
try {
- // FIXME: no create job for immediate download
- const payload = await createJob(jobParams, context, req);
const {
content_type: jobOutputContentType,
content: jobOutputContent,
size: jobOutputSize,
- }: TaskRunResult = await runTaskFn(null, payload, context, req);
+ }: TaskRunResult = await runTaskFn(null, req.body, context, req);
logger.info(`Job output size: ${jobOutputSize} bytes`);
diff --git a/x-pack/plugins/reporting/server/routes/generation.ts b/x-pack/plugins/reporting/server/routes/generation.ts
index c7ebd969fc01a..c4016e3f81edd 100644
--- a/x-pack/plugins/reporting/server/routes/generation.ts
+++ b/x-pack/plugins/reporting/server/routes/generation.ts
@@ -13,7 +13,7 @@ import { API_BASE_URL } from '../../common/constants';
import { LevelLogger as Logger } from '../lib';
import { enqueueJobFactory } from '../lib/enqueue_job';
import { registerGenerateFromJobParams } from './generate_from_jobparams';
-import { registerGenerateCsvFromSavedObjectImmediate } from './generate_from_savedobject_immediate';
+import { registerGenerateCsvFromSavedObjectImmediate } from './csv_searchsource_immediate';
import { HandlerFunction } from './types';
const esErrors = elasticsearchErrors as Record;
diff --git a/x-pack/plugins/reporting/server/routes/lib/get_document_payload.ts b/x-pack/plugins/reporting/server/routes/lib/get_document_payload.ts
index 26f1a64a7ef63..7ae6159863970 100644
--- a/x-pack/plugins/reporting/server/routes/lib/get_document_payload.ts
+++ b/x-pack/plugins/reporting/server/routes/lib/get_document_payload.ts
@@ -8,7 +8,7 @@
// @ts-ignore
import contentDisposition from 'content-disposition';
import { get } from 'lodash';
-import { CSV_JOB_TYPE_DEPRECATED } from '../../../common/constants';
+import { CSV_JOB_TYPE, CSV_JOB_TYPE_DEPRECATED } from '../../../common/constants';
import { ExportTypesRegistry, statuses } from '../../lib';
import { ReportDocument } from '../../lib/store';
import { TaskRunResult } from '../../lib/tasks';
@@ -34,11 +34,13 @@ const getTitle = (exportType: ExportTypeDefinition, title?: string): string =>
const getReportingHeaders = (output: TaskRunResult, exportType: ExportTypeDefinition) => {
const metaDataHeaders: Record = {};
- if (exportType.jobType === CSV_JOB_TYPE_DEPRECATED) {
+ if (exportType.jobType === CSV_JOB_TYPE || exportType.jobType === CSV_JOB_TYPE_DEPRECATED) {
const csvContainsFormulas = get(output, 'csv_contains_formulas', false);
+ const csvRowCount = get(output, 'csv_row_count', false);
const maxSizedReach = get(output, 'max_size_reached', false);
metaDataHeaders['kbn-csv-contains-formulas'] = csvContainsFormulas;
+ metaDataHeaders['kbn-csv-num-rows'] = csvRowCount;
metaDataHeaders['kbn-max-size-reached'] = maxSizedReach;
}
diff --git a/x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts b/x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts
deleted file mode 100644
index 8dce491e3df09..0000000000000
--- a/x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { JobParamsPanelCsv } from '../../export_types/csv_from_savedobject/types';
-import { CsvFromSavedObjectRequest } from '../generate_from_savedobject_immediate';
-
-export function getJobParamsFromRequest(request: CsvFromSavedObjectRequest): JobParamsPanelCsv {
- const { savedObjectType, savedObjectId } = request.params;
- const { timerange, state } = request.body;
-
- const post = timerange || state ? { timerange, state } : undefined;
-
- return {
- savedObjectType,
- savedObjectId,
- post,
- };
-}
diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts
index ea8480ef3493d..5341db12f6548 100644
--- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts
+++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts
@@ -12,7 +12,10 @@ jest.mock('../lib/create_queue');
import _ from 'lodash';
import * as Rx from 'rxjs';
-import { coreMock } from 'src/core/server/mocks';
+import { coreMock, elasticsearchServiceMock } from 'src/core/server/mocks';
+import { fieldFormats } from 'src/plugins/data/server';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { dataPluginMock } from 'src/plugins/data/server/mocks';
import { ReportingConfig, ReportingCore } from '../';
import { featuresPluginMock } from '../../../features/server/mocks';
import {
@@ -23,6 +26,7 @@ import {
import { ReportingConfigType } from '../config';
import { ReportingInternalSetup, ReportingInternalStart } from '../core';
import { ReportingStore } from '../lib';
+import { setFieldFormats } from '../services';
import { createMockLevelLogger } from './create_mock_levellogger';
(initializeBrowserDriverFactory as jest.Mock<
@@ -45,16 +49,23 @@ export const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup =
const logger = createMockLevelLogger();
-const createMockPluginStart = (
- mockReportingCore: ReportingCore,
+const createMockReportingStore = () => ({} as ReportingStore);
+
+export const createMockPluginStart = (
+ mockReportingCore: ReportingCore | undefined,
startMock?: any
): ReportingInternalStart => {
- const store = new ReportingStore(mockReportingCore, logger);
+ const store = mockReportingCore
+ ? new ReportingStore(mockReportingCore, logger)
+ : createMockReportingStore();
+
return {
browserDriverFactory: startMock.browserDriverFactory,
esqueue: startMock.esqueue,
+ esClient: elasticsearchServiceMock.createClusterClient(),
savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() },
uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) },
+ data: startMock.data || dataPluginMock.createStartContract(),
store,
...startMock,
};
@@ -121,11 +132,18 @@ export const createMockReportingCore = async (
setupDepsMock: ReportingInternalSetup | undefined = undefined,
startDepsMock: ReportingInternalStart | undefined = undefined
) => {
- config = config || {};
+ const mockReportingCore = ({
+ getConfig: () => config,
+ getElasticsearchService: () => setupDepsMock?.elasticsearch,
+ getSearchService: () => startDepsMock?.data?.search,
+ } as unknown) as ReportingCore;
if (!setupDepsMock) {
setupDepsMock = createMockPluginSetup({});
}
+ if (!startDepsMock) {
+ startDepsMock = createMockPluginStart(mockReportingCore, {});
+ }
const context = coreMock.createPluginInitializerContext(createMockConfigSchema());
const core = new ReportingCore(logger);
@@ -140,5 +158,12 @@ export const createMockReportingCore = async (
await core.pluginStart(startDepsMock);
await core.pluginStartsUp();
+ setFieldFormats({
+ fieldFormatServiceFactory() {
+ const fieldFormatsRegistry = new fieldFormats.FieldFormatsRegistry();
+ return Promise.resolve(fieldFormatsRegistry);
+ },
+ });
+
return core;
};
diff --git a/x-pack/plugins/reporting/server/types.ts b/x-pack/plugins/reporting/server/types.ts
index b395738ad4445..f84d2fbdf4712 100644
--- a/x-pack/plugins/reporting/server/types.ts
+++ b/x-pack/plugins/reporting/server/types.ts
@@ -80,13 +80,16 @@ export type RunTaskFnFactory = (
logger: LevelLogger
) => RunTaskFnType;
-export interface ExportTypeDefinition {
+export interface ExportTypeDefinition<
+ CreateJobFnType = CreateJobFn | null,
+ RunTaskFnType = RunTaskFn
+> {
id: string;
name: string;
jobType: string;
jobContentEncoding?: string;
jobContentExtension: string;
- createJobFnFactory: CreateJobFnFactory;
+ createJobFnFactory: CreateJobFnFactory | null; // immediate job does not have a "create" phase
runTaskFnFactory: RunTaskFnFactory;
validLicenses: string[];
}
diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap
index ed2637d7a1bcb..150154fa996c5 100644
--- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap
+++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap
@@ -13,6 +13,10 @@ Object {
"available": true,
"total": 0,
},
+ "csv_searchsource": Object {
+ "available": true,
+ "total": 0,
+ },
"enabled": true,
"last7Days": Object {
"PNG": Object {
@@ -24,6 +28,10 @@ Object {
"available": true,
"total": 0,
},
+ "csv_searchsource": Object {
+ "available": true,
+ "total": 0,
+ },
"printable_pdf": Object {
"app": Object {
"dashboard": 0,
@@ -75,6 +83,10 @@ Object {
"available": true,
"total": 0,
},
+ "csv_searchsource": Object {
+ "available": true,
+ "total": 0,
+ },
"enabled": true,
"last7Days": Object {
"PNG": Object {
@@ -86,6 +98,10 @@ Object {
"available": true,
"total": 0,
},
+ "csv_searchsource": Object {
+ "available": true,
+ "total": 0,
+ },
"printable_pdf": Object {
"app": Object {
"dashboard": 0,
@@ -166,6 +182,10 @@ Object {
"available": true,
"total": 1,
},
+ "csv_searchsource": Object {
+ "available": true,
+ "total": 0,
+ },
"enabled": true,
"last7Days": Object {
"PNG": Object {
@@ -177,6 +197,10 @@ Object {
"available": true,
"total": 1,
},
+ "csv_searchsource": Object {
+ "available": true,
+ "total": 0,
+ },
"printable_pdf": Object {
"app": Object {
"canvas workpad": 1,
diff --git a/x-pack/plugins/reporting/server/usage/decorate_range_stats.ts b/x-pack/plugins/reporting/server/usage/decorate_range_stats.ts
index a750e9e196b20..9fc0141ab742e 100644
--- a/x-pack/plugins/reporting/server/usage/decorate_range_stats.ts
+++ b/x-pack/plugins/reporting/server/usage/decorate_range_stats.ts
@@ -6,7 +6,12 @@
*/
import { uniq } from 'lodash';
-import { CSV_JOB_TYPE_DEPRECATED, PDF_JOB_TYPE, PNG_JOB_TYPE } from '../../common/constants';
+import {
+ CSV_JOB_TYPE,
+ CSV_JOB_TYPE_DEPRECATED,
+ PDF_JOB_TYPE,
+ PNG_JOB_TYPE,
+} from '../../common/constants';
import { AvailableTotal, ExportType, FeatureAvailabilityMap, RangeStats } from './types';
function getForFeature(
@@ -55,6 +60,7 @@ export const decorateRangeStats = (
// combine the known types with any unknown type found in reporting data
const keysBasic = uniq([
+ CSV_JOB_TYPE,
CSV_JOB_TYPE_DEPRECATED,
PNG_JOB_TYPE,
...Object.keys(rangeStatsBasic),
diff --git a/x-pack/test/functional/apps/dashboard/reporting/__snapshots__/download_csv.snap b/x-pack/test/functional/apps/dashboard/reporting/__snapshots__/download_csv.snap
new file mode 100644
index 0000000000000..5df05de3b9fd9
--- /dev/null
+++ b/x-pack/test/functional/apps/dashboard/reporting/__snapshots__/download_csv.snap
@@ -0,0 +1,47 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`dashboard Reporting Download CSV E-Commerce Data Downloads a CSV export of a saved search panel 1`] = `
+"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"order_date\\",\\"products.created_on\\",sku
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
+"
+`;
+
+exports[`dashboard Reporting Download CSV E-Commerce Data Downloads a filtered CSV export of a saved search panel 1`] = `
+"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"order_date\\",\\"products.created_on\\",sku
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
+"
+`;
+
+exports[`dashboard Reporting Download CSV Field Formatters and Scripted Fields Downloads a CSV export of a saved search panel 1`] = `
+"date,\\"_id\\",name,gender,value,year,\\"years_ago\\",\\"date_informal\\"
+\\"Jan 1, 1984 @ 00:00:00.000\\",\\"1984-Fethany-F\\",Fethany,F,5,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"Jan 1, 1983 @ 00:00:00.000\\",\\"1983-Fethany-F\\",Fethany,F,\\"1,043\\",1983,\\"36.00000000000000000000\\",\\"Jan 1st 83\\"
+\\"Jan 1, 1982 @ 00:00:00.000\\",\\"1982-Fethany-F\\",Fethany,F,780,1982,\\"37.00000000000000000000\\",\\"Jan 1st 82\\"
+\\"Jan 1, 1981 @ 00:00:00.000\\",\\"1981-Fethany-F\\",Fethany,F,655,1981,\\"38.00000000000000000000\\",\\"Jan 1st 81\\"
+\\"Jan 1, 1980 @ 00:00:00.000\\",\\"1980-Fethany-F\\",Fethany,F,702,1980,\\"39.00000000000000000000\\",\\"Jan 1st 80\\"
+"
+`;
diff --git a/x-pack/test/functional/apps/dashboard/reporting/download_csv.ts b/x-pack/test/functional/apps/dashboard/reporting/download_csv.ts
index 72f07ef90d703..126e0a7006403 100644
--- a/x-pack/test/functional/apps/dashboard/reporting/download_csv.ts
+++ b/x-pack/test/functional/apps/dashboard/reporting/download_csv.ts
@@ -9,91 +9,139 @@ import { REPO_ROOT } from '@kbn/utils';
import expect from '@kbn/expect';
import fs from 'fs';
import path from 'path';
-import * as Rx from 'rxjs';
-import { filter, first, map, timeout } from 'rxjs/operators';
import { FtrProviderContext } from '../../../ftr_provider_context';
-const csvPath = path.resolve(REPO_ROOT, 'target/functional-tests/downloads/Ecommerce Data.csv');
-
-// checks every 100ms for the file to exist in the download dir
-// just wait up to 5 seconds
-const getDownload$ = (filePath: string) => {
- return Rx.interval(100).pipe(
- map(() => fs.existsSync(filePath)),
- filter((value) => value === true),
- first(),
- timeout(5000)
- );
-};
-
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const browser = getService('browser');
const dashboardPanelActions = getService('dashboardPanelActions');
const log = getService('log');
const testSubjects = getService('testSubjects');
+ const filterBar = getService('filterBar');
const find = getService('find');
- const PageObjects = getPageObjects(['reporting', 'common', 'dashboard']);
+ const retry = getService('retry');
+ const PageObjects = getPageObjects(['reporting', 'common', 'dashboard', 'timePicker']);
+
+ const getCsvPath = (name: string) =>
+ path.resolve(REPO_ROOT, `target/functional-tests/downloads/${name}.csv`);
+
+ // checks every 100ms for the file to exist in the download dir
+ // just wait up to 5 seconds
+ const getDownload = (filePath: string) => {
+ return retry.tryForTime(5000, async () => {
+ expect(fs.existsSync(filePath)).to.be(true);
+ return fs.readFileSync(filePath).toString();
+ });
+ };
+
+ const clickActionsMenu = async (headingTestSubj: string) => {
+ const savedSearchPanel = await testSubjects.find('embeddablePanelHeading-' + headingTestSubj);
+ await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
+ };
+
+ const clickDownloadCsv = async () => {
+ log.debug('click "More"');
+ await dashboardPanelActions.clickContextMenuMoreItem();
+
+ const actionItemTestSubj = 'embeddablePanelAction-downloadCsvReport';
+ await testSubjects.existOrFail(actionItemTestSubj); // wait for the full panel to display or else the test runner could click the wrong option!
+ log.debug('click "Download CSV"');
+ await testSubjects.click(actionItemTestSubj);
+ await testSubjects.existOrFail('csvDownloadStarted'); // validate toast panel
+ };
describe('Download CSV', () => {
before('initialize tests', async () => {
log.debug('ReportingPage:initTests');
- await esArchiver.loadIfNeeded('reporting/ecommerce');
- await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
await browser.setWindowSize(1600, 850);
});
- after('clean up archives and previous file download', async () => {
- await esArchiver.unload('reporting/ecommerce');
- await esArchiver.unload('reporting/ecommerce_kibana');
- });
-
afterEach('remove download', () => {
try {
- fs.unlinkSync(csvPath);
+ fs.unlinkSync(getCsvPath('Ecommerce Data'));
} catch (e) {
// it might not have been there to begin with
}
});
- it('Downloads a CSV export of a saved search panel', async function () {
- await PageObjects.common.navigateToApp('dashboard');
- await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
- const savedSearchPanel = await testSubjects.find('embeddablePanelHeading-EcommerceData');
- await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
+ describe('E-Commerce Data', () => {
+ before(async () => {
+ await esArchiver.load('reporting/ecommerce');
+ await esArchiver.load('reporting/ecommerce_kibana');
+ });
+ after(async () => {
+ await esArchiver.unload('reporting/ecommerce');
+ await esArchiver.unload('reporting/ecommerce_kibana');
+ });
- const actionExists = await testSubjects.exists('embeddablePanelAction-downloadCsvReport');
- if (!actionExists) {
- await dashboardPanelActions.clickContextMenuMoreItem();
- }
- await testSubjects.existOrFail('embeddablePanelAction-downloadCsvReport'); // wait for the full panel to display or else the test runner could click the wrong option!
- await testSubjects.click('embeddablePanelAction-downloadCsvReport');
- await testSubjects.existOrFail('csvDownloadStarted'); // validate toast panel
+ it('Downloads a CSV export of a saved search panel', async function () {
+ await PageObjects.common.navigateToApp('dashboard');
+ await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+ await clickActionsMenu('EcommerceData');
+ await clickDownloadCsv();
+
+ const csvFile = await getDownload(getCsvPath('Ecommerce Data'));
+ expectSnapshot(csvFile).toMatch();
+ });
- const fileExists = await getDownload$(csvPath).toPromise();
- expect(fileExists).to.be(true);
+ it('Downloads a filtered CSV export of a saved search panel', async function () {
+ await PageObjects.common.navigateToApp('dashboard');
+ await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
- // no need to validate download contents, API Integration tests do that some different variations
+ // add a filter
+ await filterBar.addFilter('currency', 'is', 'EUR');
+
+ await clickActionsMenu('EcommerceData');
+ await clickDownloadCsv();
+
+ const csvFile = await getDownload(getCsvPath('Ecommerce Data'));
+ expectSnapshot(csvFile).toMatch();
+ });
+
+ it('Gets the correct filename if panel titles are hidden', async () => {
+ await PageObjects.common.navigateToApp('dashboard');
+ await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard Hidden Panel Titles');
+ const savedSearchPanel = await find.byCssSelector(
+ '[data-test-embeddable-id="94eab06f-60ac-4a85-b771-3a8ed475c9bb"]'
+ ); // panel title is hidden
+ await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
+
+ await clickDownloadCsv();
+ await testSubjects.existOrFail('csvDownloadStarted');
+
+ const csvFile = await getDownload(getCsvPath('Ecommerce Data')); // file exists with proper name
+ expect(csvFile).to.not.be(null);
+ });
});
- it('Gets the correct filename if panel titles are hidden', async () => {
- await PageObjects.common.navigateToApp('dashboard');
- await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard Hidden Panel Titles');
- const savedSearchPanel = await find.byCssSelector(
- '[data-test-embeddable-id="94eab06f-60ac-4a85-b771-3a8ed475c9bb"]'
- ); // panel title is hidden
- await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
-
- const actionExists = await testSubjects.exists('embeddablePanelAction-downloadCsvReport');
- if (!actionExists) {
- await dashboardPanelActions.clickContextMenuMoreItem();
- }
- await testSubjects.existOrFail('embeddablePanelAction-downloadCsvReport');
- await testSubjects.click('embeddablePanelAction-downloadCsvReport');
- await testSubjects.existOrFail('csvDownloadStarted');
+ describe('Field Formatters and Scripted Fields', () => {
+ before(async () => {
+ await esArchiver.load('reporting/hugedata');
+ });
+ after(async () => {
+ await esArchiver.unload('reporting/hugedata');
+ });
+
+ it('Downloads a CSV export of a saved search panel', async () => {
+ await PageObjects.common.navigateToApp('dashboard');
+ await PageObjects.dashboard.loadSavedDashboard('names dashboard');
+ await PageObjects.timePicker.setAbsoluteRange(
+ 'Jan 01, 1980 @ 00:00:00.000',
+ 'Dec 31, 1984 @ 23:59:59.000'
+ );
+
+ await PageObjects.common.sleep(1000);
+
+ await filterBar.addFilter('name.keyword', 'is', 'Fethany');
+
+ await PageObjects.common.sleep(1000);
+
+ await clickActionsMenu('namessearch');
+ await clickDownloadCsv();
- const fileExists = await getDownload$(csvPath).toPromise(); // file exists with proper name
- expect(fileExists).to.be(true);
+ const csvFile = await getDownload(getCsvPath('namessearch'));
+ expectSnapshot(csvFile).toMatch();
+ });
});
});
}
diff --git a/x-pack/test/functional/apps/discover/__snapshots__/reporting.snap b/x-pack/test/functional/apps/discover/__snapshots__/reporting.snap
index 43771b00525cc..3c2fd087cfd66 100644
--- a/x-pack/test/functional/apps/discover/__snapshots__/reporting.snap
+++ b/x-pack/test/functional/apps/discover/__snapshots__/reporting.snap
@@ -2,39 +2,48 @@
exports[`discover Discover Generate CSV: archived search generates a report with data 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"order_date\\",\\"products.created_on\\",sku
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\",\\"\\"Women's Accessories\\"\\",\\"\\"Men's Accessories\\"\\"]\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0687606876\\"\\",\\"\\"ZO0290502905\\"\\",\\"\\"ZO0126701267\\"\\",\\"\\"ZO0308503085\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Shoes\\"\\",\\"\\"Women's Clothing\\"\\"]\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0006400064\\"\\",\\"\\"ZO0150601506\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0638206382\\"\\",\\"\\"ZO0038800388\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0297602976\\"\\",\\"\\"ZO0565605656\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0561405614\\"\\",\\"\\"ZO0281602816\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\"]\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0385003850\\"\\",\\"\\"ZO0408604086\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0505605056\\"\\",\\"\\"ZO0513605136\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0276702767\\"\\",\\"\\"ZO0291702917\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0046600466\\"\\",\\"\\"ZO0050800508\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Clothing\\"\\",\\"\\"Men's Shoes\\"\\"]\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0455604556\\"\\",\\"\\"ZO0680806808\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Clothing\\"\\",\\"\\"Women's Shoes\\"\\"]\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0229002290\\"\\",\\"\\"ZO0674406744\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0529905299\\"\\",\\"\\"ZO0617006170\\"\\"]\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
"
`;
exports[`discover Discover Generate CSV: archived search generates a report with filtered data 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"order_date\\",\\"products.created_on\\",sku
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\",\\"\\"Women's Accessories\\"\\",\\"\\"Men's Accessories\\"\\"]\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0687606876\\"\\",\\"\\"ZO0290502905\\"\\",\\"\\"ZO0126701267\\"\\",\\"\\"ZO0308503085\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Shoes\\"\\",\\"\\"Women's Clothing\\"\\"]\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0006400064\\"\\",\\"\\"ZO0150601506\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0638206382\\"\\",\\"\\"ZO0038800388\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0297602976\\"\\",\\"\\"ZO0565605656\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0561405614\\"\\",\\"\\"ZO0281602816\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\"]\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0385003850\\"\\",\\"\\"ZO0408604086\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0505605056\\"\\",\\"\\"ZO0513605136\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0276702767\\"\\",\\"\\"ZO0291702917\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0046600466\\"\\",\\"\\"ZO0050800508\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Clothing\\"\\",\\"\\"Men's Shoes\\"\\"]\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0455604556\\"\\",\\"\\"ZO0680806808\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Clothing\\"\\",\\"\\"Women's Shoes\\"\\"]\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0229002290\\"\\",\\"\\"ZO0674406744\\"\\"]\\"
-\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0529905299\\"\\",\\"\\"ZO0617006170\\"\\"]\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
+\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
"
`;
exports[`discover Discover Generate CSV: new search generates a report with data 1`] = `
-"\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
+"\\"products.manufacturer\\",\\"products.discount_amount\\",\\"products.base_unit_price\\",type,\\"products.discount_percentage\\",\\"products._id.keyword\\",\\"day_of_week_i\\",\\"total_quantity\\",\\"taxless_total_price\\",\\"total_unique_products\\",\\"geoip.continent_name\\",sku,\\"customer_full_name.keyword\\",\\"category.keyword\\",\\"products.taxless_price\\",\\"products.quantity\\",\\"products.price\\",\\"customer_first_name\\",\\"customer_phone\\",\\"geoip.region_name\\",\\"customer_full_name\\",\\"geoip.country_iso_code\\",\\"order_id\\",\\"products._id\\",\\"products.product_name.keyword\\",\\"products.product_id\\",\\"products.category\\",\\"products.manufacturer.keyword\\",manufacturer,\\"products.unit_discount_amount\\",\\"customer_last_name\\",\\"geoip.location\\",\\"products.tax_amount\\",\\"products.product_name\\",\\"products.min_price\\",\\"manufacturer.keyword\\",\\"products.taxful_price\\",currency,\\"products.base_price\\",email,\\"day_of_week\\",\\"products.sku\\",\\"customer_last_name.keyword\\",\\"products.category.keyword\\",\\"geoip.city_name\\",\\"customer_first_name.keyword\\",\\"order_date\\",\\"products.created_on\\",category,\\"customer_id\\",user,\\"customer_gender\\",\\"taxful_total_price\\",\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\"
+\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"0, 0, 0, 0\\",\\"79.99, 59.99, 21.99, 11.99\\",order,\\"0, 0, 0, 0\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",5,4,\\"173.96\\",4,Asia,\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"Sultan Al Boone\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"1, 1, 1, 1\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"Sultan Al\\",,\\"Abu Dhabi\\",\\"Sultan Al Boone\\",AE,716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"0, 0, 0, 0\\",Boone,\\"{
+ \\"\\"coordinates\\"\\": [
+ 54.4,
+ 24.5
+ ],
+ \\"\\"type\\"\\": \\"\\"Point\\"\\"
+}\\",\\"0, 0, 0, 0\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"42.39, 32.99, 10.34, 6.11\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"79.99, 59.99, 21.99, 11.99\\",EUR,\\"79.99, 59.99, 21.99, 11.99\\",\\"sultan al@boone-family.zzz\\",Saturday,\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",Boone,\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Abu Dhabi\\",\\"Sultan Al\\",\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",19,sultan,MALE,\\"173.96\\",3AMtOW0BH63Xcmy432DJ,ecommerce,,\\" - \\"
"
`;
diff --git a/x-pack/test/functional/es_archives/reporting/hugedata/data.json.gz b/x-pack/test/functional/es_archives/reporting/hugedata/data.json.gz
index 83da642be8762..e5fb8a73234e4 100644
Binary files a/x-pack/test/functional/es_archives/reporting/hugedata/data.json.gz and b/x-pack/test/functional/es_archives/reporting/hugedata/data.json.gz differ
diff --git a/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json b/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json
index d36bbc72f4ffa..8580f216a06f6 100644
--- a/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json
+++ b/x-pack/test/functional/es_archives/reporting/hugedata/mappings.json
@@ -7,86 +7,16 @@
},
"index": ".kibana_1",
"mappings": {
- "_meta": {
- "migrationMappingPropertyHashes": {
- "apm-telemetry": "0383a570af33654a51c8a1352417bc6b",
- "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231",
- "config": "87aca8fdb053154f11383fce3dbf3edf",
- "dashboard": "eb3789e1af878e73f85304333240f65f",
- "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1",
- "index-pattern": "66eccb05066c5a89924f48a9e9736499",
- "infrastructure-ui-source": "10acdf67d9a06d462e198282fd6d4b81",
- "kql-telemetry": "d12a98a6f19a2d273696597547e064ee",
- "map": "23d7aa4a720d4938ccde3983f87bd58d",
- "maps-telemetry": "a4229f8b16a6820c6d724b7e0c1f729d",
- "migrationVersion": "4a1746014a75ade3a714e1db5763276f",
- "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9",
- "namespace": "2f4316de49999235636386fe51dc06c1",
- "references": "7997cf5a56cc02bdc9c93361bde732b0",
- "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4",
- "search": "181661168bbadd1eff5902361e2a0d5c",
- "server": "ec97f1c5da1a19609a60874e5af1100c",
- "space": "0d5011d73a0ef2f0f615bb42f26f187e",
- "telemetry": "e1c8bc94e443aefd9458932cc0697a4d",
- "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf",
- "type": "2f4316de49999235636386fe51dc06c1",
- "updated_at": "00da57df13e94e9d98437d13ace4bfe0",
- "upgrade-assistant-reindex-operation": "a53a20fe086b72c9a86da3cc12dad8a6",
- "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b",
- "url": "c7f66a0df8b1b52f17c28c4adb111105",
- "user-action": "0d409297dc5ebe1e3a1da691c6ee32e3",
- "visualization": "52d7a13ad68a150c4525b292d23e12cc"
- }
- },
"dynamic": "strict",
"properties": {
- "apm-telemetry": {
- "properties": {
- "has_any_services": {
- "type": "boolean"
- },
- "services_per_agent": {
- "properties": {
- "go": {
- "null_value": 0,
- "type": "long"
- },
- "java": {
- "null_value": 0,
- "type": "long"
- },
- "js-base": {
- "null_value": 0,
- "type": "long"
- },
- "nodejs": {
- "null_value": 0,
- "type": "long"
- },
- "python": {
- "null_value": 0,
- "type": "long"
- },
- "ruby": {
- "null_value": 0,
- "type": "long"
- },
- "rum-js": {
- "null_value": 0,
- "type": "long"
- }
- }
- }
- }
- },
- "canvas-workpad": {
- "dynamic": "false",
+ "action": {
"properties": {
- "@created": {
- "type": "date"
+ "actionTypeId": {
+ "type": "keyword"
},
- "@timestamp": {
- "type": "date"
+ "config": {
+ "enabled": false,
+ "type": "object"
},
"name": {
"fields": {
@@ -95,391 +25,2208 @@
}
},
"type": "text"
+ },
+ "secrets": {
+ "type": "binary"
}
}
},
- "config": {
- "dynamic": "true",
+ "action_task_params": {
"properties": {
- "buildNum": {
+ "actionId": {
"type": "keyword"
},
- "dateFormat:tz": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "defaultIndex": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
+ "apiKey": {
+ "type": "binary"
},
- "search:queryLanguage": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
+ "params": {
+ "enabled": false,
+ "type": "object"
}
}
},
- "dashboard": {
+ "alert": {
"properties": {
- "description": {
- "type": "text"
- },
- "hits": {
- "type": "integer"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
- },
- "optionsJSON": {
- "type": "text"
- },
- "panelsJSON": {
- "type": "text"
- },
- "refreshInterval": {
+ "actions": {
"properties": {
- "display": {
+ "actionRef": {
"type": "keyword"
},
- "pause": {
- "type": "boolean"
+ "actionTypeId": {
+ "type": "keyword"
},
- "section": {
- "type": "integer"
+ "group": {
+ "type": "keyword"
},
- "value": {
- "type": "integer"
+ "params": {
+ "enabled": false,
+ "type": "object"
}
- }
+ },
+ "type": "nested"
},
- "timeFrom": {
+ "alertTypeId": {
"type": "keyword"
},
- "timeRestore": {
- "type": "boolean"
+ "apiKey": {
+ "type": "binary"
},
- "timeTo": {
+ "apiKeyOwner": {
"type": "keyword"
},
- "title": {
- "type": "text"
+ "consumer": {
+ "type": "keyword"
},
- "uiStateJSON": {
- "type": "text"
+ "createdAt": {
+ "type": "date"
},
- "version": {
- "type": "integer"
- }
- }
- },
- "graph-workspace": {
- "properties": {
- "description": {
- "type": "text"
+ "createdBy": {
+ "type": "keyword"
},
- "kibanaSavedObjectMeta": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "executionStatus": {
"properties": {
- "searchSourceJSON": {
- "type": "text"
+ "error": {
+ "properties": {
+ "message": {
+ "type": "keyword"
+ },
+ "reason": {
+ "type": "keyword"
+ }
+ }
+ },
+ "lastExecutionDate": {
+ "type": "date"
+ },
+ "status": {
+ "type": "keyword"
}
}
},
- "numLinks": {
- "type": "integer"
- },
- "numVertices": {
- "type": "integer"
- },
- "title": {
- "type": "text"
+ "meta": {
+ "properties": {
+ "versionApiKeyLastmodified": {
+ "type": "keyword"
+ }
+ }
},
- "version": {
- "type": "integer"
+ "muteAll": {
+ "type": "boolean"
},
- "wsState": {
- "type": "text"
- }
- }
- },
- "index-pattern": {
- "properties": {
- "fieldFormatMap": {
- "type": "text"
+ "mutedInstanceIds": {
+ "type": "keyword"
},
- "fields": {
+ "name": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
"type": "text"
},
- "intervalName": {
+ "notifyWhen": {
"type": "keyword"
},
- "notExpandable": {
- "type": "boolean"
+ "params": {
+ "enabled": false,
+ "type": "object"
},
- "sourceFilters": {
- "type": "text"
+ "schedule": {
+ "properties": {
+ "interval": {
+ "type": "keyword"
+ }
+ }
},
- "timeFieldName": {
+ "scheduledTaskId": {
"type": "keyword"
},
- "title": {
- "type": "text"
+ "tags": {
+ "type": "keyword"
},
- "type": {
+ "throttle": {
"type": "keyword"
},
- "typeMeta": {
+ "updatedAt": {
+ "type": "date"
+ },
+ "updatedBy": {
"type": "keyword"
}
}
},
- "infrastructure-ui-source": {
+ "api_key_pending_invalidation": {
"properties": {
- "description": {
- "type": "text"
+ "apiKeyId": {
+ "type": "keyword"
},
- "fields": {
+ "createdAt": {
+ "type": "date"
+ }
+ }
+ },
+ "apm-indices": {
+ "properties": {
+ "apm_oss": {
"properties": {
- "container": {
+ "errorIndices": {
"type": "keyword"
},
- "host": {
+ "metricsIndices": {
"type": "keyword"
},
- "pod": {
+ "onboardingIndices": {
"type": "keyword"
},
- "tiebreaker": {
+ "sourcemapIndices": {
"type": "keyword"
},
- "timestamp": {
+ "spanIndices": {
+ "type": "keyword"
+ },
+ "transactionIndices": {
"type": "keyword"
}
}
- },
- "logAlias": {
- "type": "keyword"
- },
- "metricAlias": {
- "type": "keyword"
- },
- "name": {
- "type": "text"
}
}
},
- "kql-telemetry": {
+ "apm-telemetry": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "app_search_telemetry": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "application_usage_daily": {
+ "dynamic": "false",
"properties": {
- "optInCount": {
- "type": "long"
- },
- "optOutCount": {
- "type": "long"
+ "timestamp": {
+ "type": "date"
}
}
},
- "map": {
+ "application_usage_totals": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "application_usage_transactional": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "canvas-element": {
+ "dynamic": "false",
"properties": {
- "bounds": {
- "type": "geo_shape"
+ "@created": {
+ "type": "date"
},
- "description": {
- "type": "text"
+ "@timestamp": {
+ "type": "date"
},
- "layerListJSON": {
+ "content": {
"type": "text"
},
- "mapStateJSON": {
+ "help": {
"type": "text"
},
- "title": {
+ "image": {
"type": "text"
},
- "uiStateJSON": {
+ "name": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
"type": "text"
+ }
+ }
+ },
+ "canvas-workpad": {
+ "dynamic": "false",
+ "properties": {
+ "@created": {
+ "type": "date"
},
- "version": {
- "type": "integer"
+ "@timestamp": {
+ "type": "date"
+ },
+ "name": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
+ "type": "text"
}
}
},
- "maps-telemetry": {
+ "canvas-workpad-template": {
+ "dynamic": "false",
"properties": {
- "attributesPerMap": {
- "properties": {
- "dataSourcesCount": {
- "properties": {
- "avg": {
- "type": "long"
+ "help": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "name": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "tags": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "template_key": {
+ "type": "keyword"
+ }
+ }
+ },
+ "cases": {
+ "properties": {
+ "closed_at": {
+ "type": "date"
+ },
+ "closed_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ },
+ "connector": {
+ "properties": {
+ "fields": {
+ "properties": {
+ "key": {
+ "type": "text"
+ },
+ "value": {
+ "type": "text"
+ }
+ }
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "text"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "created_at": {
+ "type": "date"
+ },
+ "created_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ },
+ "description": {
+ "type": "text"
+ },
+ "external_service": {
+ "properties": {
+ "connector_id": {
+ "type": "keyword"
+ },
+ "connector_name": {
+ "type": "keyword"
+ },
+ "external_id": {
+ "type": "keyword"
+ },
+ "external_title": {
+ "type": "text"
+ },
+ "external_url": {
+ "type": "text"
+ },
+ "pushed_at": {
+ "type": "date"
+ },
+ "pushed_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "settings": {
+ "properties": {
+ "syncAlerts": {
+ "type": "boolean"
+ }
+ }
+ },
+ "status": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
+ },
+ "title": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "updated_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "cases-comments": {
+ "properties": {
+ "alertId": {
+ "type": "keyword"
+ },
+ "comment": {
+ "type": "text"
+ },
+ "created_at": {
+ "type": "date"
+ },
+ "created_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ },
+ "index": {
+ "type": "keyword"
+ },
+ "pushed_at": {
+ "type": "date"
+ },
+ "pushed_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "updated_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "cases-configure": {
+ "properties": {
+ "closure_type": {
+ "type": "keyword"
+ },
+ "connector": {
+ "properties": {
+ "fields": {
+ "properties": {
+ "key": {
+ "type": "text"
+ },
+ "value": {
+ "type": "text"
+ }
+ }
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "text"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "created_at": {
+ "type": "date"
+ },
+ "created_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "updated_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "cases-connector-mappings": {
+ "properties": {
+ "mappings": {
+ "properties": {
+ "action_type": {
+ "type": "keyword"
+ },
+ "source": {
+ "type": "keyword"
+ },
+ "target": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "cases-user-actions": {
+ "properties": {
+ "action": {
+ "type": "keyword"
+ },
+ "action_at": {
+ "type": "date"
+ },
+ "action_by": {
+ "properties": {
+ "email": {
+ "type": "keyword"
+ },
+ "full_name": {
+ "type": "keyword"
+ },
+ "username": {
+ "type": "keyword"
+ }
+ }
+ },
+ "action_field": {
+ "type": "keyword"
+ },
+ "new_value": {
+ "type": "text"
+ },
+ "old_value": {
+ "type": "text"
+ }
+ }
+ },
+ "config": {
+ "dynamic": "false",
+ "properties": {
+ "buildNum": {
+ "type": "keyword"
+ }
+ }
+ },
+ "core-usage-stats": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "dashboard": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "hits": {
+ "doc_values": false,
+ "index": false,
+ "type": "integer"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "index": false,
+ "type": "text"
+ }
+ }
+ },
+ "optionsJSON": {
+ "index": false,
+ "type": "text"
+ },
+ "panelsJSON": {
+ "index": false,
+ "type": "text"
+ },
+ "refreshInterval": {
+ "properties": {
+ "display": {
+ "doc_values": false,
+ "index": false,
+ "type": "keyword"
+ },
+ "pause": {
+ "doc_values": false,
+ "index": false,
+ "type": "boolean"
+ },
+ "section": {
+ "doc_values": false,
+ "index": false,
+ "type": "integer"
+ },
+ "value": {
+ "doc_values": false,
+ "index": false,
+ "type": "integer"
+ }
+ }
+ },
+ "timeFrom": {
+ "doc_values": false,
+ "index": false,
+ "type": "keyword"
+ },
+ "timeRestore": {
+ "doc_values": false,
+ "index": false,
+ "type": "boolean"
+ },
+ "timeTo": {
+ "doc_values": false,
+ "index": false,
+ "type": "keyword"
+ },
+ "title": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "endpoint:user-artifact": {
+ "properties": {
+ "body": {
+ "type": "binary"
+ },
+ "compressionAlgorithm": {
+ "index": false,
+ "type": "keyword"
+ },
+ "created": {
+ "index": false,
+ "type": "date"
+ },
+ "decodedSha256": {
+ "index": false,
+ "type": "keyword"
+ },
+ "decodedSize": {
+ "index": false,
+ "type": "long"
+ },
+ "encodedSha256": {
+ "type": "keyword"
+ },
+ "encodedSize": {
+ "index": false,
+ "type": "long"
+ },
+ "encryptionAlgorithm": {
+ "index": false,
+ "type": "keyword"
+ },
+ "identifier": {
+ "type": "keyword"
+ }
+ }
+ },
+ "endpoint:user-artifact-manifest": {
+ "properties": {
+ "created": {
+ "index": false,
+ "type": "date"
+ },
+ "ids": {
+ "index": false,
+ "type": "keyword"
+ },
+ "schemaVersion": {
+ "type": "keyword"
+ },
+ "semanticVersion": {
+ "index": false,
+ "type": "keyword"
+ }
+ }
+ },
+ "enterprise_search_telemetry": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "epm-packages": {
+ "properties": {
+ "es_index_patterns": {
+ "enabled": false,
+ "type": "object"
+ },
+ "install_source": {
+ "type": "keyword"
+ },
+ "install_started_at": {
+ "type": "date"
+ },
+ "install_status": {
+ "type": "keyword"
+ },
+ "install_version": {
+ "type": "keyword"
+ },
+ "installed_es": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ },
+ "type": "nested"
+ },
+ "installed_kibana": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ },
+ "type": "nested"
+ },
+ "internal": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "package_assets": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ },
+ "type": "nested"
+ },
+ "removable": {
+ "type": "boolean"
+ },
+ "version": {
+ "type": "keyword"
+ }
+ }
+ },
+ "epm-packages-assets": {
+ "properties": {
+ "asset_path": {
+ "type": "keyword"
+ },
+ "data_base64": {
+ "type": "binary"
+ },
+ "data_utf8": {
+ "index": false,
+ "type": "text"
+ },
+ "install_source": {
+ "type": "keyword"
+ },
+ "media_type": {
+ "type": "keyword"
+ },
+ "package_name": {
+ "type": "keyword"
+ },
+ "package_version": {
+ "type": "keyword"
+ }
+ }
+ },
+ "exception-list": {
+ "properties": {
+ "_tags": {
+ "type": "keyword"
+ },
+ "comments": {
+ "properties": {
+ "comment": {
+ "type": "keyword"
+ },
+ "created_at": {
+ "type": "keyword"
+ },
+ "created_by": {
+ "type": "keyword"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "keyword"
+ },
+ "updated_by": {
+ "type": "keyword"
+ }
+ }
+ },
+ "created_at": {
+ "type": "keyword"
+ },
+ "created_by": {
+ "type": "keyword"
+ },
+ "description": {
+ "type": "keyword"
+ },
+ "entries": {
+ "properties": {
+ "entries": {
+ "properties": {
+ "field": {
+ "type": "keyword"
+ },
+ "operator": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "value": {
+ "fields": {
+ "text": {
+ "type": "text"
+ }
+ },
+ "type": "keyword"
+ }
+ }
+ },
+ "field": {
+ "type": "keyword"
+ },
+ "list": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "operator": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "value": {
+ "fields": {
+ "text": {
+ "type": "text"
+ }
+ },
+ "type": "keyword"
+ }
+ }
+ },
+ "immutable": {
+ "type": "boolean"
+ },
+ "item_id": {
+ "type": "keyword"
+ },
+ "list_id": {
+ "type": "keyword"
+ },
+ "list_type": {
+ "type": "keyword"
+ },
+ "meta": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "os_types": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
+ },
+ "tie_breaker_id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "updated_by": {
+ "type": "keyword"
+ },
+ "version": {
+ "type": "keyword"
+ }
+ }
+ },
+ "exception-list-agnostic": {
+ "properties": {
+ "_tags": {
+ "type": "keyword"
+ },
+ "comments": {
+ "properties": {
+ "comment": {
+ "type": "keyword"
+ },
+ "created_at": {
+ "type": "keyword"
+ },
+ "created_by": {
+ "type": "keyword"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "keyword"
+ },
+ "updated_by": {
+ "type": "keyword"
+ }
+ }
+ },
+ "created_at": {
+ "type": "keyword"
+ },
+ "created_by": {
+ "type": "keyword"
+ },
+ "description": {
+ "type": "keyword"
+ },
+ "entries": {
+ "properties": {
+ "entries": {
+ "properties": {
+ "field": {
+ "type": "keyword"
+ },
+ "operator": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "value": {
+ "fields": {
+ "text": {
+ "type": "text"
+ }
+ },
+ "type": "keyword"
+ }
+ }
+ },
+ "field": {
+ "type": "keyword"
+ },
+ "list": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "operator": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "value": {
+ "fields": {
+ "text": {
+ "type": "text"
+ }
+ },
+ "type": "keyword"
+ }
+ }
+ },
+ "immutable": {
+ "type": "boolean"
+ },
+ "item_id": {
+ "type": "keyword"
+ },
+ "list_id": {
+ "type": "keyword"
+ },
+ "list_type": {
+ "type": "keyword"
+ },
+ "meta": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "os_types": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
+ },
+ "tie_breaker_id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "updated_by": {
+ "type": "keyword"
+ },
+ "version": {
+ "type": "keyword"
+ }
+ }
+ },
+ "file-upload-telemetry": {
+ "properties": {
+ "filesUploadedTotalCount": {
+ "type": "long"
+ }
+ }
+ },
+ "file-upload-usage-collection-telemetry": {
+ "properties": {
+ "file_upload": {
+ "properties": {
+ "index_creation_count": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
+ "fleet-agent-actions": {
+ "properties": {
+ "ack_data": {
+ "type": "text"
+ },
+ "agent_id": {
+ "type": "keyword"
+ },
+ "created_at": {
+ "type": "date"
+ },
+ "data": {
+ "type": "binary"
+ },
+ "policy_id": {
+ "type": "keyword"
+ },
+ "policy_revision": {
+ "type": "integer"
+ },
+ "sent_at": {
+ "type": "date"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "fleet-agent-events": {
+ "properties": {
+ "action_id": {
+ "type": "keyword"
+ },
+ "agent_id": {
+ "type": "keyword"
+ },
+ "data": {
+ "type": "text"
+ },
+ "message": {
+ "type": "text"
+ },
+ "payload": {
+ "type": "text"
+ },
+ "policy_id": {
+ "type": "keyword"
+ },
+ "stream_id": {
+ "type": "keyword"
+ },
+ "subtype": {
+ "type": "keyword"
+ },
+ "timestamp": {
+ "type": "date"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "fleet-agents": {
+ "properties": {
+ "access_api_key_id": {
+ "type": "keyword"
+ },
+ "active": {
+ "type": "boolean"
+ },
+ "current_error_events": {
+ "index": false,
+ "type": "text"
+ },
+ "default_api_key": {
+ "type": "binary"
+ },
+ "default_api_key_id": {
+ "type": "keyword"
+ },
+ "enrolled_at": {
+ "type": "date"
+ },
+ "last_checkin": {
+ "type": "date"
+ },
+ "last_checkin_status": {
+ "type": "keyword"
+ },
+ "last_updated": {
+ "type": "date"
+ },
+ "local_metadata": {
+ "type": "flattened"
+ },
+ "packages": {
+ "type": "keyword"
+ },
+ "policy_id": {
+ "type": "keyword"
+ },
+ "policy_revision": {
+ "type": "integer"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "unenrolled_at": {
+ "type": "date"
+ },
+ "unenrollment_started_at": {
+ "type": "date"
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "upgrade_started_at": {
+ "type": "date"
+ },
+ "upgraded_at": {
+ "type": "date"
+ },
+ "user_provided_metadata": {
+ "type": "flattened"
+ },
+ "version": {
+ "type": "keyword"
+ }
+ }
+ },
+ "fleet-enrollment-api-keys": {
+ "properties": {
+ "active": {
+ "type": "boolean"
+ },
+ "api_key": {
+ "type": "binary"
+ },
+ "api_key_id": {
+ "type": "keyword"
+ },
+ "created_at": {
+ "type": "date"
+ },
+ "expire_at": {
+ "type": "date"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "policy_id": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "date"
+ }
+ }
+ },
+ "graph-workspace": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "type": "text"
+ }
+ }
+ },
+ "legacyIndexPatternRef": {
+ "index": false,
+ "type": "text"
+ },
+ "numLinks": {
+ "type": "integer"
+ },
+ "numVertices": {
+ "type": "integer"
+ },
+ "title": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ },
+ "wsState": {
+ "type": "text"
+ }
+ }
+ },
+ "index-pattern": {
+ "dynamic": "false",
+ "properties": {
+ "title": {
+ "type": "text"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "infrastructure-ui-source": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "ingest-agent-policies": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "is_default": {
+ "type": "boolean"
+ },
+ "is_managed": {
+ "type": "boolean"
+ },
+ "monitoring_enabled": {
+ "index": false,
+ "type": "keyword"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "namespace": {
+ "type": "keyword"
+ },
+ "package_policies": {
+ "type": "keyword"
+ },
+ "revision": {
+ "type": "integer"
+ },
+ "status": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "updated_by": {
+ "type": "keyword"
+ }
+ }
+ },
+ "ingest-outputs": {
+ "properties": {
+ "ca_sha256": {
+ "index": false,
+ "type": "keyword"
+ },
+ "config": {
+ "type": "flattened"
+ },
+ "config_yaml": {
+ "type": "text"
+ },
+ "fleet_enroll_password": {
+ "type": "binary"
+ },
+ "fleet_enroll_username": {
+ "type": "binary"
+ },
+ "hosts": {
+ "type": "keyword"
+ },
+ "is_default": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "ingest-package-policies": {
+ "properties": {
+ "created_at": {
+ "type": "date"
+ },
+ "created_by": {
+ "type": "keyword"
+ },
+ "description": {
+ "type": "text"
+ },
+ "enabled": {
+ "type": "boolean"
+ },
+ "inputs": {
+ "enabled": false,
+ "properties": {
+ "compiled_input": {
+ "type": "flattened"
+ },
+ "config": {
+ "type": "flattened"
+ },
+ "enabled": {
+ "type": "boolean"
+ },
+ "streams": {
+ "properties": {
+ "compiled_stream": {
+ "type": "flattened"
+ },
+ "config": {
+ "type": "flattened"
+ },
+ "data_stream": {
+ "properties": {
+ "dataset": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "enabled": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "vars": {
+ "type": "flattened"
+ }
+ },
+ "type": "nested"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "vars": {
+ "type": "flattened"
+ }
+ },
+ "type": "nested"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "namespace": {
+ "type": "keyword"
+ },
+ "output_id": {
+ "type": "keyword"
+ },
+ "package": {
+ "properties": {
+ "name": {
+ "type": "keyword"
+ },
+ "title": {
+ "type": "keyword"
+ },
+ "version": {
+ "type": "keyword"
+ }
+ }
+ },
+ "policy_id": {
+ "type": "keyword"
+ },
+ "revision": {
+ "type": "integer"
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "updated_by": {
+ "type": "keyword"
+ }
+ }
+ },
+ "ingest_manager_settings": {
+ "properties": {
+ "agent_auto_upgrade": {
+ "type": "keyword"
+ },
+ "has_seen_add_data_notice": {
+ "index": false,
+ "type": "boolean"
+ },
+ "kibana_ca_sha256": {
+ "type": "keyword"
+ },
+ "kibana_urls": {
+ "type": "keyword"
+ },
+ "package_auto_upgrade": {
+ "type": "keyword"
+ }
+ }
+ },
+ "inventory-view": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "kql-telemetry": {
+ "properties": {
+ "optInCount": {
+ "type": "long"
+ },
+ "optOutCount": {
+ "type": "long"
+ }
+ }
+ },
+ "legacy-url-alias": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "lens": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "expression": {
+ "doc_values": false,
+ "index": false,
+ "type": "keyword"
+ },
+ "state": {
+ "type": "flattened"
+ },
+ "title": {
+ "type": "text"
+ },
+ "visualizationType": {
+ "type": "keyword"
+ }
+ }
+ },
+ "lens-ui-telemetry": {
+ "properties": {
+ "count": {
+ "type": "integer"
+ },
+ "date": {
+ "type": "date"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "map": {
+ "properties": {
+ "bounds": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "description": {
+ "type": "text"
+ },
+ "layerListJSON": {
+ "type": "text"
+ },
+ "mapStateJSON": {
+ "type": "text"
+ },
+ "title": {
+ "type": "text"
+ },
+ "uiStateJSON": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "maps-telemetry": {
+ "enabled": false,
+ "type": "object"
+ },
+ "metrics-explorer-view": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "ml-job": {
+ "properties": {
+ "datafeed_id": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "job_id": {
+ "fields": {
+ "keyword": {
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "ml-telemetry": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "monitoring-telemetry": {
+ "properties": {
+ "reportedClusterUuids": {
+ "type": "keyword"
+ }
+ }
+ },
+ "namespace": {
+ "type": "keyword"
+ },
+ "namespaces": {
+ "type": "keyword"
+ },
+ "originId": {
+ "type": "keyword"
+ },
+ "query": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "filters": {
+ "enabled": false,
+ "type": "object"
+ },
+ "query": {
+ "properties": {
+ "language": {
+ "type": "keyword"
+ },
+ "query": {
+ "index": false,
+ "type": "keyword"
+ }
+ }
+ },
+ "timefilter": {
+ "enabled": false,
+ "type": "object"
+ },
+ "title": {
+ "type": "text"
+ }
+ }
+ },
+ "references": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ },
+ "type": "nested"
+ },
+ "sample-data-telemetry": {
+ "properties": {
+ "installCount": {
+ "type": "long"
+ },
+ "unInstallCount": {
+ "type": "long"
+ }
+ }
+ },
+ "search": {
+ "properties": {
+ "columns": {
+ "doc_values": false,
+ "index": false,
+ "type": "keyword"
+ },
+ "description": {
+ "type": "text"
+ },
+ "grid": {
+ "enabled": false,
+ "type": "object"
+ },
+ "hits": {
+ "doc_values": false,
+ "index": false,
+ "type": "integer"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "index": false,
+ "type": "text"
+ }
+ }
+ },
+ "pre712": {
+ "type": "boolean"
+ },
+ "sort": {
+ "doc_values": false,
+ "index": false,
+ "type": "keyword"
+ },
+ "title": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "search-session": {
+ "properties": {
+ "appId": {
+ "type": "keyword"
+ },
+ "created": {
+ "type": "date"
+ },
+ "expires": {
+ "type": "date"
+ },
+ "idMapping": {
+ "enabled": false,
+ "type": "object"
+ },
+ "initialState": {
+ "enabled": false,
+ "type": "object"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "persisted": {
+ "type": "boolean"
+ },
+ "restoreState": {
+ "enabled": false,
+ "type": "object"
+ },
+ "sessionId": {
+ "type": "keyword"
+ },
+ "status": {
+ "type": "keyword"
+ },
+ "touched": {
+ "type": "date"
+ },
+ "urlGeneratorId": {
+ "type": "keyword"
+ }
+ }
+ },
+ "search-telemetry": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "security-solution-signals-migration": {
+ "properties": {
+ "created": {
+ "index": false,
+ "type": "date"
+ },
+ "createdBy": {
+ "index": false,
+ "type": "text"
+ },
+ "destinationIndex": {
+ "index": false,
+ "type": "keyword"
+ },
+ "error": {
+ "index": false,
+ "type": "text"
+ },
+ "sourceIndex": {
+ "type": "keyword"
+ },
+ "status": {
+ "index": false,
+ "type": "keyword"
+ },
+ "taskId": {
+ "index": false,
+ "type": "keyword"
+ },
+ "updated": {
+ "index": false,
+ "type": "date"
+ },
+ "updatedBy": {
+ "index": false,
+ "type": "text"
+ },
+ "version": {
+ "type": "long"
+ }
+ }
+ },
+ "server": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "siem-detection-engine-rule-actions": {
+ "properties": {
+ "actions": {
+ "properties": {
+ "action_type_id": {
+ "type": "keyword"
+ },
+ "group": {
+ "type": "keyword"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "params": {
+ "enabled": false,
+ "type": "object"
+ }
+ }
+ },
+ "alertThrottle": {
+ "type": "keyword"
+ },
+ "ruleAlertId": {
+ "type": "keyword"
+ },
+ "ruleThrottle": {
+ "type": "keyword"
+ }
+ }
+ },
+ "siem-detection-engine-rule-status": {
+ "properties": {
+ "alertId": {
+ "type": "keyword"
+ },
+ "bulkCreateTimeDurations": {
+ "type": "float"
+ },
+ "gap": {
+ "type": "text"
+ },
+ "lastFailureAt": {
+ "type": "date"
+ },
+ "lastFailureMessage": {
+ "type": "text"
+ },
+ "lastLookBackDate": {
+ "type": "date"
+ },
+ "lastSuccessAt": {
+ "type": "date"
+ },
+ "lastSuccessMessage": {
+ "type": "text"
+ },
+ "searchAfterTimeDurations": {
+ "type": "float"
+ },
+ "status": {
+ "type": "keyword"
+ },
+ "statusDate": {
+ "type": "date"
+ }
+ }
+ },
+ "siem-ui-timeline": {
+ "properties": {
+ "columns": {
+ "properties": {
+ "aggregatable": {
+ "type": "boolean"
+ },
+ "category": {
+ "type": "keyword"
+ },
+ "columnHeaderType": {
+ "type": "keyword"
+ },
+ "description": {
+ "type": "text"
+ },
+ "example": {
+ "type": "text"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "indexes": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "text"
+ },
+ "placeholder": {
+ "type": "text"
+ },
+ "searchable": {
+ "type": "boolean"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "created": {
+ "type": "date"
+ },
+ "createdBy": {
+ "type": "text"
+ },
+ "dataProviders": {
+ "properties": {
+ "and": {
+ "properties": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "excluded": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "kqlQuery": {
+ "type": "text"
+ },
+ "name": {
+ "type": "text"
+ },
+ "queryMatch": {
+ "properties": {
+ "displayField": {
+ "type": "text"
+ },
+ "displayValue": {
+ "type": "text"
+ },
+ "field": {
+ "type": "text"
+ },
+ "operator": {
+ "type": "text"
+ },
+ "value": {
+ "type": "text"
+ }
+ }
+ },
+ "type": {
+ "type": "text"
+ }
+ }
+ },
+ "enabled": {
+ "type": "boolean"
+ },
+ "excluded": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "kqlQuery": {
+ "type": "text"
+ },
+ "name": {
+ "type": "text"
+ },
+ "queryMatch": {
+ "properties": {
+ "displayField": {
+ "type": "text"
+ },
+ "displayValue": {
+ "type": "text"
+ },
+ "field": {
+ "type": "text"
+ },
+ "operator": {
+ "type": "text"
+ },
+ "value": {
+ "type": "text"
+ }
+ }
+ },
+ "type": {
+ "type": "text"
+ }
+ }
+ },
+ "dateRange": {
+ "properties": {
+ "end": {
+ "type": "date"
+ },
+ "start": {
+ "type": "date"
+ }
+ }
+ },
+ "description": {
+ "type": "text"
+ },
+ "eventType": {
+ "type": "keyword"
+ },
+ "excludedRowRendererIds": {
+ "type": "text"
+ },
+ "favorite": {
+ "properties": {
+ "favoriteDate": {
+ "type": "date"
+ },
+ "fullName": {
+ "type": "text"
+ },
+ "keySearch": {
+ "type": "text"
+ },
+ "userName": {
+ "type": "text"
+ }
+ }
+ },
+ "filters": {
+ "properties": {
+ "exists": {
+ "type": "text"
+ },
+ "match_all": {
+ "type": "text"
+ },
+ "meta": {
+ "properties": {
+ "alias": {
+ "type": "text"
+ },
+ "controlledBy": {
+ "type": "text"
+ },
+ "disabled": {
+ "type": "boolean"
+ },
+ "field": {
+ "type": "text"
+ },
+ "formattedValue": {
+ "type": "text"
+ },
+ "index": {
+ "type": "keyword"
+ },
+ "key": {
+ "type": "keyword"
+ },
+ "negate": {
+ "type": "boolean"
+ },
+ "params": {
+ "type": "text"
},
- "max": {
- "type": "long"
+ "type": {
+ "type": "keyword"
},
- "min": {
- "type": "long"
+ "value": {
+ "type": "text"
}
}
},
- "emsVectorLayersCount": {
- "dynamic": "true",
- "type": "object"
+ "missing": {
+ "type": "text"
},
- "layerTypesCount": {
- "dynamic": "true",
- "type": "object"
+ "query": {
+ "type": "text"
+ },
+ "range": {
+ "type": "text"
},
- "layersCount": {
+ "script": {
+ "type": "text"
+ }
+ }
+ },
+ "indexNames": {
+ "type": "text"
+ },
+ "kqlMode": {
+ "type": "keyword"
+ },
+ "kqlQuery": {
+ "properties": {
+ "filterQuery": {
"properties": {
- "avg": {
- "type": "long"
- },
- "max": {
- "type": "long"
+ "kuery": {
+ "properties": {
+ "expression": {
+ "type": "text"
+ },
+ "kind": {
+ "type": "keyword"
+ }
+ }
},
- "min": {
- "type": "long"
+ "serializedQuery": {
+ "type": "text"
}
}
}
}
},
- "mapsTotalCount": {
- "type": "long"
+ "savedQueryId": {
+ "type": "keyword"
},
- "timeCaptured": {
- "type": "date"
- }
- }
- },
- "migrationVersion": {
- "dynamic": "true",
- "properties": {
- "dashboard": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
+ "sort": {
+ "dynamic": "false",
+ "properties": {
+ "columnId": {
"type": "keyword"
- }
- },
- "type": "text"
- },
- "index-pattern": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
+ },
+ "columnType": {
+ "type": "keyword"
+ },
+ "sortDirection": {
"type": "keyword"
}
- },
+ }
+ },
+ "status": {
+ "type": "keyword"
+ },
+ "templateTimelineId": {
"type": "text"
},
- "search": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
+ "templateTimelineVersion": {
+ "type": "integer"
+ },
+ "timelineType": {
+ "type": "keyword"
+ },
+ "title": {
"type": "text"
},
- "visualization": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
+ "updated": {
+ "type": "date"
+ },
+ "updatedBy": {
"type": "text"
}
}
},
- "ml-telemetry": {
- "properties": {
- "file_data_visualizer": {
- "properties": {
- "index_creation_count": {
- "type": "long"
- }
- }
- }
- }
- },
- "namespace": {
- "type": "keyword"
- },
- "references": {
+ "siem-ui-timeline-note": {
"properties": {
- "id": {
- "type": "keyword"
+ "created": {
+ "type": "date"
},
- "name": {
+ "createdBy": {
+ "type": "text"
+ },
+ "eventId": {
"type": "keyword"
},
- "type": {
+ "note": {
+ "type": "text"
+ },
+ "timelineId": {
"type": "keyword"
- }
- },
- "type": "nested"
- },
- "sample-data-telemetry": {
- "properties": {
- "installCount": {
- "type": "long"
},
- "unInstallCount": {
- "type": "long"
+ "updated": {
+ "type": "date"
+ },
+ "updatedBy": {
+ "type": "text"
}
}
},
- "search": {
+ "siem-ui-timeline-pinned-event": {
"properties": {
- "columns": {
- "type": "keyword"
+ "created": {
+ "type": "date"
},
- "description": {
+ "createdBy": {
"type": "text"
},
- "hits": {
- "type": "integer"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
+ "eventId": {
+ "type": "keyword"
},
- "sort": {
+ "timelineId": {
"type": "keyword"
},
- "title": {
- "type": "text"
+ "updated": {
+ "type": "date"
},
- "version": {
- "type": "integer"
- }
- }
- },
- "server": {
- "properties": {
- "uuid": {
- "type": "keyword"
+ "updatedBy": {
+ "type": "text"
}
}
},
@@ -488,9 +2235,6 @@
"_reserved": {
"type": "boolean"
},
- "disabledFeatures": {
- "type": "keyword"
- },
"color": {
"type": "keyword"
},
@@ -500,6 +2244,10 @@
"disabledFeatures": {
"type": "keyword"
},
+ "imageUrl": {
+ "index": false,
+ "type": "text"
+ },
"initials": {
"type": "keyword"
},
@@ -514,10 +2262,48 @@
}
}
},
+ "spaces-usage-stats": {
+ "dynamic": "false",
+ "type": "object"
+ },
+ "tag": {
+ "properties": {
+ "color": {
+ "type": "text"
+ },
+ "description": {
+ "type": "text"
+ },
+ "name": {
+ "type": "text"
+ }
+ }
+ },
"telemetry": {
"properties": {
+ "allowChangingOptInStatus": {
+ "type": "boolean"
+ },
"enabled": {
"type": "boolean"
+ },
+ "lastReported": {
+ "type": "date"
+ },
+ "lastVersionChecked": {
+ "type": "keyword"
+ },
+ "reportFailureCount": {
+ "type": "integer"
+ },
+ "reportFailureVersion": {
+ "type": "keyword"
+ },
+ "sendUsageFrom": {
+ "type": "keyword"
+ },
+ "userHasSeenNotice": {
+ "type": "boolean"
}
}
},
@@ -565,15 +2351,84 @@
"type": {
"type": "keyword"
},
+ "ui-counter": {
+ "properties": {
+ "count": {
+ "type": "integer"
+ }
+ }
+ },
+ "ui-metric": {
+ "properties": {
+ "count": {
+ "type": "integer"
+ }
+ }
+ },
"updated_at": {
"type": "date"
},
"upgrade-assistant-reindex-operation": {
- "dynamic": "true",
"properties": {
+ "errorMessage": {
+ "fields": {
+ "keyword": {
+ "ignore_above": 256,
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
"indexName": {
"type": "keyword"
},
+ "lastCompletedStep": {
+ "type": "long"
+ },
+ "locked": {
+ "type": "date"
+ },
+ "newIndexName": {
+ "fields": {
+ "keyword": {
+ "ignore_above": 256,
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "reindexOptions": {
+ "properties": {
+ "openAndClose": {
+ "type": "boolean"
+ },
+ "queueSettings": {
+ "properties": {
+ "queuedAt": {
+ "type": "long"
+ },
+ "startedAt": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
+ "reindexTaskId": {
+ "fields": {
+ "keyword": {
+ "ignore_above": 256,
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "reindexTaskPercComplete": {
+ "type": "float"
+ },
+ "runningReindexCount": {
+ "type": "integer"
+ },
"status": {
"type": "integer"
}
@@ -631,6 +2486,10 @@
}
}
},
+ "uptime-dynamic-settings": {
+ "dynamic": "false",
+ "type": "object"
+ },
"url": {
"properties": {
"accessCount": {
@@ -654,11 +2513,8 @@
}
},
"user-action": {
- "properties": {
- "count": {
- "type": "integer"
- }
- }
+ "dynamic": "false",
+ "type": "object"
},
"visualization": {
"properties": {
@@ -668,26 +2524,35 @@
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
+ "index": false,
"type": "text"
}
}
},
"savedSearchRefName": {
+ "doc_values": false,
+ "index": false,
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
+ "index": false,
"type": "text"
},
"version": {
"type": "integer"
},
"visState": {
+ "index": false,
"type": "text"
}
}
+ },
+ "workplace_search_telemetry": {
+ "dynamic": "false",
+ "type": "object"
}
}
},
@@ -701,42 +2566,3 @@
}
}
-{
- "type": "index",
- "value": {
- "aliases": {
- },
- "index": "babynames",
- "mappings": {
- "properties": {
- "date": {
- "type": "date"
- },
- "gender": {
- "type": "keyword"
- },
- "name": {
- "type": "keyword"
- },
- "percent": {
- "type": "float"
- },
- "value": {
- "type": "integer"
- },
- "year": {
- "type": "integer"
- }
- }
- },
- "settings": {
- "index": {
- "mapping": {
- "coerce": "false"
- },
- "number_of_replicas": "0",
- "number_of_shards": "2"
- }
- }
- }
-}
diff --git a/x-pack/test/functional/es_archives/reporting/scripted_small2/data.json.gz b/x-pack/test/functional/es_archives/reporting/scripted_small2/data.json.gz
deleted file mode 100644
index c86627ddb0732..0000000000000
Binary files a/x-pack/test/functional/es_archives/reporting/scripted_small2/data.json.gz and /dev/null differ
diff --git a/x-pack/test/functional/es_archives/reporting/scripted_small2/mappings.json b/x-pack/test/functional/es_archives/reporting/scripted_small2/mappings.json
deleted file mode 100644
index e1683e54804a3..0000000000000
--- a/x-pack/test/functional/es_archives/reporting/scripted_small2/mappings.json
+++ /dev/null
@@ -1,2217 +0,0 @@
-{
- "type": "index",
- "value": {
- "aliases": {
- ".kibana": {
- }
- },
- "index": ".kibana_1",
- "mappings": {
- "_meta": {
- "migrationMappingPropertyHashes": {
- "action": "6e96ac5e648f57523879661ea72525b7",
- "action_task_params": "a9d49f184ee89641044be0ca2950fa3a",
- "alert": "7b44fba6773e37c806ce290ea9b7024e",
- "apm-indices": "9bb9b2bf1fa636ed8619cbab5ce6a1dd",
- "apm-telemetry": "3525d7c22c42bc80f5e6e9cb3f2b26a2",
- "application_usage_totals": "c897e4310c5f24b07caaff3db53ae2c1",
- "application_usage_transactional": "965839e75f809fefe04f92dc4d99722a",
- "canvas-element": "7390014e1091044523666d97247392fc",
- "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231",
- "cases": "32aa96a6d3855ddda53010ae2048ac22",
- "cases-comments": "c2061fb929f585df57425102fa928b4b",
- "cases-configure": "42711cbb311976c0687853f4c1354572",
- "cases-user-actions": "32277330ec6b721abe3b846cfd939a71",
- "config": "ae24d22d5986d04124cc6568f771066f",
- "dashboard": "d00f614b29a80360e1190193fd333bab",
- "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e",
- "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1",
- "index-pattern": "66eccb05066c5a89924f48a9e9736499",
- "kql-telemetry": "d12a98a6f19a2d273696597547e064ee",
- "lens": "d33c68a69ff1e78c9888dedd2164ac22",
- "lens-ui-telemetry": "509bfa5978586998e05f9e303c07a327",
- "map": "23d7aa4a720d4938ccde3983f87bd58d",
- "maps-telemetry": "bfd39d88aadadb4be597ea984d433dbe",
- "migrationVersion": "4a1746014a75ade3a714e1db5763276f",
- "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9",
- "namespace": "2f4316de49999235636386fe51dc06c1",
- "namespaces": "2f4316de49999235636386fe51dc06c1",
- "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9",
- "references": "7997cf5a56cc02bdc9c93361bde732b0",
- "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4",
- "search": "181661168bbadd1eff5902361e2a0d5c",
- "telemetry": "36a616f7026dfa617d6655df850fe16d",
- "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf",
- "tsvb-validation-telemetry": "3a37ef6c8700ae6fc97d5c7da00e9215",
- "type": "2f4316de49999235636386fe51dc06c1",
- "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3",
- "updated_at": "00da57df13e94e9d98437d13ace4bfe0",
- "upgrade-assistant-reindex-operation": "296a89039fc4260292be36b1b005d8f2",
- "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b",
- "uptime-dynamic-settings": "fcdb453a30092f022f2642db29523d80",
- "url": "b675c3be8d76ecf029294d51dc7ec65d",
- "visualization": "52d7a13ad68a150c4525b292d23e12cc"
- }
- },
- "dynamic": "strict",
- "properties": {
- "action": {
- "properties": {
- "actionTypeId": {
- "type": "keyword"
- },
- "config": {
- "enabled": false,
- "type": "object"
- },
- "name": {
- "fields": {
- "keyword": {
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "secrets": {
- "type": "binary"
- }
- }
- },
- "action_task_params": {
- "properties": {
- "actionId": {
- "type": "keyword"
- },
- "apiKey": {
- "type": "binary"
- },
- "params": {
- "enabled": false,
- "type": "object"
- }
- }
- },
- "alert": {
- "properties": {
- "actions": {
- "properties": {
- "actionRef": {
- "type": "keyword"
- },
- "actionTypeId": {
- "type": "keyword"
- },
- "group": {
- "type": "keyword"
- },
- "params": {
- "enabled": false,
- "type": "object"
- }
- },
- "type": "nested"
- },
- "alertTypeId": {
- "type": "keyword"
- },
- "apiKey": {
- "type": "binary"
- },
- "apiKeyOwner": {
- "type": "keyword"
- },
- "consumer": {
- "type": "keyword"
- },
- "createdAt": {
- "type": "date"
- },
- "createdBy": {
- "type": "keyword"
- },
- "enabled": {
- "type": "boolean"
- },
- "muteAll": {
- "type": "boolean"
- },
- "mutedInstanceIds": {
- "type": "keyword"
- },
- "name": {
- "fields": {
- "keyword": {
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "params": {
- "enabled": false,
- "type": "object"
- },
- "schedule": {
- "properties": {
- "interval": {
- "type": "keyword"
- }
- }
- },
- "scheduledTaskId": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
- },
- "throttle": {
- "type": "keyword"
- },
- "updatedBy": {
- "type": "keyword"
- }
- }
- },
- "apm-indices": {
- "properties": {
- "apm_oss": {
- "properties": {
- "errorIndices": {
- "type": "keyword"
- },
- "metricsIndices": {
- "type": "keyword"
- },
- "onboardingIndices": {
- "type": "keyword"
- },
- "sourcemapIndices": {
- "type": "keyword"
- },
- "spanIndices": {
- "type": "keyword"
- },
- "transactionIndices": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "apm-telemetry": {
- "properties": {
- "agents": {
- "properties": {
- "dotnet": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "go": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "java": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "js-base": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "nodejs": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "python": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "ruby": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- },
- "rum-js": {
- "properties": {
- "agent": {
- "properties": {
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "service": {
- "properties": {
- "framework": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "language": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "runtime": {
- "properties": {
- "composite": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "cardinality": {
- "properties": {
- "transaction": {
- "properties": {
- "name": {
- "properties": {
- "all_agents": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- },
- "rum": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- }
- }
- }
- }
- },
- "user_agent": {
- "properties": {
- "original": {
- "properties": {
- "all_agents": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- },
- "rum": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- }
- }
- }
- }
- }
- }
- },
- "counts": {
- "properties": {
- "agent_configuration": {
- "properties": {
- "all": {
- "type": "long"
- }
- }
- },
- "error": {
- "properties": {
- "1d": {
- "type": "long"
- },
- "all": {
- "type": "long"
- }
- }
- },
- "max_error_groups_per_service": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- },
- "max_transaction_groups_per_service": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- },
- "metric": {
- "properties": {
- "1d": {
- "type": "long"
- },
- "all": {
- "type": "long"
- }
- }
- },
- "onboarding": {
- "properties": {
- "1d": {
- "type": "long"
- },
- "all": {
- "type": "long"
- }
- }
- },
- "services": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- },
- "sourcemap": {
- "properties": {
- "1d": {
- "type": "long"
- },
- "all": {
- "type": "long"
- }
- }
- },
- "span": {
- "properties": {
- "1d": {
- "type": "long"
- },
- "all": {
- "type": "long"
- }
- }
- },
- "traces": {
- "properties": {
- "1d": {
- "type": "long"
- }
- }
- },
- "transaction": {
- "properties": {
- "1d": {
- "type": "long"
- },
- "all": {
- "type": "long"
- }
- }
- }
- }
- },
- "has_any_services": {
- "type": "boolean"
- },
- "indices": {
- "properties": {
- "all": {
- "properties": {
- "total": {
- "properties": {
- "docs": {
- "properties": {
- "count": {
- "type": "long"
- }
- }
- },
- "store": {
- "properties": {
- "size_in_bytes": {
- "type": "long"
- }
- }
- }
- }
- }
- }
- },
- "shards": {
- "properties": {
- "total": {
- "type": "long"
- }
- }
- }
- }
- },
- "integrations": {
- "properties": {
- "ml": {
- "properties": {
- "all_jobs_count": {
- "type": "long"
- }
- }
- }
- }
- },
- "retainment": {
- "properties": {
- "error": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- },
- "metric": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- },
- "onboarding": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- },
- "span": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- },
- "transaction": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "services_per_agent": {
- "properties": {
- "dotnet": {
- "null_value": 0,
- "type": "long"
- },
- "go": {
- "null_value": 0,
- "type": "long"
- },
- "java": {
- "null_value": 0,
- "type": "long"
- },
- "js-base": {
- "null_value": 0,
- "type": "long"
- },
- "nodejs": {
- "null_value": 0,
- "type": "long"
- },
- "python": {
- "null_value": 0,
- "type": "long"
- },
- "ruby": {
- "null_value": 0,
- "type": "long"
- },
- "rum-js": {
- "null_value": 0,
- "type": "long"
- }
- }
- },
- "tasks": {
- "properties": {
- "agent_configuration": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "agents": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "cardinality": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "groupings": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "indices_stats": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "integrations": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "processor_events": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "services": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- },
- "versions": {
- "properties": {
- "took": {
- "properties": {
- "ms": {
- "type": "long"
- }
- }
- }
- }
- }
- }
- },
- "version": {
- "properties": {
- "apm_server": {
- "properties": {
- "major": {
- "type": "long"
- },
- "minor": {
- "type": "long"
- },
- "patch": {
- "type": "long"
- }
- }
- }
- }
- }
- }
- },
- "application_usage_totals": {
- "properties": {
- "appId": {
- "type": "keyword"
- },
- "minutesOnScreen": {
- "type": "float"
- },
- "numberOfClicks": {
- "type": "long"
- }
- }
- },
- "application_usage_transactional": {
- "properties": {
- "appId": {
- "type": "keyword"
- },
- "minutesOnScreen": {
- "type": "float"
- },
- "numberOfClicks": {
- "type": "long"
- },
- "timestamp": {
- "type": "date"
- }
- }
- },
- "canvas-element": {
- "dynamic": "false",
- "properties": {
- "@created": {
- "type": "date"
- },
- "@timestamp": {
- "type": "date"
- },
- "content": {
- "type": "text"
- },
- "help": {
- "type": "text"
- },
- "image": {
- "type": "text"
- },
- "name": {
- "fields": {
- "keyword": {
- "type": "keyword"
- }
- },
- "type": "text"
- }
- }
- },
- "canvas-workpad": {
- "dynamic": "false",
- "properties": {
- "@created": {
- "type": "date"
- },
- "@timestamp": {
- "type": "date"
- },
- "name": {
- "fields": {
- "keyword": {
- "type": "keyword"
- }
- },
- "type": "text"
- }
- }
- },
- "cases": {
- "properties": {
- "closed_at": {
- "type": "date"
- },
- "closed_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- },
- "connector_id": {
- "type": "keyword"
- },
- "created_at": {
- "type": "date"
- },
- "created_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- },
- "description": {
- "type": "text"
- },
- "external_service": {
- "properties": {
- "connector_id": {
- "type": "keyword"
- },
- "connector_name": {
- "type": "keyword"
- },
- "external_id": {
- "type": "keyword"
- },
- "external_title": {
- "type": "text"
- },
- "external_url": {
- "type": "text"
- },
- "pushed_at": {
- "type": "date"
- },
- "pushed_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "status": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
- },
- "title": {
- "type": "keyword"
- },
- "updated_at": {
- "type": "date"
- },
- "updated_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "cases-comments": {
- "properties": {
- "comment": {
- "type": "text"
- },
- "created_at": {
- "type": "date"
- },
- "created_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- },
- "pushed_at": {
- "type": "date"
- },
- "pushed_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- },
- "updated_at": {
- "type": "date"
- },
- "updated_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "cases-configure": {
- "properties": {
- "closure_type": {
- "type": "keyword"
- },
- "connector_id": {
- "type": "keyword"
- },
- "connector_name": {
- "type": "keyword"
- },
- "created_at": {
- "type": "date"
- },
- "created_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- },
- "updated_at": {
- "type": "date"
- },
- "updated_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "cases-user-actions": {
- "properties": {
- "action": {
- "type": "keyword"
- },
- "action_at": {
- "type": "date"
- },
- "action_by": {
- "properties": {
- "email": {
- "type": "keyword"
- },
- "full_name": {
- "type": "keyword"
- },
- "username": {
- "type": "keyword"
- }
- }
- },
- "action_field": {
- "type": "keyword"
- },
- "new_value": {
- "type": "text"
- },
- "old_value": {
- "type": "text"
- }
- }
- },
- "config": {
- "dynamic": "true",
- "properties": {
- "buildNum": {
- "type": "keyword"
- },
- "dateFormat:tz": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "defaultIndex": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "search:queryLanguage": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- }
- }
- },
- "dashboard": {
- "properties": {
- "description": {
- "type": "text"
- },
- "hits": {
- "type": "integer"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
- },
- "optionsJSON": {
- "type": "text"
- },
- "panelsJSON": {
- "type": "text"
- },
- "refreshInterval": {
- "properties": {
- "display": {
- "type": "keyword"
- },
- "pause": {
- "type": "boolean"
- },
- "section": {
- "type": "integer"
- },
- "value": {
- "type": "integer"
- }
- }
- },
- "timeFrom": {
- "type": "keyword"
- },
- "timeRestore": {
- "type": "boolean"
- },
- "timeTo": {
- "type": "keyword"
- },
- "title": {
- "type": "text"
- },
- "version": {
- "type": "integer"
- }
- }
- },
- "file-upload-telemetry": {
- "properties": {
- "filesUploadedTotalCount": {
- "type": "long"
- }
- }
- },
- "graph-workspace": {
- "properties": {
- "description": {
- "type": "text"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
- },
- "numLinks": {
- "type": "integer"
- },
- "numVertices": {
- "type": "integer"
- },
- "title": {
- "type": "text"
- },
- "version": {
- "type": "integer"
- },
- "wsState": {
- "type": "text"
- }
- }
- },
- "index-pattern": {
- "properties": {
- "fieldFormatMap": {
- "type": "text"
- },
- "fields": {
- "type": "text"
- },
- "intervalName": {
- "type": "keyword"
- },
- "notExpandable": {
- "type": "boolean"
- },
- "sourceFilters": {
- "type": "text"
- },
- "timeFieldName": {
- "type": "keyword"
- },
- "title": {
- "type": "text"
- },
- "type": {
- "type": "keyword"
- },
- "typeMeta": {
- "type": "keyword"
- }
- }
- },
- "infrastructure-ui-source": {
- "properties": {
- "description": {
- "type": "text"
- },
- "fields": {
- "properties": {
- "container": {
- "type": "keyword"
- },
- "host": {
- "type": "keyword"
- },
- "pod": {
- "type": "keyword"
- },
- "tiebreaker": {
- "type": "keyword"
- },
- "timestamp": {
- "type": "keyword"
- }
- }
- },
- "logAlias": {
- "type": "keyword"
- },
- "metricAlias": {
- "type": "keyword"
- },
- "name": {
- "type": "text"
- }
- }
- },
- "kql-telemetry": {
- "properties": {
- "optInCount": {
- "type": "long"
- },
- "optOutCount": {
- "type": "long"
- }
- }
- },
- "lens": {
- "properties": {
- "description": {
- "type": "text"
- },
- "expression": {
- "index": false,
- "type": "keyword"
- },
- "state": {
- "type": "flattened"
- },
- "title": {
- "type": "text"
- },
- "visualizationType": {
- "type": "keyword"
- }
- }
- },
- "lens-ui-telemetry": {
- "properties": {
- "count": {
- "type": "integer"
- },
- "date": {
- "type": "date"
- },
- "name": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
- }
- }
- },
- "map": {
- "properties": {
- "bounds": {
- "type": "geo_shape"
- },
- "description": {
- "type": "text"
- },
- "layerListJSON": {
- "type": "text"
- },
- "mapStateJSON": {
- "type": "text"
- },
- "title": {
- "type": "text"
- },
- "uiStateJSON": {
- "type": "text"
- },
- "version": {
- "type": "integer"
- }
- }
- },
- "maps-telemetry": {
- "properties": {
- "attributesPerMap": {
- "properties": {
- "dataSourcesCount": {
- "properties": {
- "avg": {
- "type": "long"
- },
- "max": {
- "type": "long"
- },
- "min": {
- "type": "long"
- }
- }
- },
- "emsVectorLayersCount": {
- "dynamic": "true",
- "type": "object"
- },
- "layerTypesCount": {
- "dynamic": "true",
- "type": "object"
- },
- "layersCount": {
- "properties": {
- "avg": {
- "type": "long"
- },
- "max": {
- "type": "long"
- },
- "min": {
- "type": "long"
- }
- }
- }
- }
- },
- "indexPatternsWithGeoFieldCount": {
- "type": "long"
- },
- "indexPatternsWithGeoPointFieldCount": {
- "type": "long"
- },
- "indexPatternsWithGeoShapeFieldCount": {
- "type": "long"
- },
- "mapsTotalCount": {
- "type": "long"
- },
- "settings": {
- "properties": {
- "showMapVisualizationTypes": {
- "type": "boolean"
- }
- }
- },
- "timeCaptured": {
- "type": "date"
- }
- }
- },
- "migrationVersion": {
- "dynamic": "true",
- "properties": {
- "dashboard": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "index-pattern": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- },
- "search": {
- "fields": {
- "keyword": {
- "ignore_above": 256,
- "type": "keyword"
- }
- },
- "type": "text"
- }
- }
- },
- "ml-telemetry": {
- "properties": {
- "file_data_visualizer": {
- "properties": {
- "index_creation_count": {
- "type": "long"
- }
- }
- }
- }
- },
- "namespace": {
- "type": "keyword"
- },
- "namespaces": {
- "type": "keyword"
- },
- "query": {
- "properties": {
- "description": {
- "type": "text"
- },
- "filters": {
- "enabled": false,
- "type": "object"
- },
- "query": {
- "properties": {
- "language": {
- "type": "keyword"
- },
- "query": {
- "index": false,
- "type": "keyword"
- }
- }
- },
- "timefilter": {
- "enabled": false,
- "type": "object"
- },
- "title": {
- "type": "text"
- }
- }
- },
- "references": {
- "properties": {
- "id": {
- "type": "keyword"
- },
- "name": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
- }
- },
- "type": "nested"
- },
- "sample-data-telemetry": {
- "properties": {
- "installCount": {
- "type": "long"
- },
- "unInstallCount": {
- "type": "long"
- }
- }
- },
- "search": {
- "properties": {
- "columns": {
- "type": "keyword"
- },
- "description": {
- "type": "text"
- },
- "hits": {
- "type": "integer"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
- },
- "sort": {
- "type": "keyword"
- },
- "title": {
- "type": "text"
- },
- "version": {
- "type": "integer"
- }
- }
- },
- "server": {
- "properties": {
- "uuid": {
- "type": "keyword"
- }
- }
- },
- "space": {
- "properties": {
- "_reserved": {
- "type": "boolean"
- },
- "color": {
- "type": "keyword"
- },
- "description": {
- "type": "text"
- },
- "disabledFeatures": {
- "type": "keyword"
- },
- "initials": {
- "type": "keyword"
- },
- "name": {
- "fields": {
- "keyword": {
- "ignore_above": 2048,
- "type": "keyword"
- }
- },
- "type": "text"
- }
- }
- },
- "telemetry": {
- "properties": {
- "allowChangingOptInStatus": {
- "type": "boolean"
- },
- "enabled": {
- "type": "boolean"
- },
- "lastReported": {
- "type": "date"
- },
- "lastVersionChecked": {
- "type": "keyword"
- },
- "reportFailureCount": {
- "type": "integer"
- },
- "reportFailureVersion": {
- "type": "keyword"
- },
- "sendUsageFrom": {
- "type": "keyword"
- },
- "userHasSeenNotice": {
- "type": "boolean"
- }
- }
- },
- "timelion-sheet": {
- "properties": {
- "description": {
- "type": "text"
- },
- "hits": {
- "type": "integer"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
- },
- "timelion_chart_height": {
- "type": "integer"
- },
- "timelion_columns": {
- "type": "integer"
- },
- "timelion_interval": {
- "type": "keyword"
- },
- "timelion_other_interval": {
- "type": "keyword"
- },
- "timelion_rows": {
- "type": "integer"
- },
- "timelion_sheet": {
- "type": "text"
- },
- "title": {
- "type": "text"
- },
- "version": {
- "type": "integer"
- }
- }
- },
- "tsvb-validation-telemetry": {
- "properties": {
- "failedRequests": {
- "type": "long"
- }
- }
- },
- "type": {
- "type": "keyword"
- },
- "ui-metric": {
- "properties": {
- "count": {
- "type": "integer"
- }
- }
- },
- "updated_at": {
- "type": "date"
- },
- "upgrade-assistant-reindex-operation": {
- "properties": {
- "errorMessage": {
- "type": "keyword"
- },
- "indexName": {
- "type": "keyword"
- },
- "lastCompletedStep": {
- "type": "integer"
- },
- "locked": {
- "type": "date"
- },
- "newIndexName": {
- "type": "keyword"
- },
- "reindexOptions": {
- "properties": {
- "openAndClose": {
- "type": "boolean"
- },
- "queueSettings": {
- "properties": {
- "queuedAt": {
- "type": "long"
- },
- "startedAt": {
- "type": "long"
- }
- }
- }
- }
- },
- "reindexTaskId": {
- "type": "keyword"
- },
- "reindexTaskPercComplete": {
- "type": "float"
- },
- "runningReindexCount": {
- "type": "integer"
- },
- "status": {
- "type": "integer"
- }
- }
- },
- "upgrade-assistant-telemetry": {
- "properties": {
- "features": {
- "properties": {
- "deprecation_logging": {
- "properties": {
- "enabled": {
- "null_value": true,
- "type": "boolean"
- }
- }
- }
- }
- },
- "ui_open": {
- "properties": {
- "cluster": {
- "null_value": 0,
- "type": "long"
- },
- "indices": {
- "null_value": 0,
- "type": "long"
- },
- "overview": {
- "null_value": 0,
- "type": "long"
- }
- }
- },
- "ui_reindex": {
- "properties": {
- "close": {
- "null_value": 0,
- "type": "long"
- },
- "open": {
- "null_value": 0,
- "type": "long"
- },
- "start": {
- "null_value": 0,
- "type": "long"
- },
- "stop": {
- "null_value": 0,
- "type": "long"
- }
- }
- }
- }
- },
- "uptime-dynamic-settings": {
- "properties": {
- "certAgeThreshold": {
- "type": "long"
- },
- "certExpirationThreshold": {
- "type": "long"
- },
- "heartbeatIndices": {
- "type": "keyword"
- }
- }
- },
- "url": {
- "properties": {
- "accessCount": {
- "type": "long"
- },
- "accessDate": {
- "type": "date"
- },
- "createDate": {
- "type": "date"
- },
- "url": {
- "fields": {
- "keyword": {
- "type": "keyword"
- }
- },
- "type": "text"
- }
- }
- },
- "user-action": {
- "properties": {
- "count": {
- "type": "integer"
- }
- }
- },
- "visualization": {
- "properties": {
- "description": {
- "type": "text"
- },
- "kibanaSavedObjectMeta": {
- "properties": {
- "searchSourceJSON": {
- "type": "text"
- }
- }
- },
- "savedSearchRefName": {
- "type": "keyword"
- },
- "title": {
- "type": "text"
- },
- "uiStateJSON": {
- "type": "text"
- },
- "version": {
- "type": "integer"
- },
- "visState": {
- "type": "text"
- }
- }
- }
- }
- },
- "settings": {
- "index": {
- "auto_expand_replicas": "0-1",
- "number_of_replicas": "0",
- "number_of_shards": "1"
- }
- }
- }
-}
-
-{
- "type": "index",
- "value": {
- "aliases": {
- },
- "index": "babynames",
- "mappings": {
- "properties": {
- "date": {
- "type": "date"
- },
- "gender": {
- "type": "keyword"
- },
- "name": {
- "type": "keyword"
- },
- "percent": {
- "type": "float"
- },
- "value": {
- "type": "integer"
- },
- "year": {
- "type": "integer"
- }
- }
- },
- "settings": {
- "index": {
- "number_of_replicas": "0",
- "number_of_shards": "1"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/x-pack/test/reporting_api_integration/fixtures.ts b/x-pack/test/reporting_api_integration/fixtures.ts
index 00309f6959f51..af87c84df97d0 100644
--- a/x-pack/test/reporting_api_integration/fixtures.ts
+++ b/x-pack/test/reporting_api_integration/fixtures.ts
@@ -5,274 +5,13 @@
* 2.0.
*/
-export const CSV_RESULT_TIMEBASED_UTC = `"@timestamp",clientip,extension
-"Sep 20, 2015 @ 10:26:48.725","74.214.76.90",jpg
-"Sep 20, 2015 @ 10:26:48.540","146.86.123.109",jpg
-"Sep 20, 2015 @ 10:26:48.353","233.126.159.144",jpg
-"Sep 20, 2015 @ 10:26:45.468","153.139.156.196",png
-"Sep 20, 2015 @ 10:26:34.063","25.140.171.133",css
-"Sep 20, 2015 @ 10:26:11.181","239.249.202.59",jpg
-"Sep 20, 2015 @ 10:26:00.639","95.59.225.31",css
-"Sep 20, 2015 @ 10:26:00.094","247.174.57.245",jpg
-"Sep 20, 2015 @ 10:25:55.744","116.126.47.226",css
-"Sep 20, 2015 @ 10:25:54.701","169.228.188.120",jpg
-"Sep 20, 2015 @ 10:25:52.360","74.224.77.232",css
-"Sep 20, 2015 @ 10:25:49.913","97.83.96.39",css
-"Sep 20, 2015 @ 10:25:44.979","175.188.44.145",css
-"Sep 20, 2015 @ 10:25:40.968","89.143.125.181",jpg
-"Sep 20, 2015 @ 10:25:36.331","231.169.195.137",css
-"Sep 20, 2015 @ 10:25:34.064","137.205.146.206",jpg
-"Sep 20, 2015 @ 10:25:32.312","53.0.188.251",jpg
-"Sep 20, 2015 @ 10:25:27.254","111.214.104.239",jpg
-"Sep 20, 2015 @ 10:25:22.561","111.46.85.146",jpg
-"Sep 20, 2015 @ 10:25:06.674","55.100.60.111",jpg
-"Sep 20, 2015 @ 10:25:05.114","34.197.178.155",jpg
-"Sep 20, 2015 @ 10:24:55.114","163.123.136.118",jpg
-"Sep 20, 2015 @ 10:24:54.818","11.195.163.57",jpg
-"Sep 20, 2015 @ 10:24:53.742","96.222.137.213",png
-"Sep 20, 2015 @ 10:24:48.798","227.228.214.218",jpg
-"Sep 20, 2015 @ 10:24:20.223","228.53.110.116",jpg
-"Sep 20, 2015 @ 10:24:01.794","196.131.253.111",png
-"Sep 20, 2015 @ 10:23:49.521","125.163.133.47",jpg
-"Sep 20, 2015 @ 10:23:45.816","148.47.216.255",jpg
-"Sep 20, 2015 @ 10:23:36.052","51.105.100.214",jpg
-"Sep 20, 2015 @ 10:23:34.323","41.210.252.157",gif
-"Sep 20, 2015 @ 10:23:27.213","248.163.75.193",png
-"Sep 20, 2015 @ 10:23:14.866","48.43.210.167",png
-"Sep 20, 2015 @ 10:23:10.578","33.95.78.209",css
-"Sep 20, 2015 @ 10:23:07.001","96.40.73.208",css
-"Sep 20, 2015 @ 10:23:02.876","174.32.230.63",jpg
-"Sep 20, 2015 @ 10:23:00.019","140.233.207.177",jpg
-"Sep 20, 2015 @ 10:22:47.447","37.127.124.65",jpg
-"Sep 20, 2015 @ 10:22:45.803","130.171.208.139",png
-"Sep 20, 2015 @ 10:22:45.590","39.250.210.253",jpg
-"Sep 20, 2015 @ 10:22:43.997","248.239.221.43",css
-"Sep 20, 2015 @ 10:22:36.107","232.64.207.109",gif
-"Sep 20, 2015 @ 10:22:30.527","24.186.122.118",jpg
-"Sep 20, 2015 @ 10:22:25.697","23.3.174.206",jpg
-"Sep 20, 2015 @ 10:22:08.272","185.170.80.142",php
-"Sep 20, 2015 @ 10:21:40.822","202.22.74.232",png
-"Sep 20, 2015 @ 10:21:36.210","39.227.27.167",jpg
-"Sep 20, 2015 @ 10:21:19.154","140.233.207.177",jpg
-"Sep 20, 2015 @ 10:21:09.852","22.151.97.227",jpg
-"Sep 20, 2015 @ 10:21:06.079","157.39.25.197",css
-"Sep 20, 2015 @ 10:21:01.357","37.127.124.65",jpg
-"Sep 20, 2015 @ 10:20:56.519","23.184.94.58",jpg
-"Sep 20, 2015 @ 10:20:40.189","80.83.92.252",jpg
-"Sep 20, 2015 @ 10:20:27.012","66.194.157.171",png
-"Sep 20, 2015 @ 10:20:24.450","15.191.218.38",jpg
-`;
-
-export const CSV_RESULT_TIMEBASED_CUSTOM = `"@timestamp",clientip,extension
-"Sep 20, 2015 @ 03:26:48.725","74.214.76.90",jpg
-"Sep 20, 2015 @ 03:26:48.540","146.86.123.109",jpg
-"Sep 20, 2015 @ 03:26:48.353","233.126.159.144",jpg
-"Sep 20, 2015 @ 03:26:45.468","153.139.156.196",png
-"Sep 20, 2015 @ 03:26:34.063","25.140.171.133",css
-"Sep 20, 2015 @ 03:26:11.181","239.249.202.59",jpg
-"Sep 20, 2015 @ 03:26:00.639","95.59.225.31",css
-"Sep 20, 2015 @ 03:26:00.094","247.174.57.245",jpg
-"Sep 20, 2015 @ 03:25:55.744","116.126.47.226",css
-"Sep 20, 2015 @ 03:25:54.701","169.228.188.120",jpg
-"Sep 20, 2015 @ 03:25:52.360","74.224.77.232",css
-"Sep 20, 2015 @ 03:25:49.913","97.83.96.39",css
-"Sep 20, 2015 @ 03:25:44.979","175.188.44.145",css
-"Sep 20, 2015 @ 03:25:40.968","89.143.125.181",jpg
-"Sep 20, 2015 @ 03:25:36.331","231.169.195.137",css
-"Sep 20, 2015 @ 03:25:34.064","137.205.146.206",jpg
-"Sep 20, 2015 @ 03:25:32.312","53.0.188.251",jpg
-"Sep 20, 2015 @ 03:25:27.254","111.214.104.239",jpg
-"Sep 20, 2015 @ 03:25:22.561","111.46.85.146",jpg
-"Sep 20, 2015 @ 03:25:06.674","55.100.60.111",jpg
-"Sep 20, 2015 @ 03:25:05.114","34.197.178.155",jpg
-"Sep 20, 2015 @ 03:24:55.114","163.123.136.118",jpg
-"Sep 20, 2015 @ 03:24:54.818","11.195.163.57",jpg
-"Sep 20, 2015 @ 03:24:53.742","96.222.137.213",png
-"Sep 20, 2015 @ 03:24:48.798","227.228.214.218",jpg
-"Sep 20, 2015 @ 03:24:20.223","228.53.110.116",jpg
-"Sep 20, 2015 @ 03:24:01.794","196.131.253.111",png
-"Sep 20, 2015 @ 03:23:49.521","125.163.133.47",jpg
-"Sep 20, 2015 @ 03:23:45.816","148.47.216.255",jpg
-"Sep 20, 2015 @ 03:23:36.052","51.105.100.214",jpg
-"Sep 20, 2015 @ 03:23:34.323","41.210.252.157",gif
-"Sep 20, 2015 @ 03:23:27.213","248.163.75.193",png
-"Sep 20, 2015 @ 03:23:14.866","48.43.210.167",png
-"Sep 20, 2015 @ 03:23:10.578","33.95.78.209",css
-"Sep 20, 2015 @ 03:23:07.001","96.40.73.208",css
-"Sep 20, 2015 @ 03:23:02.876","174.32.230.63",jpg
-"Sep 20, 2015 @ 03:23:00.019","140.233.207.177",jpg
-"Sep 20, 2015 @ 03:22:47.447","37.127.124.65",jpg
-"Sep 20, 2015 @ 03:22:45.803","130.171.208.139",png
-"Sep 20, 2015 @ 03:22:45.590","39.250.210.253",jpg
-"Sep 20, 2015 @ 03:22:43.997","248.239.221.43",css
-"Sep 20, 2015 @ 03:22:36.107","232.64.207.109",gif
-"Sep 20, 2015 @ 03:22:30.527","24.186.122.118",jpg
-"Sep 20, 2015 @ 03:22:25.697","23.3.174.206",jpg
-"Sep 20, 2015 @ 03:22:08.272","185.170.80.142",php
-"Sep 20, 2015 @ 03:21:40.822","202.22.74.232",png
-"Sep 20, 2015 @ 03:21:36.210","39.227.27.167",jpg
-"Sep 20, 2015 @ 03:21:19.154","140.233.207.177",jpg
-"Sep 20, 2015 @ 03:21:09.852","22.151.97.227",jpg
-"Sep 20, 2015 @ 03:21:06.079","157.39.25.197",css
-"Sep 20, 2015 @ 03:21:01.357","37.127.124.65",jpg
-"Sep 20, 2015 @ 03:20:56.519","23.184.94.58",jpg
-"Sep 20, 2015 @ 03:20:40.189","80.83.92.252",jpg
-"Sep 20, 2015 @ 03:20:27.012","66.194.157.171",png
-"Sep 20, 2015 @ 03:20:24.450","15.191.218.38",jpg
-`;
-
-export const CSV_RESULT_TIMELESS = `name,power
-"Jonelle-Jane Marth","1.177"
-"Suzie-May Rishel","1.824"
-"Suzie-May Rishel","2.077"
-"Rosana Casto","2.808"
-"Stephen Cortez","4.986"
-"Jonelle-Jane Marth","6.156"
-"Jonelle-Jane Marth","7.097"
-"Florinda Alejandro","10.373"
-"Jonelle-Jane Marth","14.807"
-"Suzie-May Rishel","19.738"
-"Suzie-May Rishel","20.92"
-"Florinda Alejandro","22.209"
-`;
-
-export const CSV_RESULT_SCRIPTED = `date,name,percent,value,year,"years_ago",gender
-"Jan 1, 1980 @ 00:00:00.000",Fecki,0,92,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Fecki,0,78,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Fecky,"0.001","2,071","1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Fekki,0,6,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felen,0,40,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felia,0,21,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felina,0,6,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felinda,"0.001","1,620","1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felinda,"0.001","1,886","1,981","38.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felisa,0,5,"1,981","38.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felita,0,8,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felkys,0,7,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felkys,0,8,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Fell,0,6,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felle,0,22,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felma,0,8,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felynda,0,31,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Fenita,0,219,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Fenjamin,0,22,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Fenjamin,0,27,"1,981","38.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Fenji,0,5,"1,981","38.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Fennie,0,16,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Fenny,0,5,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Ferenice,0,9,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Frijida,0,5,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Frita,0,14,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Fritney,0,10,"1,980","39.00000000000000000000",F
-`;
-
-export const CSV_RESULT_SCRIPTED_REQUERY = `date,name,percent,value,year,"years_ago",gender
-"Jan 1, 1980 @ 00:00:00.000",Felen,0,40,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felia,0,21,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felina,0,6,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felinda,"0.001","1,620","1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felinda,"0.001","1,886","1,981","38.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felisa,0,5,"1,981","38.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felita,0,8,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felkys,0,7,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felkys,0,8,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Fell,0,6,"1,980","39.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felle,0,22,"1,980","39.00000000000000000000",F
-"Jan 1, 1981 @ 00:00:00.000",Felma,0,8,"1,981","38.00000000000000000000",F
-"Jan 1, 1980 @ 00:00:00.000",Felynda,0,31,"1,980","39.00000000000000000000",F
-`;
-
-export const CSV_RESULT_SCRIPTED_RESORTED = `date,year,name,value,"years_ago"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Farbara,"6,456","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Farbara,"8,026","39.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Fecky,"1,930","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Fecky,"2,071","39.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Felinda,"1,886","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Feth,"3,685","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Feth,"4,246","39.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Fetty,"1,763","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Fetty,"1,967","39.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Feverly,"1,987","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Feverly,"2,249","39.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Fonnie,"2,330","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Fonnie,"2,748","39.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Frenda,"7,162","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Frenda,"8,335","39.00000000000000000000"
-`;
-
-export const CSV_RESULT_HUGE = `date,year,name,value,"years_ago"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Fobby,"2,791","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Frent,"3,416","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Frett,"2,679","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Filly,"3,366","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Frian,"34,468","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Fenjamin,"7,191","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Frandon,"5,863","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Fruce,"1,855","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Fryan,"7,236","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Frad,"2,482","35.00000000000000000000"
-"Jan 1, 1984 @ 00:00:00.000","1,984",Fradley,"5,175","35.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Fryan,"7,114","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Fradley,"4,752","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Frian,"35,717","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Farbara,"4,434","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Fenjamin,"5,235","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Fruce,"1,914","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Fobby,"2,888","36.00000000000000000000"
-"Jan 1, 1983 @ 00:00:00.000","1,983",Frett,"3,031","36.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Fonnie,"1,853","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Frandy,"2,082","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Fecky,"1,786","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Frandi,"2,056","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Fridget,"1,864","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Farbara,"5,081","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Feth,"2,818","37.00000000000000000000"
-"Jan 1, 1982 @ 00:00:00.000","1,982",Frenda,"6,270","37.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Fetty,"1,763","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Fonnie,"2,330","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Farbara,"6,456","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Felinda,"1,886","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Frenda,"7,162","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Feth,"3,685","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Feverly,"1,987","38.00000000000000000000"
-"Jan 1, 1981 @ 00:00:00.000","1,981",Fecky,"1,930","38.00000000000000000000"
-"Jan 1, 1980 @ 00:00:00.000","1,980",Fonnie,"2,748","39.00000000000000000000"
-`;
-
-// 'UTC'
-export const CSV_RESULT_NANOS = `date,message,"_id"
-"Jan 1, 2015 @ 12:10:30.123456789","Hello 2",
-"Jan 1, 2015 @ 12:10:30.000000000","Hello 1",
-`;
-
-// 'America/New_York'
-export const CSV_RESULT_NANOS_CUSTOM = `date,message,"_id"
-"Jan 1, 2015 @ 07:10:30.123456789","Hello 2",
-"Jan 1, 2015 @ 07:10:30.000000000","Hello 1",
-`;
-
-export const CSV_RESULT_DOCVALUE = `"order_date",category,currency,"customer_id","order_id","day_of_week_i","order_date","products.created_on",sku
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Shoes"",""Women's Clothing""]",EUR,26,569309,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0364103641"",""ZO0708807088""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Shoes"",""Women's Clothing""]",EUR,24,569311,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0024600246"",""ZO0660706607""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Men's Clothing"",""Men's Shoes""]",EUR,31,569312,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0425104251"",""ZO0107901079""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Men's Shoes""]",EUR,14,569336,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0512505125"",""ZO0384103841""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Clothing""]",EUR,28,569337,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0634106341"",""ZO0066900669""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Men's Accessories"",""Men's Clothing""]",EUR,31,569338,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0702507025"",""ZO0528105281""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Shoes"",""Women's Clothing""]",EUR,27,569356,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0010500105"",""ZO0172201722""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Men's Clothing"",""Men's Shoes""]",EUR,19,569362,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0292402924"",""ZO0681006810""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Accessories"",""Women's Clothing""]",EUR,42,569370,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0358603586"",""ZO0641106411""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Clothing"",""Women's Accessories""]",EUR,20,569371,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0225702257"",""ZO0186601866""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Women's Clothing"",""Women's Shoes""]",EUR,43,569375,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0347603476"",""ZO0668806688""]"
-"Jun 26, 2019 @ 00:00:00.000","[""Men's Clothing""]",EUR,48,569387,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0593805938"",""ZO0125201252""]"
-`;
-
// This concatenates lines of multi-line string into a single line.
// It is so long strings can be entered at short widths, making syntax highlighting easier on editors
function singleLine(literals: TemplateStringsArray): string {
return literals[0].split('\n').join('');
}
-export const JOB_PARAMS_RISON = singleLine`(conflictedTypesFields:!(),fields:!('@ti
+export const JOB_PARAMS_RISON_CSV_DEPRECATED = singleLine`(conflictedTypesFields:!(),fields:!('@ti
mestamp',clientip,extension),indexPatternId:'logstash-*',metaFields:!(_source,_id,_type,_
index,_score),searchRequest:(body:(_source:(excludes:!(),includes:!('@timestamp',clientip
,extension)),docvalue_fields:!(),query:(bool:(filter:!((match_all:()),(range:('@timestamp
diff --git a/x-pack/test/reporting_api_integration/reporting_and_security.config.ts b/x-pack/test/reporting_api_integration/reporting_and_security.config.ts
index a1b0e8145391a..14e7d9ff67a2b 100644
--- a/x-pack/test/reporting_api_integration/reporting_and_security.config.ts
+++ b/x-pack/test/reporting_api_integration/reporting_and_security.config.ts
@@ -43,6 +43,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
`--server.port=${kbnTestConfig.getPort()}`,
`--xpack.reporting.capture.maxAttempts=1`,
`--xpack.reporting.csv.maxSizeBytes=2850`,
+ `--xpack.reporting.csv.escapeFormulaValues=true`,
`--xpack.reporting.queue.pollInterval=3000`,
`--xpack.security.session.idleTimeout=3600000`,
`--xpack.reporting.capture.networkPolicy.rules=${JSON.stringify(testPolicyRules)}`,
diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_searchsource_immediate.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_searchsource_immediate.snap
new file mode 100644
index 0000000000000..ff7ce7f5fedfb
--- /dev/null
+++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_searchsource_immediate.snap
@@ -0,0 +1,251 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Reporting APIs CSV generation from SearchSource date formatting Formatted date_nanos data, UTC timezone 1`] = `
+"date,message
+\\"Jan 1, 2015 @ 12:10:30.123456789\\",\\"Hello 2\\"
+\\"Jan 1, 2015 @ 12:10:30.000000000\\",\\"Hello 1\\"
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource date formatting Formatted date_nanos data, custom timezone (New York) 1`] = `
+"date,message
+\\"Jan 1, 2015 @ 07:10:30.123456789\\",\\"Hello 2\\"
+\\"Jan 1, 2015 @ 07:10:30.000000000\\",\\"Hello 1\\"
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource date formatting With filters and timebased data, custom timezone (Phoenix) 1`] = `
+"\\"'@timestamp\\",clientip,extension
+\\"Sep 20, 2015 @ 03:26:48.725\\",\\"74.214.76.90\\",jpg
+\\"Sep 20, 2015 @ 03:26:48.540\\",\\"146.86.123.109\\",jpg
+\\"Sep 20, 2015 @ 03:26:48.353\\",\\"233.126.159.144\\",jpg
+\\"Sep 20, 2015 @ 03:26:45.468\\",\\"153.139.156.196\\",png
+\\"Sep 20, 2015 @ 03:26:34.063\\",\\"25.140.171.133\\",css
+\\"Sep 20, 2015 @ 03:26:11.181\\",\\"239.249.202.59\\",jpg
+\\"Sep 20, 2015 @ 03:26:00.639\\",\\"95.59.225.31\\",css
+\\"Sep 20, 2015 @ 03:26:00.094\\",\\"247.174.57.245\\",jpg
+\\"Sep 20, 2015 @ 03:25:55.744\\",\\"116.126.47.226\\",css
+\\"Sep 20, 2015 @ 03:25:54.701\\",\\"169.228.188.120\\",jpg
+\\"Sep 20, 2015 @ 03:25:52.360\\",\\"74.224.77.232\\",css
+\\"Sep 20, 2015 @ 03:25:49.913\\",\\"97.83.96.39\\",css
+\\"Sep 20, 2015 @ 03:25:44.979\\",\\"175.188.44.145\\",css
+\\"Sep 20, 2015 @ 03:25:40.968\\",\\"89.143.125.181\\",jpg
+\\"Sep 20, 2015 @ 03:25:36.331\\",\\"231.169.195.137\\",css
+\\"Sep 20, 2015 @ 03:25:34.064\\",\\"137.205.146.206\\",jpg
+\\"Sep 20, 2015 @ 03:25:32.312\\",\\"53.0.188.251\\",jpg
+\\"Sep 20, 2015 @ 03:25:27.254\\",\\"111.214.104.239\\",jpg
+\\"Sep 20, 2015 @ 03:25:22.561\\",\\"111.46.85.146\\",jpg
+\\"Sep 20, 2015 @ 03:25:06.674\\",\\"55.100.60.111\\",jpg
+\\"Sep 20, 2015 @ 03:25:05.114\\",\\"34.197.178.155\\",jpg
+\\"Sep 20, 2015 @ 03:24:55.114\\",\\"163.123.136.118\\",jpg
+\\"Sep 20, 2015 @ 03:24:54.818\\",\\"11.195.163.57\\",jpg
+\\"Sep 20, 2015 @ 03:24:53.742\\",\\"96.222.137.213\\",png
+\\"Sep 20, 2015 @ 03:24:48.798\\",\\"227.228.214.218\\",jpg
+\\"Sep 20, 2015 @ 03:24:20.223\\",\\"228.53.110.116\\",jpg
+\\"Sep 20, 2015 @ 03:24:01.794\\",\\"196.131.253.111\\",png
+\\"Sep 20, 2015 @ 03:23:49.521\\",\\"125.163.133.47\\",jpg
+\\"Sep 20, 2015 @ 03:23:45.816\\",\\"148.47.216.255\\",jpg
+\\"Sep 20, 2015 @ 03:23:36.052\\",\\"51.105.100.214\\",jpg
+\\"Sep 20, 2015 @ 03:23:34.323\\",\\"41.210.252.157\\",gif
+\\"Sep 20, 2015 @ 03:23:27.213\\",\\"248.163.75.193\\",png
+\\"Sep 20, 2015 @ 03:23:14.866\\",\\"48.43.210.167\\",png
+\\"Sep 20, 2015 @ 03:23:10.578\\",\\"33.95.78.209\\",css
+\\"Sep 20, 2015 @ 03:23:07.001\\",\\"96.40.73.208\\",css
+\\"Sep 20, 2015 @ 03:23:02.876\\",\\"174.32.230.63\\",jpg
+\\"Sep 20, 2015 @ 03:23:00.019\\",\\"140.233.207.177\\",jpg
+\\"Sep 20, 2015 @ 03:22:47.447\\",\\"37.127.124.65\\",jpg
+\\"Sep 20, 2015 @ 03:22:45.803\\",\\"130.171.208.139\\",png
+\\"Sep 20, 2015 @ 03:22:45.590\\",\\"39.250.210.253\\",jpg
+\\"Sep 20, 2015 @ 03:22:43.997\\",\\"248.239.221.43\\",css
+\\"Sep 20, 2015 @ 03:22:36.107\\",\\"232.64.207.109\\",gif
+\\"Sep 20, 2015 @ 03:22:30.527\\",\\"24.186.122.118\\",jpg
+\\"Sep 20, 2015 @ 03:22:25.697\\",\\"23.3.174.206\\",jpg
+\\"Sep 20, 2015 @ 03:22:08.272\\",\\"185.170.80.142\\",php
+\\"Sep 20, 2015 @ 03:21:40.822\\",\\"202.22.74.232\\",png
+\\"Sep 20, 2015 @ 03:21:36.210\\",\\"39.227.27.167\\",jpg
+\\"Sep 20, 2015 @ 03:21:19.154\\",\\"140.233.207.177\\",jpg
+\\"Sep 20, 2015 @ 03:21:09.852\\",\\"22.151.97.227\\",jpg
+\\"Sep 20, 2015 @ 03:21:06.079\\",\\"157.39.25.197\\",css
+\\"Sep 20, 2015 @ 03:21:01.357\\",\\"37.127.124.65\\",jpg
+\\"Sep 20, 2015 @ 03:20:56.519\\",\\"23.184.94.58\\",jpg
+\\"Sep 20, 2015 @ 03:20:40.189\\",\\"80.83.92.252\\",jpg
+\\"Sep 20, 2015 @ 03:20:27.012\\",\\"66.194.157.171\\",png
+\\"Sep 20, 2015 @ 03:20:24.450\\",\\"15.191.218.38\\",jpg
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource date formatting With filters and timebased data, default to UTC 1`] = `
+"\\"'@timestamp\\",clientip,extension
+\\"Sep 20, 2015 @ 10:26:48.725\\",\\"74.214.76.90\\",jpg
+\\"Sep 20, 2015 @ 10:26:48.540\\",\\"146.86.123.109\\",jpg
+\\"Sep 20, 2015 @ 10:26:48.353\\",\\"233.126.159.144\\",jpg
+\\"Sep 20, 2015 @ 10:26:45.468\\",\\"153.139.156.196\\",png
+\\"Sep 20, 2015 @ 10:26:34.063\\",\\"25.140.171.133\\",css
+\\"Sep 20, 2015 @ 10:26:11.181\\",\\"239.249.202.59\\",jpg
+\\"Sep 20, 2015 @ 10:26:00.639\\",\\"95.59.225.31\\",css
+\\"Sep 20, 2015 @ 10:26:00.094\\",\\"247.174.57.245\\",jpg
+\\"Sep 20, 2015 @ 10:25:55.744\\",\\"116.126.47.226\\",css
+\\"Sep 20, 2015 @ 10:25:54.701\\",\\"169.228.188.120\\",jpg
+\\"Sep 20, 2015 @ 10:25:52.360\\",\\"74.224.77.232\\",css
+\\"Sep 20, 2015 @ 10:25:49.913\\",\\"97.83.96.39\\",css
+\\"Sep 20, 2015 @ 10:25:44.979\\",\\"175.188.44.145\\",css
+\\"Sep 20, 2015 @ 10:25:40.968\\",\\"89.143.125.181\\",jpg
+\\"Sep 20, 2015 @ 10:25:36.331\\",\\"231.169.195.137\\",css
+\\"Sep 20, 2015 @ 10:25:34.064\\",\\"137.205.146.206\\",jpg
+\\"Sep 20, 2015 @ 10:25:32.312\\",\\"53.0.188.251\\",jpg
+\\"Sep 20, 2015 @ 10:25:27.254\\",\\"111.214.104.239\\",jpg
+\\"Sep 20, 2015 @ 10:25:22.561\\",\\"111.46.85.146\\",jpg
+\\"Sep 20, 2015 @ 10:25:06.674\\",\\"55.100.60.111\\",jpg
+\\"Sep 20, 2015 @ 10:25:05.114\\",\\"34.197.178.155\\",jpg
+\\"Sep 20, 2015 @ 10:24:55.114\\",\\"163.123.136.118\\",jpg
+\\"Sep 20, 2015 @ 10:24:54.818\\",\\"11.195.163.57\\",jpg
+\\"Sep 20, 2015 @ 10:24:53.742\\",\\"96.222.137.213\\",png
+\\"Sep 20, 2015 @ 10:24:48.798\\",\\"227.228.214.218\\",jpg
+\\"Sep 20, 2015 @ 10:24:20.223\\",\\"228.53.110.116\\",jpg
+\\"Sep 20, 2015 @ 10:24:01.794\\",\\"196.131.253.111\\",png
+\\"Sep 20, 2015 @ 10:23:49.521\\",\\"125.163.133.47\\",jpg
+\\"Sep 20, 2015 @ 10:23:45.816\\",\\"148.47.216.255\\",jpg
+\\"Sep 20, 2015 @ 10:23:36.052\\",\\"51.105.100.214\\",jpg
+\\"Sep 20, 2015 @ 10:23:34.323\\",\\"41.210.252.157\\",gif
+\\"Sep 20, 2015 @ 10:23:27.213\\",\\"248.163.75.193\\",png
+\\"Sep 20, 2015 @ 10:23:14.866\\",\\"48.43.210.167\\",png
+\\"Sep 20, 2015 @ 10:23:10.578\\",\\"33.95.78.209\\",css
+\\"Sep 20, 2015 @ 10:23:07.001\\",\\"96.40.73.208\\",css
+\\"Sep 20, 2015 @ 10:23:02.876\\",\\"174.32.230.63\\",jpg
+\\"Sep 20, 2015 @ 10:23:00.019\\",\\"140.233.207.177\\",jpg
+\\"Sep 20, 2015 @ 10:22:47.447\\",\\"37.127.124.65\\",jpg
+\\"Sep 20, 2015 @ 10:22:45.803\\",\\"130.171.208.139\\",png
+\\"Sep 20, 2015 @ 10:22:45.590\\",\\"39.250.210.253\\",jpg
+\\"Sep 20, 2015 @ 10:22:43.997\\",\\"248.239.221.43\\",css
+\\"Sep 20, 2015 @ 10:22:36.107\\",\\"232.64.207.109\\",gif
+\\"Sep 20, 2015 @ 10:22:30.527\\",\\"24.186.122.118\\",jpg
+\\"Sep 20, 2015 @ 10:22:25.697\\",\\"23.3.174.206\\",jpg
+\\"Sep 20, 2015 @ 10:22:08.272\\",\\"185.170.80.142\\",php
+\\"Sep 20, 2015 @ 10:21:40.822\\",\\"202.22.74.232\\",png
+\\"Sep 20, 2015 @ 10:21:36.210\\",\\"39.227.27.167\\",jpg
+\\"Sep 20, 2015 @ 10:21:19.154\\",\\"140.233.207.177\\",jpg
+\\"Sep 20, 2015 @ 10:21:09.852\\",\\"22.151.97.227\\",jpg
+\\"Sep 20, 2015 @ 10:21:06.079\\",\\"157.39.25.197\\",css
+\\"Sep 20, 2015 @ 10:21:01.357\\",\\"37.127.124.65\\",jpg
+\\"Sep 20, 2015 @ 10:20:56.519\\",\\"23.184.94.58\\",jpg
+\\"Sep 20, 2015 @ 10:20:40.189\\",\\"80.83.92.252\\",jpg
+\\"Sep 20, 2015 @ 10:20:27.012\\",\\"66.194.157.171\\",png
+\\"Sep 20, 2015 @ 10:20:24.450\\",\\"15.191.218.38\\",jpg
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource date formatting With filters and timebased data, explicit UTC format 1`] = `
+"\\"'@timestamp\\",clientip,extension
+\\"Sep 20, 2015 @ 10:26:48.725\\",\\"74.214.76.90\\",jpg
+\\"Sep 20, 2015 @ 10:26:48.540\\",\\"146.86.123.109\\",jpg
+\\"Sep 20, 2015 @ 10:26:48.353\\",\\"233.126.159.144\\",jpg
+\\"Sep 20, 2015 @ 10:26:45.468\\",\\"153.139.156.196\\",png
+\\"Sep 20, 2015 @ 10:26:34.063\\",\\"25.140.171.133\\",css
+\\"Sep 20, 2015 @ 10:26:11.181\\",\\"239.249.202.59\\",jpg
+\\"Sep 20, 2015 @ 10:26:00.639\\",\\"95.59.225.31\\",css
+\\"Sep 20, 2015 @ 10:26:00.094\\",\\"247.174.57.245\\",jpg
+\\"Sep 20, 2015 @ 10:25:55.744\\",\\"116.126.47.226\\",css
+\\"Sep 20, 2015 @ 10:25:54.701\\",\\"169.228.188.120\\",jpg
+\\"Sep 20, 2015 @ 10:25:52.360\\",\\"74.224.77.232\\",css
+\\"Sep 20, 2015 @ 10:25:49.913\\",\\"97.83.96.39\\",css
+\\"Sep 20, 2015 @ 10:25:44.979\\",\\"175.188.44.145\\",css
+\\"Sep 20, 2015 @ 10:25:40.968\\",\\"89.143.125.181\\",jpg
+\\"Sep 20, 2015 @ 10:25:36.331\\",\\"231.169.195.137\\",css
+\\"Sep 20, 2015 @ 10:25:34.064\\",\\"137.205.146.206\\",jpg
+\\"Sep 20, 2015 @ 10:25:32.312\\",\\"53.0.188.251\\",jpg
+\\"Sep 20, 2015 @ 10:25:27.254\\",\\"111.214.104.239\\",jpg
+\\"Sep 20, 2015 @ 10:25:22.561\\",\\"111.46.85.146\\",jpg
+\\"Sep 20, 2015 @ 10:25:06.674\\",\\"55.100.60.111\\",jpg
+\\"Sep 20, 2015 @ 10:25:05.114\\",\\"34.197.178.155\\",jpg
+\\"Sep 20, 2015 @ 10:24:55.114\\",\\"163.123.136.118\\",jpg
+\\"Sep 20, 2015 @ 10:24:54.818\\",\\"11.195.163.57\\",jpg
+\\"Sep 20, 2015 @ 10:24:53.742\\",\\"96.222.137.213\\",png
+\\"Sep 20, 2015 @ 10:24:48.798\\",\\"227.228.214.218\\",jpg
+\\"Sep 20, 2015 @ 10:24:20.223\\",\\"228.53.110.116\\",jpg
+\\"Sep 20, 2015 @ 10:24:01.794\\",\\"196.131.253.111\\",png
+\\"Sep 20, 2015 @ 10:23:49.521\\",\\"125.163.133.47\\",jpg
+\\"Sep 20, 2015 @ 10:23:45.816\\",\\"148.47.216.255\\",jpg
+\\"Sep 20, 2015 @ 10:23:36.052\\",\\"51.105.100.214\\",jpg
+\\"Sep 20, 2015 @ 10:23:34.323\\",\\"41.210.252.157\\",gif
+\\"Sep 20, 2015 @ 10:23:27.213\\",\\"248.163.75.193\\",png
+\\"Sep 20, 2015 @ 10:23:14.866\\",\\"48.43.210.167\\",png
+\\"Sep 20, 2015 @ 10:23:10.578\\",\\"33.95.78.209\\",css
+\\"Sep 20, 2015 @ 10:23:07.001\\",\\"96.40.73.208\\",css
+\\"Sep 20, 2015 @ 10:23:02.876\\",\\"174.32.230.63\\",jpg
+\\"Sep 20, 2015 @ 10:23:00.019\\",\\"140.233.207.177\\",jpg
+\\"Sep 20, 2015 @ 10:22:47.447\\",\\"37.127.124.65\\",jpg
+\\"Sep 20, 2015 @ 10:22:45.803\\",\\"130.171.208.139\\",png
+\\"Sep 20, 2015 @ 10:22:45.590\\",\\"39.250.210.253\\",jpg
+\\"Sep 20, 2015 @ 10:22:43.997\\",\\"248.239.221.43\\",css
+\\"Sep 20, 2015 @ 10:22:36.107\\",\\"232.64.207.109\\",gif
+\\"Sep 20, 2015 @ 10:22:30.527\\",\\"24.186.122.118\\",jpg
+\\"Sep 20, 2015 @ 10:22:25.697\\",\\"23.3.174.206\\",jpg
+\\"Sep 20, 2015 @ 10:22:08.272\\",\\"185.170.80.142\\",php
+\\"Sep 20, 2015 @ 10:21:40.822\\",\\"202.22.74.232\\",png
+\\"Sep 20, 2015 @ 10:21:36.210\\",\\"39.227.27.167\\",jpg
+\\"Sep 20, 2015 @ 10:21:19.154\\",\\"140.233.207.177\\",jpg
+\\"Sep 20, 2015 @ 10:21:09.852\\",\\"22.151.97.227\\",jpg
+\\"Sep 20, 2015 @ 10:21:06.079\\",\\"157.39.25.197\\",css
+\\"Sep 20, 2015 @ 10:21:01.357\\",\\"37.127.124.65\\",jpg
+\\"Sep 20, 2015 @ 10:20:56.519\\",\\"23.184.94.58\\",jpg
+\\"Sep 20, 2015 @ 10:20:40.189\\",\\"80.83.92.252\\",jpg
+\\"Sep 20, 2015 @ 10:20:27.012\\",\\"66.194.157.171\\",png
+\\"Sep 20, 2015 @ 10:20:24.450\\",\\"15.191.218.38\\",jpg
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource non-timebased Handle _id and _index columns 1`] = `
+"date,message,\\"_id\\",\\"_index\\"
+\\"Jan 1, 2015 @ 12:10:30.123456789\\",\\"Hello 2\\",2,nanos
+\\"Jan 1, 2015 @ 12:10:30.000000000\\",\\"Hello 1\\",1,nanos
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource non-timebased With filters and non-timebased data 1`] = `
+"name,power
+\\"Jonelle-Jane Marth\\",1
+\\"Suzie-May Rishel\\",1
+\\"Suzie-May Rishel\\",2
+\\"Rosana Casto\\",2
+\\"Stephen Cortez\\",4
+\\"Jonelle-Jane Marth\\",6
+\\"Jonelle-Jane Marth\\",7
+\\"Florinda Alejandro\\",10
+\\"Jonelle-Jane Marth\\",14
+\\"Suzie-May Rishel\\",19
+\\"Suzie-May Rishel\\",20
+\\"Florinda Alejandro\\",22
+"
+`;
+
+exports[`Reporting APIs CSV generation from SearchSource validation Searches "huge" data, stops at Max Size Reached 1`] = `
+"\\"_id\\",date,name,gender,value,year,\\"years_ago\\",\\"date_informal\\"
+\\"1984-Fiff-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fiff,F,5,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Filal-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Filal,F,15,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fillie-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fillie,F,135,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fillyjack-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fillyjack,F,6,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fing-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fing,F,6,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Firan-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Firan,F,9,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fishara-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fishara,F,5,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fishop-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fishop,F,9,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fj-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fj,F,12,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fjorn-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fjorn,F,22,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Flaine-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Flaine,F,234,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Flake-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Flake,F,420,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Flas-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Flas,F,38,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fo-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fo,F,45,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Foaz-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Foaz,F,13,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fob-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fob,F,170,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fobby-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fobby,F,\\"2,791\\",1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fodie-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fodie,F,12,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Foe-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Foe,F,15,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fomani-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fomani,F,7,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Ferley-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Ferley,F,6,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fernardino-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fernardino,F,30,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fernardo-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fernardo,F,115,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fernerd-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fernerd,F,6,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fernice-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fernice,F,7,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Ferry-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Ferry,F,47,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+\\"1984-Fert-F\\",\\"Jan 1, 1984 @ 00:00:00.000\\",Fert,F,140,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
+"
+`;
diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/csv_job_params.ts b/x-pack/test/reporting_api_integration/reporting_and_security/csv_job_params.ts
index 732e640f29a4d..b3fa9ebe46f8c 100644
--- a/x-pack/test/reporting_api_integration/reporting_and_security/csv_job_params.ts
+++ b/x-pack/test/reporting_api_integration/reporting_and_security/csv_job_params.ts
@@ -7,7 +7,7 @@
import expect from '@kbn/expect';
import supertest from 'supertest';
-import { JOB_PARAMS_RISON } from '../fixtures';
+import { JOB_PARAMS_RISON_CSV_DEPRECATED } from '../fixtures';
import { FtrProviderContext } from '../ftr_provider_context';
// eslint-disable-next-line import/no-default-export
@@ -63,7 +63,7 @@ export default function ({ getService }: FtrProviderContext) {
it('Accepts jobParams in POST payload', async () => {
const { status: resStatus, text: resText } = (await generateAPI.getCsvFromParamsInPayload({
- jobParams: JOB_PARAMS_RISON,
+ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED,
})) as supertest.Response;
expect(resText).to.match(/"jobtype":"csv"/);
expect(resStatus).to.eql(200);
@@ -71,7 +71,7 @@ export default function ({ getService }: FtrProviderContext) {
it('Accepts jobParams in query string', async () => {
const { status: resStatus, text: resText } = (await generateAPI.getCsvFromParamsInQueryString(
- JOB_PARAMS_RISON
+ JOB_PARAMS_RISON_CSV_DEPRECATED
)) as supertest.Response;
expect(resText).to.match(/"jobtype":"csv"/);
expect(resStatus).to.eql(200);
diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts b/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts
deleted file mode 100644
index bb70924f67b75..0000000000000
--- a/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import expect from '@kbn/expect';
-import supertest from 'supertest';
-import * as fixtures from '../fixtures';
-import { FtrProviderContext } from '../ftr_provider_context';
-
-interface GenerateOpts {
- timerange?: {
- timezone: string;
- min?: number | string | Date;
- max?: number | string | Date;
- };
- state: any;
-}
-
-// eslint-disable-next-line import/no-default-export
-export default function ({ getService }: FtrProviderContext) {
- const esArchiver = getService('esArchiver');
- const supertestSvc = getService('supertest');
- const reportingAPI = getService('reportingAPI');
-
- const generateAPI = {
- getCsvFromSavedSearch: async (
- id: string,
- { timerange, state }: GenerateOpts,
- isImmediate = true
- ) => {
- return await supertestSvc
- .post(`/api/reporting/v1/generate/${isImmediate ? 'immediate/' : ''}csv/saved-object/${id}`)
- .set('kbn-xsrf', 'xxx')
- .send({ timerange, state });
- },
- };
-
- describe('Generation from Saved Search ID', () => {
- after(async () => {
- await reportingAPI.deleteAllReports();
- });
-
- describe('Saved Search Features', () => {
- it('With filters and timebased data, explicit UTC format', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/logs');
- await esArchiver.load('logstash_functional');
-
- const res = (await generateAPI.getCsvFromSavedSearch(
- 'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
- {
- timerange: {
- timezone: 'UTC',
- min: '2015-09-19T10:00:00.000Z',
- max: '2015-09-21T10:00:00.000Z',
- },
- state: {},
- }
- )) as supertest.Response;
- const { status: resStatus, text: resText, type: resType } = res;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_TIMEBASED_UTC);
-
- await esArchiver.unload('reporting/logs');
- await esArchiver.unload('logstash_functional');
- });
-
- it('With filters and timebased data, default to UTC', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/logs');
- await esArchiver.load('logstash_functional');
-
- const res = (await generateAPI.getCsvFromSavedSearch(
- 'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
- {
- // @ts-expect-error: timerange.timezone is missing from post params
- timerange: {
- min: '2015-09-19T10:00:00.000Z',
- max: '2015-09-21T10:00:00.000Z',
- },
- state: {},
- }
- )) as supertest.Response;
- const { status: resStatus, text: resText, type: resType } = res;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_TIMEBASED_UTC);
-
- await esArchiver.unload('reporting/logs');
- await esArchiver.unload('logstash_functional');
- });
-
- it('With filters and timebased data, custom timezone', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/logs');
- await esArchiver.load('logstash_functional');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
- {
- timerange: {
- timezone: 'America/Phoenix',
- min: '2015-09-19T10:00:00.000Z',
- max: '2015-09-21T10:00:00.000Z',
- },
- state: {},
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_TIMEBASED_CUSTOM);
-
- await esArchiver.unload('reporting/logs');
- await esArchiver.unload('logstash_functional');
- });
-
- it('With filters and non-timebased data', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/sales');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:71e3ee20-3f99-11e9-b8ee-6b9604f2f877',
- {
- state: {},
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_TIMELESS);
-
- await esArchiver.unload('reporting/sales');
- });
-
- it('With scripted fields and field formatters', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/scripted_small2');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:a6d51430-ace2-11ea-815f-39e12f89a8c2',
- {
- timerange: {
- timezone: 'UTC',
- min: '1979-01-01T10:00:00Z',
- max: '1981-01-01T10:00:00Z',
- },
- state: {},
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_SCRIPTED);
-
- await esArchiver.unload('reporting/scripted_small2');
- });
-
- it('Formatted date_nanos data, UTC timezone', async () => {
- await esArchiver.load('reporting/nanos');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:e4035040-a295-11e9-a900-ef10e0ac769e',
- {
- state: {},
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_NANOS);
-
- await esArchiver.unload('reporting/nanos');
- });
-
- it('Formatted date_nanos data, custom time zone', async () => {
- await esArchiver.load('reporting/nanos');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:e4035040-a295-11e9-a900-ef10e0ac769e',
- {
- state: {},
- timerange: { timezone: 'America/New_York' },
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_NANOS_CUSTOM);
-
- await esArchiver.unload('reporting/nanos');
- });
- });
-
- describe('API Features', () => {
- it('Return a 404', async () => {
- const { body } = (await generateAPI.getCsvFromSavedSearch('search:gobbledygook', {
- timerange: { timezone: 'UTC', min: 63097200000, max: 126255599999 },
- state: {},
- })) as supertest.Response;
- const expectedBody = {
- error: 'Not Found',
- message: 'Saved object [search/gobbledygook] not found',
- statusCode: 404,
- };
- expect(body).to.eql(expectedBody);
- });
-
- it('Return 400 if time range param is needed but missing', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/logs');
- await esArchiver.load('logstash_functional');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
- { state: {} }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(400);
- expect(resType).to.eql('application/json');
- const { message: errorMessage } = JSON.parse(resText);
- expect(errorMessage).to.eql(
- 'Time range params are required for index pattern [logstash-*], using time field [@timestamp]'
- );
-
- await esArchiver.unload('reporting/logs');
- await esArchiver.unload('logstash_functional');
- });
-
- it('Stops at Max Size Reached', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/hugedata');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24',
- {
- timerange: {
- timezone: 'UTC',
- min: '1960-01-01T10:00:00Z',
- max: '1999-01-01T10:00:00Z',
- },
- state: {},
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_HUGE);
-
- await esArchiver.unload('reporting/hugedata');
- });
- });
-
- describe('Merge user state into the query', () => {
- it('for query', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/scripted_small2');
-
- const params = {
- searchId: 'search:a6d51430-ace2-11ea-815f-39e12f89a8c2',
- postPayload: {
- timerange: { timezone: 'UTC', min: '1979-01-01T10:00:00Z', max: '1981-01-01T10:00:00Z' }, // prettier-ignore
- state: { query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fel*' } }] } } ] } } ] } } }, // prettier-ignore
- },
- isImmediate: true,
- };
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- params.searchId,
- params.postPayload,
- params.isImmediate
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_SCRIPTED_REQUERY);
-
- await esArchiver.unload('reporting/scripted_small2');
- });
-
- it('for sort', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/hugedata');
-
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24',
- {
- timerange: {
- timezone: 'UTC',
- min: '1979-01-01T10:00:00Z',
- max: '1981-01-01T10:00:00Z',
- },
- state: { sort: [{ name: { order: 'asc', unmapped_type: 'boolean' } }] },
- }
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_SCRIPTED_RESORTED);
-
- await esArchiver.unload('reporting/hugedata');
- });
-
- it('for docvalue_fields', async () => {
- // load test data that contains a saved search and documents
- await esArchiver.load('reporting/ecommerce');
- await esArchiver.load('reporting/ecommerce_kibana');
-
- const params = {
- searchId: 'search:6091ead0-1c6d-11ea-a100-8589bb9d7c6b',
- postPayload: {
- timerange: {
- min: '2019-05-28T00:00:00Z',
- max: '2019-06-26T00:00:00Z',
- timezone: 'UTC',
- },
- state: {
- sort: [
- { order_date: { order: 'desc', unmapped_type: 'boolean' } },
- { order_id: { order: 'asc', unmapped_type: 'boolean' } },
- ],
- docvalue_fields: [
- { field: 'customer_birth_date', format: 'date_time' },
- { field: 'order_date', format: 'date_time' },
- { field: 'products.created_on', format: 'date_time' },
- ],
- query: {
- bool: {
- must: [],
- filter: [
- { match_all: {} },
- { match_all: {} },
- {
- range: {
- order_date: {
- gte: '2019-05-28T00:00:00.000Z',
- lte: '2019-06-26T00:00:00.000Z',
- format: 'strict_date_optional_time',
- },
- },
- },
- ],
- should: [],
- must_not: [],
- },
- },
- },
- },
- isImmediate: true,
- };
- const {
- status: resStatus,
- text: resText,
- type: resType,
- } = (await generateAPI.getCsvFromSavedSearch(
- params.searchId,
- params.postPayload,
- params.isImmediate
- )) as supertest.Response;
-
- expect(resStatus).to.eql(200);
- expect(resType).to.eql('text/csv');
- expect(resText).to.eql(fixtures.CSV_RESULT_DOCVALUE);
-
- await esArchiver.unload('reporting/ecommerce');
- await esArchiver.unload('reporting/ecommerce_kibana');
- });
- });
- });
-}
diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/csv_searchsource_immediate.ts b/x-pack/test/reporting_api_integration/reporting_and_security/csv_searchsource_immediate.ts
new file mode 100644
index 0000000000000..7f2a13b7cfe45
--- /dev/null
+++ b/x-pack/test/reporting_api_integration/reporting_and_security/csv_searchsource_immediate.ts
@@ -0,0 +1,344 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import supertest from 'supertest';
+import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+const getMockJobParams = (obj: Partial): JobParamsDownloadCSV => ({
+ title: `Mock CSV Title`,
+ ...(obj as any),
+});
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getService }: FtrProviderContext) {
+ const kibanaServer = getService('kibanaServer');
+ const esArchiver = getService('esArchiver');
+ const supertestSvc = getService('supertest');
+ const reportingAPI = getService('reportingAPI');
+
+ const generateAPI = {
+ getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => {
+ return await supertestSvc
+ .post(`/api/reporting/v1/generate/immediate/csv_searchsource`)
+ .set('kbn-xsrf', 'xxx')
+ .send(job);
+ },
+ };
+
+ describe('CSV generation from SearchSource', () => {
+ before(async () => {
+ await kibanaServer.uiSettings.update({
+ 'csv:quoteValues': false,
+ 'dateFormat:tz': 'UTC',
+ defaultIndex: 'logstash-*',
+ });
+ });
+ after(async () => {
+ await reportingAPI.deleteAllReports();
+ });
+
+ describe('date formatting', () => {
+ before(async () => {
+ // load test data that contains a saved search and documents
+ await esArchiver.load('reporting/logs');
+ await esArchiver.load('logstash_functional');
+ });
+ after(async () => {
+ await esArchiver.unload('reporting/logs');
+ await esArchiver.unload('logstash_functional');
+ });
+
+ it('With filters and timebased data, explicit UTC format', async () => {
+ const res = (await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ browserTimezone: 'UTC',
+ searchSource: {
+ fields: ['@timestamp', 'clientip', 'extension'],
+ filter: [
+ {
+ range: {
+ '@timestamp': {
+ gte: '2015-09-20T10:19:40.307Z',
+ lt: '2015-09-20T10:26:56.221Z',
+ },
+ },
+ },
+ {
+ range: {
+ '@timestamp': {
+ format: 'strict_date_optional_time',
+ gte: '2015-01-12T07:00:55.654Z',
+ lte: '2016-01-29T21:08:10.881Z',
+ },
+ },
+ },
+ ],
+ index: 'logstash-*',
+ query: { language: 'kuery', query: '' },
+ sort: [{ '@timestamp': 'desc' }],
+ },
+ })
+ )) as supertest.Response;
+ const { status: resStatus, text: resText, type: resType } = res;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+ });
+
+ it('With filters and timebased data, default to UTC', async () => {
+ const res = (await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ searchSource: {
+ fields: ['@timestamp', 'clientip', 'extension'],
+ filter: [
+ {
+ range: {
+ '@timestamp': {
+ gte: '2015-09-20T10:19:40.307Z',
+ lt: '2015-09-20T10:26:56.221Z',
+ },
+ },
+ },
+ {
+ range: {
+ '@timestamp': {
+ format: 'strict_date_optional_time',
+ gte: '2015-01-12T07:00:55.654Z',
+ lte: '2016-01-29T21:08:10.881Z',
+ },
+ },
+ },
+ ],
+ index: 'logstash-*',
+ query: { language: 'kuery', query: '' },
+ sort: [{ '@timestamp': 'desc' }],
+ },
+ })
+ )) as supertest.Response;
+ const { status: resStatus, text: resText, type: resType } = res;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+ });
+
+ it('With filters and timebased data, custom timezone (Phoenix)', async () => {
+ const res = (await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ browserTimezone: 'America/Phoenix',
+ searchSource: {
+ fields: ['@timestamp', 'clientip', 'extension'],
+ filter: [
+ {
+ range: {
+ '@timestamp': {
+ gte: '2015-09-20T10:19:40.307Z',
+ lt: '2015-09-20T10:26:56.221Z',
+ },
+ },
+ },
+ {
+ range: {
+ '@timestamp': {
+ format: 'strict_date_optional_time',
+ gte: '2015-01-12T07:00:55.654Z',
+ lte: '2016-01-29T21:08:10.881Z',
+ },
+ },
+ },
+ ],
+ index: 'logstash-*',
+ query: { language: 'kuery', query: '' },
+ sort: [{ '@timestamp': 'desc' }],
+ },
+ })
+ )) as supertest.Response;
+ const { status: resStatus, text: resText, type: resType } = res;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+ });
+
+ it('Formatted date_nanos data, UTC timezone', async () => {
+ await esArchiver.load('reporting/nanos');
+
+ const res = await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ searchSource: {
+ query: { query: '', language: 'kuery' },
+ version: true,
+ index: '907bc200-a294-11e9-a900-ef10e0ac769e',
+ sort: [{ date: 'desc' }],
+ fields: ['date', 'message'],
+ filter: [],
+ },
+ })
+ );
+ const { status: resStatus, text: resText, type: resType } = res;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+
+ await esArchiver.unload('reporting/nanos');
+ });
+
+ it('Formatted date_nanos data, custom timezone (New York)', async () => {
+ await esArchiver.load('reporting/nanos');
+
+ const res = await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ browserTimezone: 'America/New_York',
+ searchSource: {
+ query: { query: '', language: 'kuery' },
+ version: true,
+ index: '907bc200-a294-11e9-a900-ef10e0ac769e',
+ sort: [{ date: 'desc' }],
+ fields: ['date', 'message'],
+ filter: [],
+ },
+ })
+ );
+ const { status: resStatus, text: resText, type: resType } = res;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+
+ await esArchiver.unload('reporting/nanos');
+ });
+ });
+
+ describe('non-timebased', () => {
+ it('Handle _id and _index columns', async () => {
+ await esArchiver.load('reporting/nanos');
+
+ const res = await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ searchSource: {
+ query: { query: '', language: 'kuery' },
+ version: true,
+ index: '907bc200-a294-11e9-a900-ef10e0ac769e',
+ sort: [{ date: 'desc' }],
+ fields: ['date', 'message', '_id', '_index'],
+ filter: [],
+ },
+ })
+ );
+ const { status: resStatus, text: resText, type: resType } = res;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+
+ await esArchiver.unload('reporting/nanos');
+ });
+
+ it('With filters and non-timebased data', async () => {
+ // load test data that contains a saved search and documents
+ await esArchiver.load('reporting/sales');
+
+ const {
+ status: resStatus,
+ text: resText,
+ type: resType,
+ } = (await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ searchSource: {
+ query: { query: '', language: 'kuery' },
+ version: true,
+ index: 'timeless-sales',
+ sort: [{ power: 'asc' }],
+ fields: ['name', 'power'],
+ filter: [
+ {
+ range: { power: { gte: 1, lt: null } },
+ },
+ ],
+ },
+ })
+ )) as supertest.Response;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+
+ await esArchiver.unload('reporting/sales');
+ });
+ });
+
+ describe('validation', () => {
+ it('Return a 404', async () => {
+ const { body } = (await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ searchSource: {
+ index: 'gobbledygook',
+ },
+ })
+ )) as supertest.Response;
+ const expectedBody = {
+ error: 'Not Found',
+ message: 'Saved object [index-pattern/gobbledygook] not found',
+ statusCode: 404,
+ };
+ expect(body).to.eql(expectedBody);
+ });
+
+ it(`Searches "huge" data, stops at Max Size Reached`, async () => {
+ // load test data that contains a saved search and documents
+ await esArchiver.load('reporting/hugedata');
+
+ const {
+ status: resStatus,
+ text: resText,
+ type: resType,
+ } = (await generateAPI.getCSVFromSearchSource(
+ getMockJobParams({
+ searchSource: {
+ query: { query: '', language: 'lucene' },
+ version: true,
+ sort: [{ date: 'desc' }],
+ index: '89655130-5013-11e9-bce7-4dabcb8bef24',
+ fields: [
+ '_id',
+ 'date',
+ 'name',
+ 'gender',
+ 'value',
+ 'year',
+ 'years_ago',
+ 'date_informal',
+ ],
+ filter: [
+ {
+ meta: { index: '89655130-5013-11e9-bce7-4dabcb8bef24', params: {} },
+ range: {
+ date: {
+ gte: '1960-01-01T10:00:00Z',
+ lte: '1999-01-01T10:00:00Z',
+ format: 'strict_date_optional_time',
+ },
+ },
+ },
+ ],
+ },
+ })
+ )) as supertest.Response;
+
+ expect(resStatus).to.eql(200);
+ expect(resType).to.eql('text/csv');
+ expectSnapshot(resText).toMatch();
+
+ await esArchiver.unload('reporting/hugedata');
+ });
+ });
+ });
+}
diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts
index 2981ff81d66eb..b4e05e37d3fda 100644
--- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts
+++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts
@@ -12,7 +12,7 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('Reporting APIs', function () {
this.tags('ciGroup2');
loadTestFile(require.resolve('./csv_job_params'));
- loadTestFile(require.resolve('./csv_saved_search'));
+ loadTestFile(require.resolve('./csv_searchsource_immediate'));
loadTestFile(require.resolve('./network_policy'));
loadTestFile(require.resolve('./spaces'));
loadTestFile(require.resolve('./usage'));
diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts
index 99832e3f4b1c2..b8ef01f316381 100644
--- a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts
+++ b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts
@@ -7,7 +7,7 @@
import expect from '@kbn/expect';
import { forOwn } from 'lodash';
-import { JOB_PARAMS_RISON } from '../fixtures';
+import { JOB_PARAMS_RISON_CSV_DEPRECATED } from '../fixtures';
import { FtrProviderContext } from '../ftr_provider_context';
// eslint-disable-next-line import/no-default-export
@@ -35,7 +35,7 @@ export default function ({ getService }: FtrProviderContext) {
const { status: resStatus, text: resText } = await supertestNoAuth
.post(`/api/reporting/generate/csv`)
.set('kbn-xsrf', 'xxx')
- .send({ jobParams: JOB_PARAMS_RISON });
+ .send({ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED });
expect(resStatus).to.be(200);
@@ -67,7 +67,7 @@ export default function ({ getService }: FtrProviderContext) {
const { status: resStatus, text: resText } = await supertestNoAuth
.post(`/api/reporting/generate/csv`)
.set('kbn-xsrf', 'xxx')
- .send({ jobParams: JOB_PARAMS_RISON });
+ .send({ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED });
expect(resStatus).to.be(200);
@@ -97,7 +97,7 @@ export default function ({ getService }: FtrProviderContext) {
const { status: resStatus, text: resText } = await supertestNoAuth
.post(`/api/reporting/generate/csv`)
.set('kbn-xsrf', 'xxx')
- .send({ jobParams: JOB_PARAMS_RISON });
+ .send({ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED });
expect(resStatus).to.be(200);