[Reporting-CSV Export] Re-write CSV Export using SearchSource

elastic · Mar 4, 2021 · de10eae · de10eae
1 parent e45718d
commit de10eae
Show file tree

Hide file tree

Showing 75 changed files with 4,727 additions and 4,758 deletions.
diff --git a/...ent/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md b/...ent/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md
@@ -25,6 +25,7 @@ export interface SearchSourceFields
 |  [highlightAll](./kibana-plugin-plugins-data-public.searchsourcefields.highlightall.md) | <code>boolean</code> |  |
 |  [index](./kibana-plugin-plugins-data-public.searchsourcefields.index.md) | <code>IndexPattern</code> |  |
 |  [parent](./kibana-plugin-plugins-data-public.searchsourcefields.parent.md) | <code>SearchSourceFields</code> |  |
+|  [pit](./kibana-plugin-plugins-data-public.searchsourcefields.pit.md) | <code>{</code><br/><code>        id: string;</code><br/><code>        keep_alive?: string;</code><br/><code>    }</code> |  |
 |  [query](./kibana-plugin-plugins-data-public.searchsourcefields.query.md) | <code>Query</code> | [Query](./kibana-plugin-plugins-data-public.query.md) |
 |  [searchAfter](./kibana-plugin-plugins-data-public.searchsourcefields.searchafter.md) | <code>EsQuerySearchAfter</code> |  |
 |  [size](./kibana-plugin-plugins-data-public.searchsourcefields.size.md) | <code>number</code> |  |

diff --git a/...plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.pit.md b/...plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.pit.md
@@ -0,0 +1,14 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [SearchSourceFields](./kibana-plugin-plugins-data-public.searchsourcefields.md) &gt; [pit](./kibana-plugin-plugins-data-public.searchsourcefields.pit.md)
+
+## SearchSourceFields.pit property
+
+<b>Signature:</b>
+
+```typescript
+pit?: {
+        id: string;
+        keep_alive?: string;
+    };
+```
diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts
@@ -439,6 +439,8 @@ export class SearchSource {
           getConfig(UI_SETTINGS.SORT_OPTIONS)
         );
         return addToBody(key, sort);
+      case 'pit':
+        return addToRoot(key, val);
       default:
         return addToBody(key, val);
     }

diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts
@@ -100,7 +100,10 @@ export interface SearchSourceFields {
   searchAfter?: EsQuerySearchAfter;
   timeout?: string;
   terminate_after?: number;
-
+  pit?: {
+    id: string;
+    keep_alive?: string;
+  };
   parent?: SearchSourceFields;
 }
 

diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md
@@ -2438,6 +2438,11 @@ export interface SearchSourceFields {
     // (undocumented)
     parent?: SearchSourceFields;
     // (undocumented)
+    pit?: {
+        id: string;
+        keep_alive?: string;
+    };
+    // (undocumented)
     query?: Query;
     // Warning: (ae-forgotten-export) The symbol "EsQuerySearchAfter" needs to be exported by the entry point index.d.ts
     //

diff --git a/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts
@@ -97,8 +97,7 @@ export const getTopNavLinks = ({
       const sharingData = await getSharingData(
         searchSource,
         state.appStateContainer.getState(),
-        services.uiSettings,
-        getFieldCounts
+        services.uiSettings
       );
       services.share.toggleShareContextMenu({
         anchorElement,

diff --git a/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts b/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts
@@ -16,54 +16,26 @@ import { SORT_DEFAULT_ORDER_SETTING } from '../../../common';
 describe('getSharingData', () => {
   test('returns valid data for sharing', async () => {
     const searchSourceMock = createSearchSourceMock({ index: indexPatternMock });
-    const result = await getSharingData(
-      searchSourceMock,
-      { columns: [] },
-      ({
-        get: (key: string) => {
-          if (key === SORT_DEFAULT_ORDER_SETTING) {
-            return 'desc';
-          }
-          return false;
-        },
-      } as unknown) as IUiSettingsClient,
-      () => Promise.resolve({})
-    );
+    const result = await getSharingData(searchSourceMock, { columns: [] }, ({
+      get: (key: string) => {
+        if (key === SORT_DEFAULT_ORDER_SETTING) {
+          return 'desc';
+        }
+        return false;
+      },
+    } as unknown) as IUiSettingsClient);
     expect(result).toMatchInlineSnapshot(`
       Object {
-        "conflictedTypesFields": Array [],
-        "fields": Array [],
-        "indexPatternId": "the-index-pattern-id",
-        "metaFields": Array [
-          "_index",
-          "_score",
-        ],
-        "searchRequest": Object {
-          "body": Object {
-            "_source": Object {},
-            "fields": Array [],
-            "query": Object {
-              "bool": Object {
-                "filter": Array [],
-                "must": Array [],
-                "must_not": Array [],
-                "should": Array [],
-              },
+        "searchSource": Object {
+          "fields": Array [
+            "*",
+          ],
+          "index": "the-index-pattern-id",
+          "sort": Array [
+            Object {
+              "_score": "desc",
             },
-            "runtime_mappings": Object {},
-            "script_fields": Object {},
-            "sort": Array [
-              Object {
-                "_score": Object {
-                  "order": "desc",
-                },
-              },
-            ],
-            "stored_fields": Array [
-              "*",
-            ],
-          },
-          "index": "the-index-pattern-title",
+          ],
         },
       }
     `);

diff --git a/src/plugins/discover/public/application/helpers/get_sharing_data.ts b/src/plugins/discover/public/application/helpers/get_sharing_data.ts
@@ -6,57 +6,24 @@
  * Side Public License, v 1.
  */
 
-import { Capabilities, IUiSettingsClient } from 'kibana/public';
+import type { Capabilities, IUiSettingsClient } from 'kibana/public';
 import { DOC_HIDE_TIME_COLUMN_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common';
 import { getSortForSearchSource } from '../angular/doc_table';
 import { ISearchSource } from '../../../../data/common';
 import { AppState } from '../angular/discover_state';
-import { SortOrder } from '../../saved_searches/types';
-
-const getSharingDataFields = async (
-  getFieldCounts: () => Promise<Record<string, number>>,
-  selectedFields: string[],
-  timeFieldName: string,
-  hideTimeColumn: boolean
-) => {
-  if (
-    selectedFields.length === 0 ||
-    (selectedFields.length === 1 && selectedFields[0] === '_source')
-  ) {
-    const fieldCounts = await getFieldCounts();
-    return {
-      searchFields: undefined,
-      selectFields: Object.keys(fieldCounts).sort(),
-    };
-  }
-
-  const fields =
-    timeFieldName && !hideTimeColumn ? [timeFieldName, ...selectedFields] : selectedFields;
-  return {
-    searchFields: fields,
-    selectFields: fields,
-  };
-};
+import { SavedSearch, SortOrder } from '../../saved_searches/types';
 
 /**
  * Preparing data to share the current state as link or CSV/Report
  */
 export async function getSharingData(
   currentSearchSource: ISearchSource,
-  state: AppState,
-  config: IUiSettingsClient,
-  getFieldCounts: () => Promise<Record<string, number>>
+  state: AppState | SavedSearch,
+  config: IUiSettingsClient
 ) {
   const searchSource = currentSearchSource.createCopy();
   const index = searchSource.getField('index')!;
 
-  const { searchFields, selectFields } = await getSharingDataFields(
-    getFieldCounts,
-    state.columns || [],
-    index.timeFieldName || '',
-    config.get(DOC_HIDE_TIME_COLUMN_SETTING)
-  );
-  searchSource.setField('fieldsFromSource', searchFields);
   searchSource.setField(
     'sort',
     getSortForSearchSource(state.sort as SortOrder[], index, config.get(SORT_DEFAULT_ORDER_SETTING))
@@ -66,20 +33,45 @@ export async function getSharingData(
   searchSource.removeField('aggs');
   searchSource.removeField('size');
 
-  const body = await searchSource.getSearchRequestBody();
+  // Set the fields of the search source to match the saved search columns
+  searchSource.removeField('fields');
+  searchSource.removeField('fieldsFromSource');
+
+  let columns = state.columns || [];
+
+  // NOTE: A newly saved search with no columns selected has a bug(?) where the
+  // column array is a single '_source' value which is invalid for CSV export
+  if (columns && columns.length === 1 && /^_source$/.test(columns.join())) {
+    columns = [];
+  }
+
+  // conditionally add the time field column
+  let timeFieldName: string | undefined;
+  const hideTimeColumn = config.get(DOC_HIDE_TIME_COLUMN_SETTING);
+  if (!hideTimeColumn && index && index.timeFieldName) {
+    timeFieldName = index.timeFieldName;
+  }
+
+  if (columns && columns.length > 0 && timeFieldName) {
+    columns = [timeFieldName, ...columns];
+  }
+
+  if (columns.length === 0) {
+    searchSource.setField('fields', ['*']);
+  } else {
+    searchSource.setField('fields', columns);
+  }
 
   return {
-    searchRequest: {
-      index: index.title,
-      body,
-    },
-    fields: selectFields,
-    metaFields: index.metaFields,
-    conflictedTypesFields: index.fields.filter((f) => f.type === 'conflict').map((f) => f.name),
-    indexPatternId: index.id,
+    searchSource: searchSource.getSerializedFields(true),
   };
 }
 
+/**
+ * makes getSharingData lazy loadable
+ */
+export function getSharingDataModule() {}
+
 export interface DiscoverCapabilities {
   createShortUrl?: boolean;
   save?: boolean;

diff --git a/src/plugins/discover/public/index.ts b/src/plugins/discover/public/index.ts
@@ -16,4 +16,5 @@ export function plugin(initializerContext: PluginInitializerContext) {
 
 export { SavedSearch, SavedSearchLoader, createSavedSearchesLoader } from './saved_searches';
 export { ISearchEmbeddable, SEARCH_EMBEDDABLE_TYPE, SearchInput } from './application/embeddable';
+export { loadSharingDataHelpers } from './shared';
 export { DISCOVER_APP_URL_GENERATOR, DiscoverUrlGeneratorState } from './url_generator';
diff --git a/src/plugins/discover/public/shared/index.ts b/src/plugins/discover/public/shared/index.ts
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/*
+ * Allows the getSharingData function to be lazy loadable
+ */
+export async function loadSharingDataHelpers() {
+  return await import('../application/helpers/get_sharing_data');
+}
diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts
@@ -50,20 +50,24 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];
 export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
 export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
 export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';
+export const UI_SETTINGS_DATEFORMAT_TZ = 'dateFormat:tz';
 
 export const LAYOUT_TYPES = {
   PRESERVE_LAYOUT: 'preserve_layout',
   PRINT: 'print',
 };
 
 // Export Type Definitions
+export const CSV_REPORT_TYPE = 'CSV';
+export const CSV_JOB_TYPE = 'csv_searchsource';
+
 export const PDF_REPORT_TYPE = 'printablePdf';
 export const PDF_JOB_TYPE = 'printable_pdf';
 
 export const PNG_REPORT_TYPE = 'PNG';
 export const PNG_JOB_TYPE = 'PNG';
 
-export const CSV_FROM_SAVEDOBJECT_JOB_TYPE = 'csv_from_savedobject';
+export const CSV_SEARCHSOURCE_IMMEDIATE_TYPE = 'csv_searchsource_immediate';
 
 // This is deprecated because it lacks support for runtime fields
 // but the extension points are still needed for pre-existing scripted automation, until 8.0
@@ -86,9 +90,9 @@ export const API_BASE_GENERATE = `${API_BASE_URL}/generate`;
 export const API_LIST_URL = `${API_BASE_URL}/jobs`;
 export const API_DIAGNOSE_URL = `${API_BASE_URL}/diagnose`;
 
-// hacky endpoint
+// hacky endpoint: download CSV without queueing a report
 export const API_BASE_URL_V1 = '/api/reporting/v1'; //
-export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv/saved-object`;
+export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv_searchsource`;
 
 // Management UI route
 export const REPORTING_MANAGEMENT_HOME = '/app/management/insightsAndAlerting/reporting';

diff --git a/x-pack/plugins/reporting/common/types.ts b/x-pack/plugins/reporting/common/types.ts
@@ -47,9 +47,10 @@ export interface ReportDocumentHead {
 export interface TaskRunResult {
   content_type: string | null;
   content: string | null;
-  csv_contains_formulas?: boolean;
   size: number;
+  csv_contains_formulas?: boolean;
   max_size_reached?: boolean;
+  needs_sorting?: boolean;
   warnings?: string[];
 }
 

diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx b/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx
@@ -11,11 +11,7 @@ import React, { Component, ReactElement } from 'react';
 import { ToastsSetup } from 'src/core/public';
 import url from 'url';
 import { toMountPoint } from '../../../../../src/plugins/kibana_react/public';
-import {
-  CSV_REPORT_TYPE_DEPRECATED,
-  PDF_REPORT_TYPE,
-  PNG_REPORT_TYPE,
-} from '../../common/constants';
+import { CSV_REPORT_TYPE, PDF_REPORT_TYPE, PNG_REPORT_TYPE } from '../../common/constants';
 import { BaseParams } from '../../common/types';
 import { ReportingAPIClient } from '../lib/reporting_api_client';
 
@@ -177,8 +173,8 @@ class ReportingPanelContentUi extends Component<Props, State> {
     switch (this.props.reportType) {
       case PDF_REPORT_TYPE:
         return 'PDF';
-      case 'csv':
-        return CSV_REPORT_TYPE_DEPRECATED;
+      case 'csv_searchsource':
+        return CSV_REPORT_TYPE;
       case 'png':
         return PNG_REPORT_TYPE;
       default:

diff --git a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts
@@ -52,7 +52,20 @@ describe('GetCsvReportPanelAction', () => {
     context = {
       embeddable: {
         type: 'search',
-        getSavedSearch: () => ({ id: 'lebowski' }),
+        getSavedSearch: () => {
+          const searchSource = {
+            createCopy: () => searchSource,
+            removeField: jest.fn(),
+            setField: jest.fn(),
+            getField: jest.fn().mockImplementation((key: string) => {
+              if (key === 'index') {
+                return 'my-test-index-*';
+              }
+            }),
+            getSerializedFields: jest.fn().mockImplementation(() => ({})),
+          };
+          return { searchSource };
+        },
         getTitle: () => `The Dude`,
         getInspectorAdapters: () => null,
         getInput: () => ({