Merge branch 'master' into bug-top-n-alerts

elastic · Mar 22, 2021 · 7301afc · 7301afc
2 parents e3c5d5e + 35af8a9
commit 7301afc
Show file tree

Hide file tree

Showing 50 changed files with 573 additions and 344 deletions.
diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc
@@ -475,7 +475,7 @@ running behind a proxy. Use the <<server-rewriteBasePath, `server.rewriteBasePat
 if it should remove the basePath from requests it receives, and to prevent a
 deprecation warning at startup. This setting cannot end in a slash (`/`).
 
-|[[server-publicBaseUrl]] `server.publicBaseUrl:`
+|[[server-publicBaseUrl]] `server.publicBaseUrl:` {ess-icon}
  | The publicly available URL that end-users access Kibana at. Must include the protocol, hostname, port
  (if different than the defaults for `http` and `https`, 80 and 443 respectively), and the
  <<server-basePath, `server.basePath`>> (if configured). This setting cannot end in a slash (`/`).

diff --git a/...ugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap b/...ugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap
diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts
@@ -26,6 +26,25 @@ function setDocsourcePayload(id: string | null, providedPayload: any) {
   object = defaults(providedPayload || {}, stubbedSavedObjectIndexPattern(id));
 }
 
+const savedObject = {
+  id: 'id',
+  version: 'version',
+  attributes: {
+    title: 'kibana-*',
+    timeFieldName: '@timestamp',
+    fields: '[]',
+    sourceFilters: '[{"value":"item1"},{"value":"item2"}]',
+    fieldFormatMap: '{"field":{}}',
+    typeMeta: '{}',
+    type: '',
+    runtimeFieldMap:
+      '{"aRuntimeField": { "type": "keyword", "script": {"source": "emit(\'hello\')"}}}',
+    fieldAttrs: '{"aRuntimeField": { "count": 5, "customLabel": "A Runtime Field"}}',
+  },
+  type: 'index-pattern',
+  references: [],
+};
+
 describe('IndexPatterns', () => {
   let indexPatterns: IndexPatternsService;
   let savedObjectsClient: SavedObjectsClientCommon;
@@ -219,23 +238,14 @@ describe('IndexPatterns', () => {
   });
 
   test('savedObjectToSpec', () => {
-    const savedObject = {
-      id: 'id',
-      version: 'version',
-      attributes: {
-        title: 'kibana-*',
-        timeFieldName: '@timestamp',
-        fields: '[]',
-        sourceFilters: '[{"value":"item1"},{"value":"item2"}]',
-        fieldFormatMap: '{"field":{}}',
-        typeMeta: '{}',
-        type: '',
-      },
-      type: 'index-pattern',
-      references: [],
-    };
+    const spec = indexPatterns.savedObjectToSpec(savedObject);
+    expect(spec).toMatchSnapshot();
+  });
 
-    expect(indexPatterns.savedObjectToSpec(savedObject)).toMatchSnapshot();
+  test('correctly composes runtime field', async () => {
+    setDocsourcePayload('id', savedObject);
+    const indexPattern = await indexPatterns.get('id');
+    expect(indexPattern.fields).toMatchSnapshot();
   });
 
   test('failed requests are not cached', async () => {

diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts
@@ -425,8 +425,9 @@ export class IndexPatternsService {
             runtimeField: value,
             aggregatable: true,
             searchable: true,
-            count: 0,
             readFromDocValues: false,
+            customLabel: spec.fieldAttrs?.[key]?.customLabel,
+            count: spec.fieldAttrs?.[key]?.count,
           };
         }
       }

diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts
@@ -260,7 +260,8 @@ export default function ({ getService }: FtrProviderContext) {
           });
         });
 
-        it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
+        // flaky https://github.com/elastic/kibana/issues/94513
+        it.skip("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
           const { body } = await supertest
             .post('/api/telemetry/v2/clusters/_stats')
             .set('kbn-xsrf', 'xxx')

diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts
@@ -5,6 +5,73 @@
  * 2.0.
  */
 
+import { ES_FIELD_TYPES } from '../../../../src/plugins/data/common';
+
+export interface InputOverrides {
+  [key: string]: string | undefined;
+}
+
+export type FormattedOverrides = InputOverrides & {
+  column_names: string[];
+  has_header_row: boolean;
+  should_trim_fields: boolean;
+};
+
+export interface AnalysisResult {
+  results: FindFileStructureResponse;
+  overrides?: FormattedOverrides;
+}
+
+export interface FindFileStructureResponse {
+  charset: string;
+  has_header_row: boolean;
+  has_byte_order_marker: boolean;
+  format: string;
+  field_stats: {
+    [fieldName: string]: {
+      count: number;
+      cardinality: number;
+      top_hits: Array<{ count: number; value: any }>;
+      mean_value?: number;
+      median_value?: number;
+      max_value?: number;
+      min_value?: number;
+      earliest?: string;
+      latest?: string;
+    };
+  };
+  sample_start: string;
+  num_messages_analyzed: number;
+  mappings: {
+    properties: {
+      [fieldName: string]: {
+        // including all possible Elasticsearch types
+        // since find_file_structure API can be enhanced to include new fields in the future
+        type: Exclude<
+          ES_FIELD_TYPES,
+          ES_FIELD_TYPES._ID | ES_FIELD_TYPES._INDEX | ES_FIELD_TYPES._SOURCE | ES_FIELD_TYPES._TYPE
+        >;
+        format?: string;
+      };
+    };
+  };
+  quote: string;
+  delimiter: string;
+  need_client_timezone: boolean;
+  num_lines_analyzed: number;
+  column_names?: string[];
+  explanation?: string[];
+  grok_pattern?: string;
+  multiline_start_pattern?: string;
+  exclude_lines_pattern?: string;
+  java_timestamp_formats?: string[];
+  joda_timestamp_formats?: string[];
+  timestamp_field?: string;
+  should_trim_fields?: boolean;
+}
+
+export type InputData = any[];
+
 export interface ImportResponse {
   success: boolean;
   id: string;

diff --git a/...e_data_visualizer/file_data_visualizer.ts → ...ugins/file_upload/server/analyze_file.tsx b/...e_data_visualizer/file_data_visualizer.ts → ...ugins/file_upload/server/analyze_file.tsx
@@ -9,41 +9,38 @@ import { IScopedClusterClient } from 'kibana/server';
 import {
   AnalysisResult,
   FormattedOverrides,
+  InputData,
   InputOverrides,
   FindFileStructureResponse,
-} from '../../../common/types/file_datavisualizer';
-
-export type InputData = any[];
-
-export function fileDataVisualizerProvider(client: IScopedClusterClient) {
-  async function analyzeFile(data: InputData, overrides: InputOverrides): Promise<AnalysisResult> {
-    overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain;
-    const {
-      body,
-    } = await client.asInternalUser.textStructure.findStructure<FindFileStructureResponse>({
-      body: data,
-      ...overrides,
-    });
-
-    const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
-
-    return {
-      ...(hasOverrides && { overrides: reducedOverrides }),
-      results: body,
-    };
-  }
+} from '../common';
+
+export async function analyzeFile(
+  client: IScopedClusterClient,
+  data: InputData,
+  overrides: InputOverrides
+): Promise<AnalysisResult> {
+  overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain;
+  const {
+    body,
+  } = await client.asInternalUser.textStructure.findStructure<FindFileStructureResponse>({
+    body: data,
+    ...overrides,
+  });
+
+  const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
 
   return {
-    analyzeFile,
+    ...(hasOverrides && { overrides: reducedOverrides }),
+    results: body,
   };
 }
 
 function formatOverrides(overrides: InputOverrides) {
   let hasOverrides = false;
 
   const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => {
-    const overrideValue: string = overrides[overrideKey];
-    if (overrideValue !== '') {
+    const overrideValue: string | undefined = overrides[overrideKey];
+    if (overrideValue !== undefined && overrideValue !== '') {
       if (overrideKey === 'column_names') {
         acc.column_names = overrideValue.split(',');
       } else if (overrideKey === 'has_header_row') {

diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts
@@ -10,13 +10,12 @@ import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
 import {
   ImportResponse,
   ImportFailure,
+  InputData,
   Settings,
   Mappings,
   IngestPipelineWrapper,
 } from '../common';
 
-export type InputData = any[];
-
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
     id: string | undefined,

diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts
@@ -5,13 +5,21 @@
  * 2.0.
  */
 
+import { schema } from '@kbn/config-schema';
 import { IRouter, IScopedClusterClient } from 'kibana/server';
-import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common';
+import {
+  MAX_FILE_SIZE_BYTES,
+  IngestPipelineWrapper,
+  InputData,
+  Mappings,
+  Settings,
+} from '../common';
 import { wrapError } from './error_wrapper';
-import { InputData, importDataProvider } from './import_data';
+import { analyzeFile } from './analyze_file';
+import { importDataProvider } from './import_data';
 
 import { updateTelemetry } from './telemetry';
-import { importFileBodySchema, importFileQuerySchema } from './schemas';
+import { analyzeFileQuerySchema, importFileBodySchema, importFileQuerySchema } from './schemas';
 
 function importData(
   client: IScopedClusterClient,
@@ -30,6 +38,44 @@ function importData(
  * Routes for the file upload.
  */
 export function fileUploadRoutes(router: IRouter) {
+  /**
+   * @apiGroup FileDataVisualizer
+   *
+   * @api {post} /api/file_upload/analyze_file Analyze file data
+   * @apiName AnalyzeFile
+   * @apiDescription Performs analysis of the file data.
+   *
+   * @apiSchema (query) analyzeFileQuerySchema
+   */
+  router.post(
+    {
+      path: '/api/file_upload/analyze_file',
+      validate: {
+        body: schema.any(),
+        query: analyzeFileQuerySchema,
+      },
+      options: {
+        body: {
+          accepts: ['text/*', 'application/json'],
+          maxBytes: MAX_FILE_SIZE_BYTES,
+        },
+        tags: ['access:fileUpload:analyzeFile'],
+      },
+    },
+    async (context, request, response) => {
+      try {
+        const result = await analyzeFile(
+          context.core.elasticsearch.client,
+          request.body,
+          request.query
+        );
+        return response.ok({ body: result });
+      } catch (e) {
+        return response.customError(wrapError(e));
+      }
+    }
+  );
+
   /**
    * @apiGroup FileDataVisualizer
    *

diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts
@@ -7,6 +7,23 @@
 
 import { schema } from '@kbn/config-schema';
 
+export const analyzeFileQuerySchema = schema.object({
+  charset: schema.maybe(schema.string()),
+  column_names: schema.maybe(schema.string()),
+  delimiter: schema.maybe(schema.string()),
+  explain: schema.maybe(schema.string()),
+  format: schema.maybe(schema.string()),
+  grok_pattern: schema.maybe(schema.string()),
+  has_header_row: schema.maybe(schema.string()),
+  line_merge_size_limit: schema.maybe(schema.string()),
+  lines_to_sample: schema.maybe(schema.string()),
+  quote: schema.maybe(schema.string()),
+  should_trim_fields: schema.maybe(schema.string()),
+  timeout: schema.maybe(schema.string()),
+  timestamp_field: schema.maybe(schema.string()),
+  timestamp_format: schema.maybe(schema.string()),
+});
+
 export const importFileQuerySchema = schema.object({
   id: schema.maybe(schema.string()),
 });