Skip to content

Commit

Permalink
[ML] Moving file data vizualizer to its own plugin (#96408)
Browse files Browse the repository at this point in the history
* [ML] Moving file data vizualizer to file upload plugin

* removing maps plug dependency

* fixing imports

* small refactor

* adding missing endpoints

* fixing translations

* fxing table controls

* fixing types and disabling geo point test

* actually disabling geo point test

* making endpoints internal

* moving UI code to separate plugin

* enabling maps integration

* cleaning up dependencies

* fixing translation ids

* moving analyze file endpoint out of file upload plugin

* fixing transtations issues

* refactor for lazy loading of component

* updating limits

* updating plugin asciidoc

* code clean up

* further clean up

* adding comment

* fixing really obvious CI error

* removing commented out include

* reenabling geo point test

* fixing incorrectly changed import

* removing ml from labels and identifiers

* renaming function

* moving analyse file endpoint to file upload plugin

* reverting import path changes

* adding esUiShared back in

* fixing navigation tabs alignment in basic license

* adding key to tab wrapper

* reverting test label

* further removal of ml references

* removing ml label from more identifiers

* fixing tests

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
jgowdyelastic and kibanamachine authored Apr 20, 2021
1 parent f1d167d commit 088a618
Show file tree
Hide file tree
Showing 219 changed files with 5,249 additions and 969 deletions.
4 changes: 4 additions & 0 deletions docs/developer/plugin-list.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,10 @@ actitivies.
|The features plugin enhance Kibana with a per-feature privilege system.
|{kib-repo}blob/{branch}/x-pack/plugins/file_data_visualizer[fileDataVisualizer]
|WARNING: Missing README.
|{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload]
|WARNING: Missing README.
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-optimizer/limits.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ pageLoadAssetSize:
indexPatternFieldEditor: 90489
osquery: 107090
fileUpload: 25664
fileDataVisualizer: 27530
banners: 17946
mapsEms: 26072
timelines: 28613
1 change: 1 addition & 0 deletions x-pack/.i18nrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
"xpack.endpoint": "plugins/endpoint",
"xpack.enterpriseSearch": "plugins/enterprise_search",
"xpack.features": "plugins/features",
"xpack.fileDataVisualizer": "plugins/file_data_visualizer",
"xpack.fileUpload": "plugins/file_upload",
"xpack.globalSearch": ["plugins/global_search"],
"xpack.globalSearchBar": ["plugins/global_search_bar"],
Expand Down
31 changes: 31 additions & 0 deletions x-pack/plugins/file_data_visualizer/common/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

export const UI_SETTING_MAX_FILE_SIZE = 'fileUpload:maxFileSize';

export const MB = Math.pow(2, 20);
export const MAX_FILE_SIZE = '100MB';
export const MAX_FILE_SIZE_BYTES = 104857600; // 100MB

export const ABSOLUTE_MAX_FILE_SIZE_BYTES = 1073741274; // 1GB
export const FILE_SIZE_DISPLAY_FORMAT = '0,0.[0] b';

// Value to use in the Elasticsearch index mapping meta data to identify the
// index as having been created by the File Data Visualizer.
export const INDEX_META_DATA_CREATED_BY = 'file-data-visualizer';

export const JOB_FIELD_TYPES = {
BOOLEAN: 'boolean',
DATE: 'date',
GEO_POINT: 'geo_point',
GEO_SHAPE: 'geo_shape',
IP: 'ip',
KEYWORD: 'keyword',
NUMBER: 'number',
TEXT: 'text',
UNKNOWN: 'unknown',
} as const;
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
* 2.0.
*/

export { createUrlOverrides, processResults, readFile, DEFAULT_LINES_TO_SAMPLE } from './utils';
export * from './constants';
export * from './types';
22 changes: 22 additions & 0 deletions x-pack/plugins/file_data_visualizer/common/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { JOB_FIELD_TYPES } from './constants';

export type InputData = any[];

export type JobFieldType = typeof JOB_FIELD_TYPES[keyof typeof JOB_FIELD_TYPES];

export interface DataVisualizerTableState {
pageSize: number;
pageIndex: number;
sortField: string;
sortDirection: string;
visibleFieldTypes: string[];
visibleFieldNames: string[];
showDistributions: boolean;
}
12 changes: 12 additions & 0 deletions x-pack/plugins/file_data_visualizer/jest.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

module.exports = {
preset: '@kbn/test',
rootDir: '../../..',
roots: ['<rootDir>/x-pack/plugins/file_data_visualizer'],
};
27 changes: 27 additions & 0 deletions x-pack/plugins/file_data_visualizer/kibana.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"id": "fileDataVisualizer",
"version": "8.0.0",
"kibanaVersion": "kibana",
"server": true,
"ui": true,
"requiredPlugins": [
"data",
"usageCollection",
"embeddable",
"share",
"discover",
"fileUpload"
],
"optionalPlugins": [
"security",
"maps"
],
"requiredBundles": [
"kibanaReact",
"maps",
"esUiShared"
],
"extraPublicDirs": [
"common"
]
}
14 changes: 14 additions & 0 deletions x-pack/plugins/file_data_visualizer/public/api/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { lazyLoadModules } from '../lazy_load_bundle';
import { FileDataVisualizer } from '../application';

export async function getFileDataVisualizerComponent(): Promise<typeof FileDataVisualizer> {
const modules = await lazyLoadModules();
return modules.FileDataVisualizer;
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ export const AboutPanel: FC<Props> = ({ onFilePickerChange }) => {
<EuiFilePicker
id="filePicker"
initialPromptText={i18n.translate(
'xpack.ml.fileDatavisualizer.aboutPanel.selectOrDragAndDropFileDescription',
'xpack.fileDataVisualizer.aboutPanel.selectOrDragAndDropFileDescription',
{
defaultMessage: 'Select or drag and drop a file',
}
Expand All @@ -70,7 +70,7 @@ export const LoadingPanel: FC = () => {
<EuiTitle size="s">
<h1 role="alert">
<FormattedMessage
id="xpack.ml.fileDatavisualizer.aboutPanel.analyzingDataTitle"
id="xpack.fileDataVisualizer.aboutPanel.analyzingDataTitle"
defaultMessage="Analyzing data"
/>
</h1>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,22 @@ import {

import { ExperimentalBadge } from '../experimental_badge';

import { useMlKibana } from '../../../../contexts/kibana';
import { useFileDataVisualizerKibana } from '../../kibana_context';

export const WelcomeContent: FC = () => {
const toolTipContent = i18n.translate(
'xpack.ml.fileDatavisualizer.welcomeContent.experimentalFeatureTooltip',
'xpack.fileDataVisualizer.welcomeContent.experimentalFeatureTooltip',
{
defaultMessage: "Experimental feature. We'd love to hear your feedback.",
}
);

const {
services: { fileUpload },
} = useMlKibana();

if (fileUpload === undefined) {
// eslint-disable-next-line no-console
console.error('File upload plugin not available');
return null;
}
const maxFileSize = fileUpload.getMaxBytesFormatted();
services: {
fileUpload: { getMaxBytesFormatted },
},
} = useFileDataVisualizerKibana();
const maxFileSize = getMaxBytesFormatted();

return (
<EuiFlexGroup gutterSize="xl" alignItems="center">
Expand All @@ -51,7 +47,7 @@ export const WelcomeContent: FC = () => {
<EuiTitle size="m">
<h1>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileTitle"
id="xpack.fileDataVisualizer.welcomeContent.visualizeDataFromLogFileTitle"
defaultMessage="Visualize data from a log file&nbsp;{experimentalBadge}"
values={{
experimentalBadge: <ExperimentalBadge tooltipContent={toolTipContent} />,
Expand All @@ -63,7 +59,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileDescription"
id="xpack.fileDataVisualizer.welcomeContent.visualizeDataFromLogFileDescription"
defaultMessage="The File Data Visualizer helps you understand the fields and metrics in a log file.
Upload your file, analyze its data, and then choose whether to import the data into an Elasticsearch index."
/>
Expand All @@ -73,7 +69,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.supportedFileFormatDescription"
id="xpack.fileDataVisualizer.welcomeContent.supportedFileFormatDescription"
defaultMessage="The File Data Visualizer supports these file formats:"
/>
</p>
Expand All @@ -87,7 +83,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.delimitedTextFilesDescription"
id="xpack.fileDataVisualizer.welcomeContent.delimitedTextFilesDescription"
defaultMessage="Delimited text files, such as CSV and TSV"
/>
</p>
Expand All @@ -103,7 +99,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.newlineDelimitedJsonDescription"
id="xpack.fileDataVisualizer.welcomeContent.newlineDelimitedJsonDescription"
defaultMessage="Newline-delimited JSON"
/>
</p>
Expand All @@ -119,7 +115,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.logFilesWithCommonFormatDescription"
id="xpack.fileDataVisualizer.welcomeContent.logFilesWithCommonFormatDescription"
defaultMessage="Log files with a common format for the timestamp"
/>
</p>
Expand All @@ -130,7 +126,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.uploadedFilesAllowedSizeDescription"
id="xpack.fileDataVisualizer.welcomeContent.uploadedFilesAllowedSizeDescription"
defaultMessage="You can upload files up to {maxFileSize}."
values={{ maxFileSize }}
/>
Expand All @@ -140,7 +136,7 @@ export const WelcomeContent: FC = () => {
<EuiText>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.welcomeContent.experimentalFeatureDescription"
id="xpack.fileDataVisualizer.welcomeContent.experimentalFeatureDescription"
defaultMessage="This feature is experimental. Got feedback? Please create an issue in&nbsp;{githubLink}."
values={{
githubLink: (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
import React, { FC } from 'react';

import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui';
import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
import { FindFileStructureResponse } from '../../../../../file_upload/common';

export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => {
const items = createDisplayItems(results);
Expand All @@ -19,7 +19,7 @@ export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ re
<EuiTitle size="s">
<h2>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.summaryTitle"
id="xpack.fileDataVisualizer.analysisSummary.summaryTitle"
defaultMessage="Summary"
/>
</h2>
Expand All @@ -37,7 +37,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
{
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.analyzedLinesNumberTitle"
id="xpack.fileDataVisualizer.analysisSummary.analyzedLinesNumberTitle"
defaultMessage="Number of lines analyzed"
/>
),
Expand All @@ -53,7 +53,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
items.push({
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.formatTitle"
id="xpack.fileDataVisualizer.analysisSummary.formatTitle"
defaultMessage="Format"
/>
),
Expand All @@ -64,7 +64,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
items.push({
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.delimiterTitle"
id="xpack.fileDataVisualizer.analysisSummary.delimiterTitle"
defaultMessage="Delimiter"
/>
),
Expand All @@ -74,7 +74,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
items.push({
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.hasHeaderRowTitle"
id="xpack.fileDataVisualizer.analysisSummary.hasHeaderRowTitle"
defaultMessage="Has header row"
/>
),
Expand All @@ -87,7 +87,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
items.push({
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.grokPatternTitle"
id="xpack.fileDataVisualizer.analysisSummary.grokPatternTitle"
defaultMessage="Grok pattern"
/>
),
Expand All @@ -99,7 +99,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
items.push({
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.timeFieldTitle"
id="xpack.fileDataVisualizer.analysisSummary.timeFieldTitle"
defaultMessage="Time field"
/>
),
Expand All @@ -111,7 +111,7 @@ function createDisplayItems(results: FindFileStructureResponse) {
items.push({
title: (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.analysisSummary.timeFormatTitle"
id="xpack.fileDataVisualizer.analysisSummary.timeFormatTitle"
defaultMessage="Time {timestampFormats, plural, zero {format} one {format} other {formats}}"
values={{
timestampFormats: results.java_timestamp_formats.length,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ export const BottomBar: FC<BottomBarProps> = ({ mode, onChangeMode, onCancel, di
content={
disableImport ? (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.bottomBar.missingImportPrivilegesMessage"
id="xpack.fileDataVisualizer.bottomBar.missingImportPrivilegesMessage"
defaultMessage="You require the ingest_admin role to enable data importing"
/>
) : null
Expand All @@ -52,7 +52,7 @@ export const BottomBar: FC<BottomBarProps> = ({ mode, onChangeMode, onCancel, di
data-test-subj="mlFileDataVisOpenImportPageButton"
>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.bottomBar.readMode.importButtonLabel"
id="xpack.fileDataVisualizer.bottomBar.readMode.importButtonLabel"
defaultMessage="Import"
/>
</EuiButton>
Expand All @@ -61,7 +61,7 @@ export const BottomBar: FC<BottomBarProps> = ({ mode, onChangeMode, onCancel, di
<EuiFlexItem grow={false}>
<EuiButtonEmpty color="ghost" onClick={() => onCancel()}>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.bottomBar.readMode.cancelButtonLabel"
id="xpack.fileDataVisualizer.bottomBar.readMode.cancelButtonLabel"
defaultMessage="Cancel"
/>
</EuiButtonEmpty>
Expand All @@ -76,15 +76,15 @@ export const BottomBar: FC<BottomBarProps> = ({ mode, onChangeMode, onCancel, di
<EuiFlexItem grow={false}>
<EuiButton color="ghost" onClick={() => onChangeMode(DATAVISUALIZER_MODE.READ)}>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.bottomBar.backButtonLabel"
id="xpack.fileDataVisualizer.bottomBar.backButtonLabel"
defaultMessage="Back"
/>
</EuiButton>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiButtonEmpty color="ghost" onClick={() => onCancel()}>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.bottomBar.cancelButtonLabel"
id="xpack.fileDataVisualizer.bottomBar.cancelButtonLabel"
defaultMessage="Cancel"
/>
</EuiButtonEmpty>
Expand Down
Loading

0 comments on commit 088a618

Please sign in to comment.