Skip to content

Commit

Permalink
[ML] Adding filebeat config to file dataviz (#58152)
Browse files Browse the repository at this point in the history
* [ML] Adding filebeat config to file dataviz

* adding extra help text

* removing commented out code

* adding extra blank line to processors section

* cleaning up types

* moving hosts line out of function

* typo in config text

* updating config based on review

* tiny refactor

* translating paths text
  • Loading branch information
jgowdyelastic authored Feb 25, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 71d6c22 commit b756cc3
Showing 15 changed files with 513 additions and 209 deletions.
31 changes: 31 additions & 0 deletions x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

export interface FindFileStructureResponse {
charset: string;
has_header_row: boolean;
has_byte_order_marker: boolean;
format: string;
field_stats: {
[fieldName: string]: {
count: number;
cardinality: number;
top_hits: Array<{ count: number; value: any }>;
};
};
sample_start: string;
num_messages_analyzed: number;
mappings: {
[fieldName: string]: {
type: string;
};
};
quote: string;
delimiter: string;
need_client_timezone: boolean;
num_lines_analyzed: number;
column_names: string[];
}
4 changes: 4 additions & 0 deletions x-pack/legacy/plugins/ml/public/application/app.tsx
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ import 'ace';
import { AppMountParameters, CoreStart } from 'kibana/public';

import { DataPublicPluginStart } from 'src/plugins/data/public';
import { SecurityPluginSetup } from '../../../../../plugins/security/public';

import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public';
import { setDependencyCache, clearCache } from './util/dependency_cache';
@@ -20,6 +21,7 @@ import { MlRouter } from './routing';

export interface MlDependencies extends AppMountParameters {
data: DataPublicPluginStart;
security: SecurityPluginSetup;
__LEGACY: {
XSRF: string;
APP_URL: string;
@@ -49,6 +51,7 @@ const App: FC<AppProps> = ({ coreStart, deps }) => {
APP_URL: deps.__LEGACY.APP_URL,
application: coreStart.application,
http: coreStart.http,
security: deps.security,
});
deps.onAppLeave(actions => {
clearCache();
@@ -64,6 +67,7 @@ const App: FC<AppProps> = ({ coreStart, deps }) => {
const services = {
appName: 'ML',
data: deps.data,
security: deps.security,
...coreStart,
};

Original file line number Diff line number Diff line change
@@ -10,9 +10,11 @@ import {
useKibana,
KibanaReactContextValue,
} from '../../../../../../../../src/plugins/kibana_react/public';
import { SecurityPluginSetup } from '../../../../../../../plugins/security/public';

interface StartPlugins {
data: DataPublicPluginStart;
security: SecurityPluginSetup;
}
export type StartServices = CoreStart & StartPlugins;
// eslint-disable-next-line react-hooks/rules-of-hooks
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { i18n } from '@kbn/i18n';
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';

export function createFilebeatConfig(
index: string,
results: FindFileStructureResponse,
ingestPipelineId: string,
username: string | null
) {
return [
'filebeat.inputs:',
'- type: log',
...getPaths(),
...getEncoding(results),
...getExcludeLines(results),
...getMultiline(results),
'',
...getProcessors(results),
'output.elasticsearch:',
' hosts: ["<es_url>"]',
...getUserDetails(username),
` index: "${index}"`,
` pipeline: "${ingestPipelineId}"`,
'',
'setup:',
' template.enabled: false',
' ilm.enabled: false',
].join('\n');
}

function getPaths() {
const txt = i18n.translate('xpack.ml.fileDatavisualizer.fileBeatConfig.paths', {
defaultMessage: 'add path to your files here',
});
return [' paths:', ` - '<${txt}>'`];
}

function getEncoding(results: any) {
return results.charset !== 'UTF-8' ? [` encoding: ${results.charset}`] : [];
}

function getExcludeLines(results: any) {
return results.exclude_lines_pattern !== undefined
? [` exclude_lines: ['${results.exclude_lines_pattern.replace(/'/g, "''")}']`]
: [];
}

function getMultiline(results: any) {
return results.multiline_start_pattern !== undefined
? [
' multiline:',
` pattern: '${results.multiline_start_pattern.replace(/'/g, "''")}'`,
' match: after',
' negate: true',
]
: [];
}

function getProcessors(results: any) {
return results.need_client_timezone === true ? ['processors:', '- add_locale: ~', ''] : [];
}

function getUserDetails(username: string | null) {
return username !== null ? [` username: "${username}"`, ' password: "<password>"'] : [];
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import React, { FC, useState, useEffect } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import {
EuiFlyout,
EuiFlyoutFooter,
EuiFlexGroup,
EuiFlexItem,
EuiButton,
EuiButtonEmpty,
EuiTitle,
EuiFlyoutBody,
EuiSpacer,
EuiCodeBlock,
EuiCode,
EuiCopy,
} from '@elastic/eui';
import { createFilebeatConfig } from './filebeat_config';
import { useMlKibana } from '../../../../contexts/kibana';
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';

export enum EDITOR_MODE {
HIDDEN,
READONLY,
EDITABLE,
}
interface Props {
index: string;
results: FindFileStructureResponse;
indexPatternId: string;
ingestPipelineId: string;
closeFlyout(): void;
}
export const FilebeatConfigFlyout: FC<Props> = ({
index,
results,
indexPatternId,
ingestPipelineId,
closeFlyout,
}) => {
const [fileBeatConfig, setFileBeatConfig] = useState('');
const [username, setUsername] = useState<string | null>(null);
const {
services: { security },
} = useMlKibana();

useEffect(() => {
security.authc.getCurrentUser().then(user => {
setUsername(user.username === undefined ? null : user.username);
});
}, []);

useEffect(() => {
const config = createFilebeatConfig(index, results, ingestPipelineId, username);
setFileBeatConfig(config);
}, [username]);

return (
<EuiFlyout onClose={closeFlyout} hideCloseButton size={'m'}>
<EuiFlyoutBody>
<EuiFlexGroup>
<Contents value={fileBeatConfig} username={username} index={index} />
</EuiFlexGroup>
</EuiFlyoutBody>
<EuiFlyoutFooter>
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
<EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left">
<FormattedMessage
id="xpack.ml.fileDatavisualizer.fileBeatConfigFlyout.closeButton"
defaultMessage="Close"
/>
</EuiButtonEmpty>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiCopy textToCopy={fileBeatConfig}>
{copy => (
<EuiButton onClick={copy}>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.fileBeatConfigFlyout.copyButton"
defaultMessage="Copy to clipboard"
/>
</EuiButton>
)}
</EuiCopy>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlyoutFooter>
</EuiFlyout>
);
};

const Contents: FC<{
value: string;
index: string;
username: string | null;
}> = ({ value, index, username }) => {
return (
<EuiFlexItem>
<EuiTitle size="s">
<h5>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTitle"
defaultMessage="Filebeat configuration"
/>
</h5>
</EuiTitle>
<EuiSpacer size="s" />
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTopText1"
defaultMessage="Additional data can be uploaded to the {index} index using Filebeat."
values={{ index: <EuiCode>{index}</EuiCode> }}
/>
</p>
<p>
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTopText2"
defaultMessage="Modify {filebeatYml} to set the connection information:"
values={{ filebeatYml: <EuiCode>filebeat.yml</EuiCode> }}
/>
</p>

<EuiSpacer size="s" />

<EuiCodeBlock language="bash">{value}</EuiCodeBlock>

<EuiSpacer size="s" />
<p>
{username === null ? (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigBottomTextNoUsername"
defaultMessage="Where {esUrl} is the URL of Elasticsearch."
values={{
esUrl: <EuiCode>{'<es_url>'}</EuiCode>,
}}
/>
) : (
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigBottomText"
defaultMessage="Where {password} is the password of the {user} user, {esUrl} is the URL of Elasticsearch."
values={{
user: <EuiCode>{username}</EuiCode>,
password: <EuiCode>{'<password>'}</EuiCode>,
esUrl: <EuiCode>{'<es_url>'}</EuiCode>,
}}
/>
)}
</p>
</EuiFlexItem>
);
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

export { FilebeatConfigFlyout } from './filebeat_config_flyout';
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@ import {
import { i18n } from '@kbn/i18n';
import { importerFactory } from './importer';
import { ResultsLinks } from '../results_links';
import { FilebeatConfigFlyout } from '../filebeat_config_flyout';
import { ImportProgress, IMPORT_STATUS } from '../import_progress';
import { ImportErrors } from '../import_errors';
import { ImportSummary } from '../import_summary';
@@ -64,6 +65,7 @@ const DEFAULT_STATE = {
indexNameError: '',
indexPatternNameError: '',
timeFieldName: undefined,
isFilebeatFlyoutVisible: false,
};

export class ImportView extends Component {
@@ -384,6 +386,16 @@ export class ImportView extends Component {
});
};

showFilebeatFlyout = () => {
this.setState({ isFilebeatFlyoutVisible: true });
this.props.hideBottomBar();
};

closeFilebeatFlyout = () => {
this.setState({ isFilebeatFlyoutVisible: false });
this.props.showBottomBar();
};

async loadIndexNames() {
const indices = await ml.getIndices();
const indexNames = indices.map(i => i.name);
@@ -424,6 +436,7 @@ export class ImportView extends Component {
indexNameError,
indexPatternNameError,
timeFieldName,
isFilebeatFlyoutVisible,
} = this.state;

const createPipeline = pipelineString !== '';
@@ -549,7 +562,18 @@ export class ImportView extends Component {
indexPatternId={indexPatternId}
timeFieldName={timeFieldName}
createIndexPattern={createIndexPattern}
showFilebeatFlyout={this.showFilebeatFlyout}
/>

{isFilebeatFlyoutVisible && (
<FilebeatConfigFlyout
index={index}
results={this.props.results}
indexPatternId={indexPatternId}
ingestPipelineId={ingestPipelineId}
closeFlyout={this.closeFilebeatFlyout}
/>
)}
</React.Fragment>
)}
</EuiPanel>
Original file line number Diff line number Diff line change
@@ -77,6 +77,11 @@ export class MessageImporter extends Importer {
if (this.multilineStartRegex === null || line.match(this.multilineStartRegex) !== null) {
this.addMessage(data, message);
message = '';
} else if (data.length === 0) {
// discard everything before the first line that is considered the first line of a message
// as it could be left over partial data from a spilt or rolled over log,
// or could be a blank line after the header in a csv file
return '';
} else {
message += '\n';
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import React, { FC, useState, useEffect } from 'react';
import moment from 'moment';
import { FormattedMessage } from '@kbn/i18n/react';
import { EuiFlexGroup, EuiFlexItem, EuiCard, EuiIcon } from '@elastic/eui';
import { ml } from '../../../../services/ml_api_service';
import { isFullLicense } from '../../../../license/check_license';
import { checkPermission } from '../../../../privilege/check_privilege';
import { mlNodesAvailable } from '../../../../ml_nodes_check/check_ml_nodes';
import { useMlKibana } from '../../../../contexts/kibana';

const RECHECK_DELAY_MS = 3000;

interface Props {
index: string;
indexPatternId: string;
timeFieldName?: string;
createIndexPattern: boolean;
showFilebeatFlyout(): void;
}

export const ResultsLinks: FC<Props> = ({
index,
indexPatternId,
timeFieldName,
createIndexPattern,
showFilebeatFlyout,
}) => {
const [duration, setDuration] = useState({
from: 'now-30m',
to: 'now',
});
const [showCreateJobLink, setShowCreateJobLink] = useState(false);
const [globalStateString, setGlobalStateString] = useState('');
const {
services: {
http: { basePath },
},
} = useMlKibana();

useEffect(() => {
setShowCreateJobLink(checkPermission('canCreateJob') && mlNodesAvailable());
updateTimeValues();
}, []);

useEffect(() => {
const _g =
timeFieldName !== undefined
? `&_g=(time:(from:'${duration.from}',mode:quick,to:'${duration.to}'))`
: '';
setGlobalStateString(_g);
}, [duration]);

async function updateTimeValues(recheck = true) {
if (timeFieldName !== undefined) {
const { from, to } = await getFullTimeRange(index, timeFieldName);
setDuration({
from: from === null ? duration.from : from,
to: to === null ? duration.to : to,
});

// these links may have been drawn too quickly for the index to be ready
// to give us the correct start and end times.
// especially if the data was small.
// so if the start and end were null, try again in 3s
if (recheck && (from === null || to === null)) {
setTimeout(() => {
updateTimeValues(false);
}, RECHECK_DELAY_MS);
}
}
}

return (
<EuiFlexGroup gutterSize="l">
{createIndexPattern && (
<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`discoverApp`} />}
title={
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.viewIndexInDiscoverTitle"
defaultMessage="View index in Discover"
/>
}
description=""
href={`${basePath.get()}/app/kibana#/discover?&_a=(index:'${indexPatternId}')${globalStateString}`}
/>
</EuiFlexItem>
)}

{isFullLicense() === true &&
timeFieldName !== undefined &&
showCreateJobLink &&
createIndexPattern && (
<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`machineLearningApp`} />}
title={
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.createNewMLJobTitle"
defaultMessage="Create new ML job"
/>
}
description=""
href={`#/jobs/new_job/step/job_type?index=${indexPatternId}${globalStateString}`}
/>
</EuiFlexItem>
)}

{createIndexPattern && (
<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`dataVisualizer`} />}
title={
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.openInDataVisualizerTitle"
defaultMessage="Open in Data Visualizer"
/>
}
description=""
href={`#/jobs/new_job/datavisualizer?index=${indexPatternId}${globalStateString}`}
/>
</EuiFlexItem>
)}

<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`managementApp`} />}
title={
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.indexManagementTitle"
defaultMessage="Index Management"
/>
}
description=""
href={`${basePath.get()}/app/kibana#/management/elasticsearch/index_management/indices/filter/${index}`}
/>
</EuiFlexItem>

<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`managementApp`} />}
title={
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.indexPatternManagementTitle"
defaultMessage="Index Pattern Management"
/>
}
description=""
href={`${basePath.get()}/app/kibana#/management/kibana/index_patterns/${
createIndexPattern ? indexPatternId : ''
}`}
/>
</EuiFlexItem>
<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`filebeatApp`} />}
title={
<FormattedMessage
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfig"
defaultMessage="Create Filebeat configuration"
/>
}
description=""
onClick={showFilebeatFlyout}
/>
</EuiFlexItem>
</EuiFlexGroup>
);
};

async function getFullTimeRange(index: string, timeFieldName: string) {
const query = { bool: { must: [{ query_string: { analyze_wildcard: true, query: '*' } }] } };
const resp = await ml.getTimeFieldRange({
index,
timeFieldName,
query,
});

return {
from: moment(resp.start.epoch).toISOString(),
to: moment(resp.end.epoch).toISOString(),
};
}
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@ import {
ChromeRecentlyAccessed,
IBasePath,
} from 'kibana/public';
import { SecurityPluginSetup } from '../../../../../../plugins/security/public';

export interface DependencyCache {
timefilter: TimefilterSetup | null;
@@ -38,6 +39,7 @@ export interface DependencyCache {
APP_URL: string | null;
application: ApplicationStart | null;
http: HttpStart | null;
security: SecurityPluginSetup | null;
}

const cache: DependencyCache = {
@@ -57,6 +59,7 @@ const cache: DependencyCache = {
APP_URL: null,
application: null,
http: null,
security: null,
};

export function setDependencyCache(deps: Partial<DependencyCache>) {
@@ -189,6 +192,13 @@ export function getHttp() {
return cache.http;
}

export function getSecurity() {
if (cache.security === null) {
throw new Error("security hasn't been initialized");
}
return cache.security;
}

export function clearCache() {
console.log('clearing dependency cache'); // eslint-disable-line no-console
Object.keys(cache).forEach(k => {
4 changes: 3 additions & 1 deletion x-pack/legacy/plugins/ml/public/legacy.ts
Original file line number Diff line number Diff line change
@@ -6,14 +6,16 @@

import chrome from 'ui/chrome';
import { npSetup, npStart } from 'ui/new_platform';

import { PluginInitializerContext } from 'src/core/public';
import { SecurityPluginSetup } from '../../../../plugins/security/public';

import { plugin } from '.';

const pluginInstance = plugin({} as PluginInitializerContext);

export const setup = pluginInstance.setup(npSetup.core, {
data: npStart.plugins.data,
security: ((npSetup.plugins as unknown) as { security: SecurityPluginSetup }).security, // security isn't in the PluginsSetup interface, but does exist
__LEGACY: {
XSRF: chrome.getXsrfToken(),
// @ts-ignore getAppUrl is missing from chrome's definition
3 changes: 2 additions & 1 deletion x-pack/legacy/plugins/ml/public/plugin.ts
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ import { Plugin, CoreStart, CoreSetup } from 'src/core/public';
import { MlDependencies } from './application/app';

export class MlPlugin implements Plugin<Setup, Start> {
setup(core: CoreSetup, { data, __LEGACY }: MlDependencies) {
setup(core: CoreSetup, { data, security, __LEGACY }: MlDependencies) {
core.application.register({
id: 'ml',
title: 'Machine learning',
@@ -21,6 +21,7 @@ export class MlPlugin implements Plugin<Setup, Start> {
onAppLeave: params.onAppLeave,
data,
__LEGACY,
security,
});
},
});
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@

import Boom from 'boom';
import { RequestHandlerContext } from 'kibana/server';
import { FindFileStructureResponse } from '../../../common/types/file_datavisualizer';

export type InputData = any[];

@@ -20,31 +21,7 @@ export type FormattedOverrides = InputOverrides & {
};

export interface AnalysisResult {
results: {
charset: string;
has_header_row: boolean;
has_byte_order_marker: boolean;
format: string;
field_stats: {
[fieldName: string]: {
count: number;
cardinality: number;
top_hits: Array<{ count: number; value: any }>;
};
};
sample_start: string;
num_messages_analyzed: number;
mappings: {
[fieldName: string]: {
type: string;
};
};
quote: string;
delimiter: string;
need_client_timezone: boolean;
num_lines_analyzed: number;
column_names: string[];
};
results: FindFileStructureResponse;
overrides?: FormattedOverrides;
}

0 comments on commit b756cc3

Please sign in to comment.