-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ML] Adding filebeat config to file dataviz (#58152)
* [ML] Adding filebeat config to file dataviz * adding extra help text * removing commented out code * adding extra blank line to processors section * cleaning up types * moving hosts line out of function * typo in config text * updating config based on review * tiny refactor * translating paths text
- Loading branch information
1 parent
71d6c22
commit b756cc3
Showing
15 changed files
with
513 additions
and
209 deletions.
There are no files selected for viewing
31 changes: 31 additions & 0 deletions
31
x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
|
||
export interface FindFileStructureResponse { | ||
charset: string; | ||
has_header_row: boolean; | ||
has_byte_order_marker: boolean; | ||
format: string; | ||
field_stats: { | ||
[fieldName: string]: { | ||
count: number; | ||
cardinality: number; | ||
top_hits: Array<{ count: number; value: any }>; | ||
}; | ||
}; | ||
sample_start: string; | ||
num_messages_analyzed: number; | ||
mappings: { | ||
[fieldName: string]: { | ||
type: string; | ||
}; | ||
}; | ||
quote: string; | ||
delimiter: string; | ||
need_client_timezone: boolean; | ||
num_lines_analyzed: number; | ||
column_names: string[]; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
71 changes: 71 additions & 0 deletions
71
...pplication/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
|
||
import { i18n } from '@kbn/i18n'; | ||
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; | ||
|
||
export function createFilebeatConfig( | ||
index: string, | ||
results: FindFileStructureResponse, | ||
ingestPipelineId: string, | ||
username: string | null | ||
) { | ||
return [ | ||
'filebeat.inputs:', | ||
'- type: log', | ||
...getPaths(), | ||
...getEncoding(results), | ||
...getExcludeLines(results), | ||
...getMultiline(results), | ||
'', | ||
...getProcessors(results), | ||
'output.elasticsearch:', | ||
' hosts: ["<es_url>"]', | ||
...getUserDetails(username), | ||
` index: "${index}"`, | ||
` pipeline: "${ingestPipelineId}"`, | ||
'', | ||
'setup:', | ||
' template.enabled: false', | ||
' ilm.enabled: false', | ||
].join('\n'); | ||
} | ||
|
||
function getPaths() { | ||
const txt = i18n.translate('xpack.ml.fileDatavisualizer.fileBeatConfig.paths', { | ||
defaultMessage: 'add path to your files here', | ||
}); | ||
return [' paths:', ` - '<${txt}>'`]; | ||
} | ||
|
||
function getEncoding(results: any) { | ||
return results.charset !== 'UTF-8' ? [` encoding: ${results.charset}`] : []; | ||
} | ||
|
||
function getExcludeLines(results: any) { | ||
return results.exclude_lines_pattern !== undefined | ||
? [` exclude_lines: ['${results.exclude_lines_pattern.replace(/'/g, "''")}']`] | ||
: []; | ||
} | ||
|
||
function getMultiline(results: any) { | ||
return results.multiline_start_pattern !== undefined | ||
? [ | ||
' multiline:', | ||
` pattern: '${results.multiline_start_pattern.replace(/'/g, "''")}'`, | ||
' match: after', | ||
' negate: true', | ||
] | ||
: []; | ||
} | ||
|
||
function getProcessors(results: any) { | ||
return results.need_client_timezone === true ? ['processors:', '- add_locale: ~', ''] : []; | ||
} | ||
|
||
function getUserDetails(username: string | null) { | ||
return username !== null ? [` username: "${username}"`, ' password: "<password>"'] : []; | ||
} |
162 changes: 162 additions & 0 deletions
162
...on/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,162 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
|
||
import React, { FC, useState, useEffect } from 'react'; | ||
import { FormattedMessage } from '@kbn/i18n/react'; | ||
import { | ||
EuiFlyout, | ||
EuiFlyoutFooter, | ||
EuiFlexGroup, | ||
EuiFlexItem, | ||
EuiButton, | ||
EuiButtonEmpty, | ||
EuiTitle, | ||
EuiFlyoutBody, | ||
EuiSpacer, | ||
EuiCodeBlock, | ||
EuiCode, | ||
EuiCopy, | ||
} from '@elastic/eui'; | ||
import { createFilebeatConfig } from './filebeat_config'; | ||
import { useMlKibana } from '../../../../contexts/kibana'; | ||
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; | ||
|
||
export enum EDITOR_MODE { | ||
HIDDEN, | ||
READONLY, | ||
EDITABLE, | ||
} | ||
interface Props { | ||
index: string; | ||
results: FindFileStructureResponse; | ||
indexPatternId: string; | ||
ingestPipelineId: string; | ||
closeFlyout(): void; | ||
} | ||
export const FilebeatConfigFlyout: FC<Props> = ({ | ||
index, | ||
results, | ||
indexPatternId, | ||
ingestPipelineId, | ||
closeFlyout, | ||
}) => { | ||
const [fileBeatConfig, setFileBeatConfig] = useState(''); | ||
const [username, setUsername] = useState<string | null>(null); | ||
const { | ||
services: { security }, | ||
} = useMlKibana(); | ||
|
||
useEffect(() => { | ||
security.authc.getCurrentUser().then(user => { | ||
setUsername(user.username === undefined ? null : user.username); | ||
}); | ||
}, []); | ||
|
||
useEffect(() => { | ||
const config = createFilebeatConfig(index, results, ingestPipelineId, username); | ||
setFileBeatConfig(config); | ||
}, [username]); | ||
|
||
return ( | ||
<EuiFlyout onClose={closeFlyout} hideCloseButton size={'m'}> | ||
<EuiFlyoutBody> | ||
<EuiFlexGroup> | ||
<Contents value={fileBeatConfig} username={username} index={index} /> | ||
</EuiFlexGroup> | ||
</EuiFlyoutBody> | ||
<EuiFlyoutFooter> | ||
<EuiFlexGroup justifyContent="spaceBetween"> | ||
<EuiFlexItem grow={false}> | ||
<EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left"> | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.fileBeatConfigFlyout.closeButton" | ||
defaultMessage="Close" | ||
/> | ||
</EuiButtonEmpty> | ||
</EuiFlexItem> | ||
<EuiFlexItem grow={false}> | ||
<EuiCopy textToCopy={fileBeatConfig}> | ||
{copy => ( | ||
<EuiButton onClick={copy}> | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.fileBeatConfigFlyout.copyButton" | ||
defaultMessage="Copy to clipboard" | ||
/> | ||
</EuiButton> | ||
)} | ||
</EuiCopy> | ||
</EuiFlexItem> | ||
</EuiFlexGroup> | ||
</EuiFlyoutFooter> | ||
</EuiFlyout> | ||
); | ||
}; | ||
|
||
const Contents: FC<{ | ||
value: string; | ||
index: string; | ||
username: string | null; | ||
}> = ({ value, index, username }) => { | ||
return ( | ||
<EuiFlexItem> | ||
<EuiTitle size="s"> | ||
<h5> | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTitle" | ||
defaultMessage="Filebeat configuration" | ||
/> | ||
</h5> | ||
</EuiTitle> | ||
<EuiSpacer size="s" /> | ||
<p> | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTopText1" | ||
defaultMessage="Additional data can be uploaded to the {index} index using Filebeat." | ||
values={{ index: <EuiCode>{index}</EuiCode> }} | ||
/> | ||
</p> | ||
<p> | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTopText2" | ||
defaultMessage="Modify {filebeatYml} to set the connection information:" | ||
values={{ filebeatYml: <EuiCode>filebeat.yml</EuiCode> }} | ||
/> | ||
</p> | ||
|
||
<EuiSpacer size="s" /> | ||
|
||
<EuiCodeBlock language="bash">{value}</EuiCodeBlock> | ||
|
||
<EuiSpacer size="s" /> | ||
<p> | ||
{username === null ? ( | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigBottomTextNoUsername" | ||
defaultMessage="Where {esUrl} is the URL of Elasticsearch." | ||
values={{ | ||
esUrl: <EuiCode>{'<es_url>'}</EuiCode>, | ||
}} | ||
/> | ||
) : ( | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigBottomText" | ||
defaultMessage="Where {password} is the password of the {user} user, {esUrl} is the URL of Elasticsearch." | ||
values={{ | ||
user: <EuiCode>{username}</EuiCode>, | ||
password: <EuiCode>{'<password>'}</EuiCode>, | ||
esUrl: <EuiCode>{'<es_url>'}</EuiCode>, | ||
}} | ||
/> | ||
)} | ||
</p> | ||
</EuiFlexItem> | ||
); | ||
}; |
7 changes: 7 additions & 0 deletions
7
...l/public/application/datavisualizer/file_based/components/filebeat_config_flyout/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
|
||
export { FilebeatConfigFlyout } from './filebeat_config_flyout'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
182 changes: 0 additions & 182 deletions
182
...ml/public/application/datavisualizer/file_based/components/results_links/results_links.js
This file was deleted.
Oops, something went wrong.
190 changes: 190 additions & 0 deletions
190
...l/public/application/datavisualizer/file_based/components/results_links/results_links.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,190 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License; | ||
* you may not use this file except in compliance with the Elastic License. | ||
*/ | ||
|
||
import React, { FC, useState, useEffect } from 'react'; | ||
import moment from 'moment'; | ||
import { FormattedMessage } from '@kbn/i18n/react'; | ||
import { EuiFlexGroup, EuiFlexItem, EuiCard, EuiIcon } from '@elastic/eui'; | ||
import { ml } from '../../../../services/ml_api_service'; | ||
import { isFullLicense } from '../../../../license/check_license'; | ||
import { checkPermission } from '../../../../privilege/check_privilege'; | ||
import { mlNodesAvailable } from '../../../../ml_nodes_check/check_ml_nodes'; | ||
import { useMlKibana } from '../../../../contexts/kibana'; | ||
|
||
const RECHECK_DELAY_MS = 3000; | ||
|
||
interface Props { | ||
index: string; | ||
indexPatternId: string; | ||
timeFieldName?: string; | ||
createIndexPattern: boolean; | ||
showFilebeatFlyout(): void; | ||
} | ||
|
||
export const ResultsLinks: FC<Props> = ({ | ||
index, | ||
indexPatternId, | ||
timeFieldName, | ||
createIndexPattern, | ||
showFilebeatFlyout, | ||
}) => { | ||
const [duration, setDuration] = useState({ | ||
from: 'now-30m', | ||
to: 'now', | ||
}); | ||
const [showCreateJobLink, setShowCreateJobLink] = useState(false); | ||
const [globalStateString, setGlobalStateString] = useState(''); | ||
const { | ||
services: { | ||
http: { basePath }, | ||
}, | ||
} = useMlKibana(); | ||
|
||
useEffect(() => { | ||
setShowCreateJobLink(checkPermission('canCreateJob') && mlNodesAvailable()); | ||
updateTimeValues(); | ||
}, []); | ||
|
||
useEffect(() => { | ||
const _g = | ||
timeFieldName !== undefined | ||
? `&_g=(time:(from:'${duration.from}',mode:quick,to:'${duration.to}'))` | ||
: ''; | ||
setGlobalStateString(_g); | ||
}, [duration]); | ||
|
||
async function updateTimeValues(recheck = true) { | ||
if (timeFieldName !== undefined) { | ||
const { from, to } = await getFullTimeRange(index, timeFieldName); | ||
setDuration({ | ||
from: from === null ? duration.from : from, | ||
to: to === null ? duration.to : to, | ||
}); | ||
|
||
// these links may have been drawn too quickly for the index to be ready | ||
// to give us the correct start and end times. | ||
// especially if the data was small. | ||
// so if the start and end were null, try again in 3s | ||
if (recheck && (from === null || to === null)) { | ||
setTimeout(() => { | ||
updateTimeValues(false); | ||
}, RECHECK_DELAY_MS); | ||
} | ||
} | ||
} | ||
|
||
return ( | ||
<EuiFlexGroup gutterSize="l"> | ||
{createIndexPattern && ( | ||
<EuiFlexItem> | ||
<EuiCard | ||
icon={<EuiIcon size="xxl" type={`discoverApp`} />} | ||
title={ | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.viewIndexInDiscoverTitle" | ||
defaultMessage="View index in Discover" | ||
/> | ||
} | ||
description="" | ||
href={`${basePath.get()}/app/kibana#/discover?&_a=(index:'${indexPatternId}')${globalStateString}`} | ||
/> | ||
</EuiFlexItem> | ||
)} | ||
|
||
{isFullLicense() === true && | ||
timeFieldName !== undefined && | ||
showCreateJobLink && | ||
createIndexPattern && ( | ||
<EuiFlexItem> | ||
<EuiCard | ||
icon={<EuiIcon size="xxl" type={`machineLearningApp`} />} | ||
title={ | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.createNewMLJobTitle" | ||
defaultMessage="Create new ML job" | ||
/> | ||
} | ||
description="" | ||
href={`#/jobs/new_job/step/job_type?index=${indexPatternId}${globalStateString}`} | ||
/> | ||
</EuiFlexItem> | ||
)} | ||
|
||
{createIndexPattern && ( | ||
<EuiFlexItem> | ||
<EuiCard | ||
icon={<EuiIcon size="xxl" type={`dataVisualizer`} />} | ||
title={ | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.openInDataVisualizerTitle" | ||
defaultMessage="Open in Data Visualizer" | ||
/> | ||
} | ||
description="" | ||
href={`#/jobs/new_job/datavisualizer?index=${indexPatternId}${globalStateString}`} | ||
/> | ||
</EuiFlexItem> | ||
)} | ||
|
||
<EuiFlexItem> | ||
<EuiCard | ||
icon={<EuiIcon size="xxl" type={`managementApp`} />} | ||
title={ | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.indexManagementTitle" | ||
defaultMessage="Index Management" | ||
/> | ||
} | ||
description="" | ||
href={`${basePath.get()}/app/kibana#/management/elasticsearch/index_management/indices/filter/${index}`} | ||
/> | ||
</EuiFlexItem> | ||
|
||
<EuiFlexItem> | ||
<EuiCard | ||
icon={<EuiIcon size="xxl" type={`managementApp`} />} | ||
title={ | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.indexPatternManagementTitle" | ||
defaultMessage="Index Pattern Management" | ||
/> | ||
} | ||
description="" | ||
href={`${basePath.get()}/app/kibana#/management/kibana/index_patterns/${ | ||
createIndexPattern ? indexPatternId : '' | ||
}`} | ||
/> | ||
</EuiFlexItem> | ||
<EuiFlexItem> | ||
<EuiCard | ||
icon={<EuiIcon size="xxl" type={`filebeatApp`} />} | ||
title={ | ||
<FormattedMessage | ||
id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfig" | ||
defaultMessage="Create Filebeat configuration" | ||
/> | ||
} | ||
description="" | ||
onClick={showFilebeatFlyout} | ||
/> | ||
</EuiFlexItem> | ||
</EuiFlexGroup> | ||
); | ||
}; | ||
|
||
async function getFullTimeRange(index: string, timeFieldName: string) { | ||
const query = { bool: { must: [{ query_string: { analyze_wildcard: true, query: '*' } }] } }; | ||
const resp = await ml.getTimeFieldRange({ | ||
index, | ||
timeFieldName, | ||
query, | ||
}); | ||
|
||
return { | ||
from: moment(resp.start.epoch).toISOString(), | ||
to: moment(resp.end.epoch).toISOString(), | ||
}; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters