From b0ba5096910f0b22ba56116b7df920f1b54ac5aa Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 24 Apr 2020 14:20:50 +0100 Subject: [PATCH 1/4] [ML] Add kibana setting for file data visualizers max file size --- .../plugins/ml/common/constants/settings.ts | 7 ++++ x-pack/plugins/ml/common/types/ml_config.ts | 16 --------- x-pack/plugins/ml/public/application/app.tsx | 7 +--- .../file_based/components/utils/utils.ts | 7 ++-- .../application/util/dependency_cache.ts | 11 ------ x-pack/plugins/ml/public/index.ts | 4 +-- x-pack/plugins/ml/public/plugin.ts | 13 +------ x-pack/plugins/ml/server/config.ts | 15 -------- x-pack/plugins/ml/server/index.ts | 2 -- .../ml/server/lib/register_settings.ts | 34 +++++++++++++++++++ x-pack/plugins/ml/server/plugin.ts | 3 ++ 11 files changed, 52 insertions(+), 67 deletions(-) create mode 100644 x-pack/plugins/ml/common/constants/settings.ts delete mode 100644 x-pack/plugins/ml/common/types/ml_config.ts delete mode 100644 x-pack/plugins/ml/server/config.ts create mode 100644 x-pack/plugins/ml/server/lib/register_settings.ts diff --git a/x-pack/plugins/ml/common/constants/settings.ts b/x-pack/plugins/ml/common/constants/settings.ts new file mode 100644 index 0000000000000..2df2ecd22e078 --- /dev/null +++ b/x-pack/plugins/ml/common/constants/settings.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const FILE_DATA_VISUALIZER_MAX_FILE_SIZE = 'ml:fileDataVisualizerMaxFileSize'; diff --git a/x-pack/plugins/ml/common/types/ml_config.ts b/x-pack/plugins/ml/common/types/ml_config.ts deleted file mode 100644 index f2ddadccb2170..0000000000000 --- a/x-pack/plugins/ml/common/types/ml_config.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { schema, TypeOf } from '@kbn/config-schema'; -import { MAX_FILE_SIZE } from '../constants/file_datavisualizer'; - -export const configSchema = schema.object({ - file_data_visualizer: schema.object({ - max_file_size: schema.string({ defaultValue: MAX_FILE_SIZE }), - }), -}); - -export type MlConfigType = TypeOf; diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index f1facd18b9da5..e9796fcbb0fe4 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -15,14 +15,10 @@ import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/p import { setDependencyCache, clearCache } from './util/dependency_cache'; import { setLicenseCache } from './license'; import { MlSetupDependencies, MlStartDependencies } from '../plugin'; -import { MlConfigType } from '../../common/types/ml_config'; import { MlRouter } from './routing'; -type MlDependencies = MlSetupDependencies & - MlStartDependencies & { - mlConfig: MlConfigType; - }; +type MlDependencies = MlSetupDependencies & MlStartDependencies; interface AppProps { coreStart: CoreStart; @@ -78,7 +74,6 @@ export const renderApp = ( http: coreStart.http, security: deps.security, urlGenerators: deps.share.urlGenerators, - mlConfig: deps.mlConfig, }); const mlLicense = setLicenseCache(deps.licensing); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index 7b6464570e55c..e0e72da27c7ee 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -13,7 +13,8 @@ import { ABSOLUTE_MAX_FILE_SIZE_BYTES, FILE_SIZE_DISPLAY_FORMAT, } from '../../../../../../common/constants/file_datavisualizer'; -import { getMlConfig } from '../../../../util/dependency_cache'; +import { getUiSettings } from '../../../../util/dependency_cache'; +import { FILE_DATA_VISUALIZER_MAX_FILE_SIZE } from '../../../../../../common/constants/settings'; const DEFAULT_LINES_TO_SAMPLE = 1000; const UPLOAD_SIZE_MB = 5; @@ -62,13 +63,13 @@ export function readFile(file: File) { } export function getMaxBytes() { - const maxFileSize = getMlConfig().file_data_visualizer.max_file_size; + const maxFileSize = getUiSettings().get(FILE_DATA_VISUALIZER_MAX_FILE_SIZE); // @ts-ignore const maxBytes = numeral(maxFileSize.toUpperCase()).value(); if (maxBytes < MAX_FILE_SIZE_BYTES) { return MAX_FILE_SIZE_BYTES; } - return maxBytes < ABSOLUTE_MAX_FILE_SIZE_BYTES ? maxBytes : ABSOLUTE_MAX_FILE_SIZE_BYTES; + return maxBytes <= ABSOLUTE_MAX_FILE_SIZE_BYTES ? maxBytes : ABSOLUTE_MAX_FILE_SIZE_BYTES; } export function getMaxBytesFormatted() { diff --git a/x-pack/plugins/ml/public/application/util/dependency_cache.ts b/x-pack/plugins/ml/public/application/util/dependency_cache.ts index 934a0a5e9ae3a..d5605d3bca65f 100644 --- a/x-pack/plugins/ml/public/application/util/dependency_cache.ts +++ b/x-pack/plugins/ml/public/application/util/dependency_cache.ts @@ -23,7 +23,6 @@ import { } from 'kibana/public'; import { SharePluginStart } from 'src/plugins/share/public'; import { SecurityPluginSetup } from '../../../../security/public'; -import { MlConfigType } from '../../../common/types/ml_config'; export interface DependencyCache { timefilter: DataPublicPluginSetup['query']['timefilter'] | null; @@ -43,7 +42,6 @@ export interface DependencyCache { security: SecurityPluginSetup | null; i18n: I18nStart | null; urlGenerators: SharePluginStart['urlGenerators'] | null; - mlConfig: MlConfigType | null; } const cache: DependencyCache = { @@ -64,7 +62,6 @@ const cache: DependencyCache = { security: null, i18n: null, urlGenerators: null, - mlConfig: null, }; export function setDependencyCache(deps: Partial) { @@ -85,7 +82,6 @@ export function setDependencyCache(deps: Partial) { cache.security = deps.security || null; cache.i18n = deps.i18n || null; cache.urlGenerators = deps.urlGenerators || null; - cache.mlConfig = deps.mlConfig || null; } export function getTimefilter() { @@ -206,13 +202,6 @@ export function getGetUrlGenerator() { return cache.urlGenerators.getUrlGenerator; } -export function getMlConfig() { - if (cache.mlConfig === null) { - throw new Error("mlConfig hasn't been initialized"); - } - return cache.mlConfig; -} - export function clearCache() { console.log('clearing dependency cache'); // eslint-disable-line no-console Object.keys(cache).forEach(k => { diff --git a/x-pack/plugins/ml/public/index.ts b/x-pack/plugins/ml/public/index.ts index 4697496270edf..8070f94a1264d 100755 --- a/x-pack/plugins/ml/public/index.ts +++ b/x-pack/plugins/ml/public/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { PluginInitializer, PluginInitializerContext } from 'kibana/public'; +import { PluginInitializer } from 'kibana/public'; import './index.scss'; import { MlPlugin, @@ -19,6 +19,6 @@ export const plugin: PluginInitializer< MlPluginStart, MlSetupDependencies, MlStartDependencies -> = (context: PluginInitializerContext) => new MlPlugin(context); +> = () => new MlPlugin(); export { MlPluginSetup, MlPluginStart }; diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts index b51be4d248683..2472c343d0510 100644 --- a/x-pack/plugins/ml/public/plugin.ts +++ b/x-pack/plugins/ml/public/plugin.ts @@ -5,13 +5,7 @@ */ import { i18n } from '@kbn/i18n'; -import { - Plugin, - CoreStart, - CoreSetup, - AppMountParameters, - PluginInitializerContext, -} from 'kibana/public'; +import { Plugin, CoreStart, CoreSetup, AppMountParameters } from 'kibana/public'; import { ManagementSetup } from 'src/plugins/management/public'; import { SharePluginStart } from 'src/plugins/share/public'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; @@ -25,7 +19,6 @@ import { LicenseManagementUIPluginSetup } from '../../license_management/public' import { setDependencyCache } from './application/util/dependency_cache'; import { PLUGIN_ID, PLUGIN_ICON } from '../common/constants/app'; import { registerFeature } from './register_feature'; -import { MlConfigType } from '../common/types/ml_config'; export interface MlStartDependencies { data: DataPublicPluginStart; @@ -41,10 +34,7 @@ export interface MlSetupDependencies { } export class MlPlugin implements Plugin { - constructor(private readonly initializerContext: PluginInitializerContext) {} - setup(core: CoreSetup, pluginsSetup: MlSetupDependencies) { - const mlConfig = this.initializerContext.config.get(); core.application.register({ id: PLUGIN_ID, title: i18n.translate('xpack.ml.plugin.title', { @@ -67,7 +57,6 @@ export class MlPlugin implements Plugin { usageCollection: pluginsSetup.usageCollection, licenseManagement: pluginsSetup.licenseManagement, home: pluginsSetup.home, - mlConfig, }, { element: params.element, diff --git a/x-pack/plugins/ml/server/config.ts b/x-pack/plugins/ml/server/config.ts deleted file mode 100644 index 7cef6f17bbefb..0000000000000 --- a/x-pack/plugins/ml/server/config.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { PluginConfigDescriptor } from 'kibana/server'; -import { MlConfigType, configSchema } from '../common/types/ml_config'; - -export const config: PluginConfigDescriptor = { - exposeToBrowser: { - file_data_visualizer: true, - }, - schema: configSchema, -}; diff --git a/x-pack/plugins/ml/server/index.ts b/x-pack/plugins/ml/server/index.ts index 6e638d647a387..175c20bf49c94 100644 --- a/x-pack/plugins/ml/server/index.ts +++ b/x-pack/plugins/ml/server/index.ts @@ -9,5 +9,3 @@ import { MlServerPlugin } from './plugin'; export { MlPluginSetup, MlPluginStart } from './plugin'; export const plugin = (ctx: PluginInitializerContext) => new MlServerPlugin(ctx); - -export { config } from './config'; diff --git a/x-pack/plugins/ml/server/lib/register_settings.ts b/x-pack/plugins/ml/server/lib/register_settings.ts new file mode 100644 index 0000000000000..64af3db323655 --- /dev/null +++ b/x-pack/plugins/ml/server/lib/register_settings.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreSetup } from 'kibana/server'; +import { i18n } from '@kbn/i18n'; +import { schema } from '@kbn/config-schema'; +import { FILE_DATA_VISUALIZER_MAX_FILE_SIZE } from '../../common/constants/settings'; +import { MAX_FILE_SIZE } from '../../common/constants/file_datavisualizer'; + +export function registerKibanaSettings(coreSetup: CoreSetup) { + coreSetup.uiSettings.register({ + [FILE_DATA_VISUALIZER_MAX_FILE_SIZE]: { + name: i18n.translate('xpack.rollupJobs.rollupIndexPatternsTitle', { + defaultMessage: 'File Data Visualizer maximum file upload size', + }), + value: MAX_FILE_SIZE, + description: i18n.translate('xpack.ml.maxFileSizeSettingsDescription', { + defaultMessage: + 'Sets the file size limit when importing data in the File Data Visualizer. The highest supported value for this setting is 1GB.', + }), + category: ['Machine Learning'], + schema: schema.string(), + validation: { + regexString: '\\d+[mMgG][bB]', + message: i18n.translate('xpack.ml.maxFileSizeSettingsError', { + defaultMessage: 'Should be a valid data size. e.g. 200MB, 1GB', + }), + }, + }, + }); +} diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 67e80a3bc44c0..64f8eb4b0acd3 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -47,6 +47,7 @@ import { MlServerLicense } from './lib/license'; import { createSharedServices, SharedServices } from './shared_services'; import { userMlCapabilities, adminMlCapabilities } from '../common/types/capabilities'; import { setupCapabilitiesSwitcher } from './lib/capabilities'; +import { registerKibanaSettings } from './lib/register_settings'; declare module 'kibana/server' { interface RequestHandlerContext { @@ -122,6 +123,8 @@ export class MlServerPlugin implements Plugin initSampleDataSets(mlLicense, plugins), ]); From 712bc256ad327fb50c492eeca4b2306824efa040 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 24 Apr 2020 14:38:37 +0100 Subject: [PATCH 2/4] adding failsafe for setting --- .../datavisualizer/file_based/components/utils/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index e0e72da27c7ee..7d966949624c1 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -9,6 +9,7 @@ import numeral from '@elastic/numeral'; import { ml } from '../../../../services/ml_api_service'; import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer'; import { + MAX_FILE_SIZE, MAX_FILE_SIZE_BYTES, ABSOLUTE_MAX_FILE_SIZE_BYTES, FILE_SIZE_DISPLAY_FORMAT, @@ -63,7 +64,7 @@ export function readFile(file: File) { } export function getMaxBytes() { - const maxFileSize = getUiSettings().get(FILE_DATA_VISUALIZER_MAX_FILE_SIZE); + const maxFileSize = getUiSettings().get(FILE_DATA_VISUALIZER_MAX_FILE_SIZE, MAX_FILE_SIZE); // @ts-ignore const maxBytes = numeral(maxFileSize.toUpperCase()).value(); if (maxBytes < MAX_FILE_SIZE_BYTES) { From d14568ce7410723c19ff6e145c0aeb810a863812 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 24 Apr 2020 16:30:50 +0100 Subject: [PATCH 3/4] fixing id --- x-pack/plugins/ml/server/lib/register_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/server/lib/register_settings.ts b/x-pack/plugins/ml/server/lib/register_settings.ts index 64af3db323655..38b1f5e3fc083 100644 --- a/x-pack/plugins/ml/server/lib/register_settings.ts +++ b/x-pack/plugins/ml/server/lib/register_settings.ts @@ -13,7 +13,7 @@ import { MAX_FILE_SIZE } from '../../common/constants/file_datavisualizer'; export function registerKibanaSettings(coreSetup: CoreSetup) { coreSetup.uiSettings.register({ [FILE_DATA_VISUALIZER_MAX_FILE_SIZE]: { - name: i18n.translate('xpack.rollupJobs.rollupIndexPatternsTitle', { + name: i18n.translate('xpack.ml.maxFileSizeSettingsName', { defaultMessage: 'File Data Visualizer maximum file upload size', }), value: MAX_FILE_SIZE, From 52cad9387be884cb11776412ba63951808ae4c59 Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 24 Apr 2020 14:12:26 -0700 Subject: [PATCH 4/4] [DOCS] Updates Data Visualizer setting --- docs/management/advanced-options.asciidoc | 7 +++++++ docs/settings/ml-settings.asciidoc | 6 ------ docs/user/ml/index.asciidoc | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/management/advanced-options.asciidoc b/docs/management/advanced-options.asciidoc index 7081590931a99..51910169e8673 100644 --- a/docs/management/advanced-options.asciidoc +++ b/docs/management/advanced-options.asciidoc @@ -142,7 +142,14 @@ This setting does not have an effect when loading a saved search. Highlighting slows requests when working on big documents. +[float] +[[kibana-ml-settings]] +==== Machine learning +[horizontal] +`ml:fileDataVisualizerMaxFileSize`:: Sets the file size limit when importing +data in the {data-viz}. The default value is `100MB`. The highest supported +value for this setting is `1GB`. [float] diff --git a/docs/settings/ml-settings.asciidoc b/docs/settings/ml-settings.asciidoc index b71e1c672756a..36578c909f513 100644 --- a/docs/settings/ml-settings.asciidoc +++ b/docs/settings/ml-settings.asciidoc @@ -19,10 +19,4 @@ instance. If `xpack.ml.enabled` is set to `true` in `elasticsearch.yml`, however you can still use the {ml} APIs. To disable {ml} entirely, see the {ref}/ml-settings.html[{es} {ml} settings]. -[[data-visualizer-settings]] -==== {data-viz} settings - -`xpack.ml.file_data_visualizer.max_file_size`:: -Sets the file size limit when importing data in the {data-viz}. The default -value is `100MB`. The highest supported value for this setting is `1GB`. diff --git a/docs/user/ml/index.asciidoc b/docs/user/ml/index.asciidoc index e9ef4a55b2b3a..6483ddde07335 100644 --- a/docs/user/ml/index.asciidoc +++ b/docs/user/ml/index.asciidoc @@ -20,7 +20,7 @@ image::user/ml/images/ml-data-visualizer-sample.jpg[{data-viz} for sample flight experimental[] You can also upload a CSV, NDJSON, or log file. The *{data-viz}* identifies the file format and field mappings. You can then optionally import that data into an {es} index. To change the default file size limit, see -<>. +<>. You need the following permissions to use the {data-viz} with file upload: