From 442609404841dee7d577bcf7401d63c1aed95319 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 1 Jul 2019 13:59:28 +0200 Subject: [PATCH] [ML] Data frames: Updated progress reporting. (#39920) - Adds a column mode to the data frame transforms list to indicate if the transform is batch or continuous. - For continuous data frames an animated horizontal loading indicator is shown in the Progress column instead of a progress bar with percentage. - In expanded rows, for Job details the section checkpointing was added. - In the transform wizard's creation step, for continuous transforms the progress bar is not shown. In a follow up we could add more useful stats there, like what's shown in the transform list's job details. --- .../components/job_create/job_create_form.tsx | 16 ++++-- .../job_details/job_details_form.tsx | 3 +- .../__mocks__/data_frame_job_list_row.json | 44 ++++++++++++++- .../__snapshots__/expanded_row.test.tsx.snap | 28 ++++++++++ .../job_details_pane.test.tsx.snap | 7 +-- .../components/job_list/columns.test.tsx | 7 ++- .../components/job_list/columns.tsx | 55 +++++++++++++++---- .../components/job_list/common.ts | 1 + .../components/job_list/expanded_row.tsx | 10 +++- .../components/job_list/job_details_pane.tsx | 14 +++-- .../job_list/job_service/get_jobs.ts | 9 ++- 11 files changed, 159 insertions(+), 35 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/data_frame/components/job_create/job_create_form.tsx b/x-pack/legacy/plugins/ml/public/data_frame/components/job_create/job_create_form.tsx index 2460ae03170b0..e560025672c85 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/components/job_create/job_create_form.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/components/job_create/job_create_form.tsx @@ -180,16 +180,20 @@ export const JobCreateForm: SFC = React.memo( } }; - if (started === true && progressPercentComplete === undefined) { + const isBatchTransform = typeof jobConfig.sync === 'undefined'; + + if (started === true && progressPercentComplete === undefined && isBatchTransform) { // wrapping in function so we can keep the interval id in local scope function startProgressBar() { const interval = setInterval(async () => { try { const stats = await ml.dataFrame.getDataFrameTransformsStats(jobId); - const percent = Math.round(stats.transforms[0].state.progress.percent_complete); - setProgressPercentComplete(percent); - if (percent >= 100) { - clearInterval(interval); + if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) { + const percent = Math.round(stats.transforms[0].state.progress.percent_complete); + setProgressPercentComplete(percent); + if (percent >= 100) { + clearInterval(interval); + } } } catch (e) { toastNotifications.addDanger( @@ -303,7 +307,7 @@ export const JobCreateForm: SFC = React.memo( - {progressPercentComplete !== undefined && ( + {progressPercentComplete !== undefined && isBatchTransform && ( diff --git a/x-pack/legacy/plugins/ml/public/data_frame/components/job_details/job_details_form.tsx b/x-pack/legacy/plugins/ml/public/data_frame/components/job_details/job_details_form.tsx index bd55f5494a6be..69e42712f2294 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/components/job_details/job_details_form.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/components/job_details/job_details_form.tsx @@ -257,8 +257,7 @@ export const JobDetailsForm: SFC = React.memo(({ overrides = {}, onChange helpText={i18n.translate( 'xpack.ml.dataframe.jobDetailsForm.continuousModeDateFieldHelpText', { - defaultMessage: - 'Pick a date field for the time based continuous data frame transform that reflects ingestion time.', + defaultMessage: 'Select the date field that can be used to identify new documents.', } )} > diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__mocks__/data_frame_job_list_row.json b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__mocks__/data_frame_job_list_row.json index c76c3db814008..f2045ea78112b 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__mocks__/data_frame_job_list_row.json +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__mocks__/data_frame_job_list_row.json @@ -1 +1,43 @@ -{"config":{"id":"fq_date_histogram_1m_1441","source":{"index":["farequote-2019"],"query":{"match_all":{}}},"dest":{"index":"fq_data_histogram_1m_1441"},"pivot":{"group_by":{"date_histogram(@timestamp)":{"date_histogram":{"field":"@timestamp","interval":"1m"}}},"aggregations":{"avg(response)":{"avg":{"field":"responsetime"}}}}},"id":"fq_date_histogram_1m_1441","state":{"task_state":"stopped","indexer_state":"stopped","current_position":{"date_histogram(@timestamp)":1549929540000},"checkpoint":1},"stats":{"pages_processed":0,"documents_processed":0,"documents_indexed":0,"trigger_count":0,"index_time_in_ms":0,"index_total":0,"index_failures":0,"search_time_in_ms":0,"search_total":0,"search_failures":0}} +{ + "config": { + "id": "fq_date_histogram_1m_1441", + "source": { "index": ["farequote-2019"], "query": { "match_all": {} } }, + "dest": { "index": "fq_data_histogram_1m_1441" }, + "pivot": { + "group_by": { + "date_histogram(@timestamp)": { + "date_histogram": { "field": "@timestamp", "interval": "1m" } + } + }, + "aggregations": { "avg(response)": { "avg": { "field": "responsetime" } } } + } + }, + "id": "fq_date_histogram_1m_1441", + "state": { + "task_state": "stopped", + "indexer_state": "stopped", + "current_position": { "date_histogram(@timestamp)": 1549929540000 }, + "checkpoint": 1 + }, + "stats": { + "pages_processed": 0, + "documents_processed": 0, + "documents_indexed": 0, + "trigger_count": 0, + "index_time_in_ms": 0, + "index_total": 0, + "index_failures": 0, + "search_time_in_ms": 0, + "search_total": 0, + "search_failures": 0 + }, + "checkpointing": { + "current": { + "timestamp": "2019-06-28T16:09:23.539Z", + "timestamp_millis": 1561738163539, + "time_upper_bound": "2019-06-28T16:09:13.539Z", + "time_upper_bound_millis": 1561738153539 + }, + "operations_behind": 0 + } +} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/expanded_row.test.tsx.snap b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/expanded_row.test.tsx.snap index 6955df73fe8af..aeca9b712e817 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/expanded_row.test.tsx.snap +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/expanded_row.test.tsx.snap @@ -31,6 +31,20 @@ exports[`Data Frame: Job List Minimal initialization 1`] = ` "position": "left", "title": "State", }, + Object { + "items": Array [ + Object { + "description": "{\\"timestamp\\":\\"2019-06-28T16:09:23.539Z\\",\\"timestamp_millis\\":1561738163539,\\"time_upper_bound\\":\\"2019-06-28T16:09:13.539Z\\",\\"time_upper_bound_millis\\":1561738153539}", + "title": "current", + }, + Object { + "description": "0", + "title": "operations_behind", + }, + ], + "position": "left", + "title": "Checkpointing", + }, Object { "items": Array [ Object { @@ -114,6 +128,20 @@ exports[`Data Frame: Job List Minimal initialization 1`] = ` "position": "left", "title": "State", }, + Object { + "items": Array [ + Object { + "description": "{\\"timestamp\\":\\"2019-06-28T16:09:23.539Z\\",\\"timestamp_millis\\":1561738163539,\\"time_upper_bound\\":\\"2019-06-28T16:09:13.539Z\\",\\"time_upper_bound_millis\\":1561738153539}", + "title": "current", + }, + Object { + "description": "0", + "title": "operations_behind", + }, + ], + "position": "left", + "title": "Checkpointing", + }, Object { "items": Array [ Object { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/job_details_pane.test.tsx.snap b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/job_details_pane.test.tsx.snap index c0a2f537bac3c..7173e982af31d 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/job_details_pane.test.tsx.snap +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/job_details_pane.test.tsx.snap @@ -7,7 +7,6 @@ exports[`Data Frame: Job List Expanded Row Minimal initializa size="s" />
Minimal initializa } /> - - - + `; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.test.tsx index 548009c3a2669..c88da3100b613 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.test.tsx @@ -10,13 +10,14 @@ describe('Data Frame: Job List Columns', () => { test('getColumns()', () => { const columns = getColumns(() => {}, [], () => {}); - expect(columns).toHaveLength(7); + expect(columns).toHaveLength(8); expect(columns[0].isExpander).toBeTruthy(); expect(columns[1].name).toBe('ID'); expect(columns[2].name).toBe('Source index'); expect(columns[3].name).toBe('Destination index'); expect(columns[4].name).toBe('Status'); - expect(columns[5].name).toBe('Progress'); - expect(columns[6].name).toBe('Actions'); + expect(columns[5].name).toBe('Mode'); + expect(columns[6].name).toBe('Progress'); + expect(columns[7].name).toBe('Actions'); }); }); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.tsx index 6922d5bb61a87..5105c4a718b58 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; +import React, { Fragment } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiBadge, @@ -91,9 +91,23 @@ export const getColumns = ( const color = item.state.task_state === 'started' ? 'primary' : 'hollow'; return {item.state.task_state}; }, + width: '100px', }, { - name: i18n.translate('xpack.ml.dataframe.progress', { defaultMessage: 'Progress' }), + name: i18n.translate('xpack.ml.dataframe.mode', { defaultMessage: 'Mode' }), + sortable: true, + truncateText: true, + render(item: DataFrameJobListRow) { + const mode = typeof item.config.sync !== 'undefined' ? 'continuous' : 'batch'; + const color = 'hollow'; + return {mode}; + }, + width: '100px', + }, + { + name: i18n.translate('xpack.ml.dataframe.progressIconTipContent', { + defaultMessage: 'Progress', + }), sortable: true, truncateText: true, render(item: DataFrameJobListRow) { @@ -103,23 +117,44 @@ export const getColumns = ( progress = Math.round(item.state.progress.percent_complete); } + const isBatchTransform = typeof item.config.sync === 'undefined'; + return ( - - - {progress}% - - - - {`${progress}%`} - + {isBatchTransform && ( + + + + {progress}% + + + + {`${progress}%`} + + + )} + {!isBatchTransform && ( + + + {item.state.task_state === 'started' && } + {item.state.task_state !== 'started' && ( + + )} + + +   + + + )} ); }, + width: '100px', }, { name: i18n.translate('xpack.ml.dataframe.tableActionLabel', { defaultMessage: 'Actions' }), actions, + width: '200px', }, ]; }; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/common.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/common.ts index 7cedb9f581194..493bb0ea498ef 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/common.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/common.ts @@ -45,6 +45,7 @@ export interface DataFrameJobStats { export interface DataFrameJobListRow { id: JobId; + checkpointing: object; state: DataFrameJobState; stats: DataFrameJobStats; config: DataFrameTransformWithId; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.tsx index 1f71ecf931268..f6fce2b21cee8 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.tsx @@ -37,6 +37,14 @@ export const ExpandedRow: SFC = ({ item, lastUpdate }) => { position: 'left', }; + const checkpointing: SectionConfig = { + title: 'Checkpointing', + items: Object.entries(item.checkpointing).map(s => { + return { title: s[0].toString(), description: getItemDescription(s[1]) }; + }), + position: 'left', + }; + const stats: SectionConfig = { title: 'Stats', items: Object.entries(item.stats).map(s => { @@ -51,7 +59,7 @@ export const ExpandedRow: SFC = ({ item, lastUpdate }) => { name: i18n.translate('xpack.ml.dataframe.jobsList.jobDetails.tabs.jobSettingsLabel', { defaultMessage: 'Job details', }), - content: , + content: , }, { id: 'job-json', diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_details_pane.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_details_pane.tsx index 86cfbd752c5b9..3f73f899f03bc 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_details_pane.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_details_pane.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { SFC } from 'react'; +import React, { Fragment, SFC } from 'react'; import { EuiDescriptionList, @@ -52,19 +52,23 @@ export const JobDetailsPane: SFC = ({ sections }) => { return ( - {sections .filter(s => s.position === 'left') .map(s => ( -
+ + +
+ ))} - {sections .filter(s => s.position === 'right') .map(s => ( -
+ + +
+ ))} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_service/get_jobs.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_service/get_jobs.ts index 678c20abfab10..c6369ac429da8 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_service/get_jobs.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_service/get_jobs.ts @@ -12,6 +12,7 @@ import { DataFrameJobListRow, DataFrameJobState, DataFrameJobStats } from '../co interface DataFrameJobStateStats { id: JobId; + checkpointing: object; state: DataFrameJobState; stats: DataFrameJobStats; } @@ -47,7 +48,13 @@ export const getJobsFactory = ( return reducedtableRows; } // Table with expandable rows requires `id` on the outer most level - reducedtableRows.push({ config, id: config.id, state: stats.state, stats: stats.stats }); + reducedtableRows.push({ + config, + id: config.id, + checkpointing: stats.checkpointing, + state: stats.state, + stats: stats.stats, + }); return reducedtableRows; }, [] as DataFrameJobListRow[]