[ML] Data frames: Updated progress reporting. (#39920)

- Adds a column mode to the data frame transforms list to indicate if the transform is batch or continuous.
- For continuous data frames an animated horizontal loading indicator is shown in the Progress column instead of a progress bar with percentage.
- In expanded rows, for Job details the section checkpointing was added.
- In the transform wizard's creation step, for continuous transforms the progress bar is not shown. In a follow up we could add more useful stats there, like what's shown in the transform list's job details.

x-pack/legacy/plugins/ml/public/data_frame/components/job_create/job_create_form.tsx

@@ -180,16 +180,20 @@ export const JobCreateForm: SFC<Props> = React.memo(
       }
     };
 
-    if (started === true && progressPercentComplete === undefined) {
+    const isBatchTransform = typeof jobConfig.sync === 'undefined';
+
+    if (started === true && progressPercentComplete === undefined && isBatchTransform) {
       // wrapping in function so we can keep the interval id in local scope
       function startProgressBar() {
         const interval = setInterval(async () => {
           try {
             const stats = await ml.dataFrame.getDataFrameTransformsStats(jobId);
-            const percent = Math.round(stats.transforms[0].state.progress.percent_complete);
-            setProgressPercentComplete(percent);
-            if (percent >= 100) {
-              clearInterval(interval);
+            if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) {
+              const percent = Math.round(stats.transforms[0].state.progress.percent_complete);
+              setProgressPercentComplete(percent);
+              if (percent >= 100) {
+                clearInterval(interval);
+              }
             }
           } catch (e) {
             toastNotifications.addDanger(
@@ -303,7 +307,7 @@ export const JobCreateForm: SFC<Props> = React.memo(
             </EuiText>
           </EuiFlexItem>
         </EuiFlexGroup>
-        {progressPercentComplete !== undefined && (
+        {progressPercentComplete !== undefined && isBatchTransform && (
           <Fragment>
             <EuiSpacer size="m" />
             <EuiText size="xs">

x-pack/legacy/plugins/ml/public/data_frame/components/job_details/job_details_form.tsx

@@ -257,8 +257,7 @@ export const JobDetailsForm: SFC<Props> = React.memo(({ overrides = {}, onChange
             helpText={i18n.translate(
               'xpack.ml.dataframe.jobDetailsForm.continuousModeDateFieldHelpText',
               {
-                defaultMessage:
-                  'Pick a date field for the time based continuous data frame transform that reflects ingestion time.',
+                defaultMessage: 'Select the date field that can be used to identify new documents.',
               }
             )}
           >

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__mocks__/data_frame_job_list_row.json

@@ -1 +1,43 @@
-{"config":{"id":"fq_date_histogram_1m_1441","source":{"index":["farequote-2019"],"query":{"match_all":{}}},"dest":{"index":"fq_data_histogram_1m_1441"},"pivot":{"group_by":{"date_histogram(@timestamp)":{"date_histogram":{"field":"@timestamp","interval":"1m"}}},"aggregations":{"avg(response)":{"avg":{"field":"responsetime"}}}}},"id":"fq_date_histogram_1m_1441","state":{"task_state":"stopped","indexer_state":"stopped","current_position":{"date_histogram(@timestamp)":1549929540000},"checkpoint":1},"stats":{"pages_processed":0,"documents_processed":0,"documents_indexed":0,"trigger_count":0,"index_time_in_ms":0,"index_total":0,"index_failures":0,"search_time_in_ms":0,"search_total":0,"search_failures":0}}
+{
+  "config": {
+    "id": "fq_date_histogram_1m_1441",
+    "source": { "index": ["farequote-2019"], "query": { "match_all": {} } },
+    "dest": { "index": "fq_data_histogram_1m_1441" },
+    "pivot": {
+      "group_by": {
+        "date_histogram(@timestamp)": {
+          "date_histogram": { "field": "@timestamp", "interval": "1m" }
+        }
+      },
+      "aggregations": { "avg(response)": { "avg": { "field": "responsetime" } } }
+    }
+  },
+  "id": "fq_date_histogram_1m_1441",
+  "state": {
+    "task_state": "stopped",
+    "indexer_state": "stopped",
+    "current_position": { "date_histogram(@timestamp)": 1549929540000 },
+    "checkpoint": 1
+  },
+  "stats": {
+    "pages_processed": 0,
+    "documents_processed": 0,
+    "documents_indexed": 0,
+    "trigger_count": 0,
+    "index_time_in_ms": 0,
+    "index_total": 0,
+    "index_failures": 0,
+    "search_time_in_ms": 0,
+    "search_total": 0,
+    "search_failures": 0
+  },
+  "checkpointing": {
+    "current": {
+      "timestamp": "2019-06-28T16:09:23.539Z",
+      "timestamp_millis": 1561738163539,
+      "time_upper_bound": "2019-06-28T16:09:13.539Z",
+      "time_upper_bound_millis": 1561738153539
+    },
+    "operations_behind": 0
+  }
+}

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.test.tsx

@@ -10,13 +10,14 @@ describe('Data Frame: Job List Columns', () => {
   test('getColumns()', () => {
     const columns = getColumns(() => {}, [], () => {});
 
-    expect(columns).toHaveLength(7);
+    expect(columns).toHaveLength(8);
     expect(columns[0].isExpander).toBeTruthy();
     expect(columns[1].name).toBe('ID');
     expect(columns[2].name).toBe('Source index');
     expect(columns[3].name).toBe('Destination index');
     expect(columns[4].name).toBe('Status');
-    expect(columns[5].name).toBe('Progress');
-    expect(columns[6].name).toBe('Actions');
+    expect(columns[5].name).toBe('Mode');
+    expect(columns[6].name).toBe('Progress');
+    expect(columns[7].name).toBe('Actions');
   });
 });

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/columns.tsx

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React from 'react';
+import React, { Fragment } from 'react';
 import { i18n } from '@kbn/i18n';
 import {
   EuiBadge,
@@ -91,9 +91,23 @@ export const getColumns = (
         const color = item.state.task_state === 'started' ? 'primary' : 'hollow';
         return <EuiBadge color={color}>{item.state.task_state}</EuiBadge>;
       },
+      width: '100px',
     },
     {
-      name: i18n.translate('xpack.ml.dataframe.progress', { defaultMessage: 'Progress' }),
+      name: i18n.translate('xpack.ml.dataframe.mode', { defaultMessage: 'Mode' }),
+      sortable: true,
+      truncateText: true,
+      render(item: DataFrameJobListRow) {
+        const mode = typeof item.config.sync !== 'undefined' ? 'continuous' : 'batch';
+        const color = 'hollow';
+        return <EuiBadge color={color}>{mode}</EuiBadge>;
+      },
+      width: '100px',
+    },
+    {
+      name: i18n.translate('xpack.ml.dataframe.progressIconTipContent', {
+        defaultMessage: 'Progress',
+      }),
       sortable: true,
       truncateText: true,
       render(item: DataFrameJobListRow) {
@@ -103,23 +117,44 @@ export const getColumns = (
           progress = Math.round(item.state.progress.percent_complete);
         }
 
+        const isBatchTransform = typeof item.config.sync === 'undefined';
+
         return (
           <EuiFlexGroup alignItems="center" gutterSize="xs">
-            <EuiFlexItem>
-              <EuiProgress value={progress} max={100} color="primary" size="m">
-                {progress}%
-              </EuiProgress>
-            </EuiFlexItem>
-            <EuiFlexItem>
-              <EuiText size="xs">{`${progress}%`}</EuiText>
-            </EuiFlexItem>
+            {isBatchTransform && (
+              <Fragment>
+                <EuiFlexItem style={{ width: '40px' }} grow={false}>
+                  <EuiProgress value={progress} max={100} color="primary" size="m">
+                    {progress}%
+                  </EuiProgress>
+                </EuiFlexItem>
+                <EuiFlexItem style={{ width: '35px' }} grow={false}>
+                  <EuiText size="xs">{`${progress}%`}</EuiText>
+                </EuiFlexItem>
+              </Fragment>
+            )}
+            {!isBatchTransform && (
+              <Fragment>
+                <EuiFlexItem style={{ width: '40px' }} grow={false}>
+                  {item.state.task_state === 'started' && <EuiProgress color="primary" size="m" />}
+                  {item.state.task_state !== 'started' && (
+                    <EuiProgress value={0} max={100} color="primary" size="m" />
+                  )}
+                </EuiFlexItem>
+                <EuiFlexItem style={{ width: '35px' }} grow={false}>
+                  &nbsp;
+                </EuiFlexItem>
+              </Fragment>
+            )}
           </EuiFlexGroup>
         );
       },
+      width: '100px',
     },
     {
       name: i18n.translate('xpack.ml.dataframe.tableActionLabel', { defaultMessage: 'Actions' }),
       actions,
+      width: '200px',
     },
   ];
 };

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/common.ts

@@ -45,6 +45,7 @@ export interface DataFrameJobStats {
 
 export interface DataFrameJobListRow {
   id: JobId;
+  checkpointing: object;
   state: DataFrameJobState;
   stats: DataFrameJobStats;
   config: DataFrameTransformWithId;

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.tsx

@@ -37,6 +37,14 @@ export const ExpandedRow: SFC<Props> = ({ item, lastUpdate }) => {
     position: 'left',
   };
 
+  const checkpointing: SectionConfig = {
+    title: 'Checkpointing',
+    items: Object.entries(item.checkpointing).map(s => {
+      return { title: s[0].toString(), description: getItemDescription(s[1]) };
+    }),
+    position: 'left',
+  };
+
   const stats: SectionConfig = {
     title: 'Stats',
     items: Object.entries(item.stats).map(s => {
@@ -51,7 +59,7 @@ export const ExpandedRow: SFC<Props> = ({ item, lastUpdate }) => {
       name: i18n.translate('xpack.ml.dataframe.jobsList.jobDetails.tabs.jobSettingsLabel', {
         defaultMessage: 'Job details',
       }),
-      content: <JobDetailsPane sections={[state, stats]} />,
+      content: <JobDetailsPane sections={[state, checkpointing, stats]} />,
     },
     {
       id: 'job-json',

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_details_pane.tsx

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { SFC } from 'react';
+import React, { Fragment, SFC } from 'react';
 
 import {
   EuiDescriptionList,
@@ -52,19 +52,23 @@ export const JobDetailsPane: SFC<JobDetailsPaneProps> = ({ sections }) => {
   return (
     <EuiFlexGroup>
       <EuiFlexItem>
-        <EuiSpacer size="s" />
         {sections
           .filter(s => s.position === 'left')
           .map(s => (
-            <Section section={s} key={s.title} />
+            <Fragment key={s.title}>
+              <EuiSpacer size="s" />
+              <Section section={s} />
+            </Fragment>
           ))}
       </EuiFlexItem>
       <EuiFlexItem>
-        <EuiSpacer size="s" />
         {sections
           .filter(s => s.position === 'right')
           .map(s => (
-            <Section section={s} key={s.title} />
+            <Fragment key={s.title}>
+              <EuiSpacer size="s" />
+              <Section section={s} />
+            </Fragment>
           ))}
       </EuiFlexItem>
     </EuiFlexGroup>

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_service/get_jobs.ts

@@ -12,6 +12,7 @@ import { DataFrameJobListRow, DataFrameJobState, DataFrameJobStats } from '../co
 
 interface DataFrameJobStateStats {
   id: JobId;
+  checkpointing: object;
   state: DataFrameJobState;
   stats: DataFrameJobStats;
 }
@@ -47,7 +48,13 @@ export const getJobsFactory = (
             return reducedtableRows;
           }
           // Table with expandable rows requires `id` on the outer most level
-          reducedtableRows.push({ config, id: config.id, state: stats.state, stats: stats.stats });
+          reducedtableRows.push({
+            config,
+            id: config.id,
+            checkpointing: stats.checkpointing,
+            state: stats.state,
+            stats: stats.stats,
+          });
           return reducedtableRows;
         },
         [] as DataFrameJobListRow[]