[Logs UI] Add ML module setup hook

x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts

@@ -6,8 +6,13 @@
 
 import { JobType } from './log_analysis';
 
+export const bucketSpan = 900000;
+
 export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
   `kibana-logs-ui-${spaceId}-${sourceId}-`;
 
 export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) =>
   `${getJobIdPrefix(spaceId, sourceId)}${jobType}`;
+
+export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) =>
+  `datafeed-${getJobId(spaceId, sourceId, jobType)}`;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { getJobIdPrefix } from '../../../../../common/log_analysis';
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+
+const MODULE_ID = 'logs_ui_analysis';
+
+export const callSetupMlModuleAPI = async (
+  spaceId: string,
+  sourceId: string,
+  indexPattern: string,
+  timeField: string,
+  bucketSpan: number
+) => {
+  const response = await kfetch({
+    method: 'POST',
+    pathname: `/api/ml/modules/setup/${MODULE_ID}`,
+    body: JSON.stringify(
+      setupMlModuleRequestPayloadRT.encode({
+        indexPatternName: indexPattern,
+        prefix: getJobIdPrefix(spaceId, sourceId),
+        startDatafeed: true,
+        jobOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            analysis_config: {
+              bucket_span: `${bucketSpan}ms`,
+            },
+            data_description: {
+              time_field: timeField,
+            },
+          },
+        ],
+        datafeedOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            aggregations: {
+              buckets: {
+                date_histogram: {
+                  field: timeField,
+                  fixed_interval: `{bucketSpan}ms`,
+                },
+                aggregations: {
+                  [timeField]: {
+                    max: {
+                      field: [timeField],
+                    },
+                  },
+                  doc_count_per_minute: {
+                    bucket_script: {
+                      script: {
+                        params: {
+                          bucket_span_in_ms: bucketSpan,
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        ],
+      })
+    ),
+  });
+
+  return setupMlModuleResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError));
+};
+
+const setupMlModuleRequestPayloadRT = rt.type({
+  indexPatternName: rt.string,
+  prefix: rt.string,
+  startDatafeed: rt.boolean,
+  jobOverrides: rt.array(rt.object),
+  datafeedOverrides: rt.array(rt.object),
+});
+
+const setupMlModuleResponsePayloadRT = rt.type({
+  datafeeds: rt.array(
+    rt.type({
+      id: rt.string,
+      started: rt.boolean,
+      success: rt.boolean,
+    })
+  ),
+  jobs: rt.array(
+    rt.type({
+      id: rt.string,
+      success: rt.boolean,
+    })
+  ),
+});

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -7,47 +7,59 @@
 import createContainer from 'constate-latest';
 import { useMemo, useEffect, useState } from 'react';
 import { values } from 'lodash';
-import { getJobId } from '../../../../common/log_analysis';
+import { bucketSpan, getJobId } from '../../../../common/log_analysis';
 import { useTrackedPromise } from '../../../utils/use_tracked_promise';
+import { callSetupMlModuleAPI } from './api/ml_setup_module_api';
 import { callJobsSummaryAPI } from './api/ml_get_jobs_summary_api';
 
-type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted';
+// combines and abstracts job and datafeed status
+type JobStatus = 'unknown' | 'missing' | 'inconsistent' | 'created' | 'started';
+
+// type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted';
+
 // type DatafeedStatus = 'unknown' | 'started' | 'starting' | 'stopped' | 'stopping' | 'deleted';
 
 export const useLogAnalysisJobs = ({
   indexPattern,
   sourceId,
   spaceId,
+  timeField,
 }: {
   indexPattern: string;
   sourceId: string;
   spaceId: string;
+  timeField: string;
 }) => {
   const [jobStatus, setJobStatus] = useState<{
     logEntryRate: JobStatus;
   }>({
     logEntryRate: 'unknown',
   });
 
-  // const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
-  //   {
-  //     cancelPreviousOn: 'resolution',
-  //     createPromise: async () => {
-  //       kfetch({
-  //         method: 'POST',
-  //         pathname: '/api/ml/modules/setup',
-  //         body: JSON.stringify(
-  //           setupMlModuleRequestPayloadRT.encode({
-  //             indexPatternName: indexPattern,
-  //             prefix: getJobIdPrefix(spaceId, sourceId),
-  //             startDatafeed: true,
-  //           })
-  //         ),
-  //       });
-  //     },
-  //   },
-  //   [indexPattern, spaceId, sourceId]
-  // );
+  const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async () => {
+        return await callSetupMlModuleAPI(spaceId, sourceId, indexPattern, timeField, bucketSpan);
+      },
+      onResolve: ({ datafeeds, jobs }) => {
+        const hasSuccessfullyCreatedJobs = jobs.every(job => job.success);
+        const hasSuccessfullyStartedDatafeeds = datafeeds.every(
+          datafeed => datafeed.success && datafeed.started
+        );
+
+        setJobStatus(currentJobStatus => ({
+          ...currentJobStatus,
+          logEntryRate: hasSuccessfullyCreatedJobs
+            ? hasSuccessfullyStartedDatafeeds
+              ? 'started'
+              : 'created'
+            : 'inconsistent',
+        }));
+      },
+    },
+    [indexPattern, spaceId, sourceId]
+  );
 
   const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise(
     {
@@ -91,6 +103,8 @@ export const useLogAnalysisJobs = ({
     jobStatus,
     isSetupRequired,
     isLoadingSetupStatus,
+    setupMlModule,
+    setupMlModuleRequest,
   };
 };
 

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx

@@ -22,6 +22,7 @@ export const AnalysisPage = () => {
     indexPattern: source ? source.configuration.logAlias : '',
     sourceId,
     spaceId,
+    timeField: source ? source.configuration.fields.timestamp : '',
   });
 
   return (