[Logs UI] Add ML module setup hook

x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts

@@ -6,5 +6,13 @@
 
 import { JobType } from './log_analysis';
 
+export const bucketSpan = 900000;
+
+export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
+  `kibana-logs-ui-${spaceId}-${sourceId}-`;
+
 export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) =>
-  `kibana-logs-ui-${spaceId}-${sourceId}-${jobType}`;
+  `${getJobIdPrefix(spaceId, sourceId)}${jobType}`;
+
+export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) =>
+  `datafeed-${getJobId(spaceId, sourceId, jobType)}`;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+import { getJobId } from '../../../../../common/log_analysis';
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+
+export const callJobsSummaryAPI = async (spaceId: string, sourceId: string) => {
+  const response = await kfetch({
+    method: 'POST',
+    pathname: '/api/ml/jobs/jobs_summary',
+    body: JSON.stringify(
+      fetchJobStatusRequestPayloadRT.encode({
+        jobIds: [getJobId(spaceId, sourceId, 'log-entry-rate')],
+      })
+    ),
+  });
+  return fetchJobStatusResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError));
+};
+
+export const fetchJobStatusRequestPayloadRT = rt.type({
+  jobIds: rt.array(rt.string),
+});
+
+export type FetchJobStatusRequestPayload = rt.TypeOf<typeof fetchJobStatusRequestPayloadRT>;
+
+// TODO: Get this to align with the payload - something is tripping it up somewhere
+// export const fetchJobStatusResponsePayloadRT = rt.array(rt.type({
+//   datafeedId: rt.string,
+//   datafeedIndices: rt.array(rt.string),
+//   datafeedState: rt.string,
+//   description: rt.string,
+//   earliestTimestampMs: rt.number,
+//   groups: rt.array(rt.string),
+//   hasDatafeed: rt.boolean,
+//   id: rt.string,
+//   isSingleMetricViewerJob: rt.boolean,
+//   jobState: rt.string,
+//   latestResultsTimestampMs: rt.number,
+//   latestTimestampMs: rt.number,
+//   memory_status: rt.string,
+//   nodeName: rt.union([rt.string, rt.undefined]),
+//   processed_record_count: rt.number,
+//   fullJob: rt.any,
+//   auditMessage: rt.any,
+//   deleting: rt.union([rt.boolean, rt.undefined]),
+// }));
+
+export const fetchJobStatusResponsePayloadRT = rt.any;
+
+export type FetchJobStatusResponsePayload = rt.TypeOf<typeof fetchJobStatusResponsePayloadRT>;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { getJobIdPrefix } from '../../../../../common/log_analysis';
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+
+export const callSetupMlModuleAPI = async (
+  spaceId: string,
+  sourceId: string,
+  indexPattern: string,
+  timeField: string,
+  bucketSpan: number
+) => {
+  const response = await kfetch({
+    method: 'POST',
+    pathname: '/api/ml/modules/setup',
+    body: JSON.stringify(
+      setupMlModuleRequestPayloadRT.encode({
+        indexPatternName: indexPattern,
+        prefix: getJobIdPrefix(spaceId, sourceId),
+        startDatafeed: true,
+        jobOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            analysis_config: {
+              bucket_span: `${bucketSpan}ms`,
+            },
+            data_description: {
+              time_field: timeField,
+            },
+          },
+        ],
+        datafeedOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            aggregations: {
+              buckets: {
+                date_histogram: {
+                  field: timeField,
+                  fixed_interval: `{bucketSpan}ms`,
+                },
+                aggregations: {
+                  [timeField]: {
+                    max: {
+                      field: [timeField],
+                    },
+                  },
+                  doc_count_per_minute: {
+                    bucket_script: {
+                      script: {
+                        params: {
+                          bucket_span_in_ms: bucketSpan,
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        ],
+      })
+    ),
+  });
+
+  return setupMlModuleResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError));
+};
+
+const setupMlModuleRequestPayloadRT = rt.type({
+  indexPatternName: rt.string,
+  prefix: rt.string,
+  startDatafeed: rt.boolean,
+  jobOverrides: rt.array(rt.object),
+  datafeedOverrides: rt.array(rt.object),
+});
+
+const setupMlModuleResponsePayloadRT = rt.type({
+  datafeeds: rt.array(
+    rt.type({
+      id: rt.string,
+      started: rt.boolean,
+      success: rt.boolean,
+    })
+  ),
+  jobs: rt.array(
+    rt.type({
+      id: rt.string,
+      success: rt.boolean,
+    })
+  ),
+});

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/index.ts

@@ -4,4 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export * from './log_analysis_capabilities';
+export * from './log_analysis_jobs';
 export * from './log_analysis_results';

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_capabilities.tsx

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useMemo, useState, useEffect } from 'react';
+import { kfetch } from 'ui/kfetch';
+
+import { useTrackedPromise } from '../../../utils/use_tracked_promise';
+import {
+  getMlCapabilitiesResponsePayloadRT,
+  GetMlCapabilitiesResponsePayload,
+} from './ml_api_types';
+import { throwErrors, createPlainError } from '../../../../common/runtime_types';
+
+export const useLogAnalysisCapabilities = () => {
+  const [mlCapabilities, setMlCapabilities] = useState<GetMlCapabilitiesResponsePayload>(
+    initialMlCapabilities
+  );
+
+  const [fetchMlCapabilitiesRequest, fetchMlCapabilities] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async () => {
+        const rawResponse = await kfetch({
+          method: 'GET',
+          pathname: '/api/ml/ml_capabilities',
+        });
+
+        return getMlCapabilitiesResponsePayloadRT
+          .decode(rawResponse)
+          .getOrElseL(throwErrors(createPlainError));
+      },
+      onResolve: response => {
+        setMlCapabilities(response);
+      },
+    },
+    []
+  );
+
+  useEffect(() => {
+    fetchMlCapabilities();
+  }, []);
+
+  const isLoading = useMemo(() => fetchMlCapabilitiesRequest.state === 'pending', [
+    fetchMlCapabilitiesRequest.state,
+  ]);
+
+  return {
+    hasLogAnalysisCapabilites: mlCapabilities.capabilities.canCreateJob,
+    isLoading,
+  };
+};
+
+const initialMlCapabilities = {
+  capabilities: {
+    canGetJobs: false,
+    canCreateJob: false,
+    canDeleteJob: false,
+    canOpenJob: false,
+    canCloseJob: false,
+    canForecastJob: false,
+    canGetDatafeeds: false,
+    canStartStopDatafeed: false,
+    canUpdateJob: false,
+    canUpdateDatafeed: false,
+    canPreviewDatafeed: false,
+    canGetCalendars: false,
+    canCreateCalendar: false,
+    canDeleteCalendar: false,
+    canGetFilters: false,
+    canCreateFilter: false,
+    canDeleteFilter: false,
+    canFindFileStructure: false,
+    canGetDataFrameJobs: false,
+    canDeleteDataFrameJob: false,
+    canPreviewDataFrameJob: false,
+    canCreateDataFrameJob: false,
+    canStartStopDataFrameJob: false,
+  },
+  isPlatinumOrTrialLicense: false,
+  mlFeatureEnabledInSpace: false,
+  upgradeInProgress: false,
+};

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import createContainer from 'constate-latest';
+import { useMemo, useEffect, useState } from 'react';
+import { values } from 'lodash';
+import { bucketSpan, getJobId } from '../../../../common/log_analysis';
+import { useTrackedPromise } from '../../../utils/use_tracked_promise';
+import { callSetupMlModuleAPI } from './api/ml_setup_module_api';
+import { callJobsSummaryAPI } from './api/ml_get_jobs_summary_api';
+
+// combines and abstracts job and datafeed status
+type JobStatus = 'unknown' | 'missing' | 'inconsistent' | 'created' | 'started';
+
+// type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted';
+// type DatafeedStatus = 'unknown' | 'started' | 'starting' | 'stopped' | 'stopping' | 'deleted';
+
+export const useLogAnalysisJobs = ({
+  indexPattern,
+  sourceId,
+  spaceId,
+  timeField,
+}: {
+  indexPattern: string;
+  sourceId: string;
+  spaceId: string;
+  timeField: string;
+}) => {
+  const [jobStatus, setJobStatus] = useState<{
+    logEntryRate: JobStatus;
+  }>({
+    logEntryRate: 'unknown',
+  });
+
+  const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async () => {
+        return await callSetupMlModuleAPI(spaceId, sourceId, indexPattern, timeField, bucketSpan);
+      },
+      onResolve: ({ datafeeds, jobs }) => {
+        const hasSuccessfullyCreatedJobs = jobs.every(job => job.success);
+        const hasSuccessfullyStartedDatafeeds = datafeeds.every(
+          datafeed => datafeed.success && datafeed.started
+        );
+
+        setJobStatus(currentJobStatus => ({
+          ...currentJobStatus,
+          logEntryRate: hasSuccessfullyCreatedJobs
+            ? hasSuccessfullyStartedDatafeeds
+              ? 'started'
+              : 'created'
+            : 'inconsistent',
+        }));
+      },
+    },
+    [indexPattern, spaceId, sourceId]
+  );
+
+  const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async () => {
+        return callJobsSummaryAPI(spaceId, sourceId);
+      },
+      onResolve: response => {
+        if (response && response.length) {
+          const logEntryRate = response.find(
+            (job: any) => job.id === getJobId(spaceId, sourceId, 'log-entry-rate')
+          );
+          setJobStatus({
+            logEntryRate: logEntryRate ? logEntryRate.jobState : 'unknown',
+          });
+        }
+      },
+      onReject: error => {
+        // TODO: Handle errors
+      },
+    },
+    [indexPattern, spaceId, sourceId]
+  );
+
+  useEffect(() => {
+    fetchJobStatus();
+  }, []);
+
+  const isSetupRequired = useMemo(() => {
+    const jobStates = values(jobStatus);
+    return (
+      jobStates.filter(state => state === 'opened' || state === 'opening').length < jobStates.length
+    );
+  }, [jobStatus]);
+
+  const isLoadingSetupStatus = useMemo(() => fetchJobStatusRequest.state === 'pending', [
+    fetchJobStatusRequest.state,
+  ]);
+
+  return {
+    jobStatus,
+    isSetupRequired,
+    isLoadingSetupStatus,
+  };
+};
+
+export const LogAnalysisJobs = createContainer(useLogAnalysisJobs);

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_results.tsx

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import createContainer from 'constate-latest/dist/ts/src';
+import createContainer from 'constate-latest';
 import { useMemo } from 'react';
 
 import { useLogEntryRate } from './log_entry_rate';