diff --git a/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts index 82cfb0f83ed69..ebdc2251891eb 100644 --- a/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts +++ b/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts @@ -6,8 +6,13 @@ import { JobType } from './log_analysis'; +export const bucketSpan = 900000; + export const getJobIdPrefix = (spaceId: string, sourceId: string) => `kibana-logs-ui-${spaceId}-${sourceId}-`; export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) => `${getJobIdPrefix(spaceId, sourceId)}${jobType}`; + +export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) => + `datafeed-${getJobId(spaceId, sourceId, jobType)}`; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts new file mode 100644 index 0000000000000..d731ad19eca46 --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { kfetch } from 'ui/kfetch'; + +import { getJobIdPrefix } from '../../../../../common/log_analysis'; +import { throwErrors, createPlainError } from '../../../../../common/runtime_types'; + +const MODULE_ID = 'logs_ui_analysis'; + +export const callSetupMlModuleAPI = async ( + spaceId: string, + sourceId: string, + indexPattern: string, + timeField: string, + bucketSpan: number +) => { + const response = await kfetch({ + method: 'POST', + pathname: `/api/ml/modules/setup/${MODULE_ID}`, + body: JSON.stringify( + setupMlModuleRequestPayloadRT.encode({ + indexPatternName: indexPattern, + prefix: getJobIdPrefix(spaceId, sourceId), + startDatafeed: true, + jobOverrides: [ + { + job_id: 'log-entry-rate', + analysis_config: { + bucket_span: `${bucketSpan}ms`, + }, + data_description: { + time_field: timeField, + }, + }, + ], + datafeedOverrides: [ + { + job_id: 'log-entry-rate', + aggregations: { + buckets: { + date_histogram: { + field: timeField, + fixed_interval: `{bucketSpan}ms`, + }, + aggregations: { + [timeField]: { + max: { + field: [timeField], + }, + }, + doc_count_per_minute: { + bucket_script: { + script: { + params: { + bucket_span_in_ms: bucketSpan, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }) + ), + }); + + return setupMlModuleResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError)); +}; + +const setupMlModuleRequestPayloadRT = rt.type({ + indexPatternName: rt.string, + prefix: rt.string, + startDatafeed: rt.boolean, + jobOverrides: rt.array(rt.object), + datafeedOverrides: rt.array(rt.object), +}); + +const setupMlModuleResponsePayloadRT = rt.type({ + datafeeds: rt.array( + rt.type({ + id: rt.string, + started: rt.boolean, + success: rt.boolean, + }) + ), + jobs: rt.array( + rt.type({ + id: rt.string, + success: rt.boolean, + }) + ), +}); diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx index 391a4b190edd2..ab5395dc76234 100644 --- a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx @@ -7,21 +7,28 @@ import createContainer from 'constate-latest'; import { useMemo, useEffect, useState } from 'react'; import { values } from 'lodash'; -import { getJobId } from '../../../../common/log_analysis'; +import { bucketSpan, getJobId } from '../../../../common/log_analysis'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; +import { callSetupMlModuleAPI } from './api/ml_setup_module_api'; import { callJobsSummaryAPI } from './api/ml_get_jobs_summary_api'; -type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted'; +// combines and abstracts job and datafeed status +type JobStatus = 'unknown' | 'missing' | 'inconsistent' | 'created' | 'started'; + +// type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted'; + // type DatafeedStatus = 'unknown' | 'started' | 'starting' | 'stopped' | 'stopping' | 'deleted'; export const useLogAnalysisJobs = ({ indexPattern, sourceId, spaceId, + timeField, }: { indexPattern: string; sourceId: string; spaceId: string; + timeField: string; }) => { const [jobStatus, setJobStatus] = useState<{ logEntryRate: JobStatus; @@ -29,25 +36,30 @@ export const useLogAnalysisJobs = ({ logEntryRate: 'unknown', }); - // const [setupMlModuleRequest, setupMlModule] = useTrackedPromise( - // { - // cancelPreviousOn: 'resolution', - // createPromise: async () => { - // kfetch({ - // method: 'POST', - // pathname: '/api/ml/modules/setup', - // body: JSON.stringify( - // setupMlModuleRequestPayloadRT.encode({ - // indexPatternName: indexPattern, - // prefix: getJobIdPrefix(spaceId, sourceId), - // startDatafeed: true, - // }) - // ), - // }); - // }, - // }, - // [indexPattern, spaceId, sourceId] - // ); + const [setupMlModuleRequest, setupMlModule] = useTrackedPromise( + { + cancelPreviousOn: 'resolution', + createPromise: async () => { + return await callSetupMlModuleAPI(spaceId, sourceId, indexPattern, timeField, bucketSpan); + }, + onResolve: ({ datafeeds, jobs }) => { + const hasSuccessfullyCreatedJobs = jobs.every(job => job.success); + const hasSuccessfullyStartedDatafeeds = datafeeds.every( + datafeed => datafeed.success && datafeed.started + ); + + setJobStatus(currentJobStatus => ({ + ...currentJobStatus, + logEntryRate: hasSuccessfullyCreatedJobs + ? hasSuccessfullyStartedDatafeeds + ? 'started' + : 'created' + : 'inconsistent', + })); + }, + }, + [indexPattern, spaceId, sourceId] + ); const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise( { @@ -91,6 +103,8 @@ export const useLogAnalysisJobs = ({ jobStatus, isSetupRequired, isLoadingSetupStatus, + setupMlModule, + setupMlModuleRequest, }; }; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx index ef4a1ffe6735c..5c97046fa9663 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx @@ -22,6 +22,7 @@ export const AnalysisPage = () => { indexPattern: source ? source.configuration.logAlias : '', sourceId, spaceId, + timeField: source ? source.configuration.fields.timestamp : '', }); return (