[ML] Adding space aware jobs

x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts

@@ -73,8 +73,14 @@ export function registerTransactionDurationAnomalyAlertType({
       }
       const alertParams = params;
       const request = {} as KibanaRequest;
-      const { mlAnomalySearch } = ml.mlSystemProvider(request);
-      const anomalyDetectors = ml.anomalyDetectorsProvider(request);
+      const { mlAnomalySearch } = ml.mlSystemProvider(
+        request,
+        services.savedObjectsClient
+      );
+      const anomalyDetectors = ml.anomalyDetectorsProvider(
+        request,
+        services.savedObjectsClient
+      );
 
       const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment);
 
@@ -94,6 +100,7 @@ export function registerTransactionDurationAnomalyAlertType({
         return {};
       }
 
+      const jobIds = mlJobs.map((job) => job.job_id);
       const anomalySearchParams = {
         terminateAfter: 1,
         body: {
@@ -102,7 +109,7 @@ export function registerTransactionDurationAnomalyAlertType({
             bool: {
               filter: [
                 { term: { result_type: 'record' } },
-                { terms: { job_id: mlJobs.map((job) => job.job_id) } },
+                { terms: { job_id: jobIds } },
                 {
                   range: {
                     timestamp: {
@@ -163,7 +170,8 @@ export function registerTransactionDurationAnomalyAlertType({
       };
 
       const response = ((await mlAnomalySearch(
-        anomalySearchParams
+        anomalySearchParams,
+        jobIds
       )) as unknown) as {
         hits: { total: { value: number } };
         aggregations?: {

x-pack/plugins/apm/server/lib/helpers/setup_request.ts

@@ -123,8 +123,8 @@ function getMlSetup(
   request: KibanaRequest
 ) {
   return {
-    mlSystem: ml.mlSystemProvider(request),
-    anomalyDetectors: ml.anomalyDetectorsProvider(request),
+    mlSystem: ml.mlSystemProvider(request, savedObjectsClient),
+    anomalyDetectors: ml.anomalyDetectorsProvider(request, savedObjectsClient),
     modules: ml.modulesProvider(request, savedObjectsClient),
   };
 }

x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts

@@ -104,7 +104,7 @@ export async function getServiceAnomalies({
     },
   };
 
-  const response = await ml.mlSystem.mlAnomalySearch(params);
+  const response = await ml.mlSystem.mlAnomalySearch(params, mlJobIds);
 
   return {
     mlJobIds,

x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts

@@ -81,7 +81,7 @@ export async function anomalySeriesFetcher({
     const response: ESSearchResponse<
       unknown,
       typeof params
-    > = (await ml.mlSystem.mlAnomalySearch(params)) as any;
+    > = (await ml.mlSystem.mlAnomalySearch(params, [jobId])) as any;
 
     return response;
   } catch (err) {

x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts

@@ -49,7 +49,7 @@ export async function getMlBucketSize({
   };
 
   try {
-    const resp = await ml.mlSystem.mlAnomalySearch<ESResponse>(params);
+    const resp = await ml.mlSystem.mlAnomalySearch<ESResponse>(params, [jobId]);
     return resp.hits.hits[0]?._source.bucket_span;
   } catch (err) {
     const isHttpError = 'statusCode' in err;

x-pack/plugins/infra/server/lib/infra_ml/common.ts

@@ -62,7 +62,8 @@ export async function getLogEntryDatasets(
           endTime,
           COMPOSITE_AGGREGATION_BATCH_SIZE,
           afterLatestBatchKey
-        )
+        ),
+        jobIds
       )
     );
 

x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts

@@ -177,7 +177,8 @@ async function fetchMetricsHostsAnomalies(
 
   const results = decodeOrThrow(metricsHostsAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination)
+      createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination),
+      jobIds
     )
   );
 

x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts

@@ -173,7 +173,8 @@ async function fetchMetricK8sAnomalies(
 
   const results = decodeOrThrow(metricsK8sAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createMetricsK8sAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination)
+      createMetricsK8sAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination),
+      jobIds
     )
   );
 

x-pack/plugins/infra/server/lib/log_analysis/common.ts

@@ -62,7 +62,8 @@ export async function getLogEntryDatasets(
           endTime,
           COMPOSITE_AGGREGATION_BATCH_SIZE,
           afterLatestBatchKey
-        )
+        ),
+        jobIds
       )
     );
 

x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts

@@ -223,7 +223,8 @@ async function fetchLogEntryAnomalies(
 
   const results = decodeOrThrow(logEntryAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createLogEntryAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination, datasets)
+      createLogEntryAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination, datasets),
+      jobIds
     )
   );
 

x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts

@@ -226,7 +226,8 @@ async function fetchTopLogEntryCategories(
         endTime,
         categoryCount,
         datasets
-      )
+      ),
+      [logEntryCategoriesCountJobId]
     )
   );
 
@@ -284,7 +285,8 @@ export async function fetchLogEntryCategories(
 
   const logEntryCategoriesResponse = decodeOrThrow(logEntryCategoriesResponseRT)(
     await context.infra.mlSystem.mlAnomalySearch(
-      createLogEntryCategoriesQuery(logEntryCategoriesCountJobId, categoryIds)
+      createLogEntryCategoriesQuery(logEntryCategoriesCountJobId, categoryIds),
+      [logEntryCategoriesCountJobId]
     )
   );
 
@@ -333,7 +335,8 @@ async function fetchTopLogEntryCategoryHistograms(
             startTime,
             endTime,
             bucketCount
-          )
+          ),
+          [logEntryCategoriesCountJobId]
         )
         .then(decodeOrThrow(logEntryCategoryHistogramsResponseRT))
         .then((response) => ({

x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts

@@ -40,7 +40,8 @@ export async function getLatestLogEntriesCategoriesDatasetsStats(
         endTime,
         COMPOSITE_AGGREGATION_BATCH_SIZE,
         afterLatestBatchKey
-      )
+      ),
+      jobIds
     );
 
     const { after_key: afterKey, buckets: latestBatchBuckets = [] } =

x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts

@@ -43,7 +43,8 @@ export async function getLogEntryRateBuckets(
         COMPOSITE_AGGREGATION_BATCH_SIZE,
         afterLatestBatchKey,
         datasets
-      )
+      ),
+      [logRateJobId]
     );
 
     const { after_key: afterKey, buckets: latestBatchBuckets = [] } =

x-pack/plugins/infra/server/plugin.ts

@@ -149,8 +149,11 @@ export class InfraServerPlugin {
     core.http.registerRouteHandlerContext(
       'infra',
       (context, request): InfraRequestHandlerContext => {
-        const mlSystem = plugins.ml?.mlSystemProvider(request);
-        const mlAnomalyDetectors = plugins.ml?.anomalyDetectorsProvider(request);
+        const mlSystem = plugins.ml?.mlSystemProvider(request, context.core.savedObjects.client);
+        const mlAnomalyDetectors = plugins.ml?.anomalyDetectorsProvider(
+          request,
+          context.core.savedObjects.client
+        );
         const spaceId = plugins.spaces?.spacesService.getSpaceId(request) || 'default';
 
         return {

x-pack/plugins/ml/common/types/calendars.ts

@@ -11,6 +11,7 @@ export interface Calendar {
   description: string;
   events: any[];
   job_ids: string[];
+  total_job_count?: number;
 }
 
 export interface UpdateCalendar extends Calendar {

x-pack/plugins/ml/common/types/capabilities.ts

@@ -78,7 +78,7 @@ export function getPluginPrivileges() {
   const adminMlCapabilitiesKeys = Object.keys(adminMlCapabilities);
   const allMlCapabilitiesKeys = [...adminMlCapabilitiesKeys, ...userMlCapabilitiesKeys];
   // TODO: include ML in base privileges for the `8.0` release: https://github.com/elastic/kibana/issues/71422
-  const savedObjects = ['index-pattern', 'dashboard', 'search', 'visualization'];
+  const savedObjects = ['index-pattern', 'dashboard', 'search', 'visualization', 'ml-job'];
   const privilege = {
     app: [PLUGIN_ID, 'kibana'],
     excludeFromBasePrivileges: true,
@@ -116,7 +116,7 @@ export function getPluginPrivileges() {
       catalogue: [],
       savedObject: {
         all: [],
-        read: [],
+        read: ['ml-job'],
       },
       api: apmUserMlCapabilitiesKeys.map((k) => `ml:${k}`),
       ui: apmUserMlCapabilitiesKeys,

x-pack/plugins/ml/common/types/saved_objects.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export type JobType = 'anomaly-detector' | 'data-frame-analytics';

x-pack/plugins/ml/public/application/components/job_spaces_list/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { JobSpacesList } from './job_spaces_list';

x-pack/plugins/ml/public/application/components/job_spaces_list/job_spaces_list.tsx

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC } from 'react';
+
+import { EuiFlexGroup, EuiFlexItem, EuiBadge } from '@elastic/eui';
+
+interface Props {
+  spaces: string[];
+}
+
+export const JobSpacesList: FC<Props> = ({ spaces }) => (
+  <EuiFlexGroup wrap responsive={false} gutterSize="xs">
+    {spaces.map((space) => (
+      <EuiFlexItem grow={false} key={space}>
+        <EuiBadge color={'hollow'}>{space}</EuiBadge>
+      </EuiFlexItem>
+    ))}
+  </EuiFlexGroup>
+);

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx

@@ -116,7 +116,8 @@ export const DataFrameAnalyticsList: FC<Props> = ({
     setAnalyticsStats,
     setErrorMessage,
     setIsInitialized,
-    blockRefresh
+    blockRefresh,
+    isManagementTable
   );
 
   const updateFilteredItems = (queryClauses: any) => {

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts

@@ -112,6 +112,7 @@ export interface DataFrameAnalyticsListRow {
   mode: string;
   state: DataFrameAnalyticsStats['state'];
   stats: DataFrameAnalyticsStats;
+  spaces?: string[];
 }
 
 // Used to pass on attribute names to table columns

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/use_columns.tsx

@@ -32,6 +32,7 @@ import {
 import { useActions } from './use_actions';
 import { useMlLink } from '../../../../../contexts/kibana';
 import { ML_PAGES } from '../../../../../../../common/constants/ml_url_generator';
+import { JobSpacesList } from '../../../../../components/job_spaces_list';
 
 enum TASK_STATE_COLOR {
   analyzing = 'primary',
@@ -278,7 +279,8 @@ export const useColumns = (
       name: i18n.translate('xpack.ml.jobsList.analyticsSpacesLabel', {
         defaultMessage: 'Spaces',
       }),
-      render: () => <EuiBadge color={'hollow'}>{'all'}</EuiBadge>,
+      render: (item: DataFrameAnalyticsListRow) =>
+        Array.isArray(item.spaces) ? <JobSpacesList spaces={item.spaces} /> : null,
       width: '75px',
     });
 

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/get_analytics.ts

@@ -106,7 +106,8 @@ export const getAnalyticsFactory = (
     React.SetStateAction<GetDataFrameAnalyticsStatsResponseError | undefined>
   >,
   setIsInitialized: React.Dispatch<React.SetStateAction<boolean>>,
-  blockRefresh: boolean
+  blockRefresh: boolean,
+  isManagementTable: boolean
 ): GetAnalytics => {
   let concurrentLoads = 0;
 
@@ -123,6 +124,12 @@ export const getAnalyticsFactory = (
         const analyticsConfigs = await ml.dataFrameAnalytics.getDataFrameAnalytics();
         const analyticsStats = await ml.dataFrameAnalytics.getDataFrameAnalyticsStats();
 
+        let spaces: { [id: string]: string[] } = {};
+        if (isManagementTable) {
+          const allSpaces = await ml.savedObjects.jobsSpaces();
+          spaces = allSpaces['data-frame-analytics'];
+        }
+
         const analyticsStatsResult = isGetDataFrameAnalyticsStatsResponseOk(analyticsStats)
           ? getAnalyticsJobsStats(analyticsStats)
           : undefined;
@@ -148,6 +155,7 @@ export const getAnalyticsFactory = (
               mode: DATA_FRAME_MODE.BATCH,
               state: stats.state,
               stats,
+              spaces: spaces[config.id] ?? [],
             });
             return reducedtableRows;
           },

x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.js

@@ -296,9 +296,12 @@ export function getTestUrl(job, customUrl) {
 
   return new Promise((resolve, reject) => {
     ml.results
-      .anomalySearch({
-        body,
-      })
+      .anomalySearch(
+        {
+          body,
+        },
+        [job.job_id]
+      )
       .then((resp) => {
         if (resp.hits.total.value > 0) {
           const record = resp.hits.hits[0]._source;

x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list/jobs_list.js

@@ -14,9 +14,10 @@ import { toLocaleString } from '../../../../util/string_utils';
 import { ResultLinks, actionsMenuContent } from '../job_actions';
 import { JobDescription } from './job_description';
 import { JobIcon } from '../../../../components/job_message_icon';
+import { JobSpacesList } from '../../../../components/job_spaces_list';
 import { TIME_FORMAT } from '../../../../../../common/constants/time_format';
 
-import { EuiBadge, EuiBasicTable, EuiButtonIcon, EuiScreenReaderOnly } from '@elastic/eui';
+import { EuiBasicTable, EuiButtonIcon, EuiScreenReaderOnly } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { AnomalyDetectionJobIdLink } from './job_id_link';
@@ -251,7 +252,7 @@ export class JobsList extends Component {
         name: i18n.translate('xpack.ml.jobsList.spacesLabel', {
           defaultMessage: 'Spaces',
         }),
-        render: () => <EuiBadge color={'hollow'}>{'all'}</EuiBadge>,
+        render: (item) => <JobSpacesList spaces={item.spaces} />,
       });
       // Remove actions if Ml not enabled in current space
       if (this.props.isMlEnabledInSpace === false) {