Merge remote-tracking branch 'upstream/main' into migrate-add-from-lib

elastic · Mar 16, 2022 · 85ad863 · 85ad863
2 parents 587aa41 + c331cae
commit 85ad863
Show file tree

Hide file tree

Showing 42 changed files with 1,270 additions and 483 deletions.
diff --git a/packages/kbn-securitysolution-utils/src/path_validations/index.ts b/packages/kbn-securitysolution-utils/src/path_validations/index.ts
@@ -28,7 +28,9 @@ export const enum OperatingSystem {
   WINDOWS = 'windows',
 }
 
-export type TrustedAppEntryTypes = 'match' | 'wildcard';
+export type EntryTypes = 'match' | 'wildcard' | 'match_any';
+export type TrustedAppEntryTypes = Extract<EntryTypes, 'match' | 'wildcard'>;
+
 /*
  * regex to match executable names
  * starts matching from the eol of the path
@@ -82,7 +84,7 @@ export const hasSimpleExecutableName = ({
   value,
 }: {
   os: OperatingSystem;
-  type: TrustedAppEntryTypes;
+  type: EntryTypes;
   value: string;
 }): boolean => {
   if (type === 'wildcard') {
@@ -99,7 +101,7 @@ export const isPathValid = ({
 }: {
   os: OperatingSystem;
   field: ConditionEntryField | 'file.path.text';
-  type: TrustedAppEntryTypes;
+  type: EntryTypes;
   value: string;
 }): boolean => {
   if (field === ConditionEntryField.PATH || field === 'file.path.text') {

diff --git a/x-pack/plugins/ml/common/types/trained_models.ts b/x-pack/plugins/ml/common/types/trained_models.ts
@@ -205,6 +205,8 @@ export interface NodeDeploymentStatsResponse {
       total: number;
       jvm: number;
     };
+    /** Max amount of memory available for ML */
+    ml_max_in_bytes: number;
     /** Open anomaly detection jobs + hardcoded overhead */
     anomaly_detection: {
       /** Total size in bytes */
@@ -226,6 +228,6 @@ export interface NodeDeploymentStatsResponse {
 }
 
 export interface NodesOverviewResponse {
-  count: number;
+  _nodes: { total: number; failed: number; successful: number };
   nodes: NodeDeploymentStatsResponse[];
 }
diff --git a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx
@@ -15,8 +15,10 @@ import {
   ScaleType,
   SeriesColorAccessor,
   Settings,
+  LineAnnotation,
+  AnnotationDomainType,
 } from '@elastic/charts';
-import { euiPaletteGray } from '@elastic/eui';
+import { EuiIcon, euiPaletteGray } from '@elastic/eui';
 import { NodeDeploymentStatsResponse } from '../../../../common/types/trained_models';
 import { useFieldFormatter } from '../../contexts/kibana/use_field_formatter';
 import { useCurrentEuiTheme } from '../../components/color_range_legend';
@@ -126,6 +128,22 @@ export const MemoryPreviewChart: FC<MemoryPreviewChartProps> = ({ memoryOverview
         tickFormat={(d: number) => bytesFormatter(d)}
       />
 
+      <LineAnnotation
+        id="line_annotation"
+        domainType={AnnotationDomainType.YDomain}
+        dataValues={[
+          {
+            dataValue: memoryOverview.ml_max_in_bytes,
+            details: bytesFormatter(memoryOverview.ml_max_in_bytes),
+            header: i18n.translate('xpack.ml.trainedModels.nodesList.mlMaxMemory', {
+              defaultMessage: 'Maximum memory permitted for ML native processes',
+            }),
+          },
+        ]}
+        marker={<EuiIcon type="arrowDown" />}
+        markerPosition={Position.Top}
+      />
+
       <BarSeries
         id="bars"
         xScaleType={ScaleType.Linear}

diff --git a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx
@@ -123,6 +123,7 @@ export const NodesList: FC<NodesListProps> = ({ compactView = false }) => {
       name: i18n.translate('xpack.ml.trainedModels.nodesList.nodeNameHeader', {
         defaultMessage: 'Name',
       }),
+      width: '200px',
       sortable: true,
       truncateText: true,
       'data-test-subj': 'mlNodesTableColumnName',

diff --git a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts
@@ -636,6 +636,9 @@ export function getMlClient(
     async validateDetector(...p: Parameters<MlClient['validateDetector']>) {
       return mlClient.validateDetector(...p);
     },
+    async getMemoryStats(...p: Parameters<MlClient['getMemoryStats']>) {
+      return mlClient.getMemoryStats(...p);
+    },
 
     ...searchProvider(client, jobSavedObjectService),
   } as MlClient;

diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.test.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.test.ts
@@ -5,129 +5,129 @@
  * 2.0.
  */
 
-import { ModelService, modelsProvider } from './models_provider';
+import { MemoryStatsResponse, ModelService, modelsProvider } from './models_provider';
 import { IScopedClusterClient } from 'kibana/server';
 import { MlClient } from '../../lib/ml_client';
 import mockResponse from './__mocks__/mock_deployment_response.json';
-import { MemoryOverviewService } from '../memory_overview/memory_overview_service';
 
 describe('Model service', () => {
   const client = {
-    asInternalUser: {
-      nodes: {
-        stats: jest.fn(() => {
-          return Promise.resolve({
-            _nodes: {
-              total: 3,
-              successful: 3,
-              failed: 0,
-            },
-            cluster_name: 'test_cluster',
-            nodes: {
-              '3qIoLFnbSi-DwVrYioUCdw': {
-                timestamp: 1635167166946,
-                name: 'node3',
-                transport_address: '10.10.10.2:9353',
-                host: '10.10.10.2',
-                ip: '10.10.10.2:9353',
-                roles: ['data', 'ingest', 'master', 'ml', 'transform'],
-                attributes: {
-                  'ml.machine_memory': '15599742976',
-                  'xpack.installed': 'true',
-                  'ml.max_jvm_size': '1073741824',
-                },
-                os: {
-                  mem: {
-                    total_in_bytes: 15599742976,
-                    adjusted_total_in_bytes: 15599742976,
-                    free_in_bytes: 376324096,
-                    used_in_bytes: 15223418880,
-                    free_percent: 2,
-                    used_percent: 98,
-                  },
-                },
-              },
-              'DpCy7SOBQla3pu0Dq-tnYw': {
-                timestamp: 1635167166946,
-                name: 'node2',
-                transport_address: '10.10.10.2:9352',
-                host: '10.10.10.2',
-                ip: '10.10.10.2:9352',
-                roles: ['data', 'master', 'ml', 'transform'],
-                attributes: {
-                  'ml.machine_memory': '15599742976',
-                  'xpack.installed': 'true',
-                  'ml.max_jvm_size': '1073741824',
-                },
-                os: {
-                  timestamp: 1635167166959,
-                  mem: {
-                    total_in_bytes: 15599742976,
-                    adjusted_total_in_bytes: 15599742976,
-                    free_in_bytes: 376324096,
-                    used_in_bytes: 15223418880,
-                    free_percent: 2,
-                    used_percent: 98,
-                  },
-                },
-              },
-              'pt7s6lKHQJaP4QHKtU-Q0Q': {
-                timestamp: 1635167166945,
-                name: 'node1',
-                transport_address: '10.10.10.2:9351',
-                host: '10.10.10.2',
-                ip: '10.10.10.2:9351',
-                roles: ['data', 'master', 'ml'],
-                attributes: {
-                  'ml.machine_memory': '15599742976',
-                  'xpack.installed': 'true',
-                  'ml.max_jvm_size': '1073741824',
-                },
-                os: {
-                  timestamp: 1635167166959,
-                  mem: {
-                    total_in_bytes: 15599742976,
-                    adjusted_total_in_bytes: 15599742976,
-                    free_in_bytes: 376324096,
-                    used_in_bytes: 15223418880,
-                    free_percent: 2,
-                    used_percent: 98,
-                  },
-                },
-              },
-            },
-          });
-        }),
-      },
-    },
+    asInternalUser: {},
   } as unknown as jest.Mocked<IScopedClusterClient>;
+
   const mlClient = {
     getTrainedModelsStats: jest.fn(() => {
       return Promise.resolve({
         trained_model_stats: mockResponse,
       });
     }),
-  } as unknown as jest.Mocked<MlClient>;
-  const memoryOverviewService = {
-    getDFAMemoryOverview: jest.fn(() => {
-      return Promise.resolve([{ job_id: '', node_id: '', model_size: 32165465 }]);
-    }),
-    getAnomalyDetectionMemoryOverview: jest.fn(() => {
-      return Promise.resolve([{ job_id: '', node_id: '', model_size: 32165465 }]);
+    getMemoryStats: jest.fn(() => {
+      return Promise.resolve({
+        _nodes: {
+          total: 3,
+          successful: 3,
+          failed: 0,
+        },
+        cluster_name: 'test_cluster',
+        nodes: {
+          '3qIoLFnbSi-DwVrYioUCdw': {
+            name: 'node3',
+            transport_address: '10.10.10.2:9353',
+            roles: ['data', 'ingest', 'master', 'ml', 'transform'],
+            attributes: {
+              'ml.machine_memory': '15599742976',
+              'ml.max_jvm_size': '1073741824',
+            },
+            jvm: {
+              heap_max_in_bytes: 1073741824,
+              java_inference_in_bytes: 0,
+              java_inference_max_in_bytes: 0,
+            },
+            mem: {
+              adjusted_total_in_bytes: 15599742976,
+              total_in_bytes: 15599742976,
+              ml: {
+                data_frame_analytics_in_bytes: 0,
+                native_code_overhead_in_bytes: 0,
+                max_in_bytes: 1073741824,
+                anomaly_detectors_in_bytes: 0,
+                native_inference_in_bytes: 1555161790,
+              },
+            },
+            ephemeral_id: '3qIoLFnbSi-DwVrYioUCdw',
+          },
+          'DpCy7SOBQla3pu0Dq-tnYw': {
+            name: 'node2',
+            transport_address: '10.10.10.2:9352',
+            roles: ['data', 'master', 'ml', 'transform'],
+            attributes: {
+              'ml.machine_memory': '15599742976',
+              'ml.max_jvm_size': '1073741824',
+            },
+            jvm: {
+              heap_max_in_bytes: 1073741824,
+              java_inference_in_bytes: 0,
+              java_inference_max_in_bytes: 0,
+            },
+            mem: {
+              adjusted_total_in_bytes: 15599742976,
+              total_in_bytes: 15599742976,
+              ml: {
+                data_frame_analytics_in_bytes: 0,
+                native_code_overhead_in_bytes: 0,
+                max_in_bytes: 1073741824,
+                anomaly_detectors_in_bytes: 0,
+                native_inference_in_bytes: 1555161790,
+              },
+            },
+            ephemeral_id: '3qIoLFnbSi-DwVrYioUCdw',
+          },
+          'pt7s6lKHQJaP4QHKtU-Q0Q': {
+            name: 'node1',
+            transport_address: '10.10.10.2:9351',
+            roles: ['data', 'master', 'ml'],
+            attributes: {
+              'ml.machine_memory': '15599742976',
+              'ml.max_jvm_size': '1073741824',
+            },
+            jvm: {
+              heap_max_in_bytes: 1073741824,
+              java_inference_in_bytes: 0,
+              java_inference_max_in_bytes: 0,
+            },
+            mem: {
+              adjusted_total_in_bytes: 15599742976,
+              total_in_bytes: 15599742976,
+              ml: {
+                data_frame_analytics_in_bytes: 0,
+                native_code_overhead_in_bytes: 0,
+                max_in_bytes: 1073741824,
+                anomaly_detectors_in_bytes: 0,
+                native_inference_in_bytes: 1555161790,
+              },
+            },
+            ephemeral_id: '3qIoLFnbSi-DwVrYioUCdw',
+          },
+        },
+      } as MemoryStatsResponse);
     }),
-  } as unknown as jest.Mocked<MemoryOverviewService>;
+  } as unknown as jest.Mocked<MlClient>;
 
   let service: ModelService;
 
   beforeEach(() => {
-    service = modelsProvider(client, mlClient, memoryOverviewService);
+    service = modelsProvider(client, mlClient);
   });
 
   afterEach(() => {});
 
   it('extract nodes list correctly', async () => {
     expect(await service.getNodesOverview()).toEqual({
-      count: 3,
+      _nodes: {
+        failed: 0,
+        successful: 3,
+        total: 3,
+      },
       nodes: [
         {
           name: 'node3',
@@ -219,6 +219,7 @@ describe('Model service', () => {
           },
           id: '3qIoLFnbSi-DwVrYioUCdw',
           memory_overview: {
+            ml_max_in_bytes: 1073741824,
             anomaly_detection: {
               total: 0,
             },
@@ -339,6 +340,7 @@ describe('Model service', () => {
           },
           id: 'DpCy7SOBQla3pu0Dq-tnYw',
           memory_overview: {
+            ml_max_in_bytes: 1073741824,
             anomaly_detection: {
               total: 0,
             },
@@ -462,6 +464,7 @@ describe('Model service', () => {
           },
           id: 'pt7s6lKHQJaP4QHKtU-Q0Q',
           memory_overview: {
+            ml_max_in_bytes: 1073741824,
             anomaly_detection: {
               total: 0,
             },