From 37be03c4270b2d2b4560c22ac1032da908140e81 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 30 Nov 2023 10:56:23 +0100 Subject: [PATCH 1/4] skip flaky tests (#172260) ## Summary Skipping several flaky tests // FLAKY: https://github.com/elastic/kibana/issues/172251 // FLAKY: https://github.com/elastic/kibana/issues/172253 // FLAKY: https://github.com/elastic/kibana/issues/172254 // FLAKY: https://github.com/elastic/kibana/issues/172255 --- .../e2e/detection_response/rule_creation/esql_rule_ess.cy.ts | 3 ++- .../cypress/e2e/detection_response/rule_edit/esql_rule.cy.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/esql_rule_ess.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/esql_rule_ess.cy.ts index 0e10557bcaf0e..f3a898e430a4e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/esql_rule_ess.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/esql_rule_ess.cy.ts @@ -38,7 +38,8 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { const rule = getEsqlRule(); const expectedNumberOfRules = 1; - describe('creation', () => { + // FLAKY: https://github.com/elastic/kibana/issues/172251 + describe.skip('creation', () => { beforeEach(() => { deleteAlertsAndRules(); login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/esql_rule.cy.ts index 20d48b211995e..445b33a682f9c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/esql_rule.cy.ts @@ -34,7 +34,10 @@ const rule = getEsqlRule(); const expectedValidEsqlQuery = 'from auditbeat* | stats count(event.category) by event.category'; -describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/172253 +// FLAKY: https://github.com/elastic/kibana/issues/172254 +// FLAKY: https://github.com/elastic/kibana/issues/172255 +describe.skip('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => { beforeEach(() => { login(); deleteAlertsAndRules(); From 71eca5b5ec4aa908866fd947fa7f468536876b5e Mon Sep 17 00:00:00 2001 From: Konrad Szwarc Date: Thu, 30 Nov 2023 11:44:53 +0100 Subject: [PATCH 2/4] [EDR Workflows] Disabled Timeout input for saved queries preloaded queries (#171931) https://github.com/elastic/kibana/issues/171906 Render timeout field on saved queries view, for preloaded queries as disabled. ![Screenshot 2023-11-24 at 12 06 17](https://github.com/elastic/kibana/assets/29123534/fe712b0b-ad39-46fe-a4c3-2e2c867ca9bc) --- x-pack/plugins/osquery/public/saved_queries/form/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/osquery/public/saved_queries/form/index.tsx b/x-pack/plugins/osquery/public/saved_queries/form/index.tsx index 8d5eea2f4ec22..4704590fc17c9 100644 --- a/x-pack/plugins/osquery/public/saved_queries/form/index.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/form/index.tsx @@ -93,7 +93,7 @@ const SavedQueryFormComponent: React.FC = ({ - + From b323fc90a81dd404686c1858a3c525ae1cc922ae Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Thu, 30 Nov 2023 11:47:46 +0100 Subject: [PATCH 3/4] Core metrics collection - collect more memory related metrics (#172146) ## Summary Part of https://github.com/elastic/kibana/issues/171060 Add additional memory metrics for collection: **process**: - `process.memory.array_buffers_in_bytes` the `process.memoryUsage().arrayBuffers` value - `process.memory.external_in_bytes` the `process.memoryUsage().external` value **cgroup**: (v2 only) - `os.cgroupMemory.current_in_bytes` value from `/sys/fs/cgroup/{group}/memory.current` - `os.cgroupMemory.swap_current_in_bytes` value from `/sys/fs/cgroup/{group}/memory.swap.current` --- .../src/status/lib/load_status.test.ts | 4 ++ .../src/cgroup/cgroup.test.ts | 5 +- .../src/cgroup/cgroup.ts | 21 +++++--- .../src/cgroup/gather_info.test.ts | 17 ++++--- .../src/cgroup/gather_info.ts | 20 ++++---- .../src/cgroup/types.ts | 2 +- .../src/cgroup/v2.test.ts | 19 +++++-- .../src/cgroup/v2.ts | 49 ++++++++++++++----- .../src/mocks_internal.ts | 2 + .../src/process.test.ts | 9 +++- .../src/process.ts | 2 + .../src/process.mocks.ts | 2 + .../src/logging/get_ops_metrics_log.test.ts | 13 ++++- .../src/logging/get_ops_metrics_log.ts | 5 ++ .../src/metrics_service.test.ts | 5 ++ .../core-metrics-server/src/metrics.ts | 12 +++++ .../ops_stats_collector.test.ts.snap | 4 ++ .../common/types/stats/core_metrics.ts | 12 +++++ .../server/collector/collector_set.test.ts | 20 ++++++++ 19 files changed, 176 insertions(+), 47 deletions(-) diff --git a/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts b/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts index 45b466225c375..ed6cc186313f3 100644 --- a/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts +++ b/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts @@ -76,6 +76,8 @@ const mockedResponse: StatusResponse = { total_in_bytes: 0, }, resident_set_size_in_bytes: 1, + array_buffers_in_bytes: 1, + external_in_bytes: 1, }, event_loop_delay: 1, event_loop_delay_histogram: mocked.createHistogram(), @@ -96,6 +98,8 @@ const mockedResponse: StatusResponse = { total_in_bytes: 0, }, resident_set_size_in_bytes: 1, + array_buffers_in_bytes: 1, + external_in_bytes: 1, }, event_loop_delay: 1, event_loop_delay_histogram: mocked.createHistogram(), diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.test.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.test.ts index 4ac8870de5051..ee85fd9b755bb 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.test.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.test.ts @@ -93,10 +93,7 @@ describe('OsCgroupMetricsCollector', () => { await collector.collect(); expect(gatherV2CgroupMetrics).toHaveBeenCalledTimes(1); - expect(gatherV2CgroupMetrics).toHaveBeenCalledWith({ - cpuAcctPath: '/groupname', - cpuPath: '/groupname', - }); + expect(gatherV2CgroupMetrics).toHaveBeenCalledWith('/groupname'); expect(gatherV1CgroupMetrics).toHaveBeenCalledTimes(0); }); diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.ts index b336bff923b25..cea83674faa9c 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/cgroup.ts @@ -42,9 +42,13 @@ export class OsCgroupMetricsCollector implements MetricsCollector { if (this.hasPaths()) return; - const { data: cgroups, v2 } = await gatherInfo(); - this.isCgroup2 = v2; - this.cpuPath = this.options.cpuPath || cgroups[GROUP_CPU]; - this.cpuAcctPath = this.options.cpuAcctPath || cgroups[GROUP_CPUACCT]; + const result = await gatherInfo(); + this.isCgroup2 = result.v2; + if (result.v2) { + this.cpuPath = result.path; + this.cpuAcctPath = result.path; + } else { + this.cpuPath = this.options.cpuPath || result.data[GROUP_CPU]; + this.cpuAcctPath = this.options.cpuAcctPath || result.data[GROUP_CPUACCT]; + } // prevents undefined cgroup paths this.noCgroupPresent = Boolean(!this.cpuPath || !this.cpuAcctPath); diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.test.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.test.ts index bf86907e656d9..8324091eaced7 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.test.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.test.ts @@ -17,11 +17,14 @@ describe('gatherInfo', () => { '/proc/self/cgroup': `0:controller:/path 1:controller2,controller3:/otherpath`, }); - const { data } = await gatherInfo(); - expect(data).toEqual({ - controller: '/path', - controller2: '/otherpath', - controller3: '/otherpath', + const result = await gatherInfo(); + expect(result).toEqual({ + v2: false, + data: { + controller: '/path', + controller2: '/otherpath', + controller3: '/otherpath', + }, }); }); @@ -30,7 +33,7 @@ describe('gatherInfo', () => { '/proc/self/cgroup': `0:controller:/path 1:controller2,controller3:/otherpath`, }); - await expect(gatherInfo()).resolves.toMatchObject({ v2: false }); + expect(await gatherInfo()).toMatchObject({ v2: false }); mockFs({ '/proc/self/cgroup': ` @@ -38,7 +41,7 @@ describe('gatherInfo', () => { `, }); - await expect(gatherInfo()).resolves.toMatchObject({ v2: true }); + expect(await gatherInfo()).toMatchObject({ v2: true }); }); test('missing cgroup file', async () => { diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.ts index 7e490a577ff07..0afbceb85d960 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/gather_info.ts @@ -7,8 +7,6 @@ */ import fs from 'fs/promises'; -import { GROUP_CPU, GROUP_CPUACCT } from './constants'; - const CONTROL_GROUP_RE = new RegExp('\\d+:([^:]+):(/.*)'); const CONTROLLER_SEPARATOR_RE = ','; const PROC_SELF_CGROUP_FILE = '/proc/self/cgroup'; @@ -27,10 +25,15 @@ async function readProcSelf(): Promise { return data.split(/\n/).filter((line) => line.trim().length > 0); } -interface Result { - data: Record; - v2: boolean; -} +type Result = + | { + v2: true; + path: string; + } + | { + v2: false; + data: Record; + }; export async function gatherInfo(): Promise { const lines = await readProcSelf(); @@ -39,11 +42,8 @@ export async function gatherInfo(): Promise { // eslint-disable-next-line prettier/prettier const [/* '0' */, /* '' */, path] = lines[0].trim().split(':'); return { - data: { - [GROUP_CPU]: path, - [GROUP_CPUACCT]: path, - }, v2: true, + path, }; } diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/types.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/types.ts index 84b03e3ba151a..133de91efb37f 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/types.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/types.ts @@ -8,4 +8,4 @@ import type { OpsOsMetrics } from '@kbn/core-metrics-server'; -export type OsCgroupMetrics = Pick; +export type OsCgroupMetrics = Pick; diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.test.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.test.ts index 96c720fe9a639..17f781a401ece 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.test.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.test.ts @@ -22,10 +22,17 @@ system_usec 125968 nr_periods 123 nr_throttled 1 throttled_usec 123123`, + '/sys/fs/cgroup/memory.current': '9000', + '/sys/fs/cgroup/memory.swap.current': '42', }); - expect(await gatherV2CgroupMetrics({ cpuAcctPath: '/', cpuPath: '/' })).toMatchInlineSnapshot(` + const metrics = await gatherV2CgroupMetrics('/'); + expect(metrics).toMatchInlineSnapshot(` Object { + "cgroup_memory": Object { + "current_in_bytes": 9000, + "swap_current_in_bytes": 42, + }, "cpu": Object { "cfs_period_micros": 100000, "cfs_quota_micros": -1, @@ -54,11 +61,17 @@ system_usec 125968 nr_periods 123 nr_throttled 1 throttled_usec 123123`, + '/sys/fs/cgroup/mypath/memory.current': '9876', + '/sys/fs/cgroup/mypath/memory.swap.current': '132645', }); - expect(await gatherV2CgroupMetrics({ cpuAcctPath: '/mypath', cpuPath: '/mypath' })) - .toMatchInlineSnapshot(` + const metrics = await gatherV2CgroupMetrics('/mypath'); + expect(metrics).toMatchInlineSnapshot(` Object { + "cgroup_memory": Object { + "current_in_bytes": 9876, + "swap_current_in_bytes": 132645, + }, "cpu": Object { "cfs_period_micros": 100000, "cfs_quota_micros": 111, diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.ts index ac7fba7cfaf4d..3d5d1f563d28f 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/cgroup/v2.ts @@ -13,37 +13,60 @@ import type { OsCgroupMetrics } from './types'; const PROC_CGROUP2_DIR = '/sys/fs/cgroup'; const CPU_STATS_FILE = 'cpu.stat'; const CPU_MAX_FILE = 'cpu.max'; +const MEMORY_CURRENT_FILE = 'memory.current'; +const MEMORY_SWAP_CURRENT_FILE = 'memory.swap.current'; -interface Arg { - cpuPath: string; - cpuAcctPath: string; -} +const getCGroupFilePath = (group: string, fileName: string): string => { + return joinPath(PROC_CGROUP2_DIR, group, fileName); +}; -export async function gatherV2CgroupMetrics(arg: Arg): Promise { - const [{ usage_nanos: usageNanos, ...stat }, cpuMax] = await Promise.all([ - readCPUStat(arg.cpuPath), - readCPUMax(arg.cpuPath), - ]); +export async function gatherV2CgroupMetrics(group: string): Promise { + const [{ usage_nanos: usageNanos, ...stat }, cpuMax, memoryCurrent, swapCurrent] = + await Promise.all([ + readCPUStat(group), + readCPUMax(group), + readMemoryCurrent(group), + readSwapCurrent(group), + ]); return { cpu: { ...cpuMax, - control_group: arg.cpuPath, + control_group: group, stat, }, cpuacct: { - control_group: arg.cpuPath, + control_group: group, usage_nanos: usageNanos, }, + cgroup_memory: { + current_in_bytes: memoryCurrent, + swap_current_in_bytes: swapCurrent, + }, }; } + interface CPUMax { cfs_period_micros: number; cfs_quota_micros: number; } +async function readMemoryCurrent(group: string): Promise { + const rawMemoryCurrent = (await fs.readFile(getCGroupFilePath(group, MEMORY_CURRENT_FILE))) + .toString() + .trim(); + return parseInt(rawMemoryCurrent, 10); +} + +async function readSwapCurrent(group: string): Promise { + const rawMemoryCurrent = (await fs.readFile(getCGroupFilePath(group, MEMORY_SWAP_CURRENT_FILE))) + .toString() + .trim(); + return parseInt(rawMemoryCurrent, 10); +} + async function readCPUMax(group: string): Promise { - const [quota, period] = (await fs.readFile(joinPath(PROC_CGROUP2_DIR, group, CPU_MAX_FILE))) + const [quota, period] = (await fs.readFile(getCGroupFilePath(group, CPU_MAX_FILE))) .toString() .trim() .split(/\s+/); @@ -62,7 +85,7 @@ async function readCPUStat(group: string): Promise { time_throttled_nanos: -1, usage_nanos: -1, }; - return (await fs.readFile(joinPath(PROC_CGROUP2_DIR, group, CPU_STATS_FILE))) + return (await fs.readFile(getCGroupFilePath(group, CPU_STATS_FILE))) .toString() .split(/\n/) .reduce((acc, line) => { diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/mocks_internal.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/mocks_internal.ts index 812b2c89ff8d4..d8659d702d0c8 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/mocks_internal.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/mocks_internal.ts @@ -37,6 +37,8 @@ function createMockOpsProcessMetrics(): OpsProcessMetrics { memory: { heap: { total_in_bytes: 1, used_in_bytes: 1, size_limit: 1 }, resident_set_size_in_bytes: 1, + array_buffers_in_bytes: 1, + external_in_bytes: 1, }, event_loop_delay: 1, event_loop_delay_histogram: histogram, diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/process.test.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/process.test.ts index dbb667aeeb043..4bad8e24b322e 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/process.test.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/process.test.ts @@ -61,12 +61,15 @@ describe('ProcessMetricsCollector', () => { const heapUsed = 4688; const heapSizeLimit = 5788; const rss = 5865; + const external = 9001; + const arrayBuffers = 42; + jest.spyOn(process, 'memoryUsage').mockImplementation(() => ({ rss, heapTotal, heapUsed, - external: 0, - arrayBuffers: 0, + external, + arrayBuffers, })); jest.spyOn(v8, 'getHeapStatistics').mockImplementation( @@ -83,6 +86,8 @@ describe('ProcessMetricsCollector', () => { expect(metrics[0].memory.heap.used_in_bytes).toEqual(heapUsed); expect(metrics[0].memory.heap.size_limit).toEqual(heapSizeLimit); expect(metrics[0].memory.resident_set_size_in_bytes).toEqual(rss); + expect(metrics[0].memory.external_in_bytes).toEqual(external); + expect(metrics[0].memory.array_buffers_in_bytes).toEqual(arrayBuffers); }); }); diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/process.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/process.ts index 399da1ad9c29e..3e35eaa108ea9 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/process.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/process.ts @@ -38,6 +38,8 @@ export class ProcessMetricsCollector implements MetricsCollector): OpsMetrics { ...testMetrics, }; } + const testMetrics = { process: { - memory: { heap: { used_in_bytes: 100 } }, + memory: { + heap: { used_in_bytes: 100, total_in_bytes: 200, size_limit: 300 }, + resident_set_size_in_bytes: 400, + external_in_bytes: 500, + array_buffers_in_bytes: 600, + }, uptime_in_millis: 1500, event_loop_delay: 50, event_loop_delay_histogram: { percentiles: { '50': 50, '75': 75, '95': 95, '99': 99 } }, @@ -127,9 +133,14 @@ describe('getEcsOpsMetricsLog', () => { "utilization": 0.6365329598160299, }, "memory": Object { + "arrayBuffersInBytes": 600, + "externalInBytes": 500, "heap": Object { + "sizeLimit": 300, + "totalInBytes": 200, "usedInBytes": 100, }, + "residentSetSizeInBytes": 400, }, "uptime": 1, }, diff --git a/packages/core/metrics/core-metrics-server-internal/src/logging/get_ops_metrics_log.ts b/packages/core/metrics/core-metrics-server-internal/src/logging/get_ops_metrics_log.ts index 78e4482b7a351..e3fffbdeb0433 100644 --- a/packages/core/metrics/core-metrics-server-internal/src/logging/get_ops_metrics_log.ts +++ b/packages/core/metrics/core-metrics-server-internal/src/logging/get_ops_metrics_log.ts @@ -86,7 +86,12 @@ export function getEcsOpsMetricsLog(metrics: OpsMetrics) { memory: { heap: { usedInBytes: processMemoryUsedInBytes, + totalInBytes: process?.memory?.heap.total_in_bytes, + sizeLimit: process?.memory?.heap.size_limit, }, + residentSetSizeInBytes: process?.memory?.resident_set_size_in_bytes, + externalInBytes: process?.memory?.external_in_bytes, + arrayBuffersInBytes: process?.memory?.array_buffers_in_bytes, }, eventLoopDelay: eventLoopDelayVal, eventLoopDelayHistogram: eventLoopDelayHistVals, diff --git a/packages/core/metrics/core-metrics-server-internal/src/metrics_service.test.ts b/packages/core/metrics/core-metrics-server-internal/src/metrics_service.test.ts index 45b3ae49a1f5e..176ad9c7fa4ce 100644 --- a/packages/core/metrics/core-metrics-server-internal/src/metrics_service.test.ts +++ b/packages/core/metrics/core-metrics-server-internal/src/metrics_service.test.ts @@ -216,9 +216,14 @@ describe('MetricsService', () => { "eventLoopDelayHistogram": undefined, "eventLoopUtilization": undefined, "memory": Object { + "arrayBuffersInBytes": undefined, + "externalInBytes": undefined, "heap": Object { + "sizeLimit": undefined, + "totalInBytes": undefined, "usedInBytes": undefined, }, + "residentSetSizeInBytes": undefined, }, "uptime": undefined, }, diff --git a/packages/core/metrics/core-metrics-server/src/metrics.ts b/packages/core/metrics/core-metrics-server/src/metrics.ts index 6f903cd66a246..615998e600a29 100644 --- a/packages/core/metrics/core-metrics-server/src/metrics.ts +++ b/packages/core/metrics/core-metrics-server/src/metrics.ts @@ -82,6 +82,10 @@ export interface OpsProcessMetrics { }; /** node rss */ resident_set_size_in_bytes: number; + /** memory usage of C++ objects bound to JavaScript objects managed by V8 */ + external_in_bytes: number; + /** memory allocated for array buffers. This is also included in the external value*/ + array_buffers_in_bytes: number; }; /** mean event loop delay since last collection*/ event_loop_delay: number; @@ -153,6 +157,14 @@ export interface OpsOsMetrics { time_throttled_nanos: number; }; }; + + /** memory cgroup metrics, undefined when not running in cgroup v2 */ + cgroup_memory?: { + /** The total amount of memory currently being used by the cgroup and its descendants. */ + current_in_bytes: number; + /** The total amount of swap currently being used by the cgroup and its descendants. */ + swap_current_in_bytes: number; + }; } /** diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap index 5f4aabf57995e..8464a88dfce5d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap @@ -32,6 +32,8 @@ Object { "utilization": 1, }, "memory": Object { + "array_buffers_in_bytes": 1, + "external_in_bytes": 1, "heap": Object { "size_limit": 1, "total_in_bytes": 1, @@ -51,6 +53,8 @@ Object { "utilization": 1, }, "memory": Object { + "array_buffers_in_bytes": 1, + "external_in_bytes": 1, "heap": Object { "size_limit": 1, "total_in_bytes": 1, diff --git a/src/plugins/usage_collection/common/types/stats/core_metrics.ts b/src/plugins/usage_collection/common/types/stats/core_metrics.ts index 64e5ef34155d9..5bc2023afc041 100644 --- a/src/plugins/usage_collection/common/types/stats/core_metrics.ts +++ b/src/plugins/usage_collection/common/types/stats/core_metrics.ts @@ -89,6 +89,10 @@ export interface OpsProcessMetrics { }; /** node rss */ resident_set_size_in_bytes: number; + /** memory usage of C++ objects bound to JavaScript objects managed by V8 */ + external_in_bytes: number; + /** memory allocated for array buffers. This is also included in the external value*/ + array_buffers_in_bytes: number; }; /** mean event loop delay since last collection*/ event_loop_delay: number; @@ -159,6 +163,14 @@ export interface OpsOsMetrics { time_throttled_nanos: number; }; }; + + /** memory cgroup metrics, undefined when not running in cgroup v2 */ + cgroup_memory?: { + /** The total amount of memory currently being used by the cgroup and its descendants. */ + current_in_bytes: number; + /** The total amount of swap currently being used by the cgroup and its descendants. */ + swap_current_in_bytes: number; + }; } /** diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts index 56b4e55eccfc4..43e063b95333e 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.test.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts @@ -237,6 +237,16 @@ describe('CollectorSet', () => { }, uptime_in_millis: 137844000, }, + process: { + heap: { + total_in_bytes: 1, + used_in_bytes: 2, + size_limit: 3, + }, + resident_set_size_in_bytes: 4, + array_buffers_in_bytes: 5, + external_in_bytes: 6, + }, daysOfTheWeek: ['monday', 'tuesday', 'wednesday'], }; @@ -247,6 +257,16 @@ describe('CollectorSet', () => { memory: { free_bytes: 458280960, total_bytes: 17179869184, used_bytes: 16721588224 }, uptime_ms: 137844000, }, + process: { + heap: { + total_bytes: 1, + used_bytes: 2, + size_limit: 3, + }, + resident_set_size_bytes: 4, + array_buffers_bytes: 5, + external_bytes: 6, + }, days_of_the_week: ['monday', 'tuesday', 'wednesday'], }); }); From a4845721d37b5b58f1616916a21e4edce5d1d173 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 30 Nov 2023 11:15:28 +0000 Subject: [PATCH 4/4] [ML] Fixing kibana object list in new job from recognised index page (#171935) Fixes issue where the list of kibana objects in a module are not being rendered correctly. The issue was caused by the kibana objects from the module not being initially added to the list. **Before** ![image](https://github.com/elastic/kibana/assets/22172091/e4253cc4-6f37-40a6-bfe5-808145769a3b) **After** ![image](https://github.com/elastic/kibana/assets/22172091/74fefafc-ab53-45db-be21-4c3b9e3bdcc0) --- .../new_job/recognize/components/kibana_objects.tsx | 10 +++++++--- .../public/application/jobs/new_job/recognize/page.tsx | 10 ++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx index 95ac0b4043f57..55692ac507195 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx @@ -23,11 +23,11 @@ import { KibanaObjectUi } from '../page'; export interface KibanaObjectItemProps { objectType: string; - kibanaObjects: KibanaObjectUi[]; + kibanaObjects: KibanaObjectUi[] | undefined; isSaving: boolean; } -export const KibanaObjects: FC = memo( +export const KibanaObjectList: FC = memo( ({ objectType, kibanaObjects, isSaving }) => { const kibanaObjectLabels: Record = { dashboard: i18n.translate('xpack.ml.newJob.recognize.dashboardsLabel', { @@ -41,6 +41,10 @@ export const KibanaObjects: FC = memo( }), }; + if (kibanaObjects === undefined) { + return null; + } + return ( <> @@ -53,7 +57,7 @@ export const KibanaObjects: FC = memo( - + {title} diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx index 8be908871fe2c..fe2c5dfa966aa 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx @@ -30,11 +30,12 @@ import { JobOverride, JobResponse, KibanaObject, + KibanaObjects, KibanaObjectResponse, ModuleJob, } from '../../../../../common/types/modules'; import { CreateResultCallout } from './components/create_result_callout'; -import { KibanaObjects } from './components/kibana_objects'; +import { KibanaObjectList } from './components/kibana_objects'; import { ModuleJobs } from './components/module_jobs'; import { JobSettingsForm, JobSettingsFormValues } from './components/job_settings_form'; import { TimeRange } from '../common/components'; @@ -50,10 +51,6 @@ export interface ModuleJobUI extends ModuleJob { export type KibanaObjectUi = KibanaObject & KibanaObjectResponse; -export interface KibanaObjects { - [objectType: string]: KibanaObjectUi[]; -} - interface PageProps { moduleId: string; existingGroupIds: string[]; @@ -111,6 +108,7 @@ export const Page: FC = ({ moduleId, existingGroupIds }) => { try { const response = await getDataRecognizerModule({ moduleId }); setJobs(response.jobs); + setKibanaObjects(response.kibana); setSaveState(SAVE_STATE.NOT_SAVED); @@ -365,7 +363,7 @@ export const Page: FC = ({ moduleId, existingGroupIds }) => { {Object.keys(kibanaObjects).map((objectType, i) => ( -