From 8eb127da9a625b55e8cd0344d0225a57f530c490 Mon Sep 17 00:00:00 2001 From: Ido Cohen <90558359+CohenIdo@users.noreply.github.com> Date: Tue, 28 Mar 2023 11:27:08 +0300 Subject: [PATCH] [Cloud Security] fetch k8s version and account id for telemetry --- .../collectors/accounts_stats_collector.ts | 17 ++ .../collectors/rules_stats_collector.ts | 287 ++++++++++-------- .../server/lib/telemetry/collectors/schema.ts | 2 + .../server/lib/telemetry/collectors/types.ts | 2 + .../schema/xpack_plugins.json | 6 + .../telemetry/data.ts | 7 +- .../telemetry/telemetry.ts | 9 +- .../config.ts | 2 +- 8 files changed, 199 insertions(+), 133 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/accounts_stats_collector.ts b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/accounts_stats_collector.ts index 69cce5ed45893..d144343807a06 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/accounts_stats_collector.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/accounts_stats_collector.ts @@ -30,6 +30,10 @@ interface BenchmarkVersion { metrics: { 'rule.benchmark.version': string }; } +interface KubernetesVersion { + metrics: { 'cloudbeat.kubernetes.version': string }; +} + interface AccountsStats { accounts: { buckets: AccountEntity[]; @@ -43,6 +47,7 @@ interface AccountEntity { benchmark_name: { top: BenchmarkName[] }; benchmark_id: { top: BenchmarkId[] }; benchmark_version: { top: BenchmarkVersion[] }; + kubernetes_version: { top: KubernetesVersion[] }; agents_count: Value; nodes_count: Value; pods_count: Value; @@ -110,6 +115,17 @@ const getAccountsStatsQuery = (): SearchRequest => ({ }, }, }, + kubernetes_version: { + top_metrics: { + metrics: { + field: 'cloudbeat.kubernetes.version', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, + }, passed_findings_count: { filter: { bool: { @@ -203,6 +219,7 @@ const getCspmAccountsStats = ( benchmark_name: account.benchmark_name.top[0].metrics['rule.benchmark.name'], benchmark_id: account.benchmark_id.top[0].metrics['rule.benchmark.id'], benchmark_version: account.benchmark_version.top[0].metrics['rule.benchmark.version'], + kubernetes_version: account.kubernetes_version.top[0].metrics['cloudbeat.kubernetes.version'], agents_count: account.agents_count.value, nodes_count: account.nodes_count.value, pods_count: account.resources.pods_count.value, diff --git a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/rules_stats_collector.ts b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/rules_stats_collector.ts index b4f21061893e5..d2b315dfeb938 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/rules_stats_collector.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/rules_stats_collector.ts @@ -10,6 +10,7 @@ import type { AggregationsMultiBucketBase, SearchRequest, } from '@elastic/elasticsearch/lib/api/types'; +import { getIdentifierRuntimeMapping } from '../../../../common/runtime_mappings/get_identifier_runtime_mapping'; import type { CspmRulesStats } from './types'; import { LATEST_FINDINGS_INDEX_DEFAULT_NS } from '../../../../common/constants'; @@ -40,7 +41,14 @@ interface PostureType { metrics: { 'rule.benchmark.posture_type': string }; } +interface Accounts { + accounts: { + buckets: RulesStats[]; + }; +} + interface RulesStats { + key: string; // account_id rules: { buckets: RuleEntity[]; }; @@ -65,141 +73,154 @@ const getRulesStatsQuery = (): SearchRequest => ({ query: { match_all: {}, }, + // generates the 'asset_identifier' field + runtime_mappings: getIdentifierRuntimeMapping(), aggs: { - rules: { + accounts: { terms: { - field: 'rule.id', + field: 'asset_identifier', order: { _count: 'desc', }, - size: 100, + size: 1000, }, aggs: { - rule_name: { - top_metrics: { - metrics: { - field: 'rule.name', - }, - size: 1, - sort: { - '@timestamp': 'desc', - }, - }, - }, - rule_section: { - top_metrics: { - metrics: { - field: 'rule.section', - }, - size: 1, - sort: { - '@timestamp': 'desc', - }, - }, - }, - rule_version: { - top_metrics: { - metrics: { - field: 'rule.version', - }, - size: 1, - sort: { - '@timestamp': 'desc', - }, - }, - }, - posture_type: { - top_metrics: { - metrics: { - field: 'rule.benchmark.posture_type', - }, - size: 1, - sort: { - '@timestamp': 'desc', + rules: { + terms: { + field: 'rule.id', + order: { + _count: 'desc', }, + size: 1000, }, - }, - rule_number: { - top_metrics: { - metrics: { - field: 'rule.benchmark.rule_number', + aggs: { + rule_name: { + top_metrics: { + metrics: { + field: 'rule.name', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - size: 1, - sort: { - '@timestamp': 'desc', + rule_section: { + top_metrics: { + metrics: { + field: 'rule.section', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - }, - }, - benchmark_id: { - top_metrics: { - metrics: { - field: 'rule.benchmark.id', + rule_version: { + top_metrics: { + metrics: { + field: 'rule.version', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - size: 1, - sort: { - '@timestamp': 'desc', + posture_type: { + top_metrics: { + metrics: { + field: 'rule.benchmark.posture_type', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - }, - }, - benchmark_version: { - top_metrics: { - metrics: { - field: 'rule.benchmark.version', + rule_number: { + top_metrics: { + metrics: { + field: 'rule.benchmark.rule_number', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - size: 1, - sort: { - '@timestamp': 'desc', + benchmark_id: { + top_metrics: { + metrics: { + field: 'rule.benchmark.id', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - }, - }, - benchmark_name: { - top_metrics: { - metrics: { - field: 'rule.benchmark.name', + benchmark_version: { + top_metrics: { + metrics: { + field: 'rule.benchmark.version', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - size: 1, - sort: { - '@timestamp': 'desc', + benchmark_name: { + top_metrics: { + metrics: { + field: 'rule.benchmark.name', + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, + }, }, - }, - }, - passed_findings_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'result.evaluation': 'passed', - }, + passed_findings_count: { + filter: { + bool: { + filter: [ + { + bool: { + should: [ + { + term: { + 'result.evaluation': 'passed', + }, + }, + ], + minimum_should_match: 1, }, - ], - minimum_should_match: 1, - }, + }, + ], }, - ], + }, }, - }, - }, - failed_findings_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'result.evaluation': 'failed', - }, + failed_findings_count: { + filter: { + bool: { + filter: [ + { + bool: { + should: [ + { + term: { + 'result.evaluation': 'failed', + }, + }, + ], + minimum_should_match: 1, }, - ], - minimum_should_match: 1, - }, + }, + ], }, - ], + }, }, }, }, @@ -211,24 +232,30 @@ const getRulesStatsQuery = (): SearchRequest => ({ _source: false, }); -const getCspmRulesStats = (aggregatedRulesStats: RulesStats, logger: Logger): CspmRulesStats[] => { - const rules = aggregatedRulesStats.rules.buckets; +const getCspmRulesStats = (aggregatedRulesStats: Accounts, logger: Logger): CspmRulesStats[] => { + const accounts = aggregatedRulesStats.accounts.buckets; - const cspmRulesStats = rules.map((rule) => ({ - rule_id: rule.key, - rule_name: rule.rule_name.top[0].metrics['rule.name'], - rule_section: rule.rule_section.top[0].metrics['rule.section'], - rule_version: rule.rule_version.top[0].metrics['rule.version'], - rule_number: rule.rule_number.top[0].metrics['rule.benchmark.rule_number'], - posture_type: rule.posture_type.top[0].metrics['rule.benchmark.posture_type'], - benchmark_name: rule.benchmark_name.top[0].metrics['rule.benchmark.name'], - benchmark_id: rule.benchmark_id.top[0].metrics['rule.benchmark.id'], - passed_findings_count: rule.passed_findings_count.doc_count, - benchmark_version: rule.benchmark_version.top[0].metrics['rule.benchmark.version'], - failed_findings_count: rule.failed_findings_count.doc_count, - })); + const cspmRulesStats = accounts.map((account) => { + const accountId = account.key; + return account.rules.buckets.map((rule) => { + return { + account_id: accountId, + rule_id: rule.key, + rule_name: rule.rule_name.top[0].metrics['rule.name'], + rule_section: rule.rule_section.top[0].metrics['rule.section'], + rule_version: rule.rule_version.top[0].metrics['rule.version'], + rule_number: rule.rule_number.top[0].metrics['rule.benchmark.rule_number'], + posture_type: rule.posture_type.top[0].metrics['rule.benchmark.posture_type'], + benchmark_name: rule.benchmark_name.top[0].metrics['rule.benchmark.name'], + benchmark_id: rule.benchmark_id.top[0].metrics['rule.benchmark.id'], + benchmark_version: rule.benchmark_version.top[0].metrics['rule.benchmark.version'], + passed_findings_count: rule.passed_findings_count.doc_count, + failed_findings_count: rule.failed_findings_count.doc_count, + }; + }); + }); - return cspmRulesStats; + return cspmRulesStats.flat(2); }; export const getRulesStats = async ( @@ -241,7 +268,7 @@ export const getRulesStats = async ( }); if (isIndexExists) { - const rulesStatsResponse = await esClient.search(getRulesStatsQuery()); + const rulesStatsResponse = await esClient.search(getRulesStatsQuery()); const cspmRulesStats = rulesStatsResponse.aggregations ? getCspmRulesStats(rulesStatsResponse.aggregations, logger) diff --git a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/schema.ts b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/schema.ts index 47bfaca74e457..fd45e02dc1402 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/schema.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/schema.ts @@ -74,6 +74,7 @@ export const cspmUsageSchema: MakeSchemaFrom = { benchmark_id: { type: 'keyword' }, benchmark_name: { type: 'keyword' }, benchmark_version: { type: 'keyword' }, + kubernetes_version: { type: 'keyword' }, passed_findings_count: { type: 'long' }, failed_findings_count: { type: 'long' }, agents_count: { type: 'short' }, @@ -84,6 +85,7 @@ export const cspmUsageSchema: MakeSchemaFrom = { rules_stats: { type: 'array', items: { + account_id: { type: 'keyword' }, rule_id: { type: 'keyword' }, rule_name: { type: 'keyword' }, rule_section: { type: 'keyword' }, diff --git a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/types.ts b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/types.ts index fcbd6cd9c36e2..d3baa5f67249c 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/types.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/types.ts @@ -41,6 +41,7 @@ export interface CspmAccountsStats { benchmark_id: string; benchmark_name: string; benchmark_version: string; + kubernetes_version: string | null; passed_findings_count: number; failed_findings_count: number; agents_count: number; @@ -48,6 +49,7 @@ export interface CspmAccountsStats { pods_count: number; } export interface CspmRulesStats { + account_id: string; rule_id: string; rule_name: string; rule_section: string; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index fc3f0ac44d7f3..a537eaba48279 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -5467,6 +5467,9 @@ "benchmark_version": { "type": "keyword" }, + "kubernetes_version": { + "type": "keyword" + }, "passed_findings_count": { "type": "long" }, @@ -5489,6 +5492,9 @@ "type": "array", "items": { "properties": { + "account_id": { + "type": "keyword" + }, "rule_id": { "type": "keyword" }, diff --git a/x-pack/test/cloud_security_posture_api/telemetry/data.ts b/x-pack/test/cloud_security_posture_api/telemetry/data.ts index 2550982ea5ac6..da50b2531e575 100644 --- a/x-pack/test/cloud_security_posture_api/telemetry/data.ts +++ b/x-pack/test/cloud_security_posture_api/telemetry/data.ts @@ -14,7 +14,8 @@ export interface MockTelemetryFindings { result: { evaluation: string }; host: { name: string }; cluster_id?: string; - cloud?: { account: { id: string } }; + cloud?: { account?: { id: string } }; + cloudbeat?: { kubernetes: { version: string } }; } export interface MockTelemetryData { @@ -81,6 +82,7 @@ export const data: MockTelemetryData = { agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' }, result: { evaluation: 'passed' }, host: { name: 'docker-fleet-agent' }, + cloudbeat: { kubernetes: { version: 'v1.23.0' } }, }, { cluster_id: 'my-k8s-cluster-5555', @@ -100,6 +102,7 @@ export const data: MockTelemetryData = { agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae3' }, result: { evaluation: 'passed' }, host: { name: 'control-plane' }, + cloudbeat: { kubernetes: { version: 'v1.23.0' } }, }, ], kspmFindingsNoPostureType: [ @@ -120,6 +123,7 @@ export const data: MockTelemetryData = { agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' }, result: { evaluation: 'passed' }, host: { name: 'docker-fleet-agent' }, + cloudbeat: { kubernetes: { version: 'v1.23.0' } }, }, { cluster_id: 'my-k8s-cluster-5555', @@ -138,6 +142,7 @@ export const data: MockTelemetryData = { agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae3' }, result: { evaluation: 'passed' }, host: { name: 'control-plane' }, + cloudbeat: { kubernetes: { version: 'v1.23.0' } }, }, ], }; diff --git a/x-pack/test/cloud_security_posture_api/telemetry/telemetry.ts b/x-pack/test/cloud_security_posture_api/telemetry/telemetry.ts index 49f1cd39ae787..ede036d93239f 100644 --- a/x-pack/test/cloud_security_posture_api/telemetry/telemetry.ts +++ b/x-pack/test/cloud_security_posture_api/telemetry/telemetry.ts @@ -82,6 +82,7 @@ export default function ({ getService }: FtrProviderContext) { failed_findings_count: 0, benchmark_name: 'CIS Kubernetes V1.23', benchmark_id: 'cis_k8s', + kubernetes_version: 'v1.23.0', benchmark_version: 'v1.0.0', agents_count: 2, nodes_count: 2, @@ -134,6 +135,7 @@ export default function ({ getService }: FtrProviderContext) { benchmark_name: 'CIS Amazon Web Services Foundations', benchmark_id: 'cis_aws', benchmark_version: 'v1.5.0', + kubernetes_version: null, agents_count: 1, nodes_count: 1, pods_count: 0, @@ -178,6 +180,7 @@ export default function ({ getService }: FtrProviderContext) { benchmark_name: 'CIS Amazon Web Services Foundations', benchmark_id: 'cis_aws', benchmark_version: 'v1.5.0', + kubernetes_version: null, agents_count: 1, nodes_count: 1, pods_count: 0, @@ -191,6 +194,7 @@ export default function ({ getService }: FtrProviderContext) { benchmark_name: 'CIS Kubernetes V1.23', benchmark_id: 'cis_k8s', benchmark_version: 'v1.0.0', + kubernetes_version: 'v1.23.0', agents_count: 2, nodes_count: 2, pods_count: 0, @@ -228,7 +232,7 @@ export default function ({ getService }: FtrProviderContext) { ]); }); - it('includes only KSPM findings without posture_type', async () => { + it(`'includes only KSPM findings without posture_type'`, async () => { await index.add(data.kspmFindingsNoPostureType); const { @@ -252,6 +256,7 @@ export default function ({ getService }: FtrProviderContext) { benchmark_name: 'CIS Kubernetes V1.23', benchmark_id: 'cis_k8s', benchmark_version: 'v1.0.0', + kubernetes_version: 'v1.23.0', agents_count: 2, nodes_count: 2, pods_count: 0, @@ -305,6 +310,7 @@ export default function ({ getService }: FtrProviderContext) { benchmark_name: 'CIS Amazon Web Services Foundations', benchmark_id: 'cis_aws', benchmark_version: 'v1.5.0', + kubernetes_version: null, agents_count: 1, nodes_count: 1, pods_count: 0, @@ -318,6 +324,7 @@ export default function ({ getService }: FtrProviderContext) { benchmark_name: 'CIS Kubernetes V1.23', benchmark_id: 'cis_k8s', benchmark_version: 'v1.0.0', + kubernetes_version: 'v1.23.0', agents_count: 2, nodes_count: 2, pods_count: 0, diff --git a/x-pack/test/cloud_security_posture_functional/config.ts b/x-pack/test/cloud_security_posture_functional/config.ts index a87722cc936fe..09e2d2308c78d 100644 --- a/x-pack/test/cloud_security_posture_functional/config.ts +++ b/x-pack/test/cloud_security_posture_functional/config.ts @@ -38,7 +38,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { * 2. merge the updated version number change to kibana */ `--xpack.fleet.packages.0.name=cloud_security_posture`, - `--xpack.fleet.packages.0.version=1.0.8`, + `--xpack.fleet.packages.0.version=1.2.10`, // `--xpack.fleet.registryUrl=https://localhost:8080`, ], },