diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts index 72568fbd4c8a0..1a5fb8e64a265 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts @@ -21,6 +21,15 @@ export const cloudSecurityMetringCallback = async ({ lastSuccessfulReport, config, }: MeteringCallbackInput): Promise => { + const projectHasCloudProductLine = config.productTypes.some( + (product) => product.product_line === ProductLine.cloud + ); + + if (!projectHasCloudProductLine) { + logger.info('No cloud product line found in the project'); + return { records: [] }; + } + const projectId = cloudSetup?.serverless?.projectId || 'missing_project_id'; const tier: Tier = getCloudProductTier(config, logger); @@ -30,8 +39,8 @@ export const cloudSecurityMetringCallback = async ({ const promiseResults = await Promise.allSettled( cloudSecuritySolutions.map((cloudSecuritySolution) => { - if (cloudSecuritySolution === CLOUD_DEFEND) { - return getCloudDefendUsageRecords({ + if (cloudSecuritySolution !== CLOUD_DEFEND) { + return getCloudSecurityUsageRecord({ esClient, projectId, logger, @@ -41,7 +50,9 @@ export const cloudSecurityMetringCallback = async ({ tier, }); } - return getCloudSecurityUsageRecord({ + + // since lastSuccessfulReport is not used by getCloudSecurityUsageRecord, we want to verify if it is used by getCloudDefendUsageRecords before getCloudSecurityUsageRecord. + return getCloudDefendUsageRecords({ esClient, projectId, logger, diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts index f775b91406c4c..6d55e52469283 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts @@ -14,16 +14,23 @@ import { } from './cloud_security_metering_task'; import type { ServerlessSecurityConfig } from '../config'; -import type { CloudSecuritySolutions } from './types'; + import type { ProductTier } from '../../common/product'; -import { CLOUD_SECURITY_TASK_TYPE, CSPM, KSPM, CNVM, CLOUD_DEFEND } from './constants'; +import { + CLOUD_SECURITY_TASK_TYPE, + CSPM, + KSPM, + CNVM, + CLOUD_DEFEND, + BILLABLE_ASSETS_CONFIG, +} from './constants'; import { getCloudDefendUsageRecords } from './defend_for_containers_metering'; const mockEsClient = elasticsearchServiceMock.createStart().client.asInternalUser; const logger: ReturnType = loggingSystemMock.createLogger(); const chance = new Chance(); -const cloudSecuritySolutions: CloudSecuritySolutions[] = [CSPM, KSPM, CNVM]; +const cloudSecuritySolutions: Array = [CSPM, KSPM]; describe('getCloudSecurityUsageRecord', () => { beforeEach(() => { @@ -54,18 +61,33 @@ describe('getCloudSecurityUsageRecord', () => { }); test.each(cloudSecuritySolutions)( - 'should return usageRecords with correct values for cspm, kspm, and cnvm when Elasticsearch response has aggregations', + 'should return usageRecords with correct values for cspm and kspm when Elasticsearch response has aggregations', async (cloudSecuritySolution) => { // @ts-ignore mockEsClient.search.mockResolvedValueOnce({ hits: { hits: [{ _id: 'someRecord', _index: 'mockIndex' }] }, // mocking for indexHasDataInDateRange }); + const randomIndex = Math.floor( + Math.random() * BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values.length + ); + const randomBillableAsset = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values[randomIndex]; // @ts-ignore mockEsClient.search.mockResolvedValueOnce({ aggregations: { - unique_assets: { - value: 10, + resource_sub_type: { + buckets: [ + { + key: randomBillableAsset, + doc_count: 100, + unique_assets: { value: 10 }, + }, + { + key: 'not_billable_asset', + doc_count: 50, + unique_assets: { value: 11 }, + }, + ], }, min_timestamp: { value_as_string: '2023-07-30T15:11:41.738Z', @@ -100,6 +122,10 @@ describe('getCloudSecurityUsageRecord', () => { sub_type: cloudSecuritySolution, quantity: 10, period_seconds: expect.any(Number), + metadata: { + [randomBillableAsset]: '10', + not_billable_asset: '11', + }, }, source: { id: taskId, @@ -113,6 +139,62 @@ describe('getCloudSecurityUsageRecord', () => { } ); + it('should return usageRecords with correct values for cnvm when Elasticsearch response has aggregations', async () => { + const cloudSecuritySolution = CNVM; + + // @ts-ignore + mockEsClient.search.mockResolvedValueOnce({ + hits: { hits: [{ _id: 'someRecord', _index: 'mockIndex' }] }, // mocking for indexHasDataInDateRange + }); + + // @ts-ignore + mockEsClient.search.mockResolvedValueOnce({ + aggregations: { + unique_assets: { + value: 10, + }, + min_timestamp: { + value_as_string: '2023-07-30T15:11:41.738Z', + }, + }, + }); + + const projectId = chance.guid(); + const taskId = chance.guid(); + + const tier = 'essentials' as ProductTier; + const result = await getCloudSecurityUsageRecord({ + esClient: mockEsClient, + projectId, + logger, + taskId, + lastSuccessfulReport: new Date(), + cloudSecuritySolution, + tier, + }); + + expect(result).toEqual([ + { + id: expect.stringContaining(`${CLOUD_SECURITY_TASK_TYPE}_cnvm_${projectId}`), + usage_timestamp: '2023-07-30T15:11:41.738Z', + creation_timestamp: expect.any(String), // Expect a valid ISO string + usage: { + type: CLOUD_SECURITY_TASK_TYPE, + sub_type: CNVM, + quantity: 10, + period_seconds: expect.any(Number), + }, + source: { + id: taskId, + instance_group_id: projectId, + metadata: { + tier: 'essentials', + }, + }, + }, + ]); + }); + it('should return undefined when Elasticsearch response does not have aggregations', async () => { // @ts-ignore mockEsClient.search.mockResolvedValue({}); @@ -162,9 +244,7 @@ describe('getCloudSecurityUsageRecord', () => { describe('getSearchQueryByCloudSecuritySolution', () => { it('should return the correct search query for CSPM', () => { - const searchFrom = new Date('2023-07-30T15:11:41.738Z'); - - const result = getSearchQueryByCloudSecuritySolution('cspm', searchFrom); + const result = getSearchQueryByCloudSecuritySolution('cspm'); expect(result).toEqual({ bool: { @@ -181,39 +261,13 @@ describe('getSearchQueryByCloudSecuritySolution', () => { 'rule.benchmark.posture_type': 'cspm', }, }, - { - terms: { - 'resource.sub_type': [ - // 'aws-ebs', we can't include EBS volumes until https://github.com/elastic/security-team/issues/9283 is resolved - // 'aws-ec2', we can't include EC2 instances until https://github.com/elastic/security-team/issues/9254 is resolved - 'aws-s3', - 'aws-rds', - 'azure-disk', - 'azure-document-db-database-account', - 'azure-flexible-mysql-server-db', - 'azure-flexible-postgresql-server-db', - 'azure-mysql-server-db', - 'azure-postgresql-server-db', - 'azure-sql-server', - 'azure-storage-account', - 'azure-vm', - 'gcp-bigquery-dataset', - 'gcp-compute-disk', - 'gcp-compute-instance', - 'gcp-sqladmin-instance', - 'gcp-storage-bucket', - ], - }, - }, ], }, }); }); it('should return the correct search query for KSPM', () => { - const searchFrom = new Date('2023-07-30T15:11:41.738Z'); - - const result = getSearchQueryByCloudSecuritySolution('kspm', searchFrom); + const result = getSearchQueryByCloudSecuritySolution('kspm'); expect(result).toEqual({ bool: { @@ -230,20 +284,13 @@ describe('getSearchQueryByCloudSecuritySolution', () => { 'rule.benchmark.posture_type': 'kspm', }, }, - { - terms: { - 'resource.sub_type': ['Node', 'node'], - }, - }, ], }, }); }); it('should return the correct search query for CNVM', () => { - const searchFrom = new Date('2023-07-30T15:11:41.738Z'); - - const result = getSearchQueryByCloudSecuritySolution(CNVM, searchFrom); + const result = getSearchQueryByCloudSecuritySolution(CNVM); expect(result).toEqual({ bool: { diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts index 7827c5d25ebe6..6687c3dfed48c 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts @@ -15,72 +15,112 @@ import { CSPM, KSPM, METERING_CONFIGS, - THRESHOLD_MINUTES, BILLABLE_ASSETS_CONFIG, } from './constants'; -import type { Tier, UsageRecord } from '../types'; +import type { ResourceSubtypeCounter, Tier, UsageRecord } from '../types'; import type { CloudSecurityMeteringCallbackInput, CloudSecuritySolutions, AssetCountAggregation, + ResourceSubtypeAggregationBucket, } from './types'; export const getUsageRecords = ( - assetCountAggregations: AssetCountAggregation[], + assetCountAggregation: AssetCountAggregation, cloudSecuritySolution: CloudSecuritySolutions, taskId: string, tier: Tier, projectId: string, periodSeconds: number, logger: Logger -): UsageRecord[] => { - const usageRecords = assetCountAggregations.map((assetCountAggregation) => { - const assetCount = assetCountAggregation.unique_assets.value; - - if (assetCount > AGGREGATION_PRECISION_THRESHOLD) { - logger.warn( - `The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.` - ); - } - - const minTimestamp = new Date( - assetCountAggregation.min_timestamp.value_as_string - ).toISOString(); - - const creationTimestamp = new Date(); - const minutes = creationTimestamp.getMinutes(); - if (minutes >= 30) { - creationTimestamp.setMinutes(30, 0, 0); - } else { - creationTimestamp.setMinutes(0, 0, 0); - } - const roundedCreationTimestamp = creationTimestamp.toISOString(); - - const usageRecord: UsageRecord = { - id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`, - usage_timestamp: minTimestamp, - creation_timestamp: creationTimestamp.toISOString(), - usage: { - type: CLOUD_SECURITY_TASK_TYPE, - sub_type: cloudSecuritySolution, - quantity: assetCount, - period_seconds: periodSeconds, - }, - source: { - id: taskId, - instance_group_id: projectId, - metadata: { tier }, +): UsageRecord => { + let assetCount; + let resourceSubtypeCounterMap; + + if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) { + const resourceSubtypeBuckets: ResourceSubtypeAggregationBucket[] = + assetCountAggregation.resource_sub_type.buckets; + + const billableAssets = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values; + assetCount = resourceSubtypeBuckets + .filter((bucket) => billableAssets.includes(bucket.key)) + .reduce((acc, bucket) => acc + bucket.unique_assets.value, 0); + + resourceSubtypeCounterMap = assetCountAggregation.resource_sub_type.buckets.reduce( + (resourceMap, item) => { + // By the usage spec, the resource subtype counter should be a string // https://github.com/elastic/usage-api/blob/main/api/user-v1-spec.yml + resourceMap[item.key] = String(item.unique_assets.value); + return resourceMap; }, - }; + {} as ResourceSubtypeCounter + ); + } else { + assetCount = assetCountAggregation.unique_assets.value; + } + + if (assetCount > AGGREGATION_PRECISION_THRESHOLD) { + logger.warn( + `The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.` + ); + } + + const minTimestamp = new Date(assetCountAggregation.min_timestamp.value_as_string).toISOString(); + + const creationTimestamp = new Date(); + const minutes = creationTimestamp.getMinutes(); + if (minutes >= 30) { + creationTimestamp.setMinutes(30, 0, 0); + } else { + creationTimestamp.setMinutes(0, 0, 0); + } + const roundedCreationTimestamp = creationTimestamp.toISOString(); + + const usageRecord: UsageRecord = { + id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`, + usage_timestamp: minTimestamp, + creation_timestamp: creationTimestamp.toISOString(), + usage: { + type: CLOUD_SECURITY_TASK_TYPE, + sub_type: cloudSecuritySolution, + quantity: assetCount, + period_seconds: periodSeconds, + ...(resourceSubtypeCounterMap && { metadata: resourceSubtypeCounterMap }), + }, + source: { + id: taskId, + instance_group_id: projectId, + metadata: { tier }, + }, + }; - return usageRecord; - }); - return usageRecords; + return usageRecord; }; export const getAggregationByCloudSecuritySolution = ( cloudSecuritySolution: CloudSecuritySolutions ) => { + if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) + return { + resource_sub_type: { + terms: { + field: BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].filter_attribute, + }, + aggs: { + unique_assets: { + cardinality: { + field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier, + precision_threshold: AGGREGATION_PRECISION_THRESHOLD, + }, + }, + }, + }, + min_timestamp: { + min: { + field: '@timestamp', + }, + }, + }; + return { unique_assets: { cardinality: { @@ -97,8 +137,7 @@ export const getAggregationByCloudSecuritySolution = ( }; export const getSearchQueryByCloudSecuritySolution = ( - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { const mustFilters = []; @@ -117,20 +156,11 @@ export const getSearchQueryByCloudSecuritySolution = ( } if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) { - const billableAssetsConfig = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution]; - mustFilters.push({ term: { 'rule.benchmark.posture_type': cloudSecuritySolution, }, }); - - // filter in only billable assets - mustFilters.push({ - terms: { - [billableAssetsConfig.filter_attribute]: billableAssetsConfig.values, - }, - }); } return { @@ -141,10 +171,9 @@ export const getSearchQueryByCloudSecuritySolution = ( }; export const getAssetAggQueryByCloudSecuritySolution = ( - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { - const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom); + const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution); const aggs = getAggregationByCloudSecuritySolution(cloudSecuritySolution); return { @@ -157,28 +186,27 @@ export const getAssetAggQueryByCloudSecuritySolution = ( export const getAssetAggByCloudSecuritySolution = async ( esClient: ElasticsearchClient, - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date -): Promise => { - const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom); + cloudSecuritySolution: CloudSecuritySolutions +): Promise => { + const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution); const response = await esClient.search(assetsAggQuery); - if (!response.aggregations) return []; - return [response.aggregations]; + if (!response.aggregations) return; + + return response.aggregations; }; const indexHasDataInDateRange = async ( esClient: ElasticsearchClient, - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { const response = await esClient.search( { index: METERING_CONFIGS[cloudSecuritySolution].index, size: 1, _source: false, - query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom), + query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution), }, { ignore: [404] } ); @@ -186,47 +214,29 @@ const indexHasDataInDateRange = async ( return response.hits?.hits.length > 0; }; -const getSearchStartDate = (lastSuccessfulReport: Date): Date => { - const initialDate = new Date(); - const thresholdDate = new Date(initialDate.getTime() - THRESHOLD_MINUTES * 60 * 1000); - - if (lastSuccessfulReport) { - const lastSuccessfulReportDate = new Date(lastSuccessfulReport); - - const searchFrom = - lastSuccessfulReport && lastSuccessfulReportDate > thresholdDate - ? lastSuccessfulReportDate - : thresholdDate; - return searchFrom; - } - return thresholdDate; -}; - export const getCloudSecurityUsageRecord = async ({ esClient, projectId, taskId, - lastSuccessfulReport, cloudSecuritySolution, tier, logger, }: CloudSecurityMeteringCallbackInput): Promise => { try { - const searchFrom = getSearchStartDate(lastSuccessfulReport); - - if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution, searchFrom))) return; + if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution))) return; // const periodSeconds = Math.floor((new Date().getTime() - searchFrom.getTime()) / 1000); const periodSeconds = 1800; // Workaround to prevent overbilling by charging for a constant time window. The issue should be resolved in https://github.com/elastic/security-team/issues/9424. - const assetCountAggregations = await getAssetAggByCloudSecuritySolution( + const assetCountAggregation = await getAssetAggByCloudSecuritySolution( esClient, - cloudSecuritySolution, - searchFrom + cloudSecuritySolution ); + if (!assetCountAggregation) return []; + const usageRecords = await getUsageRecords( - assetCountAggregations, + assetCountAggregation, cloudSecuritySolution, taskId, tier, @@ -235,7 +245,7 @@ export const getCloudSecurityUsageRecord = async ({ logger ); - return usageRecords; + return [usageRecords]; } catch (err) { logger.error(`Failed to fetch ${cloudSecuritySolution} metering data ${err}`); } diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts index 62ded11d5ad1e..0ca9a7b5b943a 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts @@ -14,12 +14,24 @@ export interface CloudDefendAssetCountAggregation { export interface AssetCountAggregationBucket { buckets: AssetCountAggregation[]; } + +export interface ResourceSubtypeAggregationBucket { + key: string; + doc_count: number; + unique_assets: { + value: number; + }; +} + export interface AssetCountAggregation { key_as_string: string; min_timestamp: MinTimestamp; unique_assets: { value: number; }; + resource_sub_type: { + buckets: ResourceSubtypeAggregationBucket[]; + }; } export interface MinTimestamp { diff --git a/x-pack/plugins/security_solution_serverless/server/types.ts b/x-pack/plugins/security_solution_serverless/server/types.ts index eec8185a28199..8f8d4b36041a1 100644 --- a/x-pack/plugins/security_solution_serverless/server/types.ts +++ b/x-pack/plugins/security_solution_serverless/server/types.ts @@ -65,17 +65,13 @@ export interface UsageMetrics { quantity: number; period_seconds?: number; cause?: string; - metadata?: unknown; + metadata?: ResourceSubtypeCounter; } export interface UsageSource { id: string; instance_group_id: string; - metadata?: UsageSourceMetadata; -} - -export interface UsageSourceMetadata { - tier?: Tier; + metadata?: { tier?: Tier }; } export type Tier = ProductTier | 'none'; @@ -125,3 +121,6 @@ export interface MetringTaskProperties { periodSeconds: number; version: string; } +export interface ResourceSubtypeCounter { + [key: string]: string; +}