diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml index 7aff13525a2fc..7697da4b3edaf 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml @@ -17,3 +17,66 @@ steps: automatic: - exit_status: '-1' limit: 1 + + - group: "API MKI - Explore" + key: api_test_explore + steps: + - label: Running explore:hosts:runner:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:hosts:runner:qa:serverless + key: explore:hosts:runner:qa:serverless + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running explore:network:runner:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:network:runner:qa:serverless + key: explore:network:runner:qa:serverless + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running explore:overview:runner:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:overview:runner:qa:serverless + key: explore:overview:runner:qa:serverless + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running explore:users:runner:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:users:runner:qa:serverless + key: explore:users:runner:qa:serverless + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml index d19d709231e31..0988bf6ecf6b8 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml @@ -17,3 +17,36 @@ steps: automatic: - exit_status: '-1' limit: 1 + + - group: "API MKI - Investigations" + key: api_test_investigations + steps: + - label: Running investigations:timeline:runner:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh investigations:timeline:runner:qa:serverless + key: investigations:timeline:runner:qa:serverless + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running investigations:saved-objects:runner:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh investigations:saved-objects:runner:qa:serverless + key: investigations:saved-objects:runner:qa:serverless + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_explore.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_explore.yml index e60f4509fcb3e..2c518fa24efab 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_explore.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_explore.yml @@ -17,3 +17,66 @@ steps: automatic: - exit_status: '-1' limit: 1 + + - group: "API MKI - Explore" + key: api_test_explore + steps: + - label: Running explore:hosts:runner:qa:serverless:release + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:hosts:runner:qa:serverless:release + key: explore:hosts:runner:qa:serverless:release + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running explore:network:runner:qa:serverless:release + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:network:runner:qa:serverless:release + key: explore:network:runner:qa:serverless:release + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running explore:overview:runner:qa:serverless:release + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:overview:runner:qa:serverless:release + key: explore:overview:runner:qa:serverless:release + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running explore:users:runner:qa:serverless:release + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh explore:users:runner:qa:serverless:release + key: explore:users:runner:qa:serverless:release + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_investigations.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_investigations.yml index ed46611989b87..d3f57e40ec2cb 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_investigations.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_quality_gate/mki_quality_gate_investigations.yml @@ -17,3 +17,36 @@ steps: automatic: - exit_status: '-1' limit: 1 + + - group: "API MKI - Investigations" + key: api_test_investigations + steps: + - label: Running investigations:timeline:runner:qa:serverless:release + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh investigations:timeline:runner:qa:serverless:release + key: investigations:timeline:runner:qa:serverless:release + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + + - label: Running investigations:saved-objects:runner:qa:serverless:release + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh investigations:saved-objects:runner:qa:serverless:release + key: investigations:saved-objects:runner:qa:serverless:release + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 diff --git a/x-pack/plugins/ml/server/routes/system.ts b/x-pack/plugins/ml/server/routes/system.ts index 1d186a66893ae..ba985c8b0d395 100644 --- a/x-pack/plugins/ml/server/routes/system.ts +++ b/x-pack/plugins/ml/server/routes/system.ts @@ -147,7 +147,7 @@ export function systemRoutes( path: `${ML_INTERNAL_BASE_PATH}/ml_node_count`, access: 'internal', options: { - tags: ['access:ml:canGetJobs', 'access:ml:canGetDatafeeds'], + tags: ['access:ml:canGetMlInfo'], }, }) .addVersion( diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts index 72568fbd4c8a0..1a5fb8e64a265 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts @@ -21,6 +21,15 @@ export const cloudSecurityMetringCallback = async ({ lastSuccessfulReport, config, }: MeteringCallbackInput): Promise => { + const projectHasCloudProductLine = config.productTypes.some( + (product) => product.product_line === ProductLine.cloud + ); + + if (!projectHasCloudProductLine) { + logger.info('No cloud product line found in the project'); + return { records: [] }; + } + const projectId = cloudSetup?.serverless?.projectId || 'missing_project_id'; const tier: Tier = getCloudProductTier(config, logger); @@ -30,8 +39,8 @@ export const cloudSecurityMetringCallback = async ({ const promiseResults = await Promise.allSettled( cloudSecuritySolutions.map((cloudSecuritySolution) => { - if (cloudSecuritySolution === CLOUD_DEFEND) { - return getCloudDefendUsageRecords({ + if (cloudSecuritySolution !== CLOUD_DEFEND) { + return getCloudSecurityUsageRecord({ esClient, projectId, logger, @@ -41,7 +50,9 @@ export const cloudSecurityMetringCallback = async ({ tier, }); } - return getCloudSecurityUsageRecord({ + + // since lastSuccessfulReport is not used by getCloudSecurityUsageRecord, we want to verify if it is used by getCloudDefendUsageRecords before getCloudSecurityUsageRecord. + return getCloudDefendUsageRecords({ esClient, projectId, logger, diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts index f775b91406c4c..6d55e52469283 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts @@ -14,16 +14,23 @@ import { } from './cloud_security_metering_task'; import type { ServerlessSecurityConfig } from '../config'; -import type { CloudSecuritySolutions } from './types'; + import type { ProductTier } from '../../common/product'; -import { CLOUD_SECURITY_TASK_TYPE, CSPM, KSPM, CNVM, CLOUD_DEFEND } from './constants'; +import { + CLOUD_SECURITY_TASK_TYPE, + CSPM, + KSPM, + CNVM, + CLOUD_DEFEND, + BILLABLE_ASSETS_CONFIG, +} from './constants'; import { getCloudDefendUsageRecords } from './defend_for_containers_metering'; const mockEsClient = elasticsearchServiceMock.createStart().client.asInternalUser; const logger: ReturnType = loggingSystemMock.createLogger(); const chance = new Chance(); -const cloudSecuritySolutions: CloudSecuritySolutions[] = [CSPM, KSPM, CNVM]; +const cloudSecuritySolutions: Array = [CSPM, KSPM]; describe('getCloudSecurityUsageRecord', () => { beforeEach(() => { @@ -54,18 +61,33 @@ describe('getCloudSecurityUsageRecord', () => { }); test.each(cloudSecuritySolutions)( - 'should return usageRecords with correct values for cspm, kspm, and cnvm when Elasticsearch response has aggregations', + 'should return usageRecords with correct values for cspm and kspm when Elasticsearch response has aggregations', async (cloudSecuritySolution) => { // @ts-ignore mockEsClient.search.mockResolvedValueOnce({ hits: { hits: [{ _id: 'someRecord', _index: 'mockIndex' }] }, // mocking for indexHasDataInDateRange }); + const randomIndex = Math.floor( + Math.random() * BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values.length + ); + const randomBillableAsset = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values[randomIndex]; // @ts-ignore mockEsClient.search.mockResolvedValueOnce({ aggregations: { - unique_assets: { - value: 10, + resource_sub_type: { + buckets: [ + { + key: randomBillableAsset, + doc_count: 100, + unique_assets: { value: 10 }, + }, + { + key: 'not_billable_asset', + doc_count: 50, + unique_assets: { value: 11 }, + }, + ], }, min_timestamp: { value_as_string: '2023-07-30T15:11:41.738Z', @@ -100,6 +122,10 @@ describe('getCloudSecurityUsageRecord', () => { sub_type: cloudSecuritySolution, quantity: 10, period_seconds: expect.any(Number), + metadata: { + [randomBillableAsset]: '10', + not_billable_asset: '11', + }, }, source: { id: taskId, @@ -113,6 +139,62 @@ describe('getCloudSecurityUsageRecord', () => { } ); + it('should return usageRecords with correct values for cnvm when Elasticsearch response has aggregations', async () => { + const cloudSecuritySolution = CNVM; + + // @ts-ignore + mockEsClient.search.mockResolvedValueOnce({ + hits: { hits: [{ _id: 'someRecord', _index: 'mockIndex' }] }, // mocking for indexHasDataInDateRange + }); + + // @ts-ignore + mockEsClient.search.mockResolvedValueOnce({ + aggregations: { + unique_assets: { + value: 10, + }, + min_timestamp: { + value_as_string: '2023-07-30T15:11:41.738Z', + }, + }, + }); + + const projectId = chance.guid(); + const taskId = chance.guid(); + + const tier = 'essentials' as ProductTier; + const result = await getCloudSecurityUsageRecord({ + esClient: mockEsClient, + projectId, + logger, + taskId, + lastSuccessfulReport: new Date(), + cloudSecuritySolution, + tier, + }); + + expect(result).toEqual([ + { + id: expect.stringContaining(`${CLOUD_SECURITY_TASK_TYPE}_cnvm_${projectId}`), + usage_timestamp: '2023-07-30T15:11:41.738Z', + creation_timestamp: expect.any(String), // Expect a valid ISO string + usage: { + type: CLOUD_SECURITY_TASK_TYPE, + sub_type: CNVM, + quantity: 10, + period_seconds: expect.any(Number), + }, + source: { + id: taskId, + instance_group_id: projectId, + metadata: { + tier: 'essentials', + }, + }, + }, + ]); + }); + it('should return undefined when Elasticsearch response does not have aggregations', async () => { // @ts-ignore mockEsClient.search.mockResolvedValue({}); @@ -162,9 +244,7 @@ describe('getCloudSecurityUsageRecord', () => { describe('getSearchQueryByCloudSecuritySolution', () => { it('should return the correct search query for CSPM', () => { - const searchFrom = new Date('2023-07-30T15:11:41.738Z'); - - const result = getSearchQueryByCloudSecuritySolution('cspm', searchFrom); + const result = getSearchQueryByCloudSecuritySolution('cspm'); expect(result).toEqual({ bool: { @@ -181,39 +261,13 @@ describe('getSearchQueryByCloudSecuritySolution', () => { 'rule.benchmark.posture_type': 'cspm', }, }, - { - terms: { - 'resource.sub_type': [ - // 'aws-ebs', we can't include EBS volumes until https://github.com/elastic/security-team/issues/9283 is resolved - // 'aws-ec2', we can't include EC2 instances until https://github.com/elastic/security-team/issues/9254 is resolved - 'aws-s3', - 'aws-rds', - 'azure-disk', - 'azure-document-db-database-account', - 'azure-flexible-mysql-server-db', - 'azure-flexible-postgresql-server-db', - 'azure-mysql-server-db', - 'azure-postgresql-server-db', - 'azure-sql-server', - 'azure-storage-account', - 'azure-vm', - 'gcp-bigquery-dataset', - 'gcp-compute-disk', - 'gcp-compute-instance', - 'gcp-sqladmin-instance', - 'gcp-storage-bucket', - ], - }, - }, ], }, }); }); it('should return the correct search query for KSPM', () => { - const searchFrom = new Date('2023-07-30T15:11:41.738Z'); - - const result = getSearchQueryByCloudSecuritySolution('kspm', searchFrom); + const result = getSearchQueryByCloudSecuritySolution('kspm'); expect(result).toEqual({ bool: { @@ -230,20 +284,13 @@ describe('getSearchQueryByCloudSecuritySolution', () => { 'rule.benchmark.posture_type': 'kspm', }, }, - { - terms: { - 'resource.sub_type': ['Node', 'node'], - }, - }, ], }, }); }); it('should return the correct search query for CNVM', () => { - const searchFrom = new Date('2023-07-30T15:11:41.738Z'); - - const result = getSearchQueryByCloudSecuritySolution(CNVM, searchFrom); + const result = getSearchQueryByCloudSecuritySolution(CNVM); expect(result).toEqual({ bool: { diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts index 7827c5d25ebe6..6687c3dfed48c 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts @@ -15,72 +15,112 @@ import { CSPM, KSPM, METERING_CONFIGS, - THRESHOLD_MINUTES, BILLABLE_ASSETS_CONFIG, } from './constants'; -import type { Tier, UsageRecord } from '../types'; +import type { ResourceSubtypeCounter, Tier, UsageRecord } from '../types'; import type { CloudSecurityMeteringCallbackInput, CloudSecuritySolutions, AssetCountAggregation, + ResourceSubtypeAggregationBucket, } from './types'; export const getUsageRecords = ( - assetCountAggregations: AssetCountAggregation[], + assetCountAggregation: AssetCountAggregation, cloudSecuritySolution: CloudSecuritySolutions, taskId: string, tier: Tier, projectId: string, periodSeconds: number, logger: Logger -): UsageRecord[] => { - const usageRecords = assetCountAggregations.map((assetCountAggregation) => { - const assetCount = assetCountAggregation.unique_assets.value; - - if (assetCount > AGGREGATION_PRECISION_THRESHOLD) { - logger.warn( - `The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.` - ); - } - - const minTimestamp = new Date( - assetCountAggregation.min_timestamp.value_as_string - ).toISOString(); - - const creationTimestamp = new Date(); - const minutes = creationTimestamp.getMinutes(); - if (minutes >= 30) { - creationTimestamp.setMinutes(30, 0, 0); - } else { - creationTimestamp.setMinutes(0, 0, 0); - } - const roundedCreationTimestamp = creationTimestamp.toISOString(); - - const usageRecord: UsageRecord = { - id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`, - usage_timestamp: minTimestamp, - creation_timestamp: creationTimestamp.toISOString(), - usage: { - type: CLOUD_SECURITY_TASK_TYPE, - sub_type: cloudSecuritySolution, - quantity: assetCount, - period_seconds: periodSeconds, - }, - source: { - id: taskId, - instance_group_id: projectId, - metadata: { tier }, +): UsageRecord => { + let assetCount; + let resourceSubtypeCounterMap; + + if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) { + const resourceSubtypeBuckets: ResourceSubtypeAggregationBucket[] = + assetCountAggregation.resource_sub_type.buckets; + + const billableAssets = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values; + assetCount = resourceSubtypeBuckets + .filter((bucket) => billableAssets.includes(bucket.key)) + .reduce((acc, bucket) => acc + bucket.unique_assets.value, 0); + + resourceSubtypeCounterMap = assetCountAggregation.resource_sub_type.buckets.reduce( + (resourceMap, item) => { + // By the usage spec, the resource subtype counter should be a string // https://github.com/elastic/usage-api/blob/main/api/user-v1-spec.yml + resourceMap[item.key] = String(item.unique_assets.value); + return resourceMap; }, - }; + {} as ResourceSubtypeCounter + ); + } else { + assetCount = assetCountAggregation.unique_assets.value; + } + + if (assetCount > AGGREGATION_PRECISION_THRESHOLD) { + logger.warn( + `The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.` + ); + } + + const minTimestamp = new Date(assetCountAggregation.min_timestamp.value_as_string).toISOString(); + + const creationTimestamp = new Date(); + const minutes = creationTimestamp.getMinutes(); + if (minutes >= 30) { + creationTimestamp.setMinutes(30, 0, 0); + } else { + creationTimestamp.setMinutes(0, 0, 0); + } + const roundedCreationTimestamp = creationTimestamp.toISOString(); + + const usageRecord: UsageRecord = { + id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`, + usage_timestamp: minTimestamp, + creation_timestamp: creationTimestamp.toISOString(), + usage: { + type: CLOUD_SECURITY_TASK_TYPE, + sub_type: cloudSecuritySolution, + quantity: assetCount, + period_seconds: periodSeconds, + ...(resourceSubtypeCounterMap && { metadata: resourceSubtypeCounterMap }), + }, + source: { + id: taskId, + instance_group_id: projectId, + metadata: { tier }, + }, + }; - return usageRecord; - }); - return usageRecords; + return usageRecord; }; export const getAggregationByCloudSecuritySolution = ( cloudSecuritySolution: CloudSecuritySolutions ) => { + if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) + return { + resource_sub_type: { + terms: { + field: BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].filter_attribute, + }, + aggs: { + unique_assets: { + cardinality: { + field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier, + precision_threshold: AGGREGATION_PRECISION_THRESHOLD, + }, + }, + }, + }, + min_timestamp: { + min: { + field: '@timestamp', + }, + }, + }; + return { unique_assets: { cardinality: { @@ -97,8 +137,7 @@ export const getAggregationByCloudSecuritySolution = ( }; export const getSearchQueryByCloudSecuritySolution = ( - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { const mustFilters = []; @@ -117,20 +156,11 @@ export const getSearchQueryByCloudSecuritySolution = ( } if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) { - const billableAssetsConfig = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution]; - mustFilters.push({ term: { 'rule.benchmark.posture_type': cloudSecuritySolution, }, }); - - // filter in only billable assets - mustFilters.push({ - terms: { - [billableAssetsConfig.filter_attribute]: billableAssetsConfig.values, - }, - }); } return { @@ -141,10 +171,9 @@ export const getSearchQueryByCloudSecuritySolution = ( }; export const getAssetAggQueryByCloudSecuritySolution = ( - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { - const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom); + const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution); const aggs = getAggregationByCloudSecuritySolution(cloudSecuritySolution); return { @@ -157,28 +186,27 @@ export const getAssetAggQueryByCloudSecuritySolution = ( export const getAssetAggByCloudSecuritySolution = async ( esClient: ElasticsearchClient, - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date -): Promise => { - const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom); + cloudSecuritySolution: CloudSecuritySolutions +): Promise => { + const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution); const response = await esClient.search(assetsAggQuery); - if (!response.aggregations) return []; - return [response.aggregations]; + if (!response.aggregations) return; + + return response.aggregations; }; const indexHasDataInDateRange = async ( esClient: ElasticsearchClient, - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { const response = await esClient.search( { index: METERING_CONFIGS[cloudSecuritySolution].index, size: 1, _source: false, - query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom), + query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution), }, { ignore: [404] } ); @@ -186,47 +214,29 @@ const indexHasDataInDateRange = async ( return response.hits?.hits.length > 0; }; -const getSearchStartDate = (lastSuccessfulReport: Date): Date => { - const initialDate = new Date(); - const thresholdDate = new Date(initialDate.getTime() - THRESHOLD_MINUTES * 60 * 1000); - - if (lastSuccessfulReport) { - const lastSuccessfulReportDate = new Date(lastSuccessfulReport); - - const searchFrom = - lastSuccessfulReport && lastSuccessfulReportDate > thresholdDate - ? lastSuccessfulReportDate - : thresholdDate; - return searchFrom; - } - return thresholdDate; -}; - export const getCloudSecurityUsageRecord = async ({ esClient, projectId, taskId, - lastSuccessfulReport, cloudSecuritySolution, tier, logger, }: CloudSecurityMeteringCallbackInput): Promise => { try { - const searchFrom = getSearchStartDate(lastSuccessfulReport); - - if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution, searchFrom))) return; + if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution))) return; // const periodSeconds = Math.floor((new Date().getTime() - searchFrom.getTime()) / 1000); const periodSeconds = 1800; // Workaround to prevent overbilling by charging for a constant time window. The issue should be resolved in https://github.com/elastic/security-team/issues/9424. - const assetCountAggregations = await getAssetAggByCloudSecuritySolution( + const assetCountAggregation = await getAssetAggByCloudSecuritySolution( esClient, - cloudSecuritySolution, - searchFrom + cloudSecuritySolution ); + if (!assetCountAggregation) return []; + const usageRecords = await getUsageRecords( - assetCountAggregations, + assetCountAggregation, cloudSecuritySolution, taskId, tier, @@ -235,7 +245,7 @@ export const getCloudSecurityUsageRecord = async ({ logger ); - return usageRecords; + return [usageRecords]; } catch (err) { logger.error(`Failed to fetch ${cloudSecuritySolution} metering data ${err}`); } diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts index 62ded11d5ad1e..0ca9a7b5b943a 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts @@ -14,12 +14,24 @@ export interface CloudDefendAssetCountAggregation { export interface AssetCountAggregationBucket { buckets: AssetCountAggregation[]; } + +export interface ResourceSubtypeAggregationBucket { + key: string; + doc_count: number; + unique_assets: { + value: number; + }; +} + export interface AssetCountAggregation { key_as_string: string; min_timestamp: MinTimestamp; unique_assets: { value: number; }; + resource_sub_type: { + buckets: ResourceSubtypeAggregationBucket[]; + }; } export interface MinTimestamp { diff --git a/x-pack/plugins/security_solution_serverless/server/types.ts b/x-pack/plugins/security_solution_serverless/server/types.ts index eec8185a28199..8f8d4b36041a1 100644 --- a/x-pack/plugins/security_solution_serverless/server/types.ts +++ b/x-pack/plugins/security_solution_serverless/server/types.ts @@ -65,17 +65,13 @@ export interface UsageMetrics { quantity: number; period_seconds?: number; cause?: string; - metadata?: unknown; + metadata?: ResourceSubtypeCounter; } export interface UsageSource { id: string; instance_group_id: string; - metadata?: UsageSourceMetadata; -} - -export interface UsageSourceMetadata { - tier?: Tier; + metadata?: { tier?: Tier }; } export type Tier = ProductTier | 'none'; @@ -125,3 +121,6 @@ export interface MetringTaskProperties { periodSeconds: number; version: string; } +export interface ResourceSubtypeCounter { + [key: string]: string; +} diff --git a/x-pack/test/api_integration/apis/ml/datafeeds/index.ts b/x-pack/test/api_integration/apis/ml/datafeeds/index.ts index 449a9b2622b8b..e7cd57640f28e 100644 --- a/x-pack/test/api_integration/apis/ml/datafeeds/index.ts +++ b/x-pack/test/api_integration/apis/ml/datafeeds/index.ts @@ -12,5 +12,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./get_with_spaces')); loadTestFile(require.resolve('./get_stats_with_spaces')); loadTestFile(require.resolve('./update')); + loadTestFile(require.resolve('./preview')); }); } diff --git a/x-pack/test/api_integration/apis/ml/datafeeds/preview.ts b/x-pack/test/api_integration/apis/ml/datafeeds/preview.ts new file mode 100644 index 0000000000000..8fd305fb5b0d5 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/datafeeds/preview.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api'; + +export default ({ getService }: FtrProviderContext) => { + const ml = getService('ml'); + const esArchiver = getService('esArchiver'); + const spacesService = getService('spaces'); + const supertest = getService('supertestWithoutAuth'); + + const jobIdSpace1 = 'fq_single_space1'; + const datafeedIdSpace1 = `datafeed-${jobIdSpace1}`; + const idSpace1 = 'space1'; + const idSpace2 = 'space2'; + + async function getDatafeedPreview( + datafeedId: string, + expectedStatusCode: number, + space?: string + ) { + const { body, status } = await supertest + .get(`${space ? `/s/${space}` : ''}/internal/ml/datafeeds/${datafeedId}/_preview`) + .auth( + USER.ML_POWERUSER_ALL_SPACES, + ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER_ALL_SPACES) + ) + .set(getCommonRequestHeader('1')); + ml.api.assertResponseStatusCode(expectedStatusCode, status, body); + + return body; + } + + describe('GET datafeed preview', () => { + before(async () => { + await spacesService.create({ id: idSpace1, name: 'space_one', disabledFeatures: [] }); + await spacesService.create({ id: idSpace2, name: 'space_two', disabledFeatures: [] }); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + + const jobConfig = ml.commonConfig.getADFqSingleMetricJobConfig(jobIdSpace1); + await ml.api.createAnomalyDetectionJob(jobConfig, idSpace1); + const datafeedConfig = ml.commonConfig.getADFqDatafeedConfig(jobIdSpace1); + await ml.api.createDatafeed(datafeedConfig, idSpace1); + + await ml.testResources.setKibanaTimeZoneToUTC(); + }); + + after(async () => { + await spacesService.delete(idSpace1); + await spacesService.delete(idSpace2); + await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); + }); + + it('should fail with non-existing datafeed', async () => { + await getDatafeedPreview('non-existing-datafeed', 404); + }); + + it('should return datafeed preview with datafeed id from correct space', async () => { + const body = await getDatafeedPreview(datafeedIdSpace1, 200, idSpace1); + expect(body.length).to.eql(1000, `response length should be 1000 (got ${body.length})`); + }); + + it('should fail with datafeed from different space', async () => { + await getDatafeedPreview(datafeedIdSpace1, 404, idSpace2); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/system/index.ts b/x-pack/test/api_integration/apis/ml/system/index.ts index 8b9aef9b813c9..5c332fb33cedc 100644 --- a/x-pack/test/api_integration/apis/ml/system/index.ts +++ b/x-pack/test/api_integration/apis/ml/system/index.ts @@ -12,5 +12,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./capabilities')); loadTestFile(require.resolve('./space_capabilities')); loadTestFile(require.resolve('./index_exists')); + loadTestFile(require.resolve('./node_count')); }); } diff --git a/x-pack/test/api_integration/apis/ml/system/node_count.ts b/x-pack/test/api_integration/apis/ml/system/node_count.ts new file mode 100644 index 0000000000000..08fa7abe482ee --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/system/node_count.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + async function runRequest(user: USER, expectedStatusCode: number) { + const { body, status } = await supertest + .get(`/internal/ml/ml_node_count`) + .auth(user, ml.securityCommon.getPasswordForUser(user)) + .set(getCommonRequestHeader('1')); + ml.api.assertResponseStatusCode(expectedStatusCode, status, body); + + return body; + } + + describe('GET ml/ml_node_count', function () { + describe('get ml node count', () => { + it('should match expected values', async () => { + const resp = await runRequest(USER.ML_POWERUSER, 200); + expect(resp.count).to.be.greaterThan(0, 'count should be greater than 0'); + expect(resp.lazyNodeCount).to.be.greaterThan( + -1, + 'lazyNodeCount should be greater or equal to 0' + ); + }); + + it('should should fail for a unauthorized user', async () => { + await runRequest(USER.ML_UNAUTHORIZED, 403); + }); + }); + }); +};