Skip to content

Commit

Permalink
[Cloud Security] Increase retention period on queries related to 3rd …
Browse files Browse the repository at this point in the history
…party data loading (elastic#195636)

## Summary

Increase retention on Cloud Security queries to accommodate a longer
retention period on third-party CDR integrations, such as Wiz and AWS
SecurityHub. This introduces regression for
elastic#142198
This is meant is a temporary workaround until we find a robust way to
get full posture for third-party CDR integrations
This change goes together with increasing retention period on Wiz:
- elastic/integrations#11393

fixes:
- elastic/security-team#10683

## How to test
The CI deployed a serverless project where I installed Wiz and CSP
integrations and ingested some data.

(cherry picked from commit e18c52e)
  • Loading branch information
maxcold committed Oct 11, 2024
1 parent bf886a5 commit ed6ff54
Show file tree
Hide file tree
Showing 9 changed files with 121 additions and 25 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ export const CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN =
export const CDR_VULNERABILITIES_INDEX_PATTERN = `${CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN},${CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN}`;
export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d';

// TODO: remove once https://github.com/elastic/security-team/issues/10801 is done
// meant as a temp workaround to get good enough posture view for 3rd party integrations, see https://github.com/elastic/security-team/issues/10683
export const CDR_3RD_PARTY_RETENTION_POLICY = '90d';

export const VULNERABILITIES_SEVERITY: Record<VulnSeverity, VulnSeverity> = {
LOW: 'LOW',
MEDIUM: 'MEDIUM',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import {
CDR_MISCONFIGURATIONS_INDEX_PATTERN,
CDR_VULNERABILITIES_INDEX_PATTERN,
LATEST_FINDINGS_RETENTION_POLICY,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
import { buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
Expand Down Expand Up @@ -103,7 +102,7 @@ const buildMisconfigurationsFindingsQueryWithFilters = (
{
range: {
'@timestamp': {
gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},
Expand Down Expand Up @@ -182,7 +181,7 @@ const buildVulnerabilityFindingsQueryWithFilters = (query: UseCspOptions['query'
{
range: {
'@timestamp': {
gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { showErrorToast } from '@kbn/cloud-security-posture';
import { MAX_FINDINGS_TO_LOAD, buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
import {
CDR_MISCONFIGURATIONS_INDEX_PATTERN,
LATEST_FINDINGS_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import type { CspFinding } from '@kbn/cloud-security-posture-common';
import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
Expand Down Expand Up @@ -77,7 +77,7 @@ export const getFindingsQuery = (
{
range: {
'@timestamp': {
gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
import { useMemo } from 'react';
import { buildEsQuery, Filter } from '@kbn/es-query';
import {
LATEST_FINDINGS_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
buildMutedRulesFilter,
} from '@kbn/cloud-security-posture-common';
import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
Expand Down Expand Up @@ -249,7 +249,7 @@ export const useLatestFindingsGrouping = ({
additionalFilters: query ? [query, additionalFilters] : [additionalFilters],
groupByField: currentSelectedGroup,
uniqueValue,
from: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
to: 'now',
pageNumber: activePageIndex * pageSize,
size: pageSize,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { EsHitRecord } from '@kbn/discover-utils/types';
import {
MAX_FINDINGS_TO_LOAD,
CDR_VULNERABILITIES_INDEX_PATTERN,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
Expand Down Expand Up @@ -90,7 +90,7 @@ export const getVulnerabilitiesQuery = (
{
range: {
'@timestamp': {
gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import {
} from '@kbn/grouping/src';
import { useMemo } from 'react';
import {
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
VULNERABILITIES_SEVERITY,
} from '@kbn/cloud-security-posture-common';
import { buildEsQuery, Filter } from '@kbn/es-query';
Expand Down Expand Up @@ -202,7 +202,7 @@ export const useLatestVulnerabilitiesGrouping = ({
additionalFilters: query ? [query, additionalFilters] : [additionalFilters],
groupByField: currentSelectedGroup,
uniqueValue,
from: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
to: 'now',
pageNumber: activePageIndex * pageSize,
size: pageSize,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_VULNERABILITIES_INDEX_PATTERN,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import type {
CspSetupStatus,
Expand Down Expand Up @@ -218,13 +219,13 @@ export const getCspStatus = async ({
checkIndexHasFindings(
esClient,
CDR_MISCONFIGURATIONS_INDEX_PATTERN,
LATEST_FINDINGS_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
logger
),
checkIndexHasFindings(
esClient,
CDR_VULNERABILITIES_INDEX_PATTERN,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
logger
),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,8 +366,7 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
});
const isLatestFindingsTableThere = async () => {
const table = await testSubjects.findAll('docTable');
const trueOrFalse = table.length > 0 ? true : false;
return trueOrFalse;
return table.length > 0;
};

const getUnprivilegedPrompt = async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,38 @@
import expect from '@kbn/expect';
import Chance from 'chance';
import type { FtrProviderContext } from '../ftr_provider_context';
import { vulnerabilitiesLatestMock } from '../mocks/vulnerabilities_latest_mock';

// eslint-disable-next-line import/no-default-export
export default function ({ getPageObjects }: FtrProviderContext) {
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const retry = getService('retry');
const pageObjects = getPageObjects(['common', 'findings', 'header']);
const chance = new Chance();
const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000;
const daysToMillisecond = (days: number) => days * 24 * 60 * 60 * 1000;
const RETENTION = 90;

const dataOldKspm = [
{
'@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
'@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
name: 'Upper case rule name',
section: 'Upper case section',
benchmark: {
id: 'cis_k8s',
posture_type: 'kspm',
name: 'CIS Kubernetes V1.23',
version: 'v1.0.0',
},
type: 'process',
},
cluster_id: 'Upper case cluster id',
},
];
const dataWithinRetentionKspm = [
{
'@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
Expand All @@ -37,7 +59,26 @@ export default function ({ getPageObjects }: FtrProviderContext) {

const dataOldCspm = [
{
'@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
'@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
name: 'Upper case rule name',
section: 'Upper case section',
benchmark: {
id: 'cis_aws',
posture_type: 'cspm',
name: 'CIS AWS V1.23',
version: 'v1.0.0',
},
type: 'process',
},
cluster_id: 'Upper case cluster id',
},
];
const dataWithinRetentionCspm = [
{
'@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
Expand All @@ -55,40 +96,92 @@ export default function ({ getPageObjects }: FtrProviderContext) {
},
];

const dataOldCnvm = [
{
...vulnerabilitiesLatestMock[0],
'@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
},
];
const dataWithinRetentionCnvm = [
{
...vulnerabilitiesLatestMock[0],
'@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
},
];

describe('Old Data', function () {
this.tags(['cloud_security_posture_findings']);
let findings: typeof pageObjects.findings;
let latestFindingsTable: typeof findings.latestFindingsTable;
let latestVulnerabilitiesTable: typeof findings.latestVulnerabilitiesTable;

before(async () => {
findings = pageObjects.findings;
latestFindingsTable = findings.latestFindingsTable;
latestVulnerabilitiesTable = findings.latestVulnerabilitiesTable;

// Before we start any test we must wait for cloud_security_posture plugin to complete its initialization
await findings.waitForPluginInitialized();
});

after(async () => {
afterEach(async () => {
await findings.index.remove();
await findings.vulnerabilitiesIndex.remove();
});

describe('Findings page with old data', () => {
it('returns no Findings KSPM', async () => {
// Prepare mocked findings
await findings.index.remove();
await findings.index.add(dataOldKspm);

await findings.navigateToLatestFindingsPage();
await pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns no Findings CSPM', async () => {
// Prepare mocked findings
await findings.index.remove();
await findings.index.add(dataOldCspm);

await findings.navigateToLatestFindingsPage();
await pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns no Findings CNVM', async () => {
await findings.vulnerabilitiesIndex.add(dataOldCnvm);

await findings.navigateToLatestVulnerabilitiesPage();
await pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns data grid with only data within retention KSPM', async () => {
await findings.index.add([...dataOldKspm, ...dataWithinRetentionKspm]);

await findings.navigateToLatestFindingsPage();
await retry.waitFor(
'Findings table to be loaded',
async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionKspm.length
);
await pageObjects.header.waitUntilLoadingHasFinished();
});
it('returns data grid with only data within retention CSPM', async () => {
await findings.index.add([...dataOldCspm, ...dataWithinRetentionCspm]);

await findings.navigateToLatestFindingsPage();
await retry.waitFor(
'Findings table to be loaded',
async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionCspm.length
);
await pageObjects.header.waitUntilLoadingHasFinished();
});
it('returns data grid with only data within retention CSPM', async () => {
await findings.vulnerabilitiesIndex.add([...dataOldCnvm, ...dataWithinRetentionCnvm]);

await findings.navigateToLatestVulnerabilitiesPage();
await retry.waitFor(
'Findings table to be loaded',
async () =>
(await latestVulnerabilitiesTable.getRowsCount()) === dataWithinRetentionCnvm.length
);
await pageObjects.header.waitUntilLoadingHasFinished();
});
});
});
}

0 comments on commit ed6ff54

Please sign in to comment.