From cd04cd305a29431200f9f8e08211ee0065fcfaca Mon Sep 17 00:00:00 2001 From: Rickyanto Ang Date: Tue, 4 Jul 2023 10:51:09 -0700 Subject: [PATCH] [Cloud Security][FTR]Refactor API FTR to use .to.eql instead of .to.be (#160694) ## Summary This PR is for refactoring current API FTR to use .to.eql instead of .to.be for more understandable error message when an error occurs. Currently when test fail due to unmatched value, the error message can be quite confusing as we don't know which one is the expected and actual value from current error message. With this change from this PR it's easier to see which is the expected value and which is the actual value Screenshot 2023-06-27 at 7 49 07 PM --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../get_csp_rule_template.ts | 35 +++++-- .../apis/cloud_security_posture/helper.ts | 14 +++ .../status/status_index_timeout.ts | 15 ++- .../status/status_indexed.ts | 15 ++- .../status/status_indexing.ts | 15 ++- .../status_not_deployed_not_installed.ts | 75 ++++++++++++--- .../status/status_unprivileged.ts | 94 ++++++++++++++----- .../status/status_waiting_for_results.ts | 15 ++- 8 files changed, 217 insertions(+), 61 deletions(-) diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/get_csp_rule_template.ts b/x-pack/test/api_integration/apis/cloud_security_posture/get_csp_rule_template.ts index 1d91efdc7fe89..99fa403c22635 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/get_csp_rule_template.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/get_csp_rule_template.ts @@ -55,8 +55,9 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(500); - expect(body.message).to.be( - 'Please provide either benchmarkId or packagePolicyId, but not both' + expect(body.message).to.eql( + 'Please provide either benchmarkId or packagePolicyId, but not both', + `expected message to be 'Please provide either benchmarkId or packagePolicyId, but not both' but got ${body.message} instead` ); }); @@ -80,8 +81,9 @@ export default function ({ getService }: FtrProviderContext) { }) .expect(500); - expect(body.message).to.be( - 'Please provide either benchmarkId or packagePolicyId, but not both' + expect(body.message).to.eql( + 'Please provide either benchmarkId or packagePolicyId, but not both', + `expected message to be 'Please provide either benchmarkId or packagePolicyId, but not both' but got ${body.message} instead` ); }); @@ -95,8 +97,14 @@ export default function ({ getService }: FtrProviderContext) { }) .expect(404); - expect(body.statusCode).to.be(404); - expect(body.error).to.be('Not Found'); + expect(body.statusCode).to.eql( + 404, + `expected status code to be 404 but got ${body.statusCode} instead` + ); + expect(body.error).to.eql( + 'Not Found', + `expected error message to be 'Not Found' but got ${body.error} instead` + ); }); it(`Should return 200 status code and filter rules by benchmarkId`, async () => { @@ -124,7 +132,10 @@ export default function ({ getService }: FtrProviderContext) { (rule: CspRuleTemplate) => rule.metadata.benchmark.id === 'cis_k8s' ); - expect(allRulesHaveCorrectBenchmarkId).to.be(true); + expect(allRulesHaveCorrectBenchmarkId).to.eql( + true, + `expected true but got ${allRulesHaveCorrectBenchmarkId} instead` + ); }); it(`Should return 200 status code, and only requested fields in the response`, async () => { @@ -157,7 +168,7 @@ export default function ({ getService }: FtrProviderContext) { ); }); - expect(fieldsMatched).to.be(true); + expect(fieldsMatched).to.eql(true, `expected true but got ${fieldsMatched} instead`); }); it(`Should return 200 status code, items sorted by metadata.section field`, async () => { @@ -188,7 +199,8 @@ export default function ({ getService }: FtrProviderContext) { const isSorted = sections.every( (section, index) => index === 0 || section >= sections[index - 1] ); - expect(isSorted).to.be(true); + + expect(isSorted).to.eql(true, `expected true but got ${isSorted} instead`); }); it(`Should return 200 status code and paginate rules with a limit of PerPage`, async () => { @@ -213,7 +225,10 @@ export default function ({ getService }: FtrProviderContext) { }) .expect(200); - expect(body.items.length).to.be(perPage); + expect(body.items.length).to.eql( + perPage, + `expected length to be ${perPage} but got ${body.items.length} instead` + ); }); }); } diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts b/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts index b659153b1bf69..79aede12385db 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts @@ -7,6 +7,8 @@ import type { SuperTest, Test } from 'supertest'; import { Client } from '@elastic/elasticsearch'; +import expect from '@kbn/expect'; +import type { IndexDetails } from '@kbn/cloud-security-posture-plugin/common/types'; import { SecurityService } from '../../../../../test/common/services/security/security'; export const deleteIndex = (es: Client, indexToBeDeleted: string[]) => { @@ -141,3 +143,15 @@ export const deleteRole = async (security: SecurityService, roleName: string) => export const deleteUser = async (security: SecurityService, userName: string) => { await security.user.delete(userName); }; + +export const assertIndexStatus = ( + indicesDetails: IndexDetails[], + indexName: string, + expectedStatus: string +) => { + const actualValue = indicesDetails.find((idx) => idx.index === indexName)?.status; + expect(actualValue).to.eql( + expectedStatus, + `expected ${indexName} status to be ${expectedStatus} but got ${actualValue} instead` + ); +}; diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_index_timeout.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_index_timeout.ts index fe52d8d3a0773..2203a6374db70 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_index_timeout.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_index_timeout.ts @@ -110,7 +110,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.kspm.status).to.be('index-timeout'); + expect(res.kspm.status).to.eql( + 'index-timeout', + `expected kspm status to be index-timeout but got ${res.kspm.status} instead` + ); }); it(`Should return index-timeout when installed cspm, has findings only on logs-cloud_security_posture.findings-default* and it has been more than 10 minutes since the installation`, async () => { @@ -137,7 +140,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.cspm.status).to.be('index-timeout'); + expect(res.cspm.status).to.eql( + 'index-timeout', + `expected cspm status to be index-timeout but got ${res.cspm.status} instead` + ); }); it(`Should return index-timeout when installed cnvm, has findings only on logs-cloud_security_posture.vulnerabilities-default* and it has been more than 4 hours minutes since the installation`, async () => { @@ -164,7 +170,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.vuln_mgmt.status).to.be('index-timeout'); + expect(res.vuln_mgmt.status).to.eql( + 'index-timeout', + `expected vuln_mgmt status to be index-timeout but got ${res.vuln_mgmt.status} instead` + ); }); }); }); diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexed.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexed.ts index aa9d6d3289e95..594babe643b05 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexed.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexed.ts @@ -76,7 +76,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.kspm.status).to.be('indexed'); + expect(res.kspm.status).to.eql( + 'indexed', + `expected kspm status to be indexed but got ${res.kspm.status} instead` + ); }); it(`Return cspm status indexed when logs-cloud_security_posture.findings_latest-default contains new cspm documents`, async () => { @@ -95,7 +98,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.cspm.status).to.be('indexed'); + expect(res.cspm.status).to.eql( + 'indexed', + `expected cspm status to be indexed but got ${res.cspm.status} instead` + ); }); it(`Return vuln status indexed when logs-cloud_security_posture.vulnerabilities_latest-default contains new documents`, async () => { @@ -114,7 +120,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.vuln_mgmt.status).to.be('indexed'); + expect(res.vuln_mgmt.status).to.eql( + 'indexed', + `expected vuln_mgmt status to be indexed but got ${res.vuln_mgmt.status} instead` + ); }); }); }); diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexing.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexing.ts index ef16eb94d8a33..ef38ab85efb04 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexing.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_indexing.ts @@ -75,7 +75,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.kspm.status).to.be('indexing'); + expect(res.kspm.status).to.eql( + 'indexing', + `expected kspm status to be indexing but got ${res.kspm.status} instead` + ); }); it(`Return cspm status indexing when logs-cloud_security_posture.findings_latest-default doesn't contain new cspm documents, but has newly connected agents `, async () => { @@ -94,7 +97,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.cspm.status).to.be('indexing'); + expect(res.cspm.status).to.eql( + 'indexing', + `expected cspm status to be indexing but got ${res.cspm.status} instead` + ); }); it(`Return vuln status indexing when logs-cloud_security_posture.vulnerabilities_latest-default doesn't contain vuln new documents, but has newly connected agents`, async () => { @@ -113,7 +119,10 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.vuln_mgmt.status).to.be('indexing'); + expect(res.vuln_mgmt.status).to.eql( + 'indexing', + `expected vuln_mgmt status to be indexing but got ${res.vuln_mgmt.status} instead` + ); }); }); }); diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_not_deployed_not_installed.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_not_deployed_not_installed.ts index d7d77c93ecad4..dcfbedae15741 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_not_deployed_not_installed.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_not_deployed_not_installed.ts @@ -55,11 +55,26 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.kspm.status).to.be('not-deployed'); - expect(res.cspm.status).to.be('not-installed'); - expect(res.vuln_mgmt.status).to.be('not-installed'); - expect(res.kspm.healthyAgents).to.be(0); - expect(res.kspm.installedPackagePolicies).to.be(1); + expect(res.kspm.status).to.eql( + 'not-deployed', + `expected kspm status to be not-deployed but got ${res.kspm.status} instead` + ); + expect(res.cspm.status).to.eql( + 'not-installed', + `expected cspm status to be not-installed but got ${res.cspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'not-installed', + `expected vuln_mgmt status to be not-installed but got ${res.vuln_mgmt.status} instead` + ); + expect(res.kspm.healthyAgents).to.eql( + 0, + `expected number of kspm healthy agents to be 0 but got ${res.kspm.healthyAgents} instead` + ); + expect(res.kspm.installedPackagePolicies).to.eql( + 1, + `expected number of kspm installed package policies to be 1 but got ${res.kspm.installedPackagePolicies} instead` + ); }); it(`Should return not-deployed when installed cspm, no findings on either indices and no healthy agents`, async () => { @@ -78,11 +93,26 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.cspm.status).to.be('not-deployed'); - expect(res.kspm.status).to.be('not-installed'); - expect(res.vuln_mgmt.status).to.be('not-installed'); - expect(res.cspm.healthyAgents).to.be(0); - expect(res.cspm.installedPackagePolicies).to.be(1); + expect(res.cspm.status).to.eql( + 'not-deployed', + `expected cspm status to be not-deployed but got ${res.cspm.status} instead` + ); + expect(res.kspm.status).to.eql( + 'not-installed', + `expected kspm status to be not-installed but got ${res.kspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'not-installed', + `expected vuln_mgmt status to be not-installed but got ${res.vuln_mgmt.status} instead` + ); + expect(res.cspm.healthyAgents).to.eql( + 0, + `expected number of cspm healthy agents to be 0 but got ${res.cspm.healthyAgents} instead` + ); + expect(res.cspm.installedPackagePolicies).to.eql( + 1, + `expected number of cspm installed package policies to be 1 but got ${res.cspm.installedPackagePolicies} instead` + ); }); it(`Should return not-deployed when installed cnvm, no findings on either indices and no healthy agents`, async () => { @@ -101,11 +131,26 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.cspm.status).to.be('not-installed'); - expect(res.kspm.status).to.be('not-installed'); - expect(res.vuln_mgmt.status).to.be('not-deployed'); - expect(res.vuln_mgmt.healthyAgents).to.be(0); - expect(res.vuln_mgmt.installedPackagePolicies).to.be(1); + expect(res.cspm.status).to.eql( + 'not-installed', + `expected cspm status to be not-installed but got ${res.cspm.status} instead` + ); + expect(res.kspm.status).to.eql( + 'not-installed', + `expected kspm status to be not-installed but got ${res.kspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'not-deployed', + `expected vuln_mgmt status to be not-deployed but got ${res.vuln_mgmt.status} instead` + ); + expect(res.vuln_mgmt.healthyAgents).to.eql( + 0, + `expected number of vuln_mgmt healthy agents to be 0 but got ${res.vuln_mgmt.healthyAgents} instead` + ); + expect(res.vuln_mgmt.installedPackagePolicies).to.eql( + 1, + `expected number of vuln_mgmt installed package policies to be 1 but got ${res.vuln_mgmt.installedPackagePolicies} instead` + ); }); }); }); diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts index 2432165566de8..7d1445932fa6c 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts @@ -11,6 +11,7 @@ import { BENCHMARK_SCORE_INDEX_DEFAULT_NS, LATEST_FINDINGS_INDEX_DEFAULT_NS, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + FINDINGS_INDEX_PATTERN, } from '@kbn/cloud-security-posture-plugin/common/constants'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { @@ -20,6 +21,7 @@ import { deleteRole, deleteUser, deleteIndex, + assertIndexStatus, } from '../helper'; const UNPRIVILEGED_ROLE = 'unprivileged_test_role'; @@ -85,9 +87,18 @@ export default function (providerContext: FtrProviderContext) { .auth(UNPRIVILEGED_USERNAME, 'changeme') .expect(200); - expect(res.kspm.status).to.be('unprivileged'); - expect(res.cspm.status).to.be('unprivileged'); - expect(res.vuln_mgmt.status).to.be('unprivileged'); + expect(res.kspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.kspm.status} instead` + ); + expect(res.cspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.cspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.vuln_mgmt.status} instead` + ); }); }); @@ -134,14 +145,27 @@ export default function (providerContext: FtrProviderContext) { .auth(UNPRIVILEGED_USERNAME, 'changeme') .expect(200); - expect(res.kspm.status).to.be('unprivileged'); - expect(res.cspm.status).to.be('unprivileged'); - expect(res.vuln_mgmt.status).to.be('unprivileged'); + expect(res.kspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.kspm.status} instead` + ); + expect(res.cspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.cspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.vuln_mgmt.status} instead` + ); - expect(res.indicesDetails[0].status).to.be('empty'); - expect(res.indicesDetails[1].status).to.be('empty'); - expect(res.indicesDetails[2].status).to.be('unprivileged'); - expect(res.indicesDetails[3].status).to.be('unprivileged'); + assertIndexStatus(res.indicesDetails, LATEST_FINDINGS_INDEX_DEFAULT_NS, 'empty'); + assertIndexStatus(res.indicesDetails, FINDINGS_INDEX_PATTERN, 'empty'); + assertIndexStatus(res.indicesDetails, BENCHMARK_SCORE_INDEX_DEFAULT_NS, 'unprivileged'); + assertIndexStatus( + res.indicesDetails, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + 'unprivileged' + ); }); it(`Return unprivileged when missing access to score index`, async () => { @@ -165,14 +189,27 @@ export default function (providerContext: FtrProviderContext) { .auth(UNPRIVILEGED_USERNAME, 'changeme') .expect(200); - expect(res.kspm.status).to.be('unprivileged'); - expect(res.cspm.status).to.be('unprivileged'); - expect(res.vuln_mgmt.status).to.be('unprivileged'); + expect(res.kspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.kspm.status} instead` + ); + expect(res.cspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.cspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.vuln_mgmt.status} instead` + ); - expect(res.indicesDetails[0].status).to.be('unprivileged'); - expect(res.indicesDetails[1].status).to.be('empty'); - expect(res.indicesDetails[2].status).to.be('empty'); - expect(res.indicesDetails[3].status).to.be('unprivileged'); + assertIndexStatus(res.indicesDetails, LATEST_FINDINGS_INDEX_DEFAULT_NS, 'unprivileged'); + assertIndexStatus(res.indicesDetails, FINDINGS_INDEX_PATTERN, 'empty'); + assertIndexStatus(res.indicesDetails, BENCHMARK_SCORE_INDEX_DEFAULT_NS, 'empty'); + assertIndexStatus( + res.indicesDetails, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + 'unprivileged' + ); }); it(`Return unprivileged when missing access to vulnerabilities_latest index`, async () => { @@ -199,14 +236,23 @@ export default function (providerContext: FtrProviderContext) { .auth(UNPRIVILEGED_USERNAME, 'changeme') .expect(200); - expect(res.kspm.status).to.be('unprivileged'); - expect(res.cspm.status).to.be('unprivileged'); - expect(res.vuln_mgmt.status).to.be('not-installed'); + expect(res.kspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.kspm.status} instead` + ); + expect(res.cspm.status).to.eql( + 'unprivileged', + `expected unprivileged but got ${res.cspm.status} instead` + ); + expect(res.vuln_mgmt.status).to.eql( + 'not-installed', + `expected not-installed but got ${res.vuln_mgmt.status} instead` + ); - expect(res.indicesDetails[0].status).to.be('unprivileged'); - expect(res.indicesDetails[1].status).to.be('empty'); - expect(res.indicesDetails[2].status).to.be('unprivileged'); - expect(res.indicesDetails[3].status).to.be('empty'); + assertIndexStatus(res.indicesDetails, LATEST_FINDINGS_INDEX_DEFAULT_NS, 'unprivileged'); + assertIndexStatus(res.indicesDetails, FINDINGS_INDEX_PATTERN, 'empty'); + assertIndexStatus(res.indicesDetails, BENCHMARK_SCORE_INDEX_DEFAULT_NS, 'unprivileged'); + assertIndexStatus(res.indicesDetails, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, 'empty'); }); }); }); diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_waiting_for_results.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_waiting_for_results.ts index 53692014f767a..bc6a44100dab0 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_waiting_for_results.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_waiting_for_results.ts @@ -91,7 +91,10 @@ export default function (providerContext: FtrProviderContext) { .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.kspm.status).to.be('waiting_for_results'); + expect(res.kspm.status).to.eql( + 'waiting_for_results', + `expected kspm status to be waiting_for_results but got ${res.kspm.status} instead` + ); }); it(`Should return waiting_for_result when installed cspm, has no findings and it has been less than 10 minutes since the installation`, async () => { @@ -117,7 +120,10 @@ export default function (providerContext: FtrProviderContext) { .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.cspm.status).to.be('waiting_for_results'); + expect(res.cspm.status).to.eql( + 'waiting_for_results', + `expected cspm status to be waiting_for_results but got ${res.cspm.status} instead` + ); }); it(`Should return waiting_for_result when installed cnvm, has no findings and it has been less than 4 hours minutes since the installation`, async () => { @@ -143,7 +149,10 @@ export default function (providerContext: FtrProviderContext) { .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set('kbn-xsrf', 'xxxx') .expect(200); - expect(res.vuln_mgmt.status).to.be('waiting_for_results'); + expect(res.vuln_mgmt.status).to.eql( + 'waiting_for_results', + `expected vuln_mgmt status to be waiting_for_results but got ${res.vuln_mgmt.status} instead` + ); }); }); });