Skip to content

Commit

Permalink
[Security Solution][Exceptions][API testing] Add skipInQA tag and app…
Browse files Browse the repository at this point in the history 
…lied for failing tests (elastic#169646)

## Summary

- In order to merge the MKI pipeline for FTR added in this PR
elastic#169422 we need to merge this one
first
- Introduce the "skipInQA" tag along with its respective commands, and
apply it to the failing tests.
- The skipped tests are failing because the `elastic` user on QA is
missing some permissions on the `restricted_indices `
<img width="1098" alt="image"
src="https://github.com/elastic/kibana/assets/12671903/2ce35392-9ec4-4448-8304-d08e26c21401">
- Once we fix this issue in MKI/QA env those tests will be unskipped
again

https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-api-integration/builds/6#018b47bd-edfc-4608-bce1-5b435b1a01d4
  • Loading branch information
@WafaaNasr
WafaaNasr authored Oct 25, 2023
1 parent 33c409b commit d91f0e8
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 9 deletions.
5 changes: 5 additions & 0 deletions x-pack/test/security_solution_api_integration/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,27 @@
"scripts": {
"exception_workflows:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts",
"exception_workflows:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert",
"exception_workflows:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert",
"exception_workflows:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts",
"exception_workflows:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts --grep @ess",
"exception_operators_date_numeric_types:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts",
"exception_operators_date_numeric_types:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert",
"exception_operators_date_numeric_types:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert",
"exception_operators_date_numeric_types:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts",
"exception_operators_date_numeric_types:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts --grep @ess",
"exception_operators_keyword_text_long:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts",
"exception_operators_keyword_text_long:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert",
"exception_operators_keyword_text_long:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert",
"exception_operators_keyword_text_long:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts",
"exception_operators_keyword_text_long:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts --grep @ess",
"exception_operators_ips_text_array:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts",
"exception_operators_ips_text_array:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert",
"exception_operators_ips_text_array:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert",
"exception_operators_ips_text_array:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts",
"exception_operators_ips_text_array:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts --grep @ess",
"rule_creation:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts",
"rule_creation:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert",
"rule_creation:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert",
"rule_creation:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts",
"rule_creation:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts --grep @ess"
}
Expand Down
18 changes: 9 additions & 9 deletions ...ons_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ export default ({ getService }: FtrProviderContext) => {
expect(bodyToCompare).toEqual(expected);
});

it('should allow removing an exception list from an immutable rule through patch', async () => {
it('@skipInQA should allow removing an exception list from an immutable rule through patch', async () => {
await installMockPrebuiltRules(supertest, es);

// This rule has an existing exceptions_list that we are going to use
Expand All @@ -199,7 +199,7 @@ export default ({ getService }: FtrProviderContext) => {
expect(immutableRuleSecondTime.exceptions_list.length).toEqual(0);
});

it('should allow adding a second exception list to an immutable rule through patch', async () => {
it('@skipInQA should allow adding a second exception list to an immutable rule through patch', async () => {
await installMockPrebuiltRules(supertest, es);

const { id, list_id, namespace_type, type } = await createExceptionList(
Expand Down Expand Up @@ -236,7 +236,7 @@ export default ({ getService }: FtrProviderContext) => {
expect(immutableRuleSecondTime.exceptions_list.length).toEqual(2);
});

it('should override any updates to pre-packaged rules if the user removes the exception list through the API but the new version of a rule has an exception list again', async () => {
it('@skipInQA should override any updates to pre-packaged rules if the user removes the exception list through the API but the new version of a rule has an exception list again', async () => {
await installMockPrebuiltRules(supertest, es);

// This rule has an existing exceptions_list that we are going to use
Expand All @@ -259,7 +259,7 @@ export default ({ getService }: FtrProviderContext) => {
expect(immutableRuleSecondTime.exceptions_list).toEqual(immutableRule.exceptions_list);
});

it('should merge back an exceptions_list if it was removed from the immutable rule through PATCH', async () => {
it('@skipInQA should merge back an exceptions_list if it was removed from the immutable rule through PATCH', async () => {
await installMockPrebuiltRules(supertest, es);

const { id, list_id, namespace_type, type } = await createExceptionList(
Expand Down Expand Up @@ -305,7 +305,7 @@ export default ({ getService }: FtrProviderContext) => {
]);
});

it('should NOT add an extra exceptions_list that already exists on a rule during an upgrade', async () => {
it('@skipInQA should NOT add an extra exceptions_list that already exists on a rule during an upgrade', async () => {
await installMockPrebuiltRules(supertest, es);

// This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule
Expand All @@ -324,7 +324,7 @@ export default ({ getService }: FtrProviderContext) => {
]);
});

it('should NOT allow updates to pre-packaged rules to overwrite existing exception based rules when the user adds an additional exception list', async () => {
it('@skipInQA should NOT allow updates to pre-packaged rules to overwrite existing exception based rules when the user adds an additional exception list', async () => {
await installMockPrebuiltRules(supertest, es);

const { id, list_id, namespace_type, type } = await createExceptionList(
Expand Down Expand Up @@ -371,7 +371,7 @@ export default ({ getService }: FtrProviderContext) => {
]);
});

it('should not remove any exceptions added to a pre-packaged/immutable rule during an update if that rule has no existing exception lists', async () => {
it('@skipInQA should not remove any exceptions added to a pre-packaged/immutable rule during an update if that rule has no existing exception lists', async () => {
await installMockPrebuiltRules(supertest, es);

// Create a new exception list
Expand Down Expand Up @@ -425,7 +425,7 @@ export default ({ getService }: FtrProviderContext) => {
]);
});

it('should not change the immutable tags when adding a second exception list to an immutable rule through patch', async () => {
it('@skipInQA should not change the immutable tags when adding a second exception list to an immutable rule through patch', async () => {
await installMockPrebuiltRules(supertest, es);

const { id, list_id, namespace_type, type } = await createExceptionList(
Expand Down Expand Up @@ -466,7 +466,7 @@ export default ({ getService }: FtrProviderContext) => {
expect(bodyToCompare.version).toEqual(immutableRule.version); // The version should never update on a patch
});

it('should not change count of prepacked rules when adding a second exception list to an immutable rule through patch. If this fails, suspect the immutable tags are not staying on the rule correctly.', async () => {
it('@skipInQA should not change count of prepacked rules when adding a second exception list to an immutable rule through patch. If this fails, suspect the immutable tags are not staying on the rule correctly.', async () => {
await installMockPrebuiltRules(supertest, es);

const { id, list_id, namespace_type, type } = await createExceptionList(
Expand Down

0 comments on commit d91f0e8

Please sign in to comment.