diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 977e46e2beaea..94f983f1d5e6f 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -87,7 +87,7 @@ steps: queue: n2-4-spot depends_on: build timeout_in_minutes: 40 - parallelism: 2 + parallelism: 4 retry: automatic: - exit_status: '*' diff --git a/package.json b/package.json index e5c77f4f844f5..6ade30114dcf0 100644 --- a/package.json +++ b/package.json @@ -1564,7 +1564,7 @@ "pixelmatch": "^5.3.0", "playwright": "=1.37.0", "pngjs": "^3.4.0", - "postcss": "^8.4.14", + "postcss": "^8.4.31", "postcss-loader": "^4.2.0", "postcss-prefix-selector": "^1.16.0", "postcss-scss": "^4.0.4", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index b5a1591e7a5bd..26ab99d41633d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "f11cc19275f4c3e4ee7c5cd6423b6706b21b989d", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "b4e636b13a5d0f89f0400fb67811d4cca4736eb0", - "ingest-package-policies": "8ec637429836f80f1fcc798bcee7c5916eceaed5", + "ingest-package-policies": "a0c9fb48e04dcd638e593db55f1c6451523f90ea", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", diff --git a/x-pack/plugins/cloud_security_posture/README.md b/x-pack/plugins/cloud_security_posture/README.md index f9c760fbeb99f..0befebb667de6 100755 --- a/x-pack/plugins/cloud_security_posture/README.md +++ b/x-pack/plugins/cloud_security_posture/README.md @@ -6,17 +6,21 @@ Cloud Posture automates the identification and remediation of risks across cloud ## Development -read [Kibana Contributing Guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for more details +Read [Kibana Contributing Guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for more details ## Testing -for general guidelines, read [Kibana Testing Guide](https://www.elastic.co/guide/en/kibana/current/development-tests.html) for more details +For general guidelines, read [Kibana Testing Guide](https://www.elastic.co/guide/en/kibana/current/development-tests.html) for more details ### Tests 1. Unit Tests (Jest) - located in sibling files to the source code -2. [Integration Tests](../../test/api_integration/apis/cloud_security_posture/index.ts) -3. [End-to-End Tests](../../test/cloud_security_posture_functional/pages/index.ts) +1. [API Integration Tests](../../test/api_integration/apis/cloud_security_posture/config.ts) +1. [Telemetry Integration Tests](../../test/cloud_security_posture_api/config.ts) +1. [End-to-End Tests](../../test/cloud_security_posture_functional/config.ts) +1. [Serverless API Integration tests](../../test_serverless/api_integration/test_suites/security/config.ts) +1. [Serverless End-to-End Tests](../../test_serverless/functional/test_suites/security/config.ts) + ### Tools @@ -32,6 +36,17 @@ Run **ESLint**: yarn lint:es x-pack/plugins/cloud_security_posture ``` +Run **i18n check**: +```bash +node scripts/i18n_check.js +``` + +> **Note** +> +> i18n should run on project scope as it checks translations files outside of our plugin. +> +> Fixes can be applied using the --fix flag + Run [**Unit Tests**](https://www.elastic.co/guide/en/kibana/current/development-tests.html#_unit_testing): ```bash @@ -39,6 +54,7 @@ yarn test:jest --config x-pack/plugins/cloud_security_posture/jest.config.js ``` > **Note** +> > for a coverage report, add the `--coverage` flag, and run `open target/kibana-coverage/jest/x-pack/plugins/cloud_security_posture/index.html` Run [**Integration Tests**](https://docs.elastic.dev/kibana-dev-docs/tutorials/testing-plugins#): @@ -50,14 +66,45 @@ yarn test:ftr --config x-pack/test/api_integration/config.ts Run [**End-to-End Tests**](https://www.elastic.co/guide/en/kibana/current/development-tests.html#_running_functional_tests): ```bash -yarn test:ftr --config x-pack/test/cloud_security_posture_functional/config.ts --debug +yarn test:ftr --config x-pack/test/cloud_security_posture_functional/config.ts +yarn test:ftr --config x-pack/test/api_integration/config.ts --include-tag=cloud_security_posture +yarn test:ftr --config x-pack/test/cloud_security_posture_api/config.ts +yarn test:ftr --config x-pack/test_serverless/api_integration/test_suites/security/config.ts --include-tag=cloud_security_posture +yarn test:ftr --config x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts ``` -
+#### Run **FTR tests (integration or e2e) for development** + +Functional test runner (FTR) can be used separately with `ftr:runner` and `ftr:server`. This is convenient while developing tests. -test runner (FTR) can be used separately with `ftr:runner` and `ftr:server`: +For example, +run ESS (stateful) api integration tests: ```bash yarn test:ftr:server --config x-pack/test/api_integration/config.ts -yarn test:ftr:runner --include-tag=cloud_security_posture --config x-pack/test/api_integration/config.ts +yarn test:ftr:runner --config x-pack/test/api_integration/apis/cloud_security_posture/config.ts +``` + +run ESS (stateful) telemetry integration tests: +```bash +yarn test:ftr:server --config x-pack/test/cloud_security_posture_api/config.ts +yarn test:ftr:runner --config x-pack/test/cloud_security_posture_api/config.ts ``` + +run ESS (stateful) e2e tests: +```bash +yarn test:ftr:server --config x-pack/test/cloud_security_posture_functional/config.ts +yarn test:ftr:runner --config x-pack/test/cloud_security_posture_functional/config.ts +``` + +run serverless api integration tests: +```bash +yarn test:ftr:server --config x-pack/test_serverless/api_integration/test_suites/security/config.ts +yarn test:ftr:runner --config x-pack/test_serverless/api_integration/test_suites/security/config.ts --include-tag=cloud_security_posture +``` + +run serverless e2e tests: +```bash +yarn test:ftr:server --config x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts +yarn test:ftr:runner ---config x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts +``` \ No newline at end of file diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index b21af06c38349..9bf37677ae905 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -30,17 +30,14 @@ import { migratePackagePolicyToV8110, } from './migrations/security_solution/to_v8_11_0'; +import { migrateCspPackagePolicyToV8110 } from './migrations/cloud_security_posture'; + import { migrateOutputEvictionsFromV8100, migrateOutputToV8100 } from './migrations/to_v8_10_0'; import { migrateSyntheticsPackagePolicyToV8100 } from './migrations/synthetics/to_v8_10_0'; import { migratePackagePolicyEvictionsFromV8100 } from './migrations/security_solution/to_v8_10_0'; -import { - migratePackagePolicyEvictionsFromV81102, - migratePackagePolicyToV81102, -} from './migrations/security_solution/to_v8_11_0_2'; - import { migrateAgentPolicyToV7100, migratePackagePolicyToV7100, @@ -78,6 +75,10 @@ import { } from './migrations/security_solution'; import { migratePackagePolicyToV880 } from './migrations/to_v8_8_0'; import { migrateAgentPolicyToV890 } from './migrations/to_v8_9_0'; +import { + migratePackagePolicyToV81102, + migratePackagePolicyEvictionsFromV81102, +} from './migrations/security_solution/to_v8_11_0_2'; /* * Saved object types and mappings @@ -351,6 +352,14 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ forwardCompatibility: migratePackagePolicyEvictionsFromV81102, }, }, + '4': { + changes: [ + { + type: 'data_backfill', + backfillFn: migrateCspPackagePolicyToV8110, + }, + ], + }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/index.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/index.ts new file mode 100644 index 0000000000000..d9f2adb59e575 --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { migrateCspPackagePolicyToV8110 } from './to_v8_11_0'; diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/to_v8_11_0.test.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/to_v8_11_0.test.ts new file mode 100644 index 0000000000000..ec6313080642b --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/to_v8_11_0.test.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectModelTransformationContext } from '@kbn/core-saved-objects-server'; + +import { migrateCspPackagePolicyToV8110 as migration } from './to_v8_11_0'; + +describe('8.11.0 Cloud Security Posture Package Policy migration', () => { + const policyDoc = ( + accountType: string, + isAccountTypeCorrect: boolean, + packageName: string + ): any => { + return { + id: 'mock-saved-csp-object-id', + attributes: { + name: 'cloud_security_posture_test', + package: { + name: packageName, + title: '', + version: '', + }, + id: 'ID_123', + policy_id: '', + enabled: true, + namespace: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: accountType, + enabled: true, + streams: [ + { + vars: { + ...(isAccountTypeCorrect && { + 'gcp.account_type': { value: 'single-account', type: 'text' }, + }), + }, + }, + ], + config: {}, + }, + ], + }, + type: ' nested', + }; + }; + + it('adds gcp.account_type to policy, set to single', () => { + const initialDoc = policyDoc('cloudbeat/cis_gcp', false, 'cloud_security_posture'); + const migratedDoc = policyDoc('cloudbeat/cis_gcp', true, 'cloud_security_posture'); + expect(migration(initialDoc, {} as SavedObjectModelTransformationContext)).toEqual({ + attributes: migratedDoc.attributes, + }); + }); + + it('if there are no type cloudbeat/cis_gcp, do not add gcp.account_type', () => { + const initialDoc = policyDoc('cloudbeat/cis_aws', false, 'cloud_security_posture'); + const migratedDoc = policyDoc('cloudbeat/cis_aws', false, 'cloud_security_posture'); + expect(migration(initialDoc, {} as SavedObjectModelTransformationContext)).toEqual({ + attributes: migratedDoc.attributes, + }); + }); + + it('if there are no cloud_security_posture package, do not change the doc', () => { + const initialDoc = policyDoc('cloudbeat/cis_gcp', false, 'NOT_cloud_security_posture'); + const migratedDoc = policyDoc('cloudbeat/cis_gcp', false, 'NOT_cloud_security_posture'); + expect(migration(initialDoc, {} as SavedObjectModelTransformationContext)).toEqual({ + attributes: migratedDoc.attributes, + }); + }); + + it('if gcp.account_type exist and already has a value, do not set it to single-account', () => { + const policyDocWithAccountType = (): any => { + return { + id: 'mock-saved-csp-object-id', + attributes: { + name: 'cloud_security_posture_test', + package: { + name: 'cloud_security_posture', + title: '', + version: '', + }, + id: 'ID_1234', + policy_id: '', + enabled: true, + namespace: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'cloudbeat/cis_gcp', + enabled: true, + streams: [ + { + vars: { + 'gcp.account_type': { value: 'single-account-MAYBE', type: 'text' }, + }, + }, + ], + config: {}, + }, + ], + }, + type: ' nested', + }; + }; + const initialDoc = policyDocWithAccountType(); + const migratedDoc = policyDocWithAccountType(); + expect(migration(initialDoc, {} as SavedObjectModelTransformationContext)).toEqual({ + attributes: migratedDoc.attributes, + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/to_v8_11_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/to_v8_11_0.ts new file mode 100644 index 0000000000000..dd6760a9dc4ac --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/to_v8_11_0.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server'; + +import type { PackagePolicy } from '../../../../common'; + +export const migrateCspPackagePolicyToV8110: SavedObjectModelDataBackfillFn< + PackagePolicy, + PackagePolicy +> = (packagePolicyDoc) => { + if (packagePolicyDoc.attributes.package?.name !== 'cloud_security_posture') { + return { attributes: packagePolicyDoc.attributes }; + } + + const updatedAttributes = packagePolicyDoc.attributes; + + const gcpPackage = updatedAttributes.inputs.find((input) => input.type === 'cloudbeat/cis_gcp'); + + if (gcpPackage) { + const isGcpAccountTypeExists = gcpPackage.streams[0]?.vars?.hasOwnProperty('gcp.account_type'); + + if (!isGcpAccountTypeExists) { + const migratedPolicy = { 'gcp.account_type': { value: 'single-account', type: 'text' } }; + gcpPackage.streams[0].vars = { ...(gcpPackage.streams[0].vars || {}), ...migratedPolicy }; + } + } + + return { + attributes: updatedAttributes, + }; +}; diff --git a/yarn.lock b/yarn.lock index 81045d21f49ac..8efceaca77015 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22688,7 +22688,7 @@ nanoid@3.3.3: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w== -nanoid@^3.3.1, nanoid@^3.3.4: +nanoid@^3.3.1: version "3.3.4" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== @@ -24625,19 +24625,10 @@ postcss@^7.0.14, postcss@^7.0.16, postcss@^7.0.26, postcss@^7.0.32, postcss@^7.0 picocolors "^0.2.1" source-map "^0.6.1" -postcss@^8.4.14: - version "8.4.14" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.14.tgz#ee9274d5622b4858c1007a74d76e42e56fd21caf" - integrity sha512-E398TUmfAYFPBSdzgeieK2Y1+1cpdxJx8yXbK/m57nRhKSmk1GB2tO4lbLBtlkfPQTDKfe4Xqv1ASWPpayPEig== - dependencies: - nanoid "^3.3.4" - picocolors "^1.0.0" - source-map-js "^1.0.2" - -postcss@^8.4.23: - version "8.4.25" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.25.tgz#4a133f5e379eda7f61e906c3b1aaa9b81292726f" - integrity sha512-7taJ/8t2av0Z+sQEvNzCkpDynl0tX3uJMCODi6nT3PfASC7dYCWV9aQ+uiCf+KBD4SEFcu+GvJdGdwzQ6OSjCw== +postcss@^8.4.14, postcss@^8.4.23, postcss@^8.4.31: + version "8.4.31" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d" + integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== dependencies: nanoid "^3.3.6" picocolors "^1.0.0"