Skip to content

Commit

Permalink
[8.16] [8.x] [Entity Store] Fix and re-enable entity store integratio…
Browse files Browse the repository at this point in the history 
…n tests (#196296) (#196656) (#196840)

# Backport

This will backport the following commits from `8.x` to `8.16`:
- [[8.x] [Entity Store] Fix and re-enable entity store integration tests
(#196296) (#196656)](#196656)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mark
Hopkin","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-18T10:03:50Z","message":"[8.x]
[Entity Store] Fix and re-enable entity store integration tests
(#196296) (#196656)\n\n# Backport\n\nThis will backport the following
commits from `main` to `8.x`:\n- [[Entity Store] Fix and re-enable
entity store integration
tests\n(#196296)](https://github.com/elastic/kibana/pull/196296)\n\n<!---
Backport version: 8.9.8 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Mark\nHopkin\",\"email\":\"[email protected]\"},\"sourceCommit\":{\"committedDate\":\"2024-10-16T09:22:18Z\",\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\",\"branchLabelMapping\":{\"^v9.0.0$\":\"main\",\"^v8.16.0$\":\"8.x\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:skip\",\"backport\nmissing\",\"v9.0.0\",\"Team:\nSecuritySolution\",\"backport:prev-minor\",\"Team:Entity\nAnalytics\"],\"number\":196296,\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v9.0.0\",\"labelRegex\":\"^v9.0.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"number\":196296,\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}}]}]
BACKPORT-->\n\n---------\n\nCo-authored-by: kibanamachine
<[email protected]>","sha":"cb119aca152821bb41efe0b47de398960a818c1a","branchLabelMapping":{"^v8.16.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["backport"],"number":196656,"url":"https://github.com/elastic/kibana/pull/196656","mergeCommit":{"message":"[8.x]
[Entity Store] Fix and re-enable entity store integration tests
(#196296) (#196656)\n\n# Backport\n\nThis will backport the following
commits from `main` to `8.x`:\n- [[Entity Store] Fix and re-enable
entity store integration
tests\n(#196296)](https://github.com/elastic/kibana/pull/196296)\n\n<!---
Backport version: 8.9.8 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Mark\nHopkin\",\"email\":\"[email protected]\"},\"sourceCommit\":{\"committedDate\":\"2024-10-16T09:22:18Z\",\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\",\"branchLabelMapping\":{\"^v9.0.0$\":\"main\",\"^v8.16.0$\":\"8.x\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:skip\",\"backport\nmissing\",\"v9.0.0\",\"Team:\nSecuritySolution\",\"backport:prev-minor\",\"Team:Entity\nAnalytics\"],\"number\":196296,\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v9.0.0\",\"labelRegex\":\"^v9.0.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"number\":196296,\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}}]}]
BACKPORT-->\n\n---------\n\nCo-authored-by: kibanamachine
<[email protected]>","sha":"cb119aca152821bb41efe0b47de398960a818c1a"}},"sourceBranch":"8.x","suggestedTargetBranches":[],"targetPullRequestStates":[]}]
BACKPORT-->

Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
@hop-dev @elasticmachine
hop-dev and elasticmachine authored Oct 18, 2024
1 parent 806796f commit f929851
Show file tree
Hide file tree
Showing 7 changed files with 80 additions and 92 deletions.
4 changes: 1 addition & 3 deletions ...rver/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,7 @@ export const getUnitedEntityDefinition = memoize(
namespace,
indexPatterns,
});
},
({ entityType, namespace, fieldHistoryLength }: Options) =>
`${entityType}-${namespace}-${fieldHistoryLength}`
}
);

export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>
Expand Down
8 changes: 4 additions & 4 deletions x-pack/test/functional/es_archives/security_solution/entity_store/data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"type": "doc",
"value": {
"id": "a4cf452c1e0375c3d4412cb550ad1783358468a3b3b777da4829d72c7d6fb74f",
"index": ".entities.v1.latest.ea_default_user_entity_store",
"index": ".entities.v1.latest.security_user_default",
"source": {
"event": {
"ingested": "2024-09-11T11:26:49.706875Z"
Expand All @@ -27,7 +27,7 @@
"id": "LBQAgKHGmpup0Kg9nlKmeQ==",
"type": "node",
"firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
"definitionId": "ea_default_user_entity_store"
"definitionId": "security_user_default"
}
}
}
Expand All @@ -37,7 +37,7 @@
"type": "doc",
"value": {
"id": "a2cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb71f",
"index": ".entities.v1.latest.ea_default_host_entity_store",
"index": ".entities.v1.latest.security_host_default",
"source": {
"event": {
"ingested": "2024-09-11T11:26:49.641707Z"
Expand Down Expand Up @@ -78,7 +78,7 @@
"id": "ZXKm6GEcUJY6NHkMgPPmGQ==",
"type": "node",
"firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
"definitionId": "ea_default_host_entity_store"
"definitionId": "security_host_default"
}
}
}
Expand Down
38 changes: 7 additions & 31 deletions ...tegration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,16 @@

import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils';
import { EntityStoreUtils } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';
export default ({ getService }: FtrProviderContext) => {
const api = getService('securitySolutionApi');
const supertest = getService('supertest');
const {
expectTransformExists,
expectTransformNotFound,
expectEnrichPolicyExists,
expectEnrichPolicyNotFound,
expectComponentTemplateExists,
expectComponentTemplateNotFound,
expectIngestPipelineExists,
expectIngestPipelineNotFound,
} = elasticAssetCheckerFactory(getService);

const utils = EntityStoreUtils(getService);

// TODO: unskip once permissions issue is resolved
describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => {
// Failing: See https://github.com/elastic/kibana/issues/196526
describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
const dataView = dataViewRouteHelpersFactory(supertest);

before(async () => {
Expand All @@ -45,20 +35,12 @@ export default ({ getService }: FtrProviderContext) => {

it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');

await expectTransformExists('entities-v1-latest-ea_default_user_entity_store');
await expectEnrichPolicyExists('entity_store_field_retention_user_default_v1');
await expectComponentTemplateExists(`ea_default_user_entity_store-latest@platform`);
await expectIngestPipelineExists(`ea_default_user_entity_store-latest@platform`);
await utils.expectEngineAssetsExist('user');
});

it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');

await expectTransformExists('entities-v1-latest-ea_default_host_entity_store');
await expectEnrichPolicyExists('entity_store_field_retention_host_default_v1');
await expectComponentTemplateExists(`ea_default_host_entity_store-latest@platform`);
await expectIngestPipelineExists(`ea_default_host_entity_store-latest@platform`);
await utils.expectEngineAssetsExist('host');
});
});

Expand Down Expand Up @@ -188,10 +170,7 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);

await expectTransformNotFound('entities-v1-latest-ea_default_host_entity_store');
await expectEnrichPolicyNotFound('entity_store_field_retention_host_default_v1');
await expectComponentTemplateNotFound(`ea_default_host_entity_store-latest@platform`);
await expectIngestPipelineNotFound(`ea_default_host_entity_store-latest@platform`);
await utils.expectEngineAssetsDoNotExist('host');
});

it('should delete the user entity engine', async () => {
Expand All @@ -204,10 +183,7 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);

await expectTransformNotFound('entities-v1-latest-ea_default_user_entity_store');
await expectEnrichPolicyNotFound('entity_store_field_retention_user_default_v1');
await expectComponentTemplateNotFound(`ea_default_user_entity_store-latest@platform`);
await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`);
await utils.expectEngineAssetsDoNotExist('user');
});
});

Expand Down
46 changes: 20 additions & 26 deletions ...tes/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,31 @@ import expect from '@kbn/expect';
import { v4 as uuidv4 } from 'uuid';
import { FtrProviderContextWithSpaces } from '../../../../ftr_provider_context_with_spaces';
import { EntityStoreUtils } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';

export default ({ getService }: FtrProviderContextWithSpaces) => {
const api = getService('securitySolutionApi');
const spaces = getService('spaces');
const namespace = uuidv4().substring(0, 8);

const supertest = getService('supertest');
const utils = EntityStoreUtils(getService, namespace);

// TODO: unskip once kibana system user has entity index privileges
// Failing: See https://github.com/elastic/kibana/issues/196546
describe.skip('@ess Entity Store Engine APIs in non-default space', () => {
const dataView = dataViewRouteHelpersFactory(supertest, namespace);

before(async () => {
await utils.cleanEngines();
await spaces.create({
id: namespace,
name: namespace,
disabledFeatures: [],
});
await dataView.create('security-solution');
});

after(async () => {
await dataView.delete('security-solution');
await spaces.delete(namespace);
});

Expand All @@ -38,18 +44,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {

it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');

const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`];

await utils.expectTransformsExist(expectedTransforms);
await utils.expectEngineAssetsExist('user');
});

it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');

const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`];

await utils.expectTransformsExist(expectedTransforms);
await utils.expectEngineAssetsExist('host');
});
});

Expand Down Expand Up @@ -79,9 +79,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
expect(getResponse.body).to.eql({
status: 'started',
type: 'host',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
});
});

Expand All @@ -98,9 +98,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
expect(getResponse.body).to.eql({
status: 'started',
type: 'user',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
});
});
});
Expand All @@ -116,16 +116,16 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
{
status: 'started',
type: 'host',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
},
{
status: 'started',
type: 'user',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
},
]);
});
Expand Down Expand Up @@ -200,10 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
)
.expect(200);

await utils.expectTransformNotFound(
`entities-v1-history-ea_${namespace}_host_entity_store`
);
await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_host_entity_store`);
await utils.expectEngineAssetsDoNotExist('host');
});

it('should delete the user entity engine', async () => {
Expand All @@ -219,10 +216,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
)
.expect(200);

await utils.expectTransformNotFound(
`entities-v1-history-ea_${namespace}_user_entity_store`
);
await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_user_entity_store`);
await utils.expectEngineAssetsDoNotExist('user');
});
});
});
Expand Down
3 changes: 1 addition & 2 deletions ...on/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
export default ({ getService }: FtrProviderContext) => {
const securitySolutionApi = getService('securitySolutionApi');

// TODO: unskip once permissions issue is resolved
describe.skip('@ess Entity store - Entities list API', () => {
describe('@ess @skipInServerlessMKI Entity store - Entities list API', () => {
describe('when the entity store is disable', () => {
it("should return response with success status when the index doesn't exist", async () => {
const { body } = await securitySolutionApi.listEntities({
Expand Down
6 changes: 3 additions & 3 deletions ...ck/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export const dataViewRouteHelpersFactory = (
) => ({
create: (name: string) => {
return supertest
.post(`/api/data_views/data_view`)
.post(`/s/${namespace}/api/data_views/data_view`)
.set('kbn-xsrf', 'foo')
.send({
data_view: {
Expand All @@ -26,13 +26,13 @@ export const dataViewRouteHelpersFactory = (
},
delete: (name: string) => {
return supertest
.delete(`/api/data_views/data_view/${name}-${namespace}`)
.delete(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`)
.set('kbn-xsrf', 'foo')
.expect(200);
},
updateIndexPattern: (name: string, indexPattern: string) => {
return supertest
.post(`/api/data_views/data_view/${name}-${namespace}`)
.post(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`)
.set('kbn-xsrf', 'foo')
.send({
data_view: {
Expand Down
67 changes: 44 additions & 23 deletions ...test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,27 @@
*/

import { EntityType } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/common.gen';

import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { elasticAssetCheckerFactory } from './elastic_asset_checker';

export const EntityStoreUtils = (
getService: FtrProviderContext['getService'],
namespace?: string
namespace: string = 'default'
) => {
const api = getService('securitySolutionApi');
const es = getService('es');
const log = getService('log');
const {
expectTransformExists,
expectTransformNotFound,
expectEnrichPolicyExists,
expectEnrichPolicyNotFound,
expectComponentTemplateExists,
expectComponentTemplateNotFound,
expectIngestPipelineExists,
expectIngestPipelineNotFound,
} = elasticAssetCheckerFactory(getService);

log.debug(`EntityStoreUtils namespace: ${namespace}`);

Expand All @@ -37,17 +48,24 @@ export const EntityStoreUtils = (
}
};

const initEntityEngineForEntityType = (entityType: EntityType) => {
log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`);
return api
.initEntityEngine(
{
params: { entityType },
body: {},
},
namespace
)
.expect(200);
const initEntityEngineForEntityType = async (entityType: EntityType) => {
log.info(
`Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}`
);
const res = await api.initEntityEngine(
{
params: { entityType },
body: {},
},
namespace
);

if (res.status !== 200) {
log.error(`Failed to initialize engine for entity type ${entityType}`);
log.error(JSON.stringify(res.body));
}

expect(res.status).to.eql(200);
};

const expectTransformStatus = async (
Expand Down Expand Up @@ -78,22 +96,25 @@ export const EntityStoreUtils = (
}
};

const expectTransformNotFound = async (transformId: string, attempts: number = 5) => {
return expectTransformStatus(transformId, false);
};
const expectTransformExists = async (transformId: string) => {
return expectTransformStatus(transformId, true);
const expectEngineAssetsExist = async (entityType: EntityType) => {
await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`);
await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`);
await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`);
await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`);
};

const expectTransformsExist = async (transformIds: string[]) =>
Promise.all(transformIds.map((id) => expectTransformExists(id)));
const expectEngineAssetsDoNotExist = async (entityType: EntityType) => {
await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`);
await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`);
await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`);
await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`);
};

return {
cleanEngines,
initEntityEngineForEntityType,
expectTransformStatus,
expectTransformNotFound,
expectTransformExists,
expectTransformsExist,
expectEngineAssetsExist,
expectEngineAssetsDoNotExist,
};
};

0 comments on commit f929851

Please sign in to comment.