Skip to content

Commit

Permalink
Adding telemetry changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathan-buttner
jonathan-buttner committed Mar 8, 2023
1 parent 3270aff commit 8268027
Show file tree
Hide file tree
Showing 7 changed files with 916 additions and 122 deletions.
13 changes: 0 additions & 13 deletions x-pack/plugins/cases/server/telemetry/constants.ts
View file

This file was deleted.

3 changes: 2 additions & 1 deletion x-pack/plugins/cases/server/telemetry/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type {
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server';
import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
import { FILE_SO_TYPE } from '@kbn/files-plugin/common';
import { collectTelemetryData } from './collect_telemetry_data';
import {
CASE_TELEMETRY_SAVED_OBJECT,
Expand Down Expand Up @@ -42,7 +43,7 @@ export const createCasesTelemetry = async ({
}: CreateCasesTelemetryArgs) => {
const getInternalSavedObjectClient = async (): Promise<ISavedObjectsRepository> => {
const [coreStart] = await core.getStartServices();
return coreStart.savedObjects.createInternalRepository(SAVED_OBJECT_TYPES);
return coreStart.savedObjects.createInternalRepository([...SAVED_OBJECT_TYPES, FILE_SO_TYPE]);
};

taskManager.registerTaskDefinitions({
Expand Down
287 changes: 203 additions & 84 deletions x-pack/plugins/cases/server/telemetry/queries/cases.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,32 @@
* 2.0.
*/

import type { ISavedObjectsRepository, SavedObjectsFindResponse } from '@kbn/core/server';
import { FILE_SO_TYPE } from '@kbn/files-plugin/common';
import { fromKueryExpression } from '@kbn/es-query';
import {
CASE_COMMENT_SAVED_OBJECT,
CASE_SAVED_OBJECT,
CASE_USER_ACTION_SAVED_OBJECT,
OWNERS,
} from '../../../common/constants';
import { ESCaseStatus } from '../../services/cases/types';
import type { ESCaseAttributes } from '../../services/cases/types';
import { OWNERS } from '../constants';
import type {
CollectTelemetryDataParams,
Buckets,
CasesTelemetry,
Cardinality,
ReferencesAggregation,
LatestDates,
CaseAggregationResult,
AttachmentAggregationResult,
FileAttachmentAggregationResult,
} from '../types';
import {
findValueInBuckets,
getAggregationsBuckets,
getCountsAggregationQuery,
getCountsFromBuckets,
getMaxBucketOnCaseAggregationQuery,
getOnlyAlertsCommentsFilter,
getOnlyConnectorsFilter,
getReferencesAggregationQuery,
Expand Down Expand Up @@ -62,7 +66,83 @@ export const getCasesTelemetryData = async ({
savedObjectsClient,
logger,
}: CollectTelemetryDataParams): Promise<CasesTelemetry['cases']> => {
const byOwnerAggregationQuery = OWNERS.reduce(
console.log('*****collecting telemetry');

try {
const [casesRes, commentsRes, totalAlertsRes, totalConnectorsRes, latestDates, filesRes] =
await Promise.all([
getCasesSavedObjectTelemetry(savedObjectsClient),
getCommentsSavedObjectTelemetry(savedObjectsClient),
getAlertsTelemetry(savedObjectsClient),
getConnectorsTelemetry(savedObjectsClient),
getLatestCasesDates({ savedObjectsClient, logger }),
getFilesTelemetry(savedObjectsClient),
]);

console.log('****finished collection telemetry');
console.log('casesRes ', JSON.stringify(casesRes, null, 2));
console.log('filesRes ', JSON.stringify(filesRes, null, 2));

const aggregationsBuckets = getAggregationsBuckets({
aggs: casesRes.aggregations,
keys: ['counts', 'syncAlerts', 'status', 'users', 'totalAssignees'],
});

return {
all: {
total: casesRes.total,
...getCountsFromBuckets(aggregationsBuckets.counts),
status: {
open: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.OPEN),
inProgress: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.IN_PROGRESS),
closed: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.CLOSED),
},
syncAlertsOn: findValueInBuckets(aggregationsBuckets.syncAlerts, 1),
syncAlertsOff: findValueInBuckets(aggregationsBuckets.syncAlerts, 0),
totalUsers: casesRes.aggregations?.users?.value ?? 0,
totalParticipants: commentsRes.aggregations?.participants?.value ?? 0,
totalTags: casesRes.aggregations?.tags?.value ?? 0,
totalWithAlerts:
totalAlertsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
totalWithConnectors:
totalConnectorsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
latestDates,
assignees: {
total: casesRes.aggregations?.totalAssignees.value ?? 0,
totalWithZero: casesRes.aggregations?.assigneeFilters.buckets.zero.doc_count ?? 0,
totalWithAtLeastOne:
casesRes.aggregations?.assigneeFilters.buckets.atLeastOne.doc_count ?? 0,
},
},
sec: getSolutionValues({
caseAggregations: casesRes.aggregations,
attachmentAggregations: commentsRes.aggregations,
filesAggregations: filesRes.aggregations,
owner: 'securitySolution',
}),
obs: getSolutionValues({
caseAggregations: casesRes.aggregations,
attachmentAggregations: commentsRes.aggregations,
filesAggregations: filesRes.aggregations,
owner: 'observability',
}),
main: getSolutionValues({
caseAggregations: casesRes.aggregations,
attachmentAggregations: commentsRes.aggregations,
filesAggregations: filesRes.aggregations,
owner: 'cases',
}),
};
} catch (error) {
logger.error(`Cases telemetry failed with error: ${error}`);
throw error;
}
};

const getCasesSavedObjectTelemetry = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<SavedObjectsFindResponse<unknown, CaseAggregationResult>> => {
const caseByOwnerAggregationQuery = OWNERS.reduce(
(aggQuery, owner) => ({
...aggQuery,
[owner]: {
Expand All @@ -80,12 +160,12 @@ export const getCasesTelemetryData = async ({
{}
);

const casesRes = await savedObjectsClient.find<unknown, CaseAggregationResult>({
return savedObjectsClient.find<unknown, CaseAggregationResult>({
page: 0,
perPage: 0,
type: CASE_SAVED_OBJECT,
aggs: {
...byOwnerAggregationQuery,
...caseByOwnerAggregationQuery,
...getCountsAggregationQuery(CASE_SAVED_OBJECT),
...getAssigneesAggregations(),
totalsByOwner: {
Expand All @@ -111,26 +191,131 @@ export const getCasesTelemetryData = async ({
},
},
});
};

const getAssigneesAggregations = () => ({
totalAssignees: {
value_count: {
field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
},
},
assigneeFilters: {
filters: {
filters: {
zero: {
bool: {
must_not: {
exists: {
field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
},
},
},
},
atLeastOne: {
bool: {
filter: {
exists: {
field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
},
},
},
},
},
},
},
});

const getCommentsSavedObjectTelemetry = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<SavedObjectsFindResponse<unknown, AttachmentAggregationResult>> => {
const attachmentsByOwnerAggregationQuery = OWNERS.reduce(
(aggQuery, owner) => ({
...aggQuery,
[owner]: {
filter: {
term: {
[`${CASE_COMMENT_SAVED_OBJECT}.attributes.owner`]: owner,
},
},
aggs: {
externalReferenceTypes: {
terms: {
field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.externalReferenceAttachmentTypeId`,
},
aggs: {
...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT),
},
},
persistableReferenceTypes: {
terms: {
field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.persistableStateAttachmentTypeId`,
},
aggs: {
...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT),
},
},
},
},
}),
{}
);

console.log('****collecting attachments telemetry');

const commentsRes = await savedObjectsClient.find<
unknown,
Record<typeof OWNERS[number], { counts: Buckets }> & {
participants: Cardinality;
} & ReferencesAggregation
>({
return savedObjectsClient.find<unknown, AttachmentAggregationResult>({
page: 0,
perPage: 0,
type: CASE_COMMENT_SAVED_OBJECT,
aggs: {
...attachmentsByOwnerAggregationQuery,
participants: {
cardinality: {
field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.created_by.username`,
},
},
},
});
};

const getFilesTelemetry = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<SavedObjectsFindResponse<unknown, FileAttachmentAggregationResult>> => {
const filesByOwnerAggregationQuery = OWNERS.reduce(
(aggQuery, owner) => ({
...aggQuery,
[owner]: {
filter: {
term: {
[`${FILE_SO_TYPE}.attributes.Meta.owner`]: owner,
},
},
aggs: {
averageSize: {
avg: {
field: `${FILE_SO_TYPE}.attributes.size`,
},
},
},
},
}),
{}
);

const totalAlertsRes = await savedObjectsClient.find<unknown, ReferencesAggregation>({
const filterCaseIdExists = fromKueryExpression(`${FILE_SO_TYPE}.attributes.Meta.caseId: *`);

return savedObjectsClient.find<unknown, FileAttachmentAggregationResult>({
page: 0,
perPage: 0,
type: FILE_SO_TYPE,
// filter: filterCaseIdExists,
aggs: filesByOwnerAggregationQuery,
});
};

const getAlertsTelemetry = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<SavedObjectsFindResponse<unknown, ReferencesAggregation>> => {
return savedObjectsClient.find<unknown, ReferencesAggregation>({
page: 0,
perPage: 0,
type: CASE_COMMENT_SAVED_OBJECT,
Expand All @@ -143,8 +328,12 @@ export const getCasesTelemetryData = async ({
}),
},
});
};

const totalConnectorsRes = await savedObjectsClient.find<unknown, ReferencesAggregation>({
const getConnectorsTelemetry = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<SavedObjectsFindResponse<unknown, ReferencesAggregation>> => {
return savedObjectsClient.find<unknown, ReferencesAggregation>({
page: 0,
perPage: 0,
type: CASE_USER_ACTION_SAVED_OBJECT,
Expand All @@ -157,74 +346,4 @@ export const getCasesTelemetryData = async ({
}),
},
});

const latestDates = await getLatestCasesDates({ savedObjectsClient, logger });

const aggregationsBuckets = getAggregationsBuckets({
aggs: casesRes.aggregations,
keys: ['counts', 'syncAlerts', 'status', 'users', 'totalAssignees'],
});

return {
all: {
total: casesRes.total,
...getCountsFromBuckets(aggregationsBuckets.counts),
status: {
open: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.OPEN),
inProgress: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.IN_PROGRESS),
closed: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.CLOSED),
},
syncAlertsOn: findValueInBuckets(aggregationsBuckets.syncAlerts, 1),
syncAlertsOff: findValueInBuckets(aggregationsBuckets.syncAlerts, 0),
totalUsers: casesRes.aggregations?.users?.value ?? 0,
totalParticipants: commentsRes.aggregations?.participants?.value ?? 0,
totalTags: casesRes.aggregations?.tags?.value ?? 0,
totalWithAlerts:
totalAlertsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
totalWithConnectors:
totalConnectorsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
latestDates,
assignees: {
total: casesRes.aggregations?.totalAssignees.value ?? 0,
totalWithZero: casesRes.aggregations?.assigneeFilters.buckets.zero.doc_count ?? 0,
totalWithAtLeastOne:
casesRes.aggregations?.assigneeFilters.buckets.atLeastOne.doc_count ?? 0,
},
},
sec: getSolutionValues(casesRes.aggregations, 'securitySolution'),
obs: getSolutionValues(casesRes.aggregations, 'observability'),
main: getSolutionValues(casesRes.aggregations, 'cases'),
};
};

const getAssigneesAggregations = () => ({
totalAssignees: {
value_count: {
field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
},
},
assigneeFilters: {
filters: {
filters: {
zero: {
bool: {
must_not: {
exists: {
field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
},
},
},
},
atLeastOne: {
bool: {
filter: {
exists: {
field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
},
},
},
},
},
},
},
});
Loading

0 comments on commit 8268027

Please sign in to comment.