Skip to content

Commit

Permalink
Report logs data telemetry events via Usage Collection plugin.
Browse files Browse the repository at this point in the history
  • Loading branch information
awahab07 committed Sep 25, 2024
1 parent 33b11ae commit ff34311
Show file tree
Hide file tree
Showing 11 changed files with 530 additions and 311 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
"dataViews",
"lens",
"fieldsMetadata",
"taskManager"
"taskManager",
"usageCollection"
],
"optionalPlugins": ["telemetry"],
"requiredBundles": ["unifiedHistogram", "discover"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export class DatasetQualityServerPlugin implements Plugin {
});

// Setup Data Telemetry Service
this.dataTelemetryService.setup(core.analytics, plugins.taskManager);
this.dataTelemetryService.setup(plugins.taskManager, plugins.usageCollection);

return {};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,12 @@
* 2.0.
*/

import { SchemaChildValue, SchemaObject } from '@elastic/ebt';
import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from '@kbn/telemetry-plugin/server/telemetry_collection/get_data_telemetry/constants';

import {
DatasetIndexPattern,
LogsDataTelemetryEventTypes,
LogsDataTelemetryEvent,
DataTelemetryEvent,
} from './types';
import { DatasetIndexPattern } from './types';

export const LOGS_DATA_TELEMETRY_TASK_TYPE = 'logs-data-telemetry';
export const LOGS_DATA_TELEMETRY_TASK_ID = 'logs-data-telemetry:collect-and-report-task-2';
export const LOGS_DATA_TELEMETRY_TASK_ID = 'logs-data-telemetry:collect-and-report-task';

export const TELEMETRY_TASK_INTERVAL = 24 * 60; // 24 hours (in minutes)
export const TELEMETRY_TASK_TIMEOUT = 10; // 10 minutes
Expand Down Expand Up @@ -104,137 +98,3 @@ export const DATA_TELEMETRY_FIELDS = [
'data_stream.namespace',
'data_stream.type',
];

const structureLevelSchema: SchemaObject<DataTelemetryEvent['structure_level']> = {
properties: {
'0': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 0',
},
},
'1': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 1',
},
},
'2': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 2',
},
},
'3': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 3',
},
},
'4': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 4',
},
},
'5': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 5',
},
},
'6': {
type: 'integer',
_meta: {
description: 'Total docs at structure level 6',
},
},
},
};

export const logsDataTelemetryEventSchema: LogsDataTelemetryEvent = {
eventType: LogsDataTelemetryEventTypes.LogsDataTelemetryEvent,
schema: {
pattern_name: {
type: 'keyword',
_meta: { description: 'Logs pattern name representing the stream of logs' },
},
shipper: {
type: 'keyword',
_meta: { description: 'Shipper if present, sending the logs', optional: true },
},
doc_count: {
type: 'integer',
_meta: { description: 'Total number of documents in the steam of logs' },
},
structure_level: structureLevelSchema,
failure_store_doc_count: {
type: 'integer',
_meta: {
description: 'Total number of documents in the failure store in the stream of logs',
},
},
index_count: {
type: 'integer',
_meta: {
description: 'Total number of indices in the stream of logs',
},
},
namespace_count: {
type: 'integer',
_meta: {
description: 'Total number of namespaces in the stream of logs',
},
},
field_count: {
type: 'integer',
_meta: {
description: 'Total number of fields in mappings of indices of the stream of logs',
},
},
field_existence: {
properties: DATA_TELEMETRY_FIELDS.reduce((acc, field) => {
acc[field] = {
type: 'integer',
_meta: { description: `Count of docs having field "${field}" mapped` },
};
return acc;
}, {} as Record<string, SchemaChildValue<number>>),
},
size_in_bytes: {
type: 'long',
_meta: {
description: 'Total size in bytes of the stream of logs',
},
},
managed_by: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description: 'Value captured in _meta.managed_by',
},
},
_meta: {
description: 'List of managed by values present in _meta.managed_by of mappings',
},
},
package_name: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description: 'Value captured in _meta.package.name',
},
},
_meta: {
description: 'List of package name values present in _meta.package.name of mappings',
},
},
beat: {
type: 'array',
items: { type: 'keyword', _meta: { description: 'Value captured in _meta.beat.name' } },
_meta: { description: 'List of beat values present in _meta.beat.name of mappings' },
},
},
};
Loading

0 comments on commit ff34311

Please sign in to comment.