-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add basic framework for maps telemetry gathering * Update mappings to cover required maps usage metrics * Update telemetry to harvest from saved maps * Consolidate. Clean up * Harvest telemetry daily at midnight using task manager. Retrieve task mgr state in usage collector * Organize, clean up * Reorg * Add usage collector tests. Add test utils * Add maps telemetry tests to verify values. Multiple structure changes to support testing * Review feedback * More review feedback * Add capture date/time to telemetry * Change date to 'type: date' * Review feedback * Fix merge issue
- Loading branch information
Showing
13 changed files
with
716 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| export { initTelemetryCollection } from './maps_usage_collector'; |
108 changes: 108 additions & 0 deletions
108
x-pack/plugins/maps/server/maps_telemetry/maps_telemetry.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import _ from 'lodash'; | ||
| import { EMS_FILE } from '../../common/constants'; | ||
|
|
||
| function getSavedObjectsClient(server, callCluster) { | ||
| const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects; | ||
| const internalRepository = getSavedObjectsRepository(callCluster); | ||
| return new SavedObjectsClient(internalRepository); | ||
| } | ||
|
|
||
| function getUniqueLayerCounts(layerCountsList, mapsCount) { | ||
| const uniqueLayerTypes = _.uniq(_.flatten( | ||
| layerCountsList.map(lTypes => Object.keys(lTypes)))); | ||
|
|
||
| return uniqueLayerTypes.reduce((accu, type) => { | ||
| const typeCounts = layerCountsList.reduce((accu, tCounts) => { | ||
| tCounts[type] && accu.push(tCounts[type]); | ||
| return accu; | ||
| }, []); | ||
| const typeCountsSum = _.sum(typeCounts); | ||
| accu[type] = { | ||
| min: typeCounts.length ? _.min(typeCounts) : 0, | ||
| max: typeCounts.length ? _.max(typeCounts) : 0, | ||
| avg: typeCountsSum ? typeCountsSum / mapsCount : 0 | ||
| }; | ||
| return accu; | ||
| }, {}); | ||
| } | ||
|
|
||
| export function buildMapsTelemetry(savedObjects) { | ||
| const layerLists = savedObjects | ||
| .map(savedMapObject => | ||
| JSON.parse(savedMapObject.attributes.layerListJSON)); | ||
| const mapsCount = layerLists.length; | ||
|
|
||
| const dataSourcesCount = layerLists.map(lList => { | ||
| const sourceIdList = lList.map(layer => layer.sourceDescriptor.id); | ||
| return _.uniq(sourceIdList).length; | ||
| }); | ||
|
|
||
| const layersCount = layerLists.map(lList => lList.length); | ||
| const layerTypesCount = layerLists.map(lList => _.countBy(lList, 'type')); | ||
|
|
||
| // Count of EMS Vector layers used | ||
| const emsLayersCount = layerLists.map(lList => _(lList) | ||
| .countBy(layer => { | ||
| const isEmsFile = _.get(layer, 'sourceDescriptor.type') === EMS_FILE; | ||
| return isEmsFile && _.get(layer, 'sourceDescriptor.id'); | ||
| }) | ||
| .pick((val, key) => key !== 'false') | ||
| .value()); | ||
|
|
||
| const dataSourcesCountSum = _.sum(dataSourcesCount); | ||
| const layersCountSum = _.sum(layersCount); | ||
| const mapsTelem = { | ||
| // Total count of maps | ||
| mapsTotalCount: mapsCount, | ||
| // Time of capture | ||
| timeCaptured: new Date(), | ||
| attributesPerMap: { | ||
| // Count of data sources per map | ||
| dataSourcesCount: { | ||
| min: dataSourcesCount.length ? _.min(dataSourcesCount) : 0, | ||
| max: dataSourcesCount.length ? _.max(dataSourcesCount) : 0, | ||
| avg: dataSourcesCountSum ? layersCountSum / mapsCount : 0 | ||
| }, | ||
| // Total count of layers per map | ||
| layersCount: { | ||
| min: layersCount.length ? _.min(layersCount) : 0, | ||
| max: layersCount.length ? _.max(layersCount) : 0, | ||
| avg: layersCountSum ? layersCountSum / mapsCount : 0 | ||
| }, | ||
| // Count of layers by type | ||
| layerTypesCount: { | ||
| ...getUniqueLayerCounts(layerTypesCount, mapsCount) | ||
| }, | ||
| // Count of layer by EMS region | ||
| emsVectorLayersCount: { | ||
| ...getUniqueLayerCounts(emsLayersCount, mapsCount) | ||
| } | ||
| } | ||
| }; | ||
| return mapsTelem; | ||
| } | ||
|
|
||
| async function getSavedObjects(savedObjectsClient) { | ||
| const gisMapsSavedObject = await savedObjectsClient.find({ | ||
| type: 'map' | ||
| }); | ||
| return _.get(gisMapsSavedObject, 'saved_objects'); | ||
| } | ||
|
|
||
| export async function getMapsTelemetry(server, callCluster) { | ||
| const savedObjectsClient = getSavedObjectsClient(server, callCluster); | ||
| const savedObjects = await getSavedObjects(savedObjectsClient); | ||
| const mapsTelemetry = buildMapsTelemetry(savedObjects); | ||
|
|
||
| return await savedObjectsClient.create('maps-telemetry', | ||
| mapsTelemetry, { | ||
| id: 'maps-telemetry', | ||
| overwrite: true, | ||
| }); | ||
| } |
88 changes: 88 additions & 0 deletions
88
x-pack/plugins/maps/server/maps_telemetry/maps_telemetry.test.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import * as savedObjectsPayload from | ||
| './test_resources/sample_saved_objects.json'; | ||
| import { buildMapsTelemetry } from './maps_telemetry'; | ||
|
|
||
| describe('buildMapsTelemetry', () => { | ||
|
|
||
| test('returns zeroed telemetry data when there are no saved objects', | ||
| async () => { | ||
|
|
||
| const gisMaps = []; | ||
| const result = buildMapsTelemetry(gisMaps); | ||
|
|
||
| expect(result).toMatchObject({ | ||
| "attributesPerMap": { | ||
| "dataSourcesCount": { | ||
| "avg": 0, | ||
| "max": 0, | ||
| "min": 0 | ||
| }, | ||
| "emsVectorLayersCount": {}, | ||
| "layerTypesCount": {}, | ||
| "layersCount": { | ||
| "avg": 0, | ||
| "max": 0, | ||
| "min": 0 | ||
| } | ||
| }, | ||
| "mapsTotalCount": 0 | ||
| }); | ||
| }); | ||
|
|
||
| test('returns expected telemetry data from saved objects', async () => { | ||
|
|
||
| const gisMaps = savedObjectsPayload.saved_objects; | ||
| const result = buildMapsTelemetry(gisMaps); | ||
|
|
||
| expect(result).toMatchObject({ | ||
| "attributesPerMap": { | ||
| "dataSourcesCount": { | ||
| "avg": 2.6666666666666665, | ||
| "max": 3, | ||
| "min": 2 | ||
| }, | ||
| "emsVectorLayersCount": { | ||
| "canada_provinces": { | ||
| "avg": 0.3333333333333333, | ||
| "max": 1, | ||
| "min": 1 | ||
| }, | ||
| "france_departments": { | ||
| "avg": 0.3333333333333333, | ||
| "max": 1, | ||
| "min": 1 | ||
| }, | ||
| "italy_provinces": { | ||
| "avg": 0.3333333333333333, | ||
| "max": 1, | ||
| "min": 1 | ||
| } | ||
| }, | ||
| "layerTypesCount": { | ||
| "TILE": { | ||
| "avg": 1, | ||
| "max": 1, | ||
| "min": 1 | ||
| }, | ||
| "VECTOR": { | ||
| "avg": 1.6666666666666667, | ||
| "max": 2, | ||
| "min": 1 | ||
| } | ||
| }, | ||
| "layersCount": { | ||
| "avg": 2.6666666666666665, | ||
| "max": 3, | ||
| "min": 2 | ||
| } | ||
| }, | ||
| "mapsTotalCount": 3 | ||
| }); | ||
| }); | ||
| }); |
Oops, something went wrong.