Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Telemetry] Pull local Kibana usage stats #26496

Merged
merged 17 commits into from
Dec 10, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,24 @@ import {
handleLocalStats,
} from '../get_local_stats';

const getMockServer = (getCluster = sinon.stub(), kibanaUsage = {}) => ({
log(tags, message) {
console.log({ tags, message });
},
usage: { collectorSet: { bulkFetch: () => kibanaUsage, toObject: data => data } },
plugins: {
xpack_main: { status: { plugin: { kbnServer: { version: '8675309-snapshot' } } } },
elasticsearch: { getCluster },
},
});

function mockGetLocalStats(callCluster, clusterInfo, clusterStats, license, usage) {
mockGetClusterInfo(callCluster, clusterInfo);
mockGetClusterStats(callCluster, clusterStats);
mockGetXPack(callCluster, license, usage);
}

function dropTimestamp(localStats) {
return omit(localStats, 'timestamp');
}

describe('get_local_stats', () => {

const clusterUuid = 'abc123';
const clusterName = 'my-cool-cluster';
const version = '2.3.4';
Expand All @@ -47,102 +53,127 @@ describe('get_local_stats', () => {
nodes: { yup: 'abc' },
random: 123
};
const license = {
fancy: 'license'
};
const usage = {
also: 'fancy'
const license = { fancy: 'license' };
const xpack = { also: 'fancy' };
const kibana = {
kibana: {
great: 'googlymoogly',
versions: [{ version: '8675309', count: 1 }]
},
kibana_stats: {
os: {
platform: 'rocky',
platformRelease: 'iv',
}
},
sun: { chances: 5 },
clouds: { chances: 95 },
rain: { chances: 2 },
snow: { chances: 0 },
};
const xpack = {
license,
stack_stats: {
xpack: usage
}
};
const localStats = {

const combinedStatsResult = {
collection: 'local',
cluster_uuid: clusterUuid,
cluster_name: clusterName,
license: {
fancy: 'license'
},
version,
cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
...xpack
stack_stats: {
kibana: {
great: 'googlymoogly',
count: 1,
indices: 1,
os: {
platforms: [{ platform: 'rocky', count: 1 }],
platformReleases: [{ platformRelease: 'iv', count: 1 }]
},
versions: [{ version: '8675309', count: 1 }],
plugins: {
sun: { chances: 5 },
clouds: { chances: 95 },
rain: { chances: 2 },
snow: { chances: 0 },
}
},
xpack: { also: 'fancy' },
}
};
const noXpackLocalStats = omit(localStats, 'license', 'stack_stats');

describe('handleLocalStats', () => {

it('returns expected object without xpack data', () => {
expect(dropTimestamp(handleLocalStats(clusterInfo, clusterStats))).to.eql(noXpackLocalStats);
expect(dropTimestamp(handleLocalStats(clusterInfo, clusterStats, { }))).to.eql(noXpackLocalStats);
it('returns expected object without xpack and kibana data', () => {
const result = handleLocalStats(getMockServer(), clusterInfo, clusterStats);
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
expect(result.version).to.be('2.3.4');
expect(result.collection).to.be('local');
expect(result.license).to.be(undefined);
expect(result.stack_stats).to.eql({ kibana: undefined, xpack: undefined });
});

it('returns expected object with xpack data', () => {
expect(dropTimestamp(handleLocalStats(clusterInfo, clusterStats, xpack))).to.eql(localStats);
it('returns expected object with xpack', () => {
const result = handleLocalStats(getMockServer(), clusterInfo, clusterStats, license, xpack);
const { stack_stats: stack, ...cluster } = result;
expect(cluster.collection).to.be(combinedStatsResult.collection);
expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name);
expect(stack.kibana).to.be(undefined); // not mocked for this test

expect(cluster.version).to.eql(combinedStatsResult.version);
expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
expect(cluster.license).to.eql(combinedStatsResult.license);
expect(stack.xpack).to.eql(combinedStatsResult.stack_stats.xpack);
});

});

describe('getLocalStatsWithCaller', () => {

it('returns expected object without xpack data when X-Pack fails to respond', async () => {
const callClusterUsageFailed = sinon.stub();
const callClusterLicenseFailed = sinon.stub();
const callClusterBothFailed = sinon.stub();

mockGetLocalStats(
callClusterUsageFailed,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.resolve(license), Promise.reject('usage failed')
Promise.resolve(license),
Promise.reject('usage failed')
);

mockGetLocalStats(
callClusterLicenseFailed,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.reject('license failed'), Promise.resolve(usage)
);
const result = await getLocalStatsWithCaller(getMockServer(), callClusterUsageFailed);
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
expect(result.version).to.be('2.3.4');
expect(result.collection).to.be('local');

mockGetLocalStats(
callClusterBothFailed,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.reject('license failed'), Promise.reject('usage failed')
);

expect(dropTimestamp(await getLocalStatsWithCaller(callClusterUsageFailed))).to.eql(noXpackLocalStats);
expect(dropTimestamp(await getLocalStatsWithCaller(callClusterLicenseFailed))).to.eql(noXpackLocalStats);
expect(dropTimestamp(await getLocalStatsWithCaller(callClusterBothFailed))).to.eql(noXpackLocalStats);
// license and xpack usage info come from the same cluster call
expect(result.license).to.be(undefined);
expect(result.stack_stats.xpack).to.be(undefined);
});

it('returns expected object with xpack data', async () => {
it('returns expected object with xpack and kibana data', async () => {
const callCluster = sinon.stub();

mockGetLocalStats(
callCluster,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.resolve(license), Promise.resolve(usage)
Promise.resolve(license),
Promise.resolve(xpack)
);

expect(dropTimestamp(await getLocalStatsWithCaller(callCluster))).to.eql(localStats);
const result = await getLocalStatsWithCaller(getMockServer(callCluster, kibana), callCluster);
expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack);
expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana);
});

});

describe('getLocalStats', () => {

it('uses callWithInternalUser from data cluster', async () => {
const getCluster = sinon.stub();
const req = {
server: {
plugins: {
elasticsearch: {
getCluster
}
}
}
};
const req = { server: getMockServer(getCluster) };
const callWithInternalUser = sinon.stub();

getCluster.withArgs('data').returns({ callWithInternalUser });
Expand All @@ -151,12 +182,15 @@ describe('get_local_stats', () => {
callWithInternalUser,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.resolve(license), Promise.resolve(usage)
Promise.resolve(license),
Promise.resolve(xpack)
);

expect(dropTimestamp(await getLocalStats(req))).to.eql(localStats);
const result = await getLocalStats(req);
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.version).to.eql(combinedStatsResult.version);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
});

});

});
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,15 @@ describe('get_xpack', () => {

it('returns the formatted response object', async () => {
const license = { fancy: 'license' };
const usage = { also: 'fancy' };
const xpack = { also: 'fancy' };

const callCluster = sinon.stub();

mockGetXPack(callCluster, Promise.resolve(license), Promise.resolve(usage));
mockGetXPack(callCluster, Promise.resolve(license), Promise.resolve(xpack));

const data = await getXPack(callCluster);

expect(data).to.eql({ license, stack_stats: { xpack: usage } });
expect(data).to.eql({ license, xpack });
});

it('returns empty object upon license failure', async () => {
Expand Down
49 changes: 49 additions & 0 deletions x-pack/plugins/xpack_main/server/lib/telemetry/local/get_kibana.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { get, omit } from 'lodash';

export function handleKibanaStats(server, response) {
if (!response) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm kind of curious how we'd get in a state where response is validly null or undefined. I could reasonably see it missing certain usage stats (e.g., from disabled or unused plugins), but not missing core data.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason for this early return is to avoid throwing exceptions for unit tests that don't pass in mock response for the handleLocalStats tests: https://github.com/elastic/kibana/pull/26496/files#diff-46ad76ce05cd14b05bdc60fc08f0001dR104

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if that's a good thing though. We're allowing for a broken state of Kibana to simplify a few tests, which means we'll have to get luckier to catch it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about add logging with a warning here?

server    log   [17:43:40.419] [warning][local-stats][telemetry] No Kibana stats returned from usage collectors

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

server.log(['warning', 'telemetry', 'local-stats'], 'No Kibana stats returned from usage collectors');
return;
}

const { kibana, kibana_stats: stats, ...plugins } = response;

const platform = get(stats, 'os.platform', 'unknown');
const platformRelease = get(stats, 'os.platformRelease', 'unknown');

let version;
const { kbnServer } = get(server, 'plugins.xpack_main.status.plugin');
if (kbnServer) {
version = kbnServer.version.replace(/-snapshot/i, '');
}

// combine core stats (os types, saved objects) with plugin usage stats
// organize the object into the same format as monitoring-enabled telemetry
return {
...omit(kibana, 'index'), // discard index
count: 1,
indices: 1,
os: {
platforms: [{ platform, count: 1 }],
platformReleases: [{ platformRelease, count: 1 }],
},
versions: [{ version, count: 1 }],
plugins,
};
}

/*
* Check user privileges for read access to monitoring
* Pass callWithInternalUser to bulkFetchUsage
*/
export async function getKibana(server, callWithInternalUser) {
const { collectorSet } = server.usage;
const usage = await collectorSet.bulkFetch(callWithInternalUser);
return collectorSet.toObject(usage);
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { get, omit } from 'lodash';
import { getClusterInfo } from './get_cluster_info';
import { getClusterStats } from './get_cluster_stats';
import { getXPack } from './get_xpack';
import { getKibana, handleKibanaStats } from './get_kibana';

/**
* Handle the separate local calls by combining them into a single object response that looks like the
Expand All @@ -18,15 +19,19 @@ import { getXPack } from './get_xpack';
* @param {Object} xpack License and X-Pack details
* @return {Object} A combined object containing the different responses.
*/
export function handleLocalStats(clusterInfo, clusterStats, xpack) {
export function handleLocalStats(server, clusterInfo, clusterStats, license, xpack, kibana) {
return {
timestamp: (new Date()).toISOString(),
cluster_uuid: get(clusterInfo, 'cluster_uuid'),
cluster_name: get(clusterInfo, 'cluster_name'),
version: get(clusterInfo, 'version.number'),
cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
collection: 'local',
...xpack
tsullivan marked this conversation as resolved.
Show resolved Hide resolved
license,
stack_stats: {
kibana: handleKibanaStats(server, kibana),
xpack,
}
};
}

Expand All @@ -37,13 +42,16 @@ export function handleLocalStats(clusterInfo, clusterStats, xpack) {
* @param {function} callCluster The callWithInternalUser handler (exposed for testing)
* @return {Promise} The object containing the current Elasticsearch cluster's telemetry.
*/
export function getLocalStatsWithCaller(callCluster) {
export function getLocalStatsWithCaller(server, callCluster) {
return Promise.all([
getClusterInfo(callCluster), // cluster info
getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
getXPack(callCluster), // license, stack_stats
])
.then(([clusterInfo, clusterStats, xpack]) => handleLocalStats(clusterInfo, clusterStats, xpack));
getXPack(callCluster), // { license, xpack }
getKibana(server, callCluster)
]).then(([clusterInfo, clusterStats, { license, xpack }, kibana]) => {
return handleLocalStats(server, clusterInfo, clusterStats, license, xpack, kibana);
}
);
}

/**
Expand All @@ -53,7 +61,7 @@ export function getLocalStatsWithCaller(callCluster) {
* @return {Promise} The cluster object containing telemetry.
*/
export function getLocalStats(req) {
const { callWithInternalUser } = req.server.plugins.elasticsearch.getCluster('data');

return getLocalStatsWithCaller(callWithInternalUser);
const { server } = req;
const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('data');
return getLocalStatsWithCaller(server, callWithInternalUser);
}
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,12 @@ export function getXPack(callCluster) {
getXPackLicense(callCluster),
getXPackUsage(callCluster),
])
.then(([license, usage]) => handleXPack(license, usage))
.then(([license, xpack]) => {
return {
license,
xpack,
};
})
.catch(() => { return {}; });
}

Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,6 @@
export default function ({ loadTestFile }) {
describe('Telemetry', () => {
loadTestFile(require.resolve('./telemetry'));
loadTestFile(require.resolve('./telemetry_local'));
});
}
Loading