Skip to content

Commit

Permalink
[Beats Management] Move tokens to use JWT, add more complete test sui…
Browse files Browse the repository at this point in the history
…te (#20317)

* inital effort to move to JWT and added jest based tests on libs

* assign beats tests all passing

* token tests now pass

* add more tests

* all tests now green

* fix broken test, this is beats CM not logstash 😊

* added readme

* move enrollment token back to a hash

* remove un-needed comment

* alias lodash get to avoid confusion

* isolated hash creation
  • Loading branch information
mattapperson committed Jul 13, 2018
1 parent 71a847d commit f130bbe
Show file tree
Hide file tree
Showing 30 changed files with 1,422 additions and 199 deletions.
6 changes: 6 additions & 0 deletions x-pack/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@
"@kbn/plugin-helpers": "link:../packages/kbn-plugin-helpers",
"@kbn/test": "link:../packages/kbn-test",
"@types/boom": "^4.3.8",
"@types/chance": "^1.0.1",
"@types/expect.js": "^0.3.29",
"@types/hapi": "15.0.1",
"@types/jest": "^22.2.3",
"@types/joi": "^10.4.0",
"@types/lodash": "^3.10.0",
"@types/pngjs": "^3.3.0",
"@types/sinon": "^5.0.1",
"abab": "^1.0.4",
"ansicolors": "0.3.2",
"aws-sdk": "2.2.33",
Expand Down Expand Up @@ -89,6 +92,8 @@
"@kbn/ui-framework": "link:../packages/kbn-ui-framework",
"@samverschueren/stream-to-observable": "^0.3.0",
"@slack/client": "^4.2.2",
"@types/elasticsearch": "^5.0.24",
"@types/jsonwebtoken": "^7.2.7",
"@types/uuid": "^3.4.3",
"angular-paging": "2.2.1",
"angular-resource": "1.4.9",
Expand Down Expand Up @@ -121,6 +126,7 @@
"isomorphic-fetch": "2.2.1",
"joi": "6.10.1",
"jquery": "^3.3.1",
"jsonwebtoken": "^8.3.0",
"jstimezonedetect": "1.0.5",
"lodash": "3.10.1",
"lodash.mean": "^4.1.0",
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/beats/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export function beats(kibana: any) {
config: () =>
Joi.object({
enabled: Joi.boolean().default(true),
encryptionKey: Joi.string(),
enrollmentTokensTtlInSeconds: Joi.number()
.integer()
.min(1)
Expand Down
7 changes: 7 additions & 0 deletions x-pack/plugins/beats/readme.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Documentation for Beats CM in x-pack kibana

### Run tests

```
node scripts/jest.js plugins/beats --watch
```
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { flatten, get, omit } from 'lodash';
import { flatten, get as _get, omit } from 'lodash';
import moment from 'moment';
import { INDEX_NAMES } from '../../../../common/constants';
import {
Expand Down Expand Up @@ -35,7 +35,7 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
return null;
}

return get(response, '_source.beat');
return _get(response, '_source.beat');
}

public async insert(beat: CMBeat) {
Expand Down Expand Up @@ -73,22 +73,6 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
const ids = beatIds.map(beatId => `beat:${beatId}`);

const params = {
_source: false,
body: {
ids,
},
index: INDEX_NAMES.BEATS,
type: '_doc',
};
const response = await this.framework.callWithRequest(req, 'mget', params);
return get(response, 'docs', []);
}

// TODO merge with getBeatsWithIds
public async getVerifiedWithIds(req: FrameworkRequest, beatIds: string[]) {
const ids = beatIds.map(beatId => `beat:${beatId}`);

const params = {
_sourceInclude: ['beat.id', 'beat.verified_on'],
body: {
Expand All @@ -98,7 +82,10 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
type: '_doc',
};
const response = await this.framework.callWithRequest(req, 'mget', params);
return get(response, 'docs', []);

return get(response, 'docs', [])
.filter((b: any) => b.found)
.map((b: any) => b._source.beat);
}

public async verifyBeats(req: FrameworkRequest, beatIds: string[]) {
Expand All @@ -115,14 +102,19 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
);

const params = {
_sourceInclude: ['beat.id', 'beat.verified_on'],
body,
index: INDEX_NAMES.BEATS,
refresh: 'wait_for',
type: '_doc',
};

const response = await this.framework.callWithRequest(req, 'bulk', params);
return get(response, 'items', []);

return _get(response, 'items', []).map(b => ({
..._get(b, 'update.get._source.beat', {}),
updateStatus: _get(b, 'update.result', 'unknown error'),
}));
}

public async getAll(req: FrameworkRequest) {
Expand Down
123 changes: 123 additions & 0 deletions x-pack/plugins/beats/server/lib/adapters/beats/memory_beats_adapter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { omit } from 'lodash';
import moment from 'moment';

import {
CMBeat,
CMBeatsAdapter,
CMTagAssignment,
FrameworkRequest,
} from '../../lib';

export class MemoryBeatsAdapter implements CMBeatsAdapter {
private beatsDB: CMBeat[];

constructor(beatsDB: CMBeat[]) {
this.beatsDB = beatsDB;
}

public async get(id: string) {
return this.beatsDB.find(beat => beat.id === id);
}

public async insert(beat: CMBeat) {
this.beatsDB.push(beat);
}

public async update(beat: CMBeat) {
const beatIndex = this.beatsDB.findIndex(b => b.id === beat.id);

this.beatsDB[beatIndex] = {
...this.beatsDB[beatIndex],
...beat,
};
}

public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
return this.beatsDB.filter(beat => beatIds.includes(beat.id));
}

public async verifyBeats(req: FrameworkRequest, beatIds: string[]) {
if (!Array.isArray(beatIds) || beatIds.length === 0) {
return [];
}

const verifiedOn = moment().toJSON();

this.beatsDB.forEach((beat, i) => {
if (beatIds.includes(beat.id)) {
this.beatsDB[i].verified_on = verifiedOn;
}
});

return this.beatsDB.filter(beat => beatIds.includes(beat.id));
}

public async getAll(req: FrameworkRequest) {
return this.beatsDB.map((beat: any) => omit(beat, ['access_token']));
}

public async removeTagsFromBeats(
req: FrameworkRequest,
removals: CMTagAssignment[]
): Promise<CMTagAssignment[]> {
const beatIds = removals.map(r => r.beatId);

const response = this.beatsDB
.filter(beat => beatIds.includes(beat.id))
.map(beat => {
const tagData = removals.find(r => r.beatId === beat.id);
if (tagData) {
if (beat.tags) {
beat.tags = beat.tags.filter(tag => tag !== tagData.tag);
}
}
return beat;
});

return response.map<any>((item: CMBeat, resultIdx: number) => ({
idxInRequest: removals[resultIdx].idxInRequest,
result: 'updated',
status: 200,
}));
}

public async assignTagsToBeats(
req: FrameworkRequest,
assignments: CMTagAssignment[]
): Promise<CMTagAssignment[]> {
const beatIds = assignments.map(r => r.beatId);

this.beatsDB.filter(beat => beatIds.includes(beat.id)).map(beat => {
// get tags that need to be assigned to this beat
const tags = assignments
.filter(a => a.beatId === beat.id)
.map((t: CMTagAssignment) => t.tag);

if (tags.length > 0) {
if (!beat.tags) {
beat.tags = [];
}
const nonExistingTags = tags.filter(
(t: string) => beat.tags && !beat.tags.includes(t)
);

if (nonExistingTags.length > 0) {
beat.tags = beat.tags.concat(nonExistingTags);
}
}
return beat;
});

return assignments.map<any>((item: CMTagAssignment, resultIdx: number) => ({
idxInRequest: assignments[resultIdx].idxInRequest,
result: 'updated',
status: 200,
}));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,25 @@ import {

export class KibanaBackendFrameworkAdapter implements BackendFrameworkAdapter {
public version: string;

private server: Server;
private cryptoHash: string | null;

constructor(hapiServer: Server) {
this.server = hapiServer;
this.version = hapiServer.plugins.kibana.status.plugin.version;
this.cryptoHash = null;

this.validateConfig();
}

public getSetting(settingPath: string) {
// TODO type check this properly
// TODO type check server properly
if (settingPath === 'xpack.beats.encryptionKey') {
// @ts-ignore
return this.server.config().get(settingPath) || this.cryptoHash;
}
// @ts-ignore
return this.server.config().get(settingPath);
return this.server.config().get(settingPath) || this.cryptoHash;
}

public exposeStaticDir(urlPath: string, dir: string): void {
Expand Down Expand Up @@ -79,4 +86,17 @@ export class KibanaBackendFrameworkAdapter implements BackendFrameworkAdapter {
const fields = await callWithRequest(internalRequest, ...rest);
return fields;
}

private validateConfig() {
// @ts-ignore
const config = this.server.config();
const encryptionKey = config.get('xpack.beats.encryptionKey');

if (!encryptionKey) {
this.server.log(
'Using a default encryption key for xpack.beats.encryptionKey. It is recommended that you set xpack.beats.encryptionKey in kibana.yml with a unique token'
);
this.cryptoHash = 'xpack_beats_default_encryptionKey';
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { Client } from 'elasticsearch';
import { Request } from 'hapi';
import { get } from 'lodash';
import {
BackendFrameworkAdapter,
FrameworkRequest,
FrameworkRouteOptions,
WrappableRequest,
} from '../../../lib';

interface TestSettings {
enrollmentTokensTtlInSeconds: number;
encryptionKey: string;
}

export class TestingBackendFrameworkAdapter implements BackendFrameworkAdapter {
public version: string;
private client: Client | null;
private settings: TestSettings;

constructor(client: Client | null, settings: TestSettings) {
this.client = client;
this.settings = settings || {
encryptionKey: 'something_who_cares',
enrollmentTokensTtlInSeconds: 10 * 60, // 10 minutes
};
this.version = 'testing';
}

public getSetting(settingPath: string) {
switch (settingPath) {
case 'xpack.beats.enrollmentTokensTtlInSeconds':
return this.settings.enrollmentTokensTtlInSeconds;
case 'xpack.beats.encryptionKey':
return this.settings.encryptionKey;
}
}

public exposeStaticDir(urlPath: string, dir: string): void {
// not yet testable
}

public registerRoute<RouteRequest extends WrappableRequest, RouteResponse>(
route: FrameworkRouteOptions<RouteRequest, RouteResponse>
) {
// not yet testable
}

public installIndexTemplate(name: string, template: {}) {
if (this.client) {
return this.client.indices.putTemplate({
body: template,
name,
});
}
}

public async callWithInternalUser(esMethod: string, options: {}) {
const api = get<any>(this.client, esMethod);

api(options);

return await api(options);
}

public async callWithRequest(
req: FrameworkRequest<Request>,
esMethod: string,
options: {}
) {
const api = get<any>(this.client, esMethod);

api(options);

return await api(options);
}
}
Loading

0 comments on commit f130bbe

Please sign in to comment.