Skip to content

Commit

Permalink
[Beats Management] add get beat endpoint (elastic#20603)
Browse files Browse the repository at this point in the history
* [Beats Management] Move tokens to use JWT, add more complete test suite (elastic#20317)

* inital effort to move to JWT and added jest based tests on libs

* assign beats tests all passing

* token tests now pass

* add more tests

* all tests now green

* fix broken test, this is beats CM not logstash 😊

* added readme

* move enrollment token back to a hash

* remove un-needed comment

* alias lodash get to avoid confusion

* isolated hash creation

* inital effort to move to JWT and added jest based tests on libs

* assign beats tests all passing

* token tests now pass

* add more tests

* all tests now green

* move enrollment token back to a hash

* remove un-needed comment

* alias lodash get to avoid confusion

* isolated hash creation

* Add initial efforts for backend framework adapter testing

* move ES code to a DatabaseAdapter from BackendAdapter and add a TON of types for ES

* re-typed

* renamed types to match pattern

* aditional renames

* adapter tests should always just use adapterSetup();

* database now uses InternalRequest

* corrected spelling of framework

* fix typings

* remove CRUFT

* RequestOrInternal

* Dont pass around request objects everywhere, just pass the user. Also, removed hapi types as they were not compatible

* fix tests, add test, removed extra comment

* Moved critical path code from route, to more easeley tested domain

* fix auth

* remove beat verification, added get beat endpoint to return configs

* fix type

* update createGetBeatConfigurationRoute URL

* rename method

* update to match PR elastic#20566

* updated lock file

* fix bad merge

* update TSLinting
  • Loading branch information
mattapperson authored and justinkambic committed Jul 23, 2018
1 parent 12ecb75 commit 4e96aea
Show file tree
Hide file tree
Showing 49 changed files with 600 additions and 746 deletions.
5 changes: 1 addition & 4 deletions x-pack/plugins/beats/common/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,4 @@

export { PLUGIN } from './plugin';
export { INDEX_NAMES } from './index_names';
export {
UNIQUENESS_ENFORCING_TYPES,
ConfigurationBlockTypes,
} from './configuration_blocks';
export { UNIQUENESS_ENFORCING_TYPES, ConfigurationBlockTypes } from './configuration_blocks';
22 changes: 10 additions & 12 deletions x-pack/plugins/beats/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,18 @@ import { initServerWithKibana } from './server/kibana.index';

const DEFAULT_ENROLLMENT_TOKENS_TTL_S = 10 * 60; // 10 minutes

export const config = Joi.object({
enabled: Joi.boolean().default(true),
encryptionKey: Joi.string(),
enrollmentTokensTtlInSeconds: Joi.number()
.integer()
.min(1)
.default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
}).default();
export const configPrefix = 'xpack.beats';

export function beats(kibana: any) {
return new kibana.Plugin({
config: () => config,
configPrefix,
config: () =>
Joi.object({
enabled: Joi.boolean().default(true),
encryptionKey: Joi.string(),
enrollmentTokensTtlInSeconds: Joi.number()
.integer()
.min(1)
.default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
}).default(),
configPrefix: 'xpack.beats',
id: PLUGIN.ID,
require: ['kibana', 'elasticsearch', 'xpack_main'],
init(server: any) {
Expand Down
10 changes: 1 addition & 9 deletions x-pack/plugins/beats/readme.md
Original file line number Diff line number Diff line change
@@ -1,15 +1,7 @@
# Documentation for Beats CM in x-pack kibana

### Run tests (from x-pack dir)

Functional tests
### Run tests

```
node scripts/jest.js plugins/beats --watch
```

Functional API tests

```
node scripts/functional_tests --config test/api_integration/config
```
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,7 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
type: '_doc',
};

const response = await this.database.get(
this.framework.internalUser,
params
);
const response = await this.database.get(this.framework.internalUser, params);
if (!response.found) {
return null;
}
Expand Down Expand Up @@ -155,13 +152,11 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
refresh: 'wait_for',
type: '_doc',
});
return _get<any>(response, 'items', []).map(
(item: any, resultIdx: number) => ({
idxInRequest: removals[resultIdx].idxInRequest,
result: item.update.result,
status: item.update.status,
})
);
return _get<any>(response, 'items', []).map((item: any, resultIdx: number) => ({
idxInRequest: removals[resultIdx].idxInRequest,
result: item.update.result,
status: item.update.status,
}));
}

public async assignTagsToBeats(
Expand Down Expand Up @@ -193,12 +188,10 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
refresh: 'wait_for',
type: '_doc',
});
return _get<any>(response, 'items', []).map(
(item: any, resultIdx: any) => ({
idxInRequest: assignments[resultIdx].idxInRequest,
result: item.update.result,
status: item.update.status,
})
);
return _get<any>(response, 'items', []).map((item: any, resultIdx: any) => ({
idxInRequest: assignments[resultIdx].idxInRequest,
result: item.update.result,
status: item.update.status,
}));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
}

public async get(id: string) {
return this.beatsDB.find(beat => beat.id === id);
return this.beatsDB.find(beat => beat.id === id) || null;
}

public async insert(beat: CMBeat) {
Expand Down Expand Up @@ -65,17 +65,15 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
): Promise<BeatsTagAssignment[]> {
const beatIds = removals.map(r => r.beatId);

const response = this.beatsDB
.filter(beat => beatIds.includes(beat.id))
.map(beat => {
const tagData = removals.find(r => r.beatId === beat.id);
if (tagData) {
if (beat.tags) {
beat.tags = beat.tags.filter(tag => tag !== tagData.tag);
}
const response = this.beatsDB.filter(beat => beatIds.includes(beat.id)).map(beat => {
const tagData = removals.find(r => r.beatId === beat.id);
if (tagData) {
if (beat.tags) {
beat.tags = beat.tags.filter(tag => tag !== tagData.tag);
}
return beat;
});
}
return beat;
});

return response.map<any>((item: CMBeat, resultIdx: number) => ({
idxInRequest: removals[resultIdx].idxInRequest,
Expand All @@ -100,9 +98,7 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
if (!beat.tags) {
beat.tags = [];
}
const nonExistingTags = tags.filter(
(t: string) => beat.tags && !beat.tags.includes(t)
);
const nonExistingTags = tags.filter((t: string) => beat.tags && !beat.tags.includes(t));

if (nonExistingTags.length > 0) {
beat.tags = beat.tags.concat(nonExistingTags);
Expand All @@ -111,12 +107,10 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
return beat;
});

return assignments.map<any>(
(item: BeatsTagAssignment, resultIdx: number) => ({
idxInRequest: assignments[resultIdx].idxInRequest,
result: 'updated',
status: 200,
})
);
return assignments.map<any>((item: BeatsTagAssignment, resultIdx: number) => ({
idxInRequest: assignments[resultIdx].idxInRequest,
result: 'updated',
status: 200,
}));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ contractTests('Kibana Database Adapter', {
return await es.cleanup();
},
adapterSetup: () => {
return new KibanaDatabaseAdapter(kbnServer.server.plugins
.elasticsearch as DatabaseKbnESPlugin);
return new KibanaDatabaseAdapter(kbnServer.server.plugins.elasticsearch as DatabaseKbnESPlugin);
},
});
34 changes: 7 additions & 27 deletions x-pack/plugins/beats/server/lib/adapters/database/adapter_types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@
*/
import { FrameworkRequest, FrameworkUser } from '../framework/adapter_types';
export interface DatabaseAdapter {
putTemplate(
user: FrameworkUser,
params: DatabasePutTemplateParams
): Promise<any>;
putTemplate(user: FrameworkUser, params: DatabasePutTemplateParams): Promise<any>;
get<Source>(
user: FrameworkUser,
params: DatabaseGetParams
Expand All @@ -25,27 +22,17 @@ export interface DatabaseAdapter {
user: FrameworkUser,
params: DatabaseDeleteDocumentParams
): Promise<DatabaseDeleteDocumentResponse>;
mget<T>(
user: FrameworkUser,
params: DatabaseMGetParams
): Promise<DatabaseMGetResponse<T>>;
mget<T>(user: FrameworkUser, params: DatabaseMGetParams): Promise<DatabaseMGetResponse<T>>;
bulk(
user: FrameworkUser,
params: DatabaseBulkIndexDocumentsParams
): Promise<DatabaseBulkResponse>;
search<T>(
user: FrameworkUser,
params: DatabaseSearchParams
): Promise<DatabaseSearchResponse<T>>;
search<T>(user: FrameworkUser, params: DatabaseSearchParams): Promise<DatabaseSearchResponse<T>>;
}

export interface DatabaseKbnESCluster {
callWithInternalUser(esMethod: string, options: {}): Promise<any>;
callWithRequest(
req: FrameworkRequest,
esMethod: string,
options: {}
): Promise<any>;
callWithRequest(req: FrameworkRequest, esMethod: string, options: {}): Promise<any>;
}

export interface DatabaseKbnESPlugin {
Expand Down Expand Up @@ -142,14 +129,11 @@ export interface DatabaseBulkResponse {
took: number;
errors: boolean;
items: Array<
| DatabaseDeleteDocumentResponse
| DatabaseIndexDocumentResponse
| DatabaseUpdateDocumentResponse
DatabaseDeleteDocumentResponse | DatabaseIndexDocumentResponse | DatabaseUpdateDocumentResponse
>;
}

export interface DatabaseBulkIndexDocumentsParams
extends DatabaseGenericParams {
export interface DatabaseBulkIndexDocumentsParams extends DatabaseGenericParams {
waitForActiveShards?: string;
refresh?: DatabaseRefresh;
routing?: string;
Expand Down Expand Up @@ -299,11 +283,7 @@ export interface DatabaseGetParams extends DatabaseGenericParams {

export type DatabaseNameList = string | string[] | boolean;
export type DatabaseRefresh = boolean | 'true' | 'false' | 'wait_for' | '';
export type DatabaseVersionType =
| 'internal'
| 'external'
| 'external_gte'
| 'force';
export type DatabaseVersionType = 'internal' | 'external' | 'external_gte' | 'force';
export type ExpandWildcards = 'open' | 'closed' | 'none' | 'all';
export type DefaultOperator = 'AND' | 'OR';
export type DatabaseConflicts = 'abort' | 'proceed';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,7 @@ export class KibanaDatabaseAdapter implements DatabaseAdapter {
constructor(kbnElasticSearch: DatabaseKbnESPlugin) {
this.es = kbnElasticSearch.getCluster('admin');
}
public async putTemplate(
user: FrameworkUser,
params: DatabasePutTemplateParams
): Promise<any> {
public async putTemplate(user: FrameworkUser, params: DatabasePutTemplateParams): Promise<any> {
const callES = this.getCallType(user);
const result = await callES('indices.putTemplate', params);
return result;
Expand All @@ -59,10 +56,7 @@ export class KibanaDatabaseAdapter implements DatabaseAdapter {
// todo
}

public async bulk(
user: FrameworkUser,
params: DatabaseBulkIndexDocumentsParams
): Promise<any> {
public async bulk(user: FrameworkUser, params: DatabaseBulkIndexDocumentsParams): Promise<any> {
const callES = this.getCallType(user);
const result = await callES('bulk', params);
return result;
Expand All @@ -76,10 +70,7 @@ export class KibanaDatabaseAdapter implements DatabaseAdapter {
const result = await callES('create', params);
return result;
}
public async index<T>(
user: FrameworkUser,
params: DatabaseIndexDocumentParams<T>
): Promise<any> {
public async index<T>(user: FrameworkUser, params: DatabaseIndexDocumentParams<T>): Promise<any> {
const callES = this.getCallType(user);
const result = await callES('index', params);
return result;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import {
BackendFrameworkAdapter,
FrameworkRequest,
FrameworkRouteOptions,
WrappableRequest,
} from '../../../lib';

import { IStrictReply, Request, Server } from 'hapi';
import { internalFrameworkRequest, wrapRequest } from '../../../../utils/wrap_request';

export class KibanaBackendFrameworkAdapter implements BackendFrameworkAdapter {
public version: string;
private server: Server;
private cryptoHash: string | null;

constructor(hapiServer: Server) {
this.server = hapiServer;
this.version = hapiServer.plugins.kibana.status.plugin.version;
this.cryptoHash = null;

this.validateConfig();
}

public getSetting(settingPath: string) {
// TODO type check server properly
if (settingPath === 'xpack.beats.encryptionKey') {
// @ts-ignore
return this.server.config().get(settingPath) || this.cryptoHash;
}
// @ts-ignore
return this.server.config().get(settingPath) || this.cryptoHash;
}

public exposeStaticDir(urlPath: string, dir: string): void {
this.server.route({
handler: {
directory: {
path: dir,
},
},
method: 'GET',
path: urlPath,
});
}

public registerRoute<RouteRequest extends WrappableRequest, RouteResponse>(
route: FrameworkRouteOptions<RouteRequest, RouteResponse>
) {
const wrappedHandler = (request: any, reply: IStrictReply<RouteResponse>) =>
route.handler(wrapRequest(request), reply);

this.server.route({
config: route.config,
handler: wrappedHandler,
method: route.method,
path: route.path,
});
}

public installIndexTemplate(name: string, template: {}) {
return this.callWithInternalUser('indices.putTemplate', {
body: template,
name,
});
}

public async callWithInternalUser(esMethod: string, options: {}) {
const { elasticsearch } = this.server.plugins;
const { callWithInternalUser } = elasticsearch.getCluster('admin');
return await callWithInternalUser(esMethod, options);
}

public async callWithRequest(req: FrameworkRequest<Request>, ...rest: any[]) {
const internalRequest = req[internalFrameworkRequest];
const { elasticsearch } = internalRequest.server.plugins;
const { callWithRequest } = elasticsearch.getCluster('data');
const fields = await callWithRequest(internalRequest, ...rest);
return fields;
}

private validateConfig() {
// @ts-ignore
const config = this.server.config();
const encryptionKey = config.get('xpack.beats.encryptionKey');

if (!encryptionKey) {
this.server.log(
'Using a default encryption key for xpack.beats.encryptionKey. It is recommended that you set xpack.beats.encryptionKey in kibana.yml with a unique token'
);
this.cryptoHash = 'xpack_beats_default_encryptionKey';
}
}
}
Loading

0 comments on commit 4e96aea

Please sign in to comment.