From c807c4ee39a99a5ead4f293c28deeff5ee031ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 19 Dec 2019 16:27:17 +0100 Subject: [PATCH 01/20] Scaffold `log_entries/entries` route --- .../common/http_api/log_entries/common.ts | 3 + .../common/http_api/log_entries/entries.ts | 31 +++++++ .../common/http_api/log_entries/index.ts | 1 + .../plugins/infra/server/infra_server.ts | 2 + .../log_entries/kibana_log_entries_adapter.ts | 32 +++++++ .../log_entries_domain/log_entries_domain.ts | 29 +++++++ .../server/routes/log_entries/entries.ts | 84 +++++++++++++++++++ .../infra/server/routes/log_entries/index.ts | 1 + 8 files changed, 183 insertions(+) create mode 100644 x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts create mode 100644 x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts index 3eb7e278bf99..9b067fc16219 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts @@ -10,3 +10,6 @@ export const logEntriesCursorRT = rt.type({ time: rt.number, tiebreaker: rt.number, }); + +export const esDateRT = rt.union([rt.string, rt.number]); +export type ESDate = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts new file mode 100644 index 000000000000..8135bc385383 --- /dev/null +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { esDateRT } from './common'; + +export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; + +export const logEntriesRequestRT = rt.intersection([ + rt.type({ + sourceId: rt.string, + startDate: esDateRT, + endDate: esDateRT, + }), + rt.partial({ + query: rt.string, + }), +]); + +export type LogEntriesRequest = rt.TypeOf; + +export const logEntriesResponseRT = rt.type({ + data: rt.type({ + entries: rt.array(rt.any), + }), +}); + +export type LogEntriesResponse = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts index 8fed914c3dc8..b85c24d4ce27 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './entries'; export * from './item'; export * from './summary'; export * from './summary_highlights'; diff --git a/x-pack/legacy/plugins/infra/server/infra_server.ts b/x-pack/legacy/plugins/infra/server/infra_server.ts index 108e1b1e3f39..f9c7280f00b7 100644 --- a/x-pack/legacy/plugins/infra/server/infra_server.ts +++ b/x-pack/legacy/plugins/infra/server/infra_server.ts @@ -20,6 +20,7 @@ import { initMetadataRoute } from './routes/metadata'; import { initSnapshotRoute } from './routes/snapshot'; import { initNodeDetailsRoute } from './routes/node_details'; import { + initLogEntriesRoute, initLogEntriesSummaryRoute, initLogEntriesSummaryHighlightsRoute, initLogEntriesItemRoute, @@ -43,6 +44,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => { initSnapshotRoute(libs); initNodeDetailsRoute(libs); initValidateLogAnalysisIndicesRoute(libs); + initLogEntriesRoute(libs); initLogEntriesSummaryRoute(libs); initLogEntriesSummaryHighlightsRoute(libs); initLogEntriesItemRoute(libs); diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index ec45171baa7b..94290d5d0116 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -82,6 +82,38 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { return direction === 'asc' ? documents : documents.reverse(); } + public async getLogEntries( + requestContext: RequestHandlerContext, + sourceConfiguration: InfraSourceConfiguration, + startTimestamp: number, + endTimestamp: number + ): Promise { + const query = { + allowNoIndices: true, + index: sourceConfiguration.logAlias, + ignoreUnavailable: true, + body: { + size: 10, + query: { + range: { + [sourceConfiguration.fields.timestamp]: { + gte: startTimestamp, + lte: endTimestamp, + }, + }, + }, + sort: { + [sourceConfiguration.fields.timestamp]: 'asc', + [sourceConfiguration.fields.tiebreaker]: 'asc', + }, + }, + }; + + const documents = await this.framework.callWithRequest(requestContext, 'search', query); + return documents.hits.hits; + } + + /** @deprecated */ public async getContainedLogEntryDocuments( requestContext: RequestHandlerContext, sourceConfiguration: InfraSourceConfiguration, diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 347f0dcf795b..4ac7f28a8c3c 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -102,6 +102,28 @@ export class InfraLogEntriesDomain { }; } + public async getLogEntries( + requestContext: RequestHandlerContext, + sourceId: string, + startTimestamp: number, + endTimestamp: number + ): Promise { + const { configuration } = await this.libs.sources.getSourceConfiguration( + requestContext, + sourceId + ); + + const documents = await this.adapter.getLogEntries( + requestContext, + configuration, + startTimestamp, + endTimestamp + ); + + return documents; + } + + /** @deprecated */ public async getLogEntriesBetween( requestContext: RequestHandlerContext, sourceId: string, @@ -324,6 +346,13 @@ export interface LogEntriesAdapter { highlightQuery?: LogEntryQuery ): Promise; + getLogEntries( + requestContext: RequestHandlerContext, + sourceConfiguration: InfraSourceConfiguration, + startTimestamp: number, + endTimestamp: number + ): Promise; + getContainedLogEntryDocuments( requestContext: RequestHandlerContext, sourceConfiguration: InfraSourceConfiguration, diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts new file mode 100644 index 000000000000..c50961948eb9 --- /dev/null +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; + +import { pipe } from 'fp-ts/lib/pipeable'; +import { fold } from 'fp-ts/lib/Either'; +import { identity } from 'fp-ts/lib/function'; +import { schema } from '@kbn/config-schema'; +import datemath from '@elastic/datemath'; + +import { throwErrors } from '../../../common/runtime_types'; + +import { InfraBackendLibs } from '../../lib/infra_types'; +import { + ESDate, + LOG_ENTRIES_PATH, + logEntriesRequestRT, + logEntriesResponseRT, +} from '../../../common/http_api/log_entries'; +import { parseFilterQuery } from '../../utils/serialized_query'; + +const escapeHatch = schema.object({}, { allowUnknowns: true }); + +export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) => { + framework.registerRoute( + { + method: 'post', + path: LOG_ENTRIES_PATH, + validate: { body: escapeHatch }, + }, + async (requestContext, request, response) => { + try { + const payload = pipe( + logEntriesRequestRT.decode(request.body), + fold(throwErrors(Boom.badRequest), identity) + ); + + const { startDate, endDate, sourceId, query } = payload; + + const startTimestamp = parseDate(startDate); + const endTimestamp = parseDate(endDate); + + if (!startTimestamp || !endTimestamp) { + return response.badRequest(); + } + + const entries = await logEntries.getLogEntries( + requestContext, + sourceId, + startTimestamp, + endTimestamp + ); + + return response.ok({ + body: logEntriesResponseRT.encode({ + data: { + entries, + }, + }), + }); + } catch (error) { + return response.internalError({ + body: error.message, + }); + } + } + ); +}; + +function parseDate(date: ESDate): number | undefined { + if (typeof date === 'number') { + return date; + } + + const parsedDate = datemath.parse(date); + + if (parsedDate) { + return parsedDate.valueOf(); + } +} diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts index 8fed914c3dc8..b85c24d4ce27 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './entries'; export * from './item'; export * from './summary'; export * from './summary_highlights'; From 1fc76638a7f1a869e4c2507ade271feba900d0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 19 Dec 2019 17:10:14 +0100 Subject: [PATCH 02/20] Scaffold a log entry response --- .../common/http_api/log_entries/entries.ts | 14 +++++++-- .../log_entries_domain/log_entries_domain.ts | 29 +++++++++++++++++-- .../server/routes/log_entries/entries.ts | 2 ++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index 8135bc385383..a3edefef673b 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -5,7 +5,7 @@ */ import * as rt from 'io-ts'; -import { esDateRT } from './common'; +import { logEntriesCursorRT, esDateRT } from './common'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; @@ -22,9 +22,19 @@ export const logEntriesRequestRT = rt.intersection([ export type LogEntriesRequest = rt.TypeOf; +export const logEntryRT = rt.type({ + id: rt.string, + cursor: logEntriesCursorRT, + columns: rt.array(rt.any), +}); + +export type LogEntry = rt.TypeOf; + export const logEntriesResponseRT = rt.type({ data: rt.type({ - entries: rt.array(rt.any), + entries: rt.array(logEntryRT), + topCursor: logEntriesCursorRT, + bottomCursor: logEntriesCursorRT, }), }); diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 4ac7f28a8c3c..b6379bb03c81 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -5,7 +5,7 @@ */ import stringify from 'json-stable-stringify'; -import { sortBy } from 'lodash'; +import { sortBy, get } from 'lodash'; import { RequestHandlerContext } from 'src/core/server'; import { TimeKey } from '../../../../common/time'; @@ -13,6 +13,7 @@ import { JsonObject } from '../../../../common/typed_json'; import { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket, + LogEntry, LogEntriesItem, } from '../../../../common/http_api'; import { InfraLogEntry, InfraLogMessageSegment } from '../../../graphql/types'; @@ -107,7 +108,7 @@ export class InfraLogEntriesDomain { sourceId: string, startTimestamp: number, endTimestamp: number - ): Promise { + ): Promise { const { configuration } = await this.libs.sources.getSourceConfiguration( requestContext, sourceId @@ -120,7 +121,29 @@ export class InfraLogEntriesDomain { endTimestamp ); - return documents; + const entries = documents.map((doc: any) => { + return { + id: doc._id, + cursor: { + time: doc.sort[0], + tiebreaker: doc.sort[1], + }, + columns: configuration.logColumns.map(column => { + if ('timestampColumn' in column) { + return doc._source[configuration.fields.timestamp]; + } + if ('messageColumn' in column) { + // FIXME + return doc._source.message; + } + if ('fieldColumn' in column) { + return get(doc._source, column.fieldColumn.field); + } + }), + }; + }); + + return entries; } /** @deprecated */ diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index c50961948eb9..501f69089558 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -59,6 +59,8 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) body: logEntriesResponseRT.encode({ data: { entries, + topCursor: entries[0].cursor, + bottomCursor: entries[entries.length - 1].cursor, }, }), }); From 3699bcfb7495efd8d56e26d96c32a426b25ad375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 20 Dec 2019 17:24:43 +0100 Subject: [PATCH 03/20] Add `after` pagination --- .../common/http_api/log_entries/common.ts | 1 + .../common/http_api/log_entries/entries.ts | 9 +++++- .../common/http_api/log_entries/index.ts | 1 + .../log_entries/kibana_log_entries_adapter.ts | 29 +++++++++++++++---- .../log_entries_domain/log_entries_domain.ts | 24 +++++++++++---- .../server/routes/log_entries/entries.ts | 15 ++++++---- 6 files changed, 61 insertions(+), 18 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts index 9b067fc16219..4cdf33329837 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts @@ -10,6 +10,7 @@ export const logEntriesCursorRT = rt.type({ time: rt.number, tiebreaker: rt.number, }); +export type LogEntriesCursor = rt.TypeOf; export const esDateRT = rt.union([rt.string, rt.number]); export type ESDate = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index a3edefef673b..558663a71630 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -9,7 +9,7 @@ import { logEntriesCursorRT, esDateRT } from './common'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; -export const logEntriesRequestRT = rt.intersection([ +export const logEntriesBaseRequestRT = rt.intersection([ rt.type({ sourceId: rt.string, startDate: esDateRT, @@ -20,6 +20,13 @@ export const logEntriesRequestRT = rt.intersection([ }), ]); +export const logEntriesAfterRequestRT = rt.intersection([ + logEntriesBaseRequestRT, + rt.type({ after: rt.union([logEntriesCursorRT, rt.literal('first')]) }), +]); + +export const logEntriesRequestRT = rt.union([logEntriesBaseRequestRT, logEntriesAfterRequestRT]); + export type LogEntriesRequest = rt.TypeOf; export const logEntryRT = rt.type({ diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts index b85c24d4ce27..e78eb605d999 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './common'; export * from './entries'; export * from './item'; export * from './summary'; diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index 94290d5d0116..ea0cb84ba9b6 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -20,6 +20,7 @@ import { compareTimeKeys, isTimeKey, TimeKey } from '../../../../common/time'; import { JsonObject } from '../../../../common/typed_json'; import { LogEntriesAdapter, + LogEntriesParams, LogEntryDocument, LogEntryQuery, LogSummaryBucket, @@ -85,27 +86,43 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { public async getLogEntries( requestContext: RequestHandlerContext, sourceConfiguration: InfraSourceConfiguration, - startTimestamp: number, - endTimestamp: number + fields: string[], + params: LogEntriesParams ): Promise { + const { startTimestamp, endTimestamp, cursor } = params; + + const sortDirection = 'asc'; + let searchAfterClause = {}; + + if (cursor?.after && cursor.after !== 'first') { + searchAfterClause = { + search_after: [cursor.after.time, cursor.after.tiebreaker], + }; + } + + const sort = { + [sourceConfiguration.fields.timestamp]: sortDirection, + [sourceConfiguration.fields.tiebreaker]: sortDirection, + }; + const query = { allowNoIndices: true, index: sourceConfiguration.logAlias, ignoreUnavailable: true, body: { size: 10, + track_total_hits: false, query: { range: { [sourceConfiguration.fields.timestamp]: { gte: startTimestamp, lte: endTimestamp, + format: TIMESTAMP_FORMAT, }, }, }, - sort: { - [sourceConfiguration.fields.timestamp]: 'asc', - [sourceConfiguration.fields.tiebreaker]: 'asc', - }, + sort, + ...searchAfterClause, }, }; diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index b6379bb03c81..86a2de93bf7f 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -15,6 +15,7 @@ import { LogEntriesSummaryHighlightsBucket, LogEntry, LogEntriesItem, + LogEntriesCursor, } from '../../../../common/http_api'; import { InfraLogEntry, InfraLogMessageSegment } from '../../../graphql/types'; import { @@ -33,6 +34,13 @@ import { compileFormattingRules, } from './message'; +export interface LogEntriesParams { + startTimestamp: number; + endTimestamp: number; + query?: JsonObject; + cursor?: { after: LogEntriesCursor | 'first' }; +} + export class InfraLogEntriesDomain { constructor( private readonly adapter: LogEntriesAdapter, @@ -106,19 +114,23 @@ export class InfraLogEntriesDomain { public async getLogEntries( requestContext: RequestHandlerContext, sourceId: string, - startTimestamp: number, - endTimestamp: number + params: LogEntriesParams ): Promise { const { configuration } = await this.libs.sources.getSourceConfiguration( requestContext, sourceId ); + const messageFormattingRules = compileFormattingRules( + getBuiltinRules(configuration.fields.message) + ); + const requiredFields = getRequiredFields(configuration, messageFormattingRules); + const documents = await this.adapter.getLogEntries( requestContext, configuration, - startTimestamp, - endTimestamp + requiredFields, + params ); const entries = documents.map((doc: any) => { @@ -372,8 +384,8 @@ export interface LogEntriesAdapter { getLogEntries( requestContext: RequestHandlerContext, sourceConfiguration: InfraSourceConfiguration, - startTimestamp: number, - endTimestamp: number + fields: string[], + params: LogEntriesParams ): Promise; getContainedLogEntryDocuments( diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index 501f69089558..75531354c414 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -41,6 +41,11 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) const { startDate, endDate, sourceId, query } = payload; + let cursor; + if ('after' in payload) { + cursor = { after: payload.after }; + } + const startTimestamp = parseDate(startDate); const endTimestamp = parseDate(endDate); @@ -48,12 +53,12 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) return response.badRequest(); } - const entries = await logEntries.getLogEntries( - requestContext, - sourceId, + const entries = await logEntries.getLogEntries(requestContext, sourceId, { startTimestamp, - endTimestamp - ); + endTimestamp, + query: parseFilterQuery(query), + cursor, + }); return response.ok({ body: logEntriesResponseRT.encode({ From 30a0a8df64fc832556fdccc9fac29a4194dd88c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 20 Dec 2019 18:06:31 +0100 Subject: [PATCH 04/20] Add `before` pagination --- .../common/http_api/log_entries/entries.ts | 11 +++++- .../log_entries/kibana_log_entries_adapter.ts | 37 ++++++++++++++----- .../log_entries_domain/log_entries_domain.ts | 2 +- .../server/routes/log_entries/entries.ts | 4 +- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index 558663a71630..fac70e7874ea 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -20,12 +20,21 @@ export const logEntriesBaseRequestRT = rt.intersection([ }), ]); +export const logEntriesBeforeRequestRT = rt.intersection([ + logEntriesBaseRequestRT, + rt.type({ before: rt.union([logEntriesCursorRT, rt.literal('last')]) }), +]); + export const logEntriesAfterRequestRT = rt.intersection([ logEntriesBaseRequestRT, rt.type({ after: rt.union([logEntriesCursorRT, rt.literal('first')]) }), ]); -export const logEntriesRequestRT = rt.union([logEntriesBaseRequestRT, logEntriesAfterRequestRT]); +export const logEntriesRequestRT = rt.union([ + logEntriesBaseRequestRT, + logEntriesBeforeRequestRT, + logEntriesAfterRequestRT, +]); export type LogEntriesRequest = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index ea0cb84ba9b6..aac32f8476ba 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -91,14 +91,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { ): Promise { const { startTimestamp, endTimestamp, cursor } = params; - const sortDirection = 'asc'; - let searchAfterClause = {}; - - if (cursor?.after && cursor.after !== 'first') { - searchAfterClause = { - search_after: [cursor.after.time, cursor.after.tiebreaker], - }; - } + const { sortDirection, searchAfterClause } = processCursor(cursor); const sort = { [sourceConfiguration.fields.timestamp]: sortDirection, @@ -127,7 +120,8 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { }; const documents = await this.framework.callWithRequest(requestContext, 'search', query); - return documents.hits.hits; + + return sortDirection === 'asc' ? documents.hits.hits : documents.hits.hits.reverse(); } /** @deprecated */ @@ -404,6 +398,31 @@ const convertDateRangeBucketToSummaryBucket = ( const createQueryFilterClauses = (filterQuery: LogEntryQuery | undefined) => filterQuery ? [filterQuery] : []; +function processCursor( + cursor: LogEntriesParams['cursor'] +): { + sortDirection: 'asc' | 'desc'; + searchAfterClause: { search_after?: readonly [number, number] }; +} { + let sortDirection: 'asc' | 'desc' = 'asc'; + let searchAfterClause = {}; + + if (!cursor) { + return { sortDirection, searchAfterClause }; + } + + if ('before' in cursor) { + sortDirection = 'desc'; + if (cursor.before !== 'last') { + searchAfterClause = { search_after: [cursor.before.time, cursor.before.tiebreaker] as const }; + } + } else if ('after' in cursor && cursor.after !== 'first') { + searchAfterClause = { search_after: [cursor.after.time, cursor.after.tiebreaker] as const }; + } + + return { sortDirection, searchAfterClause }; +} + const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([ runtimeTypes.type({ doc_count: runtimeTypes.number, diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 86a2de93bf7f..cbed02a082f0 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -38,7 +38,7 @@ export interface LogEntriesParams { startTimestamp: number; endTimestamp: number; query?: JsonObject; - cursor?: { after: LogEntriesCursor | 'first' }; + cursor?: { before: LogEntriesCursor | 'last' } | { after: LogEntriesCursor | 'first' }; } export class InfraLogEntriesDomain { diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index 75531354c414..bb2a9994a78f 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -42,7 +42,9 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) const { startDate, endDate, sourceId, query } = payload; let cursor; - if ('after' in payload) { + if ('before' in payload) { + cursor = { before: payload.before }; + } else if ('after' in payload) { cursor = { after: payload.after }; } From 38435abe2069529895ee61d75ca1fce9f9b90020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 23 Dec 2019 15:12:00 +0100 Subject: [PATCH 05/20] Process `query` parameter --- .../log_entries/kibana_log_entries_adapter.ts | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index aac32f8476ba..b32dedb8065d 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -89,7 +89,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { fields: string[], params: LogEntriesParams ): Promise { - const { startTimestamp, endTimestamp, cursor } = params; + const { startTimestamp, endTimestamp, query, cursor } = params; const { sortDirection, searchAfterClause } = processCursor(cursor); @@ -98,7 +98,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { [sourceConfiguration.fields.tiebreaker]: sortDirection, }; - const query = { + const esQuery = { allowNoIndices: true, index: sourceConfiguration.logAlias, ignoreUnavailable: true, @@ -106,12 +106,19 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { size: 10, track_total_hits: false, query: { - range: { - [sourceConfiguration.fields.timestamp]: { - gte: startTimestamp, - lte: endTimestamp, - format: TIMESTAMP_FORMAT, - }, + bool: { + filter: [ + ...createQueryFilterClauses(query), + { + range: { + [sourceConfiguration.fields.timestamp]: { + gte: startTimestamp, + lte: endTimestamp, + format: TIMESTAMP_FORMAT, + }, + }, + }, + ], }, }, sort, @@ -119,7 +126,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { }, }; - const documents = await this.framework.callWithRequest(requestContext, 'search', query); + const documents = await this.framework.callWithRequest(requestContext, 'search', esQuery); return sortDirection === 'asc' ? documents.hits.hits : documents.hits.hits.reverse(); } From c57e50d8a55db3587678973f3814acafd95b5278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 23 Dec 2019 16:42:27 +0100 Subject: [PATCH 06/20] Use pre-existing structure for the columns --- .../log_entries/kibana_log_entries_adapter.ts | 42 ++++++++++++++++--- .../log_entries_domain/log_entries_domain.ts | 30 ++++++++----- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index b32dedb8065d..de1da7d6d87f 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -17,7 +17,7 @@ import { map, fold } from 'fp-ts/lib/Either'; import { identity, constant } from 'fp-ts/lib/function'; import { RequestHandlerContext } from 'src/core/server'; import { compareTimeKeys, isTimeKey, TimeKey } from '../../../../common/time'; -import { JsonObject } from '../../../../common/typed_json'; +import { JsonObject, JsonValue } from '../../../../common/typed_json'; import { LogEntriesAdapter, LogEntriesParams, @@ -88,7 +88,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { sourceConfiguration: InfraSourceConfiguration, fields: string[], params: LogEntriesParams - ): Promise { + ): Promise { const { startTimestamp, endTimestamp, query, cursor } = params; const { sortDirection, searchAfterClause } = processCursor(cursor); @@ -126,9 +126,13 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { }, }; - const documents = await this.framework.callWithRequest(requestContext, 'search', esQuery); - - return sortDirection === 'asc' ? documents.hits.hits : documents.hits.hits.reverse(); + const esResult = await this.framework.callWithRequest( + requestContext, + 'search', + esQuery + ); + const hits = sortDirection === 'asc' ? esResult.hits.hits : esResult.hits.hits.reverse(); + return mapHitsToLogEntryDocuments(hits, sourceConfiguration.fields.timestamp, fields); } /** @deprecated */ @@ -369,6 +373,34 @@ function getLookupIntervals(start: number, direction: 'asc' | 'desc'): Array<[nu return intervals; } +function mapHitsToLogEntryDocuments( + hits: LogItemHit[], + timestampField: string, + fields: string[] +): LogEntryDocument[] { + return hits.map(hit => { + const logFields = fields.reduce<{ [fieldName: string]: JsonValue }>( + (flattenedFields, field) => { + if (has(field, hit._source)) { + flattenedFields[field] = get(field, hit._source); + } + return flattenedFields; + }, + {} + ); + + return { + gid: hit._id, + // timestamp: hit._source[timestampField], + // FIXME s/key/cursor/g + key: { time: hit.sort[0], tiebreaker: hit.sort[1] }, + fields: logFields, + highlights: {}, + }; + }); +} + +/** @deprecated */ const convertHitToLogEntryDocument = (fields: string[]) => ( hit: SortedSearchHit ): LogEntryDocument => ({ diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index cbed02a082f0..58459d7371fb 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -124,6 +124,7 @@ export class InfraLogEntriesDomain { const messageFormattingRules = compileFormattingRules( getBuiltinRules(configuration.fields.message) ); + const requiredFields = getRequiredFields(configuration, messageFormattingRules); const documents = await this.adapter.getLogEntries( @@ -133,23 +134,29 @@ export class InfraLogEntriesDomain { params ); - const entries = documents.map((doc: any) => { + const entries = documents.map(doc => { return { - id: doc._id, - cursor: { - time: doc.sort[0], - tiebreaker: doc.sort[1], - }, + id: doc.gid, + cursor: doc.key, columns: configuration.logColumns.map(column => { if ('timestampColumn' in column) { - return doc._source[configuration.fields.timestamp]; + return { + columnId: column.timestampColumn.id, + timestamp: doc.key.time, + }; } if ('messageColumn' in column) { - // FIXME - return doc._source.message; + return { + columnId: column.messageColumn.id, + message: messageFormattingRules.format(doc.fields, doc.highlights), + }; } if ('fieldColumn' in column) { - return get(doc._source, column.fieldColumn.field); + return { + columnId: column.fieldColumn.id, + field: column.fieldColumn.field, + value: doc.fields[column.fieldColumn.field], + }; } }), }; @@ -386,7 +393,7 @@ export interface LogEntriesAdapter { sourceConfiguration: InfraSourceConfiguration, fields: string[], params: LogEntriesParams - ): Promise; + ): Promise; getContainedLogEntryDocuments( requestContext: RequestHandlerContext, @@ -430,6 +437,7 @@ export interface LogSummaryBucket { topEntryKeys: TimeKey[]; } +/** @deprecated */ const convertLogDocumentToEntry = ( sourceId: string, logColumns: InfraSourceConfiguration['logColumns'], From 57ad6d59ee201b383dd1e170fefb92cd0f51c6d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 24 Dec 2019 13:43:33 +0100 Subject: [PATCH 07/20] Change type of date ranges We will move the responsibility to parse the dates to the client. The API will only take timestamps --- .../common/http_api/log_entries/common.ts | 3 --- .../common/http_api/log_entries/entries.ts | 6 ++--- .../log_entries/kibana_log_entries_adapter.ts | 6 ++--- .../log_entries_domain/log_entries_domain.ts | 6 ++--- .../server/routes/log_entries/entries.ts | 25 ++----------------- 5 files changed, 11 insertions(+), 35 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts index 4cdf33329837..0b3122232200 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts @@ -11,6 +11,3 @@ export const logEntriesCursorRT = rt.type({ tiebreaker: rt.number, }); export type LogEntriesCursor = rt.TypeOf; - -export const esDateRT = rt.union([rt.string, rt.number]); -export type ESDate = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index fac70e7874ea..b941a93da068 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -5,15 +5,15 @@ */ import * as rt from 'io-ts'; -import { logEntriesCursorRT, esDateRT } from './common'; +import { logEntriesCursorRT } from './common'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; export const logEntriesBaseRequestRT = rt.intersection([ rt.type({ sourceId: rt.string, - startDate: esDateRT, - endDate: esDateRT, + startDate: rt.number, + endDate: rt.number, }), rt.partial({ query: rt.string, diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index de1da7d6d87f..fe846e0e3069 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -89,7 +89,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { fields: string[], params: LogEntriesParams ): Promise { - const { startTimestamp, endTimestamp, query, cursor } = params; + const { startDate, endDate, query, cursor } = params; const { sortDirection, searchAfterClause } = processCursor(cursor); @@ -112,8 +112,8 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { { range: { [sourceConfiguration.fields.timestamp]: { - gte: startTimestamp, - lte: endTimestamp, + gte: startDate, + lte: endDate, format: TIMESTAMP_FORMAT, }, }, diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 58459d7371fb..81b138f5e54d 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -5,7 +5,7 @@ */ import stringify from 'json-stable-stringify'; -import { sortBy, get } from 'lodash'; +import { sortBy } from 'lodash'; import { RequestHandlerContext } from 'src/core/server'; import { TimeKey } from '../../../../common/time'; @@ -35,8 +35,8 @@ import { } from './message'; export interface LogEntriesParams { - startTimestamp: number; - endTimestamp: number; + startDate: number; + endDate: number; query?: JsonObject; cursor?: { before: LogEntriesCursor | 'last' } | { after: LogEntriesCursor | 'first' }; } diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index bb2a9994a78f..1c34d3b02ab8 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -10,13 +10,11 @@ import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; import { schema } from '@kbn/config-schema'; -import datemath from '@elastic/datemath'; import { throwErrors } from '../../../common/runtime_types'; import { InfraBackendLibs } from '../../lib/infra_types'; import { - ESDate, LOG_ENTRIES_PATH, logEntriesRequestRT, logEntriesResponseRT, @@ -48,16 +46,9 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) cursor = { after: payload.after }; } - const startTimestamp = parseDate(startDate); - const endTimestamp = parseDate(endDate); - - if (!startTimestamp || !endTimestamp) { - return response.badRequest(); - } - const entries = await logEntries.getLogEntries(requestContext, sourceId, { - startTimestamp, - endTimestamp, + startDate, + endDate, query: parseFilterQuery(query), cursor, }); @@ -79,15 +70,3 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) } ); }; - -function parseDate(date: ESDate): number | undefined { - if (typeof date === 'number') { - return date; - } - - const parsedDate = datemath.parse(date); - - if (parsedDate) { - return parsedDate.valueOf(); - } -} From 54dfd0f6faa9aaa5609ecf7878c1948faf0ca91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 27 Dec 2019 12:30:04 +0100 Subject: [PATCH 08/20] Add `center` parameter Allows consumers of the API to get log items around a certain cursor --- .../common/http_api/log_entries/entries.ts | 6 +++ .../log_entries_domain/log_entries_domain.ts | 44 +++++++++++++++++++ .../server/routes/log_entries/entries.ts | 22 +++++++--- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index b941a93da068..288df085ba8e 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -30,10 +30,16 @@ export const logEntriesAfterRequestRT = rt.intersection([ rt.type({ after: rt.union([logEntriesCursorRT, rt.literal('first')]) }), ]); +export const logEntriesCenteredRT = rt.intersection([ + logEntriesBaseRequestRT, + rt.type({ center: logEntriesCursorRT }), +]); + export const logEntriesRequestRT = rt.union([ logEntriesBaseRequestRT, logEntriesBeforeRequestRT, logEntriesAfterRequestRT, + logEntriesCenteredRT, ]); export type LogEntriesRequest = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 81b138f5e54d..af84161befdc 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -40,6 +40,12 @@ export interface LogEntriesParams { query?: JsonObject; cursor?: { before: LogEntriesCursor | 'last' } | { after: LogEntriesCursor | 'first' }; } +export interface LogEntriesAroundParams { + startDate: number; + endDate: number; + center: LogEntriesCursor; + query?: JsonObject; +} export class InfraLogEntriesDomain { constructor( @@ -47,6 +53,44 @@ export class InfraLogEntriesDomain { private readonly libs: { sources: InfraSources } ) {} + /* Name is temporary until we can clean up the GraphQL implementation */ + /* eslint-disable-next-line @typescript-eslint/camelcase */ + public async getLogEntriesAround__new( + requestContext: RequestHandlerContext, + sourceId: string, + params: LogEntriesAroundParams + ) { + const { startDate, endDate, center, query } = params; + + const entriesBefore = await this.getLogEntries(requestContext, sourceId, { + startDate, + endDate, + query, + cursor: { before: center }, + }); + + /* Elasticsearch's `search_after` returns documents after the specified cursor. + * - If we have documents before the center, we search after the last of + * those. The first document of the new group is the center. + * - If there were no documents, we search one milisecond before the + * center. It then becomes the first document. + */ + const cursorAfter = + entriesBefore.length > 0 + ? entriesBefore[entriesBefore.length - 1].cursor + : { time: center.time - 1, tiebreaker: 0 }; + + const entriesAfter = await this.getLogEntries(requestContext, sourceId, { + startDate, + endDate, + query, + cursor: { after: cursorAfter }, + }); + + return [...entriesBefore, ...entriesAfter]; + } + + /** @deprecated */ public async getLogEntriesAround( requestContext: RequestHandlerContext, sourceId: string, diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index 1c34d3b02ab8..00d95fa50699 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -46,12 +46,22 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) cursor = { after: payload.after }; } - const entries = await logEntries.getLogEntries(requestContext, sourceId, { - startDate, - endDate, - query: parseFilterQuery(query), - cursor, - }); + let entries; + if ('center' in payload) { + entries = await logEntries.getLogEntriesAround__new(requestContext, sourceId, { + startDate, + endDate, + query: parseFilterQuery(query), + center: payload.center, + }); + } else { + entries = await logEntries.getLogEntries(requestContext, sourceId, { + startDate, + endDate, + query: parseFilterQuery(query), + cursor, + }); + } return response.ok({ body: logEntriesResponseRT.encode({ From 97f87ec6ad86cfe9be42d3ab767a44e718d3894d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 27 Dec 2019 13:00:59 +0100 Subject: [PATCH 09/20] Change default page size --- .../log_entries/kibana_log_entries_adapter.ts | 5 +++-- .../log_entries_domain/log_entries_domain.ts | 21 +++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index fe846e0e3069..48c3e901bb9e 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -24,6 +24,7 @@ import { LogEntryDocument, LogEntryQuery, LogSummaryBucket, + LOG_ENTRIES_PAGE_SIZE, } from '../../domains/log_entries_domain'; import { InfraSourceConfiguration } from '../../sources'; import { SortedSearchHit } from '../framework'; @@ -89,7 +90,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { fields: string[], params: LogEntriesParams ): Promise { - const { startDate, endDate, query, cursor } = params; + const { startDate, endDate, query, cursor, size } = params; const { sortDirection, searchAfterClause } = processCursor(cursor); @@ -103,7 +104,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { index: sourceConfiguration.logAlias, ignoreUnavailable: true, body: { - size: 10, + size: size ? size : LOG_ENTRIES_PAGE_SIZE, track_total_hits: false, query: { bool: { diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index af84161befdc..9e2f487bdad7 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -37,16 +37,20 @@ import { export interface LogEntriesParams { startDate: number; endDate: number; + size?: number; query?: JsonObject; cursor?: { before: LogEntriesCursor | 'last' } | { after: LogEntriesCursor | 'first' }; } export interface LogEntriesAroundParams { startDate: number; endDate: number; + size?: number; center: LogEntriesCursor; query?: JsonObject; } +export const LOG_ENTRIES_PAGE_SIZE = 200; + export class InfraLogEntriesDomain { constructor( private readonly adapter: LogEntriesAdapter, @@ -60,16 +64,28 @@ export class InfraLogEntriesDomain { sourceId: string, params: LogEntriesAroundParams ) { - const { startDate, endDate, center, query } = params; + const { startDate, endDate, center, query, size } = params; + + /* + * For odd sizes we will round this value down for the first half, and up + * for the second. This keeps the center cursor right in the center. + * + * For even sizes the half before is one entry bigger than the half after. + * [1, 2, 3, 4, 5, *6*, 7, 8, 9, 10] + * | 5 entries | |4 entries| + */ + const halfSize = (size || LOG_ENTRIES_PAGE_SIZE) / 2; const entriesBefore = await this.getLogEntries(requestContext, sourceId, { startDate, endDate, query, cursor: { before: center }, + size: Math.floor(halfSize), }); - /* Elasticsearch's `search_after` returns documents after the specified cursor. + /* + * Elasticsearch's `search_after` returns documents after the specified cursor. * - If we have documents before the center, we search after the last of * those. The first document of the new group is the center. * - If there were no documents, we search one milisecond before the @@ -85,6 +101,7 @@ export class InfraLogEntriesDomain { endDate, query, cursor: { after: cursorAfter }, + size: Math.ceil(halfSize), }); return [...entriesBefore, ...entriesAfter]; From 6b6b15b62735035a7b942d7e80cc26aea9367fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 30 Dec 2019 08:31:32 +0100 Subject: [PATCH 10/20] Test the defaults of the API --- .../api_integration/apis/infra/log_entries.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/x-pack/test/api_integration/apis/infra/log_entries.ts b/x-pack/test/api_integration/apis/infra/log_entries.ts index 4020acc00c61..05f5113e367b 100644 --- a/x-pack/test/api_integration/apis/infra/log_entries.ts +++ b/x-pack/test/api_integration/apis/infra/log_entries.ts @@ -9,6 +9,21 @@ import { ascending, pairs } from 'd3-array'; import gql from 'graphql-tag'; import { v4 as uuidv4 } from 'uuid'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { identity } from 'fp-ts/lib/function'; +import { fold } from 'fp-ts/lib/Either'; + +import { + createPlainError, + throwErrors, +} from '../../../../legacy/plugins/infra/common/runtime_types'; + +import { + LOG_ENTRIES_PATH, + logEntriesRequestRT, + logEntriesResponseRT, +} from '../../../../legacy/plugins/infra/common/http_api'; + import { sharedFragments } from '../../../../legacy/plugins/infra/common/graphql/shared'; import { InfraTimeKey } from '../../../../legacy/plugins/infra/public/graphql/types'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -88,15 +103,62 @@ const logEntriesBetweenQuery = gql` ${sharedFragments.InfraLogEntryFields} `; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + export default function({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const client = getService('infraOpsGraphQLClient'); + const supertest = getService('supertest'); const sourceConfigurationService = getService('infraOpsSourceConfiguration'); describe('log entry apis', () => { before(() => esArchiver.load('infra/metrics_and_logs')); after(() => esArchiver.unload('infra/metrics_and_logs')); + describe('/log_entries/entries', () => { + describe('with the default source', () => { + before(() => esArchiver.load('empty_kibana')); + after(() => esArchiver.unload('empty_kibana')); + + it('works', async () => { + const { body } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: EARLIEST_KEY_WITH_DATA.time, + endDate: KEY_WITHIN_DATA_RANGE.time, + }) + ) + .expect(200); + + const logEntriesResponse = pipe( + logEntriesResponseRT.decode(body), + fold(throwErrors(createPlainError), identity) + ); + + const entries = logEntriesResponse.data.entries; + const firstEntry = entries[0]; + const lastEntry = entries[entries.length - 1]; + + // Has the default page size + expect(entries).to.have.length(200); + + // Cursors are set correctly + expect(firstEntry.cursor).to.eql(logEntriesResponse.data.topCursor); + expect(lastEntry.cursor).to.eql(logEntriesResponse.data.bottomCursor); + + // Entries fall within range + // @kbn/expect doesn't have a `lessOrEqualThan` or `moreOrEqualThan` comparators + expect(firstEntry.cursor.time >= EARLIEST_KEY_WITH_DATA.time).to.be(true); + expect(lastEntry.cursor.time <= KEY_WITHIN_DATA_RANGE.time).to.be(true); + }); + }); + }); + describe('logEntriesAround', () => { describe('with the default source', () => { before(() => esArchiver.load('empty_kibana')); From c52b511b2b8dd90ec2ee351fc0c4abaa5f4cd1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 30 Dec 2019 09:03:42 +0100 Subject: [PATCH 11/20] Add optional `size` parameter This makes easier to test the pagination. By default it returns a 200 size page. --- .../plugins/infra/common/http_api/log_entries/entries.ts | 1 + .../legacy/plugins/infra/server/routes/log_entries/entries.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index 288df085ba8e..3d9b32f47f5e 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -17,6 +17,7 @@ export const logEntriesBaseRequestRT = rt.intersection([ }), rt.partial({ query: rt.string, + size: rt.number, }), ]); diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index 00d95fa50699..35380048061c 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -37,7 +37,7 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) fold(throwErrors(Boom.badRequest), identity) ); - const { startDate, endDate, sourceId, query } = payload; + const { startDate, endDate, sourceId, query, size } = payload; let cursor; if ('before' in payload) { @@ -53,6 +53,7 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) endDate, query: parseFilterQuery(query), center: payload.center, + size, }); } else { entries = await logEntries.getLogEntries(requestContext, sourceId, { @@ -60,6 +61,7 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) endDate, query: parseFilterQuery(query), cursor, + size, }); } From 54efba4e0d32d29c1133a7e8e5b8286c5adfe11a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 30 Dec 2019 09:04:58 +0100 Subject: [PATCH 12/20] Test the pagination --- .../api_integration/apis/infra/log_entries.ts | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/x-pack/test/api_integration/apis/infra/log_entries.ts b/x-pack/test/api_integration/apis/infra/log_entries.ts index 05f5113e367b..3433566de7b8 100644 --- a/x-pack/test/api_integration/apis/infra/log_entries.ts +++ b/x-pack/test/api_integration/apis/infra/log_entries.ts @@ -156,6 +156,126 @@ export default function({ getService }: FtrProviderContext) { expect(firstEntry.cursor.time >= EARLIEST_KEY_WITH_DATA.time).to.be(true); expect(lastEntry.cursor.time <= KEY_WITHIN_DATA_RANGE.time).to.be(true); }); + + it('Paginates correctly with `after`', async () => { + const { body: firstPageBody } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: EARLIEST_KEY_WITH_DATA.time, + endDate: KEY_WITHIN_DATA_RANGE.time, + size: 10, + }) + ); + const firstPage = pipe( + logEntriesResponseRT.decode(firstPageBody), + fold(throwErrors(createPlainError), identity) + ); + + const { body: secondPageBody } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: EARLIEST_KEY_WITH_DATA.time, + endDate: KEY_WITHIN_DATA_RANGE.time, + after: firstPage.data.bottomCursor, + size: 10, + }) + ); + const secondPage = pipe( + logEntriesResponseRT.decode(secondPageBody), + fold(throwErrors(createPlainError), identity) + ); + + const { body: bothPagesBody } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: EARLIEST_KEY_WITH_DATA.time, + endDate: KEY_WITHIN_DATA_RANGE.time, + size: 20, + }) + ); + const bothPages = pipe( + logEntriesResponseRT.decode(bothPagesBody), + fold(throwErrors(createPlainError), identity) + ); + + expect(bothPages.data.entries).to.eql([ + ...firstPage.data.entries, + ...secondPage.data.entries, + ]); + + expect(bothPages.data.topCursor).to.eql(firstPage.data.topCursor); + expect(bothPages.data.bottomCursor).to.eql(secondPage.data.bottomCursor); + }); + + it('Paginates correctly with `before`', async () => { + const { body: lastPageBody } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: KEY_WITHIN_DATA_RANGE.time, + endDate: LATEST_KEY_WITH_DATA.time, + before: 'last', + size: 10, + }) + ); + const lastPage = pipe( + logEntriesResponseRT.decode(lastPageBody), + fold(throwErrors(createPlainError), identity) + ); + + const { body: secondToLastPageBody } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: KEY_WITHIN_DATA_RANGE.time, + endDate: LATEST_KEY_WITH_DATA.time, + before: lastPage.data.topCursor, + size: 10, + }) + ); + const secondToLastPage = pipe( + logEntriesResponseRT.decode(secondToLastPageBody), + fold(throwErrors(createPlainError), identity) + ); + + const { body: bothPagesBody } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: KEY_WITHIN_DATA_RANGE.time, + endDate: LATEST_KEY_WITH_DATA.time, + before: 'last', + size: 20, + }) + ); + const bothPages = pipe( + logEntriesResponseRT.decode(bothPagesBody), + fold(throwErrors(createPlainError), identity) + ); + + expect(bothPages.data.entries).to.eql([ + ...secondToLastPage.data.entries, + ...lastPage.data.entries, + ]); + + expect(bothPages.data.topCursor).to.eql(secondToLastPage.data.topCursor); + expect(bothPages.data.bottomCursor).to.eql(lastPage.data.bottomCursor); + }); }); }); From 6e17c53f71d576f7685b8f2dcf547a6fb832ab57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 30 Dec 2019 09:24:29 +0100 Subject: [PATCH 13/20] Test centering around a point --- .../api_integration/apis/infra/log_entries.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/x-pack/test/api_integration/apis/infra/log_entries.ts b/x-pack/test/api_integration/apis/infra/log_entries.ts index 3433566de7b8..8db1426a219d 100644 --- a/x-pack/test/api_integration/apis/infra/log_entries.ts +++ b/x-pack/test/api_integration/apis/infra/log_entries.ts @@ -276,6 +276,33 @@ export default function({ getService }: FtrProviderContext) { expect(bothPages.data.topCursor).to.eql(secondToLastPage.data.topCursor); expect(bothPages.data.bottomCursor).to.eql(lastPage.data.bottomCursor); }); + + it('centers entries around a point', async () => { + const { body } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startDate: EARLIEST_KEY_WITH_DATA.time, + endDate: LATEST_KEY_WITH_DATA.time, + center: KEY_WITHIN_DATA_RANGE, + }) + ) + .expect(200); + const logEntriesResponse = pipe( + logEntriesResponseRT.decode(body), + fold(throwErrors(createPlainError), identity) + ); + + const entries = logEntriesResponse.data.entries; + const firstEntry = entries[0]; + const lastEntry = entries[entries.length - 1]; + + expect(entries).to.have.length(200); + expect(firstEntry.cursor.time >= EARLIEST_KEY_WITH_DATA.time).to.be(true); + expect(lastEntry.cursor.time <= LATEST_KEY_WITH_DATA.time).to.be(true); + }); }); }); From d04f15e9f4eb3bba67c2ddfc1be084e3c87d78ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez?= Date: Tue, 31 Dec 2019 10:37:02 +0100 Subject: [PATCH 14/20] Handle `0` sizes Co-Authored-By: Zacqary Adam Xeper --- .../lib/adapters/log_entries/kibana_log_entries_adapter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index 48c3e901bb9e..72169f427779 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -104,7 +104,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { index: sourceConfiguration.logAlias, ignoreUnavailable: true, body: { - size: size ? size : LOG_ENTRIES_PAGE_SIZE, + size: typeof size !== 'undefined' ? size : LOG_ENTRIES_PAGE_SIZE, track_total_hits: false, query: { bool: { From 582945635b9f62ae47080d244f06344b59c24127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 2 Jan 2020 11:15:55 +0100 Subject: [PATCH 15/20] Add highlights endpoint --- .../common/http_api/log_entries/highlights.ts | 62 ++++++++++++ .../common/http_api/log_entries/index.ts | 1 + .../plugins/infra/server/infra_server.ts | 2 + .../log_entries/kibana_log_entries_adapter.ts | 63 +++++++++++- .../log_entries_domain/log_entries_domain.ts | 7 +- .../server/routes/log_entries/highlights.ts | 95 +++++++++++++++++++ .../infra/server/routes/log_entries/index.ts | 1 + 7 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 x-pack/legacy/plugins/infra/common/http_api/log_entries/highlights.ts create mode 100644 x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/highlights.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/highlights.ts new file mode 100644 index 000000000000..516cd67f2764 --- /dev/null +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/highlights.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { + logEntriesBaseRequestRT, + logEntriesBeforeRequestRT, + logEntriesAfterRequestRT, + logEntriesCenteredRT, + logEntryRT, +} from './entries'; +import { logEntriesCursorRT } from './common'; + +export const LOG_ENTRIES_HIGHLIGHTS_PATH = '/api/log_entries/highlights'; + +const highlightsRT = rt.type({ + highlightTerms: rt.array(rt.string), +}); + +export const logEntriesHighlightsBaseRequestRT = rt.intersection([ + logEntriesBaseRequestRT, + highlightsRT, +]); + +export const logEntriesHighlightsBeforeRequestRT = rt.intersection([ + logEntriesBeforeRequestRT, + highlightsRT, +]); + +export const logEntriesHighlightsAfterRequestRT = rt.intersection([ + logEntriesAfterRequestRT, + highlightsRT, +]); + +export const logEntriesHighlightsCenteredRequestRT = rt.intersection([ + logEntriesCenteredRT, + highlightsRT, +]); + +export const logEntriesHighlightsRequestRT = rt.union([ + logEntriesHighlightsBaseRequestRT, + logEntriesHighlightsBeforeRequestRT, + logEntriesHighlightsAfterRequestRT, + logEntriesHighlightsCenteredRequestRT, +]); + +export type LogEntriesHighlightsRequest = rt.TypeOf; + +export const logEntriesHighlightsResponseRT = rt.type({ + data: rt.array( + rt.type({ + topCursor: logEntriesCursorRT, + bottomCursor: logEntriesCursorRT, + entries: rt.array(logEntryRT), + }) + ), +}); + +export type LogEntriesHighlightsResponse = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts index e78eb605d999..490f295cbff6 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts @@ -6,6 +6,7 @@ export * from './common'; export * from './entries'; +export * from './highlights'; export * from './item'; export * from './summary'; export * from './summary_highlights'; diff --git a/x-pack/legacy/plugins/infra/server/infra_server.ts b/x-pack/legacy/plugins/infra/server/infra_server.ts index f9c7280f00b7..f99589e1b52b 100644 --- a/x-pack/legacy/plugins/infra/server/infra_server.ts +++ b/x-pack/legacy/plugins/infra/server/infra_server.ts @@ -21,6 +21,7 @@ import { initSnapshotRoute } from './routes/snapshot'; import { initNodeDetailsRoute } from './routes/node_details'; import { initLogEntriesRoute, + initLogEntriesHighlightsRoute, initLogEntriesSummaryRoute, initLogEntriesSummaryHighlightsRoute, initLogEntriesItemRoute, @@ -45,6 +46,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => { initNodeDetailsRoute(libs); initValidateLogAnalysisIndicesRoute(libs); initLogEntriesRoute(libs); + initLogEntriesHighlightsRoute(libs); initLogEntriesSummaryRoute(libs); initLogEntriesSummaryHighlightsRoute(libs); initLogEntriesItemRoute(libs); diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index 72169f427779..6bae46c60969 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -8,6 +8,7 @@ import { timeMilliseconds } from 'd3-time'; import * as runtimeTypes from 'io-ts'; +import { compact } from 'lodash'; import first from 'lodash/fp/first'; import get from 'lodash/fp/get'; import has from 'lodash/fp/has'; @@ -90,10 +91,32 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { fields: string[], params: LogEntriesParams ): Promise { - const { startDate, endDate, query, cursor, size } = params; + const { startDate, endDate, query, cursor, size, highlightTerm } = params; const { sortDirection, searchAfterClause } = processCursor(cursor); + const highlightQuery = createHighlightQuery(highlightTerm, fields); + + const highlightClause = highlightQuery + ? { + highlight: { + boundary_scanner: 'word', + fields: fields.reduce( + (highlightFieldConfigs, fieldName) => ({ + ...highlightFieldConfigs, + [fieldName]: {}, + }), + {} + ), + fragment_size: 1, + number_of_fragments: 100, + post_tags: [''], + pre_tags: [''], + highlight_query: highlightQuery, + }, + } + : {}; + const sort = { [sourceConfiguration.fields.timestamp]: sortDirection, [sourceConfiguration.fields.tiebreaker]: sortDirection, @@ -106,10 +129,11 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { body: { size: typeof size !== 'undefined' ? size : LOG_ENTRIES_PAGE_SIZE, track_total_hits: false, + _source: fields, query: { bool: { filter: [ - ...createQueryFilterClauses(query), + ...createFilterClauses(query, highlightQuery), { range: { [sourceConfiguration.fields.timestamp]: { @@ -123,15 +147,17 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { }, }, sort, + ...highlightClause, ...searchAfterClause, }, }; - const esResult = await this.framework.callWithRequest( + const esResult = await this.framework.callWithRequest( requestContext, 'search', esQuery ); + const hits = sortDirection === 'asc' ? esResult.hits.hits : esResult.hits.hits.reverse(); return mapHitsToLogEntryDocuments(hits, sourceConfiguration.fields.timestamp, fields); } @@ -375,7 +401,7 @@ function getLookupIntervals(start: number, direction: 'asc' | 'desc'): Array<[nu } function mapHitsToLogEntryDocuments( - hits: LogItemHit[], + hits: SortedSearchHit[], timestampField: string, fields: string[] ): LogEntryDocument[] { @@ -396,7 +422,7 @@ function mapHitsToLogEntryDocuments( // FIXME s/key/cursor/g key: { time: hit.sort[0], tiebreaker: hit.sort[1] }, fields: logFields, - highlights: {}, + highlights: hit.highlight || {}, }; }); } @@ -435,6 +461,33 @@ const convertDateRangeBucketToSummaryBucket = ( })), }); +const createHighlightQuery = ( + highlightTerm: string | undefined, + fields: string[] +): LogEntryQuery | undefined => { + if (highlightTerm) { + return { + multi_match: { + fields, + lenient: true, + query: highlightTerm, + type: 'phrase', + }, + }; + } +}; + +const createFilterClauses = ( + filterQuery?: LogEntryQuery, + highlightQuery?: LogEntryQuery +): LogEntryQuery[] => { + if (filterQuery && highlightQuery) { + return [{ bool: { filter: [filterQuery, highlightQuery] } }]; + } + + return compact([filterQuery, highlightQuery]) as LogEntryQuery[]; +}; + const createQueryFilterClauses = (filterQuery: LogEntryQuery | undefined) => filterQuery ? [filterQuery] : []; diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 9e2f487bdad7..78317fdaa823 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -40,6 +40,7 @@ export interface LogEntriesParams { size?: number; query?: JsonObject; cursor?: { before: LogEntriesCursor | 'last' } | { after: LogEntriesCursor | 'first' }; + highlightTerm?: string; } export interface LogEntriesAroundParams { startDate: number; @@ -47,6 +48,7 @@ export interface LogEntriesAroundParams { size?: number; center: LogEntriesCursor; query?: JsonObject; + highlightTerm?: string; } export const LOG_ENTRIES_PAGE_SIZE = 200; @@ -64,7 +66,7 @@ export class InfraLogEntriesDomain { sourceId: string, params: LogEntriesAroundParams ) { - const { startDate, endDate, center, query, size } = params; + const { startDate, endDate, center, query, size, highlightTerm } = params; /* * For odd sizes we will round this value down for the first half, and up @@ -82,6 +84,7 @@ export class InfraLogEntriesDomain { query, cursor: { before: center }, size: Math.floor(halfSize), + highlightTerm, }); /* @@ -102,6 +105,7 @@ export class InfraLogEntriesDomain { query, cursor: { after: cursorAfter }, size: Math.ceil(halfSize), + highlightTerm, }); return [...entriesBefore, ...entriesAfter]; @@ -258,6 +262,7 @@ export class InfraLogEntriesDomain { return entries; } + /** @deprecated */ public async getLogEntryHighlights( requestContext: RequestHandlerContext, sourceId: string, diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts new file mode 100644 index 000000000000..fa674cd0618e --- /dev/null +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; + +import { pipe } from 'fp-ts/lib/pipeable'; +import { fold } from 'fp-ts/lib/Either'; +import { identity } from 'fp-ts/lib/function'; +import { schema } from '@kbn/config-schema'; + +import { throwErrors } from '../../../common/runtime_types'; + +import { InfraBackendLibs } from '../../lib/infra_types'; +import { + LOG_ENTRIES_HIGHLIGHTS_PATH, + logEntriesHighlightsRequestRT, + logEntriesHighlightsResponseRT, +} from '../../../common/http_api/log_entries'; +import { parseFilterQuery } from '../../utils/serialized_query'; + +const escapeHatch = schema.object({}, { allowUnknowns: true }); + +export const initLogEntriesHighlightsRoute = ({ framework, logEntries }: InfraBackendLibs) => { + framework.registerRoute( + { + method: 'post', + path: LOG_ENTRIES_HIGHLIGHTS_PATH, + validate: { body: escapeHatch }, + }, + async (requestContext, request, response) => { + try { + const payload = pipe( + logEntriesHighlightsRequestRT.decode(request.body), + fold(throwErrors(Boom.badRequest), identity) + ); + + const { startDate, endDate, sourceId, query, size, highlightTerms } = payload; + + let entriesPerHighlightTerm; + + if ('center' in payload) { + entriesPerHighlightTerm = await Promise.all( + highlightTerms.map(highlightTerm => + logEntries.getLogEntriesAround__new(requestContext, sourceId, { + startDate, + endDate, + query: parseFilterQuery(query), + center: payload.center, + size, + highlightTerm, + }) + ) + ); + } else { + let cursor; + if ('before' in payload) { + cursor = { before: payload.before }; + } else if ('after' in payload) { + cursor = { after: payload.after }; + } + + entriesPerHighlightTerm = await Promise.all( + highlightTerms.map(highlightTerm => + logEntries.getLogEntries(requestContext, sourceId, { + startDate, + endDate, + query: parseFilterQuery(query), + cursor, + size, + highlightTerm, + }) + ) + ); + } + + return response.ok({ + body: logEntriesHighlightsResponseRT.encode({ + data: entriesPerHighlightTerm.map(entries => ({ + entries, + topCursor: entries[0].cursor, + bottomCursor: entries[entries.length - 1].cursor, + })), + }), + }); + } catch (error) { + return response.internalError({ + body: error.message, + }); + } + } + ); +}; diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts index b85c24d4ce27..1090d35d89b8 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/index.ts @@ -5,6 +5,7 @@ */ export * from './entries'; +export * from './highlights'; export * from './item'; export * from './summary'; export * from './summary_highlights'; From c454479ddd0d26d8f032764fafa6a6090143089c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 2 Jan 2020 14:16:02 +0100 Subject: [PATCH 16/20] Refactor `processCursor` --- .../log_entries/kibana_log_entries_adapter.ts | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index 6bae46c60969..b936d79a8edc 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -497,23 +497,24 @@ function processCursor( sortDirection: 'asc' | 'desc'; searchAfterClause: { search_after?: readonly [number, number] }; } { - let sortDirection: 'asc' | 'desc' = 'asc'; - let searchAfterClause = {}; - - if (!cursor) { - return { sortDirection, searchAfterClause }; - } - - if ('before' in cursor) { - sortDirection = 'desc'; - if (cursor.before !== 'last') { - searchAfterClause = { search_after: [cursor.before.time, cursor.before.tiebreaker] as const }; + if (cursor) { + if ('before' in cursor) { + return { + sortDirection: 'desc', + searchAfterClause: + cursor.before !== 'last' + ? { search_after: [cursor.before.time, cursor.before.tiebreaker] as const } + : {}, + }; + } else if (cursor.after !== 'first') { + return { + sortDirection: 'asc', + searchAfterClause: { search_after: [cursor.after.time, cursor.after.tiebreaker] as const }, + }; } - } else if ('after' in cursor && cursor.after !== 'first') { - searchAfterClause = { search_after: [cursor.after.time, cursor.after.tiebreaker] as const }; } - return { sortDirection, searchAfterClause }; + return { sortDirection: 'asc', searchAfterClause: {} }; } const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([ From c5e231a2e0e4a27f4101c70fc123287f3c904370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 2 Jan 2020 15:45:49 +0100 Subject: [PATCH 17/20] Tweak cursor handling in the routes --- .../infra/server/routes/log_entries/entries.ts | 15 ++++++++------- .../infra/server/routes/log_entries/highlights.ts | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts index 35380048061c..361535886ab2 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/entries.ts @@ -20,6 +20,7 @@ import { logEntriesResponseRT, } from '../../../common/http_api/log_entries'; import { parseFilterQuery } from '../../utils/serialized_query'; +import { LogEntriesParams } from '../../lib/domains/log_entries_domain'; const escapeHatch = schema.object({}, { allowUnknowns: true }); @@ -39,13 +40,6 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) const { startDate, endDate, sourceId, query, size } = payload; - let cursor; - if ('before' in payload) { - cursor = { before: payload.before }; - } else if ('after' in payload) { - cursor = { after: payload.after }; - } - let entries; if ('center' in payload) { entries = await logEntries.getLogEntriesAround__new(requestContext, sourceId, { @@ -56,6 +50,13 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) size, }); } else { + let cursor: LogEntriesParams['cursor']; + if ('before' in payload) { + cursor = { before: payload.before }; + } else if ('after' in payload) { + cursor = { after: payload.after }; + } + entries = await logEntries.getLogEntries(requestContext, sourceId, { startDate, endDate, diff --git a/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts b/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts index fa674cd0618e..8af81a6ee313 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_entries/highlights.ts @@ -20,6 +20,7 @@ import { logEntriesHighlightsResponseRT, } from '../../../common/http_api/log_entries'; import { parseFilterQuery } from '../../utils/serialized_query'; +import { LogEntriesParams } from '../../lib/domains/log_entries_domain'; const escapeHatch = schema.object({}, { allowUnknowns: true }); @@ -55,7 +56,7 @@ export const initLogEntriesHighlightsRoute = ({ framework, logEntries }: InfraBa ) ); } else { - let cursor; + let cursor: LogEntriesParams['cursor']; if ('before' in payload) { cursor = { before: payload.before }; } else if ('after' in payload) { From 1971e5c90e746388bcbb1f1efe7a98427fcea6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 2 Jan 2020 17:47:56 +0100 Subject: [PATCH 18/20] Refine `LogEntry` type --- .../common/http_api/log_entries/entries.ts | 31 +++++++++++++- .../log_entries_domain/log_entries_domain.ts | 41 ++++++++++--------- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index 3d9b32f47f5e..bc9db5722aa9 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -45,12 +45,41 @@ export const logEntriesRequestRT = rt.union([ export type LogEntriesRequest = rt.TypeOf; +// JSON value +const valueRT = rt.union([rt.string, rt.number, rt.boolean, rt.object, rt.null, rt.undefined]); + +export const logMessagePartRT = rt.union([ + rt.type({ + constant: rt.string, + }), + rt.type({ + field: rt.string, + value: valueRT, + highlights: rt.array(rt.string), + }), +]); + +export const logColumnRT = rt.union([ + rt.type({ columnId: rt.string, timestamp: rt.number }), + rt.type({ + columnId: rt.string, + field: rt.string, + value: valueRT, + }), + rt.type({ + columnId: rt.string, + message: rt.array(logMessagePartRT), + }), +]); + export const logEntryRT = rt.type({ id: rt.string, cursor: logEntriesCursorRT, - columns: rt.array(rt.any), + columns: rt.array(logColumnRT), }); +export type LogMessagepart = rt.TypeOf; +export type LogColumn = rt.TypeOf; export type LogEntry = rt.TypeOf; export const logEntriesResponseRT = rt.type({ diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 78317fdaa823..2d5aacb1e484 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -16,6 +16,7 @@ import { LogEntry, LogEntriesItem, LogEntriesCursor, + LogColumn, } from '../../../../common/http_api'; import { InfraLogEntry, InfraLogMessageSegment } from '../../../graphql/types'; import { @@ -203,27 +204,27 @@ export class InfraLogEntriesDomain { return { id: doc.gid, cursor: doc.key, - columns: configuration.logColumns.map(column => { - if ('timestampColumn' in column) { - return { - columnId: column.timestampColumn.id, - timestamp: doc.key.time, - }; - } - if ('messageColumn' in column) { - return { - columnId: column.messageColumn.id, - message: messageFormattingRules.format(doc.fields, doc.highlights), - }; - } - if ('fieldColumn' in column) { - return { - columnId: column.fieldColumn.id, - field: column.fieldColumn.field, - value: doc.fields[column.fieldColumn.field], - }; + columns: configuration.logColumns.map( + (column): LogColumn => { + if ('timestampColumn' in column) { + return { + columnId: column.timestampColumn.id, + timestamp: doc.key.time, + }; + } else if ('messageColumn' in column) { + return { + columnId: column.messageColumn.id, + message: messageFormattingRules.format(doc.fields, doc.highlights), + }; + } else { + return { + columnId: column.fieldColumn.id, + field: column.fieldColumn.field, + value: doc.fields[column.fieldColumn.field], + }; + } } - }), + ), }; }); From 1836ba314fea61d28869986371c138698753325a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 3 Jan 2020 15:01:22 +0100 Subject: [PATCH 19/20] Add tests for highlights endpoint --- .../common/http_api/log_entries/entries.ts | 3 +- .../log_entries_domain/log_entries_domain.ts | 3 +- .../apis/infra/log_entry_highlights.ts | 152 ++++++++++++++++++ 3 files changed, 156 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index bc9db5722aa9..ae335ec58c9d 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -64,7 +64,8 @@ export const logColumnRT = rt.union([ rt.type({ columnId: rt.string, field: rt.string, - value: valueRT, + value: rt.string, + highlights: rt.array(rt.string), }), rt.type({ columnId: rt.string, diff --git a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 2d5aacb1e484..2f71d56e1e0e 100644 --- a/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/legacy/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -220,7 +220,8 @@ export class InfraLogEntriesDomain { return { columnId: column.fieldColumn.id, field: column.fieldColumn.field, - value: doc.fields[column.fieldColumn.field], + value: stringify(doc.fields[column.fieldColumn.field]), + highlights: doc.highlights[column.fieldColumn.field] || [], }; } } diff --git a/x-pack/test/api_integration/apis/infra/log_entry_highlights.ts b/x-pack/test/api_integration/apis/infra/log_entry_highlights.ts index 88d7d8716d0f..67529c77f70f 100644 --- a/x-pack/test/api_integration/apis/infra/log_entry_highlights.ts +++ b/x-pack/test/api_integration/apis/infra/log_entry_highlights.ts @@ -8,6 +8,21 @@ import expect from '@kbn/expect'; import { ascending, pairs } from 'd3-array'; import gql from 'graphql-tag'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { identity } from 'fp-ts/lib/function'; +import { fold } from 'fp-ts/lib/Either'; + +import { + createPlainError, + throwErrors, +} from '../../../../legacy/plugins/infra/common/runtime_types'; + +import { + LOG_ENTRIES_HIGHLIGHTS_PATH, + logEntriesHighlightsRequestRT, + logEntriesHighlightsResponseRT, +} from '../../../../legacy/plugins/infra/common/http_api'; + import { FtrProviderContext } from '../../ftr_provider_context'; import { sharedFragments } from '../../../../legacy/plugins/infra/common/graphql/shared'; import { InfraTimeKey } from '../../../../legacy/plugins/infra/public/graphql/types'; @@ -29,14 +44,151 @@ const KEY_AFTER_END = { tiebreaker: 0, }; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + export default function({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); const client = getService('infraOpsGraphQLClient'); describe('log highlight apis', () => { before(() => esArchiver.load('infra/simple_logs')); after(() => esArchiver.unload('infra/simple_logs')); + describe('/log_entries/highlights', () => { + describe('with the default source', () => { + before(() => esArchiver.load('empty_kibana')); + after(() => esArchiver.unload('empty_kibana')); + + it('highlights built-in message column', async () => { + const { body } = await supertest + .post(LOG_ENTRIES_HIGHLIGHTS_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesHighlightsRequestRT.encode({ + sourceId: 'default', + startDate: KEY_BEFORE_START.time, + endDate: KEY_AFTER_END.time, + highlightTerms: ['message of document 0'], + }) + ) + .expect(200); + + const logEntriesHighlightsResponse = pipe( + logEntriesHighlightsResponseRT.decode(body), + fold(throwErrors(createPlainError), identity) + ); + + expect(logEntriesHighlightsResponse.data).to.have.length(1); + + const data = logEntriesHighlightsResponse.data[0]; + const entries = data.entries; + const firstEntry = entries[0]; + const lastEntry = entries[entries.length - 1]; + + // Finds expected entries + expect(entries).to.have.length(10); + + // Cursors are set correctly + expect(firstEntry.cursor).to.eql(data.topCursor); + expect(lastEntry.cursor).to.eql(data.bottomCursor); + + // Entries fall within range + // @kbn/expect doesn't have a `lessOrEqualThan` or `moreOrEqualThan` comparators + expect(firstEntry.cursor.time >= KEY_BEFORE_START.time).to.be(true); + expect(lastEntry.cursor.time <= KEY_AFTER_END.time).to.be(true); + + // All entries contain the highlights + entries.forEach(entry => { + entry.columns.forEach(column => { + if ('message' in column && 'highlights' in column.message[0]) { + expect(column.message[0].highlights).to.eql(['message', 'of', 'document', '0']); + } + }); + }); + }); + + it('highlights field columns', async () => { + const { body } = await supertest + .post(LOG_ENTRIES_HIGHLIGHTS_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesHighlightsRequestRT.encode({ + sourceId: 'default', + startDate: KEY_BEFORE_START.time, + endDate: KEY_AFTER_END.time, + highlightTerms: ['generate_test_data/simple_logs'], + }) + ) + .expect(200); + + const logEntriesHighlightsResponse = pipe( + logEntriesHighlightsResponseRT.decode(body), + fold(throwErrors(createPlainError), identity) + ); + + expect(logEntriesHighlightsResponse.data).to.have.length(1); + + const entries = logEntriesHighlightsResponse.data[0].entries; + + // Finds expected entries + expect(entries).to.have.length(50); + + // All entries contain the highlights + entries.forEach(entry => { + entry.columns.forEach(column => { + if ('field' in column && 'highlights' in column && column.highlights.length > 0) { + // https://github.com/elastic/kibana/issues/49959 + // expect(column.highlights).to.eql(['generate_test_data/simple_logs']); + expect(column.highlights).to.eql(['generate_test_data']); + } + }); + }); + }); + + it('applies the query as well as the highlight', async () => { + const { body } = await supertest + .post(LOG_ENTRIES_HIGHLIGHTS_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesHighlightsRequestRT.encode({ + sourceId: 'default', + startDate: KEY_BEFORE_START.time, + endDate: KEY_AFTER_END.time, + query: JSON.stringify({ + multi_match: { query: 'host-a', type: 'phrase', lenient: true }, + }), + highlightTerms: ['message'], + }) + ) + .expect(200); + + const logEntriesHighlightsResponse = pipe( + logEntriesHighlightsResponseRT.decode(body), + fold(throwErrors(createPlainError), identity) + ); + + expect(logEntriesHighlightsResponse.data).to.have.length(1); + + const entries = logEntriesHighlightsResponse.data[0].entries; + + // Finds expected entries + expect(entries).to.have.length(25); + + // All entries contain the highlights + entries.forEach(entry => { + entry.columns.forEach(column => { + if ('message' in column && 'highlights' in column.message[0]) { + expect(column.message[0].highlights).to.eql(['message', 'message']); + } + }); + }); + }); + }); + }); + describe('logEntryHighlights', () => { describe('with the default source', () => { before(() => esArchiver.load('empty_kibana')); From 451be6282363739683e014be92f002fbeb9d12a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 3 Jan 2020 16:42:28 +0100 Subject: [PATCH 20/20] Tweak the types for the LogEntry --- .../legacy/plugins/infra/common/http_api/log_entries/entries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts index ae335ec58c9d..97bdad23beb2 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts @@ -64,7 +64,7 @@ export const logColumnRT = rt.union([ rt.type({ columnId: rt.string, field: rt.string, - value: rt.string, + value: rt.union([rt.string, rt.undefined]), highlights: rt.array(rt.string), }), rt.type({