From 378d89b5cd2b017121edd6904320d59f713cb3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 23 Nov 2020 17:30:02 +0100 Subject: [PATCH] [Logs UI] Allow custom columns in the `` component (#83802) --- .../common/http_api/log_entries/entries.ts | 2 + .../log_sources/log_source_configuration.ts | 2 +- .../public/components/log_stream/README.md | 30 +++++ .../public/components/log_stream/index.tsx | 34 +++++- .../containers/logs/log_stream/index.ts | 4 + .../log_entries_domain/log_entries_domain.ts | 14 ++- .../server/routes/log_entries/entries.ts | 7 +- .../apis/metrics_ui/log_entries.ts | 112 +++++++++--------- 8 files changed, 142 insertions(+), 63 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts index 48790c3faca52..5f35eb89774fa 100644 --- a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts @@ -7,6 +7,7 @@ import * as rt from 'io-ts'; import { jsonArrayRT } from '../../typed_json'; import { logEntriesCursorRT } from './common'; +import { logSourceColumnConfigurationRT } from '../log_sources'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; @@ -19,6 +20,7 @@ export const logEntriesBaseRequestRT = rt.intersection([ rt.partial({ query: rt.union([rt.string, rt.null]), size: rt.number, + columns: rt.array(logSourceColumnConfigurationRT), }), ]); diff --git a/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts b/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts index 3fc42b661ddab..7581e29692356 100644 --- a/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts +++ b/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts @@ -48,7 +48,7 @@ const logSourceFieldColumnConfigurationRT = rt.strict({ ]), }); -const logSourceColumnConfigurationRT = rt.union([ +export const logSourceColumnConfigurationRT = rt.union([ logSourceTimestampColumnConfigurationRT, logSourceMessageColumnConfigurationRT, logSourceFieldColumnConfigurationRT, diff --git a/x-pack/plugins/infra/public/components/log_stream/README.md b/x-pack/plugins/infra/public/components/log_stream/README.md index 59b3edfab736c..514ffbccbff28 100644 --- a/x-pack/plugins/infra/public/components/log_stream/README.md +++ b/x-pack/plugins/infra/public/components/log_stream/README.md @@ -68,6 +68,36 @@ By default the `` uses the `"default"` source confiuration, but if ``` +### Custom columns + +It is possible to change what columns are loaded without creating a whole new source configuration. To do so the component supports the `columns` prop. The default configuration can be replicated as follows. + +```tsx + +``` + +There are three column types: + + + + + +
`type: "timestamp"` + The configured timestamp field. Defaults to `@timestamp`. +
`type: "message"` + The value of the `message` field if it exists. If it doesn't, the component will try to recompose the original log line using values of other fields. +
`type: "field"` + A specific field specified in the `field` property. +
+ ### Considerations As mentioned in the prerequisites, the component relies on `kibana-react` to access kibana's core services. If this is not the case the component will throw an exception when rendering. We advise to use an `` in your component hierarchy to catch this error if necessary. diff --git a/x-pack/plugins/infra/public/components/log_stream/index.tsx b/x-pack/plugins/infra/public/components/log_stream/index.tsx index a880996daaade..c4e6bbe094642 100644 --- a/x-pack/plugins/infra/public/components/log_stream/index.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/index.tsx @@ -11,13 +11,18 @@ import { euiStyled } from '../../../../observability/public'; import { LogEntriesCursor } from '../../../common/http_api'; import { useKibana } from '../../../../../../src/plugins/kibana_react/public'; -import { useLogSource } from '../../containers/logs/log_source'; +import { LogSourceConfigurationProperties, useLogSource } from '../../containers/logs/log_source'; import { useLogStream } from '../../containers/logs/log_stream'; import { ScrollableLogTextStreamView } from '../logging/log_text_stream'; const PAGE_THRESHOLD = 2; +type LogColumnDefinition = + | { type: 'timestamp' } + | { type: 'message' } + | { type: 'field'; field: string }; + export interface LogStreamProps { sourceId?: string; startTimestamp: number; @@ -26,6 +31,7 @@ export interface LogStreamProps { center?: LogEntriesCursor; highlight?: string; height?: string | number; + columns?: LogColumnDefinition[]; } export const LogStream: React.FC = ({ @@ -36,7 +42,13 @@ export const LogStream: React.FC = ({ center, highlight, height = '400px', + columns, }) => { + const customColumns = useMemo( + () => (columns ? convertLogColumnDefinitionToLogSourceColumnDefinition(columns) : undefined), + [columns] + ); + // source boilerplate const { services } = useKibana(); if (!services?.http?.fetch) { @@ -74,6 +86,7 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re endTimestamp, query, center, + columns: customColumns, }); // Derived state @@ -83,8 +96,8 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re const isLoadingMore = pageLoadingState === 'loading'; const columnConfigurations = useMemo(() => { - return sourceConfiguration ? sourceConfiguration.configuration.logColumns : []; - }, [sourceConfiguration]); + return sourceConfiguration ? customColumns ?? sourceConfiguration.configuration.logColumns : []; + }, [sourceConfiguration, customColumns]); const streamItems = useMemo( () => @@ -163,6 +176,21 @@ const LogStreamContent = euiStyled.div<{ height: string }>` height: ${(props) => props.height}; `; +function convertLogColumnDefinitionToLogSourceColumnDefinition( + columns: LogColumnDefinition[] +): LogSourceConfigurationProperties['logColumns'] { + return columns.map((column) => { + switch (column.type) { + case 'timestamp': + return { timestampColumn: { id: '___#timestamp' } }; + case 'message': + return { messageColumn: { id: '___#message' } }; + case 'field': + return { fieldColumn: { id: `___#${column.field}`, field: column.field } }; + } + }); +} + // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default LogStream; diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 566edcce91318..b0b09c76f4d85 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -12,6 +12,7 @@ import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; import { LogEntry, LogEntriesCursor } from '../../../../common/http_api'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; +import { LogSourceConfigurationProperties } from '../log_source'; interface LogStreamProps { sourceId: string; @@ -19,6 +20,7 @@ interface LogStreamProps { endTimestamp: number; query?: string; center?: LogEntriesCursor; + columns?: LogSourceConfigurationProperties['logColumns']; } interface LogStreamState { @@ -60,6 +62,7 @@ export function useLogStream({ endTimestamp, query, center, + columns, }: LogStreamProps): LogStreamReturn { const { services } = useKibanaContextForPlugin(); const [state, setState] = useSetState(INITIAL_STATE); @@ -100,6 +103,7 @@ export function useLogStream({ startTimestamp, endTimestamp, query: parsedQuery, + columns, ...fetchPosition, }, services.http.fetch diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 1cf0afd50b80c..e10eb1d7e8aad 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -15,6 +15,7 @@ import { LogEntriesItem, LogEntriesCursor, LogColumn, + LogEntriesRequest, } from '../../../../common/http_api'; import { InfraSourceConfiguration, @@ -73,7 +74,8 @@ export class InfraLogEntriesDomain { public async getLogEntriesAround( requestContext: RequestHandlerContext, sourceId: string, - params: LogEntriesAroundParams + params: LogEntriesAroundParams, + columnOverrides?: LogEntriesRequest['columns'] ): Promise<{ entries: LogEntry[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> { const { startTimestamp, endTimestamp, center, query, size, highlightTerm } = params; @@ -97,7 +99,8 @@ export class InfraLogEntriesDomain { cursor: { before: center }, size: Math.floor(halfSize), highlightTerm, - } + }, + columnOverrides ); /* @@ -131,13 +134,16 @@ export class InfraLogEntriesDomain { public async getLogEntries( requestContext: RequestHandlerContext, sourceId: string, - params: LogEntriesParams + params: LogEntriesParams, + columnOverrides?: LogEntriesRequest['columns'] ): Promise<{ entries: LogEntry[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> { const { configuration } = await this.libs.sources.getSourceConfiguration( requestContext.core.savedObjects.client, sourceId ); + const columnDefinitions = columnOverrides ?? configuration.logColumns; + const messageFormattingRules = compileFormattingRules( getBuiltinRules(configuration.fields.message) ); @@ -155,7 +161,7 @@ export class InfraLogEntriesDomain { return { id: doc.id, cursor: doc.cursor, - columns: configuration.logColumns.map( + columns: columnDefinitions.map( (column): LogColumn => { if ('timestampColumn' in column) { return { diff --git a/x-pack/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/plugins/infra/server/routes/log_entries/entries.ts index 2baf3fd7aa990..67083ee9d6c0d 100644 --- a/x-pack/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/plugins/infra/server/routes/log_entries/entries.ts @@ -31,6 +31,7 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) sourceId, query, size, + columns, } = payload; let entries; @@ -47,7 +48,8 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) query: parseFilterQuery(query), center: payload.center, size, - } + }, + columns )); } else { let cursor: LogEntriesParams['cursor']; @@ -66,7 +68,8 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) query: parseFilterQuery(query), cursor, size, - } + }, + columns )); } diff --git a/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts index 2928027e88f33..2d148f4c2c0f7 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts @@ -7,11 +7,7 @@ import expect from '@kbn/expect'; import { v4 as uuidv4 } from 'uuid'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { identity } from 'fp-ts/lib/function'; -import { fold } from 'fp-ts/lib/Either'; - -import { createPlainError, throwErrors } from '../../../../plugins/infra/common/runtime_types'; +import { decodeOrThrow } from '../../../../plugins/infra/common/runtime_types'; import { LOG_ENTRIES_PATH, @@ -68,10 +64,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(200); - const logEntriesResponse = pipe( - logEntriesResponseRT.decode(body), - fold(throwErrors(createPlainError), identity) - ); + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); const entries = logEntriesResponse.data.entries; const firstEntry = entries[0]; @@ -104,10 +97,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(200); - const logEntriesResponse = pipe( - logEntriesResponseRT.decode(body), - fold(throwErrors(createPlainError), identity) - ); + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); const entries = logEntriesResponse.data.entries; const entry = entries[0]; @@ -126,6 +116,52 @@ export default function ({ getService }: FtrProviderContext) { expect(messageColumn.message.length).to.be.greaterThan(0); }); + it('Returns custom column configurations', async () => { + const customColumns = [ + { timestampColumn: { id: uuidv4() } }, + { fieldColumn: { id: uuidv4(), field: 'host.name' } }, + { fieldColumn: { id: uuidv4(), field: 'event.dataset' } }, + { messageColumn: { id: uuidv4() } }, + ]; + + const { body } = await supertest + .post(LOG_ENTRIES_PATH) + .set(COMMON_HEADERS) + .send( + logEntriesRequestRT.encode({ + sourceId: 'default', + startTimestamp: EARLIEST_KEY_WITH_DATA.time, + endTimestamp: LATEST_KEY_WITH_DATA.time, + center: KEY_WITHIN_DATA_RANGE, + columns: customColumns, + }) + ) + .expect(200); + + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); + + const entries = logEntriesResponse.data.entries; + const entry = entries[0]; + expect(entry.columns).to.have.length(4); + + const timestampColumn = entry.columns[0] as LogTimestampColumn; + expect(timestampColumn).to.have.property('timestamp'); + + const hostNameColumn = entry.columns[1] as LogFieldColumn; + expect(hostNameColumn).to.have.property('field'); + expect(hostNameColumn.field).to.be('host.name'); + expect(hostNameColumn).to.have.property('value'); + + const eventDatasetColumn = entry.columns[2] as LogFieldColumn; + expect(eventDatasetColumn).to.have.property('field'); + expect(eventDatasetColumn.field).to.be('event.dataset'); + expect(eventDatasetColumn).to.have.property('value'); + + const messageColumn = entry.columns[3] as LogMessageColumn; + expect(messageColumn).to.have.property('message'); + expect(messageColumn.message.length).to.be.greaterThan(0); + }); + it('Does not build context if entry does not have all fields', async () => { const { body } = await supertest .post(LOG_ENTRIES_PATH) @@ -140,10 +176,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(200); - const logEntriesResponse = pipe( - logEntriesResponseRT.decode(body), - fold(throwErrors(createPlainError), identity) - ); + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); const entries = logEntriesResponse.data.entries; const entry = entries[0]; @@ -162,10 +195,7 @@ export default function ({ getService }: FtrProviderContext) { size: 10, }) ); - const firstPage = pipe( - logEntriesResponseRT.decode(firstPageBody), - fold(throwErrors(createPlainError), identity) - ); + const firstPage = decodeOrThrow(logEntriesResponseRT)(firstPageBody); const { body: secondPageBody } = await supertest .post(LOG_ENTRIES_PATH) @@ -179,10 +209,7 @@ export default function ({ getService }: FtrProviderContext) { size: 10, }) ); - const secondPage = pipe( - logEntriesResponseRT.decode(secondPageBody), - fold(throwErrors(createPlainError), identity) - ); + const secondPage = decodeOrThrow(logEntriesResponseRT)(secondPageBody); const { body: bothPagesBody } = await supertest .post(LOG_ENTRIES_PATH) @@ -195,10 +222,7 @@ export default function ({ getService }: FtrProviderContext) { size: 20, }) ); - const bothPages = pipe( - logEntriesResponseRT.decode(bothPagesBody), - fold(throwErrors(createPlainError), identity) - ); + const bothPages = decodeOrThrow(logEntriesResponseRT)(bothPagesBody); expect(bothPages.data.entries).to.eql([ ...firstPage.data.entries, @@ -222,10 +246,7 @@ export default function ({ getService }: FtrProviderContext) { size: 10, }) ); - const lastPage = pipe( - logEntriesResponseRT.decode(lastPageBody), - fold(throwErrors(createPlainError), identity) - ); + const lastPage = decodeOrThrow(logEntriesResponseRT)(lastPageBody); const { body: secondToLastPageBody } = await supertest .post(LOG_ENTRIES_PATH) @@ -239,10 +260,7 @@ export default function ({ getService }: FtrProviderContext) { size: 10, }) ); - const secondToLastPage = pipe( - logEntriesResponseRT.decode(secondToLastPageBody), - fold(throwErrors(createPlainError), identity) - ); + const secondToLastPage = decodeOrThrow(logEntriesResponseRT)(secondToLastPageBody); const { body: bothPagesBody } = await supertest .post(LOG_ENTRIES_PATH) @@ -256,10 +274,7 @@ export default function ({ getService }: FtrProviderContext) { size: 20, }) ); - const bothPages = pipe( - logEntriesResponseRT.decode(bothPagesBody), - fold(throwErrors(createPlainError), identity) - ); + const bothPages = decodeOrThrow(logEntriesResponseRT)(bothPagesBody); expect(bothPages.data.entries).to.eql([ ...secondToLastPage.data.entries, @@ -283,10 +298,7 @@ export default function ({ getService }: FtrProviderContext) { }) ) .expect(200); - const logEntriesResponse = pipe( - logEntriesResponseRT.decode(body), - fold(throwErrors(createPlainError), identity) - ); + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); const entries = logEntriesResponse.data.entries; const firstEntry = entries[0]; @@ -313,10 +325,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(200); - const logEntriesResponse = pipe( - logEntriesResponseRT.decode(body), - fold(throwErrors(createPlainError), identity) - ); + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); expect(logEntriesResponse.data.entries).to.have.length(0); expect(logEntriesResponse.data.topCursor).to.be(null); @@ -371,10 +380,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(200); - const logEntriesResponse = pipe( - logEntriesResponseRT.decode(body), - fold(throwErrors(createPlainError), identity) - ); + const logEntriesResponse = decodeOrThrow(logEntriesResponseRT)(body); const entries = logEntriesResponse.data.entries; const entry = entries[0];