[Logs UI] Allow custom columns in the <LogStream /> component (#83802)

x-pack/plugins/infra/common/http_api/log_entries/entries.ts

@@ -7,6 +7,7 @@
 import * as rt from 'io-ts';
 import { jsonArrayRT } from '../../typed_json';
 import { logEntriesCursorRT } from './common';
+import { logSourceColumnConfigurationRT } from '../log_sources';
 
 export const LOG_ENTRIES_PATH = '/api/log_entries/entries';
 
@@ -19,6 +20,7 @@ export const logEntriesBaseRequestRT = rt.intersection([
   rt.partial({
     query: rt.union([rt.string, rt.null]),
     size: rt.number,
+    columns: rt.array(logSourceColumnConfigurationRT),
   }),
 ]);
 

x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts

@@ -48,7 +48,7 @@ const logSourceFieldColumnConfigurationRT = rt.strict({
   ]),
 });
 
-const logSourceColumnConfigurationRT = rt.union([
+export const logSourceColumnConfigurationRT = rt.union([
   logSourceTimestampColumnConfigurationRT,
   logSourceMessageColumnConfigurationRT,
   logSourceFieldColumnConfigurationRT,

x-pack/plugins/infra/public/components/log_stream/README.md

@@ -68,6 +68,36 @@ By default the `<LogStream />` uses the `"default"` source confiuration, but if
 <LogStream startTimestamp={startTimestamp} endTimestamp={endTimestamp} sourceId="my_source" />
 ```
 
+### Custom columns
+
+It is possible to change what columns are loaded without creating a whole new source configuration. To do so the component supports the `columns` prop. The default configuration can be replicated as follows.
+
+```tsx
+<LogStream
+  startTimestamp={...}
+  endTimetsamp={...}
+  columns={[
+    { type: 'timestamp' },
+    { type: 'field', field: 'event.dataset' }
+    { type: 'message' },
+  ]}
+/>
+```
+
+There are three column types:
+
+<table>
+  <tr>
+    <td>`type: "timestamp"`
+    <td>The configured timestamp field. Defaults to `@timestamp`.
+  <tr>
+    <td>`type: "message"`
+    <td>The value of the `message` field if it exists. If it doesn't, the component will try to recompose the original log line using values of other fields.
+  <tr>
+    <td>`type: "field"`
+    <td>A specific field specified in the `field` property.
+</table>
+
 ### Considerations
 
 As mentioned in the prerequisites, the component relies on `kibana-react` to access kibana's core services. If this is not the case the component will throw an exception when rendering. We advise to use an `<EuiErrorBoundary>` in your component hierarchy to catch this error if necessary.

x-pack/plugins/infra/public/components/log_stream/index.tsx

@@ -11,13 +11,18 @@ import { euiStyled } from '../../../../observability/public';
 import { LogEntriesCursor } from '../../../common/http_api';
 
 import { useKibana } from '../../../../../../src/plugins/kibana_react/public';
-import { useLogSource } from '../../containers/logs/log_source';
+import { LogSourceConfigurationProperties, useLogSource } from '../../containers/logs/log_source';
 import { useLogStream } from '../../containers/logs/log_stream';
 
 import { ScrollableLogTextStreamView } from '../logging/log_text_stream';
 
 const PAGE_THRESHOLD = 2;
 
+type LogColumnDefinition =
+  | { type: 'timestamp' }
+  | { type: 'message' }
+  | { type: 'field'; field: string };
+
 export interface LogStreamProps {
   sourceId?: string;
   startTimestamp: number;
@@ -26,6 +31,7 @@ export interface LogStreamProps {
   center?: LogEntriesCursor;
   highlight?: string;
   height?: string | number;
+  columns?: LogColumnDefinition[];
 }
 
 export const LogStream: React.FC<LogStreamProps> = ({
@@ -36,7 +42,13 @@ export const LogStream: React.FC<LogStreamProps> = ({
   center,
   highlight,
   height = '400px',
+  columns,
 }) => {
+  const customColumns = useMemo(
+    () => (columns ? convertLogColumnDefinitionToLogSourceColumnDefinition(columns) : undefined),
+    [columns]
+  );
+
   // source boilerplate
   const { services } = useKibana();
   if (!services?.http?.fetch) {
@@ -74,6 +86,7 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re
     endTimestamp,
     query,
     center,
+    columns: customColumns,
   });
 
   // Derived state
@@ -83,8 +96,8 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re
   const isLoadingMore = pageLoadingState === 'loading';
 
   const columnConfigurations = useMemo(() => {
-    return sourceConfiguration ? sourceConfiguration.configuration.logColumns : [];
-  }, [sourceConfiguration]);
+    return sourceConfiguration ? customColumns ?? sourceConfiguration.configuration.logColumns : [];
+  }, [sourceConfiguration, customColumns]);
 
   const streamItems = useMemo(
     () =>
@@ -163,6 +176,21 @@ const LogStreamContent = euiStyled.div<{ height: string }>`
   height: ${(props) => props.height};
 `;
 
+function convertLogColumnDefinitionToLogSourceColumnDefinition(
+  columns: LogColumnDefinition[]
+): LogSourceConfigurationProperties['logColumns'] {
+  return columns.map((column) => {
+    switch (column.type) {
+      case 'timestamp':
+        return { timestampColumn: { id: '___#timestamp' } };
+      case 'message':
+        return { messageColumn: { id: '___#message' } };
+      case 'field':
+        return { fieldColumn: { id: `___#${column.field}`, field: column.field } };
+    }
+  });
+}
+
 // Allow for lazy loading
 // eslint-disable-next-line import/no-default-export
 export default LogStream;

x-pack/plugins/infra/public/containers/logs/log_stream/index.ts

@@ -12,13 +12,15 @@ import { fetchLogEntries } from '../log_entries/api/fetch_log_entries';
 import { useTrackedPromise } from '../../../utils/use_tracked_promise';
 import { LogEntry, LogEntriesCursor } from '../../../../common/http_api';
 import { useKibanaContextForPlugin } from '../../../hooks/use_kibana';
+import { LogSourceConfigurationProperties } from '../log_source';
 
 interface LogStreamProps {
   sourceId: string;
   startTimestamp: number;
   endTimestamp: number;
   query?: string;
   center?: LogEntriesCursor;
+  columns?: LogSourceConfigurationProperties['logColumns'];
 }
 
 interface LogStreamState {
@@ -60,6 +62,7 @@ export function useLogStream({
   endTimestamp,
   query,
   center,
+  columns,
 }: LogStreamProps): LogStreamReturn {
   const { services } = useKibanaContextForPlugin();
   const [state, setState] = useSetState<LogStreamState>(INITIAL_STATE);
@@ -100,6 +103,7 @@ export function useLogStream({
             startTimestamp,
             endTimestamp,
             query: parsedQuery,
+            columns,
             ...fetchPosition,
           },
           services.http.fetch

x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts

@@ -15,6 +15,7 @@ import {
   LogEntriesItem,
   LogEntriesCursor,
   LogColumn,
+  LogEntriesRequest,
 } from '../../../../common/http_api';
 import {
   InfraSourceConfiguration,
@@ -73,7 +74,8 @@ export class InfraLogEntriesDomain {
   public async getLogEntriesAround(
     requestContext: RequestHandlerContext,
     sourceId: string,
-    params: LogEntriesAroundParams
+    params: LogEntriesAroundParams,
+    columnOverrides?: LogEntriesRequest['columns']
   ): Promise<{ entries: LogEntry[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> {
     const { startTimestamp, endTimestamp, center, query, size, highlightTerm } = params;
 
@@ -97,7 +99,8 @@ export class InfraLogEntriesDomain {
         cursor: { before: center },
         size: Math.floor(halfSize),
         highlightTerm,
-      }
+      },
+      columnOverrides
     );
 
     /*
@@ -131,13 +134,16 @@ export class InfraLogEntriesDomain {
   public async getLogEntries(
     requestContext: RequestHandlerContext,
     sourceId: string,
-    params: LogEntriesParams
+    params: LogEntriesParams,
+    columnOverrides?: LogEntriesRequest['columns']
   ): Promise<{ entries: LogEntry[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> {
     const { configuration } = await this.libs.sources.getSourceConfiguration(
       requestContext.core.savedObjects.client,
       sourceId
     );
 
+    const columnDefinitions = columnOverrides ?? configuration.logColumns;
+
     const messageFormattingRules = compileFormattingRules(
       getBuiltinRules(configuration.fields.message)
     );
@@ -155,7 +161,7 @@ export class InfraLogEntriesDomain {
       return {
         id: doc.id,
         cursor: doc.cursor,
-        columns: configuration.logColumns.map(
+        columns: columnDefinitions.map(
           (column): LogColumn => {
             if ('timestampColumn' in column) {
               return {

x-pack/plugins/infra/server/routes/log_entries/entries.ts

@@ -31,6 +31,7 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs)
           sourceId,
           query,
           size,
+          columns,
         } = payload;
 
         let entries;
@@ -47,7 +48,8 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs)
               query: parseFilterQuery(query),
               center: payload.center,
               size,
-            }
+            },
+            columns
           ));
         } else {
           let cursor: LogEntriesParams['cursor'];
@@ -66,7 +68,8 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs)
               query: parseFilterQuery(query),
               cursor,
               size,
-            }
+            },
+            columns
           ));
         }