From aee099b4096f2800a097abd767ae7f8a53cdc050 Mon Sep 17 00:00:00 2001 From: Volker Stampa Date: Tue, 31 Oct 2023 16:52:24 +0100 Subject: [PATCH] Add support for Task parsing --- log-viewer/src/lib/logfile.parser.test.ts | 88 +++++---- log-viewer/src/lib/logfile.parser.ts | 209 ++++++++++++++-------- 2 files changed, 187 insertions(+), 110 deletions(-) diff --git a/log-viewer/src/lib/logfile.parser.test.ts b/log-viewer/src/lib/logfile.parser.test.ts index d7f0e0bed..9d51d460f 100644 --- a/log-viewer/src/lib/logfile.parser.test.ts +++ b/log-viewer/src/lib/logfile.parser.test.ts @@ -1,40 +1,62 @@ import { expect, test } from 'vitest'; -import { parseLogLines } from './logfile.parser'; -import { randomLogEntry, randomSpan } from './log.test_utils'; +import { parseLogLines, type LogLine } from './logfile.parser'; +import { randomLogger } from './log.test_utils'; import { faker } from '@faker-js/faker'; +import { isLogEntry, type Entry, type TaskSpan } from './log'; test('parseLogLines can parse start task entries', () => { const rootId = faker.string.uuid(); - const span = randomSpan(); - const spanId = faker.string.uuid(); - const entry = randomLogEntry(); - const entries = [ - { - entry_type: 'SpanStart' as const, - entry: { parent: rootId, uuid: spanId, name: span.name, start: span.start_timestamp } - }, - { - entry_type: 'LogEntry' as const, - entry: { - parent: spanId, - message: entry.message, - value: entry.value, - timestamp: entry.timestamp - } - }, - { entry_type: 'SpanEnd' as const, entry: { uuid: spanId, end: span.end_timestamp } } - ]; - const debugLogger = parseLogLines(entries); + const expected = randomLogger(); + expected.name = rootId; + const actual = parseLogLines(toFlatEntries(rootId, expected.logs)); - expect(debugLogger).toStrictEqual({ - name: rootId, - logs: [ - { - name: span.name, - start_timestamp: span.start_timestamp, - end_timestamp: span.end_timestamp, - logs: [entry] - } - ] - }); + expect(actual).toStrictEqual(expected); }); + +function toFlatEntries(parent: string, logs: Entry[]): LogLine[] { + return logs.flatMap((entry) => { + if (isLogEntry(entry)) { + return [ + { + entry_type: 'LogEntry', + entry: { parent, message: entry.message, timestamp: entry.timestamp, value: entry.value } + } + ]; + } else if (isTaskSpan(entry)) { + const uuid = faker.string.uuid(); + const logs: LogLine[] = [ + { + entry_type: 'StartTask', + entry: { + parent, + name: entry.name, + start: entry.start_timestamp, + uuid, + input: entry.input + } + } + ]; + logs.push(...toFlatEntries(uuid, entry.logs)); + logs.push({ + entry_type: 'EndTask', + entry: { end: entry.end_timestamp, uuid, output: entry.output } + }); + return logs; + } else { + const uuid = faker.string.uuid(); + const logs: LogLine[] = [ + { + entry_type: 'StartSpan', + entry: { parent, name: entry.name, start: entry.start_timestamp, uuid } + } + ]; + logs.push(...toFlatEntries(uuid, entry.logs)); + logs.push({ entry_type: 'EndSpan', entry: { end: entry.end_timestamp, uuid } }); + return logs; + } + }); +} + +function isTaskSpan(entry: Entry): entry is TaskSpan { + return 'input' in entry; +} diff --git a/log-viewer/src/lib/logfile.parser.ts b/log-viewer/src/lib/logfile.parser.ts index 58e5ce931..33398c193 100644 --- a/log-viewer/src/lib/logfile.parser.ts +++ b/log-viewer/src/lib/logfile.parser.ts @@ -1,104 +1,159 @@ import { z } from 'zod'; -import type { DebugLog, Span } from './log'; -import { en } from '@faker-js/faker'; +import type { DebugLog, Span, TaskSpan } from './log'; const logEntry = z.object({ - parent: z.string(), - message: z.string(), - value: z.any(), - timestamp: z.string() + parent: z.string(), + message: z.string(), + value: z.any(), + timestamp: z.string() }); type LogEntry = z.infer; const spanStart = z.object({ - uuid: z.string(), - parent: z.string(), - name: z.string(), - start: z.string() + uuid: z.string(), + parent: z.string(), + name: z.string(), + start: z.string() }); type SpanStart = z.infer; const spanEnd = z.object({ - uuid: z.string(), - end: z.string() + uuid: z.string(), + end: z.string() }); type SpanEnd = z.infer; +const taskStart = z.object({ + uuid: z.string(), + parent: z.string(), + name: z.string(), + start: z.string(), + input: z.any() +}); + +type TaskStart = z.infer; + +const taskEnd = z.object({ + uuid: z.string(), + end: z.string(), + output: z.any() +}); + +type TaskEnd = z.infer; + const logLine = z.discriminatedUnion('entry_type', [ - z.object({ - entry_type: z.literal('LogEntry'), - entry: logEntry - }), - z.object({ - entry_type: z.literal('SpanStart'), - entry: spanStart - }), - z.object({ - entry_type: z.literal('SpanEnd'), - entry: spanEnd - }) + z.object({ + entry_type: z.literal('LogEntry'), + entry: logEntry + }), + z.object({ + entry_type: z.literal('StartSpan'), + entry: spanStart + }), + z.object({ + entry_type: z.literal('EndSpan'), + entry: spanEnd + }), + z.object({ + entry_type: z.literal('StartTask'), + entry: taskStart + }), + z.object({ + entry_type: z.literal('EndTask'), + entry: taskEnd + }) ]); export type LogLine = z.infer; export function parseLogLines(lines: LogLine[]): DebugLog { - const builder = new LogBuilder(); - for (const line of lines) { - switch (line.entry_type) { - case 'LogEntry': - builder.addLogEntry(line.entry); - break; - case 'SpanEnd': - builder.endSpan(line.entry); - break; - case 'SpanStart': - builder.startSpan(line.entry); - break; - } - } - return builder.root; + const builder = new LogBuilder(); + for (const line of lines) { + switch (line.entry_type) { + case 'LogEntry': + builder.addLogEntry(line.entry); + break; + case 'StartSpan': + builder.startSpan(line.entry); + break; + case 'EndSpan': + builder.endSpan(line.entry); + break; + case 'StartTask': + builder.startTask(line.entry); + break; + case 'EndTask': + builder.endTask(line.entry); + break; + } + } + return builder.root; } class LogBuilder { - root: DebugLog = { name: '', logs: [] }; - private loggers: Map = new Map(); - private spans: Map = new Map(); - - addLogEntry(entry: LogEntry) { - const parent = this.parentLogger(entry.parent); - // entry.value is any, but value is "Json" - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - parent.logs.push({ message: entry.message, value: entry.value, timestamp: entry.timestamp }); - } - - startSpan(entry: SpanStart) { - const parent = this.parentLogger(entry.parent); - const span: Span = { - name: entry.name, - start_timestamp: entry.start, - end_timestamp: entry.start, - logs: [] - }; - parent.logs.push(span); - this.spans.set(entry.uuid, span); - this.loggers.set(entry.uuid, span); - } - - endSpan(entry: SpanEnd) { - const span = this.spans.get(entry.uuid); - span!.end_timestamp = entry.end; - } - - parentLogger(uuid: string): DebugLog { - const parent = this.loggers.get(uuid); - if (parent) { - return parent; - } - this.root = { name: uuid, logs: [] }; - this.loggers.set(uuid, this.root); - return this.root; - } + root: DebugLog = { name: '', logs: [] }; + private loggers: Map = new Map(); + private spans: Map = new Map(); + private tasks: Map = new Map(); + + addLogEntry(entry: LogEntry) { + const parent = this.parentLogger(entry.parent); + // entry.value is any, but value is "Json" + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + parent.logs.push({ message: entry.message, value: entry.value, timestamp: entry.timestamp }); + } + + startSpan(entry: SpanStart) { + const parent = this.parentLogger(entry.parent); + const span: Span = { + name: entry.name, + start_timestamp: entry.start, + end_timestamp: entry.start, + logs: [] + }; + parent.logs.push(span); + this.spans.set(entry.uuid, span); + this.loggers.set(entry.uuid, span); + } + + endSpan(entry: SpanEnd) { + const span = this.spans.get(entry.uuid); + span!.end_timestamp = entry.end; + } + + startTask(entry: TaskStart) { + const parent = this.parentLogger(entry.parent); + const task: TaskSpan = { + name: entry.name, + start_timestamp: entry.start, + end_timestamp: entry.start, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + input: entry.input, + output: undefined, + logs: [] + }; + parent.logs.push(task); + this.tasks.set(entry.uuid, task); + this.loggers.set(entry.uuid, task); + } + + endTask(entry: TaskEnd) { + const task = this.tasks.get(entry.uuid); + task!.end_timestamp = entry.end; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + task!.output = entry.output; + } + + parentLogger(uuid: string): DebugLog { + const parent = this.loggers.get(uuid); + if (parent) { + return parent; + } + this.root = { name: uuid, logs: [] }; + this.loggers.set(uuid, this.root); + return this.root; + } }