diff --git a/new-log-viewer/package-lock.json b/new-log-viewer/package-lock.json index 1cb00d53..682c74d0 100644 --- a/new-log-viewer/package-lock.json +++ b/new-log-viewer/package-lock.json @@ -10,6 +10,7 @@ "license": "Apache-2.0", "dependencies": { "axios": "^1.7.2", + "clp-ffi-js": "^0.1.0", "dayjs": "^1.11.11", "monaco-editor": "^0.50.0", "react": "^18.3.1", @@ -3996,6 +3997,12 @@ "node": ">=6" } }, + "node_modules/clp-ffi-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/clp-ffi-js/-/clp-ffi-js-0.1.0.tgz", + "integrity": "sha512-/g1EBxKDd6syknCGj7c/pM4tl1nEhLfRRf8zwaAfDQBxWcO0isXREFya8+TBVm2KTuik9O8hb9HidR17LtI3jg==", + "license": "Apache-2.0" + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", diff --git a/new-log-viewer/package.json b/new-log-viewer/package.json index c236450b..607f0756 100644 --- a/new-log-viewer/package.json +++ b/new-log-viewer/package.json @@ -19,6 +19,7 @@ "homepage": "https://github.com/y-scope/yscope-log-viewer#readme", "dependencies": { "axios": "^1.7.2", + "clp-ffi-js": "^0.1.0", "dayjs": "^1.11.11", "monaco-editor": "^0.50.0", "react": "^18.3.1", diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index 2dcfbf62..37b31290 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -1,6 +1,7 @@ import { Decoder, DecoderOptionsType, + LOG_EVENT_FILE_END_IDX, } from "../typings/decoders"; import {MAX_V8_STRING_LENGTH} from "../typings/js"; import { @@ -13,6 +14,7 @@ import {getUint8ArrayFrom} from "../utils/http"; import {getChunkNum} from "../utils/math"; import {formatSizeInBytes} from "../utils/units"; import {getBasenameFromUrlOrDefault} from "../utils/url"; +import ClpIrDecoder from "./decoders/ClpIrDecoder"; import JsonlDecoder from "./decoders/JsonlDecoder"; @@ -48,8 +50,6 @@ const loadFile = async (fileSrc: FileSrcType) class LogFileManager { readonly #pageSize: number; - readonly #fileData: Uint8Array; - readonly #fileName: string; #decoder: Decoder; @@ -60,21 +60,27 @@ class LogFileManager { * Private constructor for LogFileManager. This is not intended to be invoked publicly. * Instead, use LogFileManager.create() to create a new instance of the class. * + * @param decoder * @param fileName - * @param fileData * @param pageSize Page size for setting up pagination. - * @param decoderOptions Initial decoder options. */ constructor ( + decoder: Decoder, fileName: string, - fileData: Uint8Array, pageSize: number, - decoderOptions: DecoderOptionsType ) { this.#fileName = fileName; - this.#fileData = fileData; this.#pageSize = pageSize; - this.#decoder = this.#initDecoder(decoderOptions); + this.#decoder = decoder; + + // Build index for the entire file + const buildIdxResult = decoder.buildIdx(0, LOG_EVENT_FILE_END_IDX); + if (null !== buildIdxResult && 0 < buildIdxResult.numInvalidEvents) { + console.error("Invalid events found in decoder.buildIdx():", buildIdxResult); + } + + this.#numEvents = decoder.getEstimatedNumEvents(); + console.log(`Found ${this.#numEvents} log events.`); } get numEvents () { @@ -96,7 +102,41 @@ class LogFileManager { decoderOptions: DecoderOptionsType ): Promise { const {fileName, fileData} = await loadFile(fileSrc); - return new LogFileManager(fileName, fileData, pageSize, decoderOptions); + const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions); + + return new LogFileManager(decoder, fileName, pageSize); + } + + /** + * Constructs a decoder instance based on the file extension. + * + * @param fileName + * @param fileData + * @param decoderOptions Initial decoder options. + * @return The constructed decoder. + * @throws {Error} if no decoder supports a file with the given extension. + */ + static async #initDecoder ( + fileName: string, + fileData: Uint8Array, + decoderOptions: DecoderOptionsType + ): Promise { + let decoder: Decoder; + if (fileName.endsWith(".jsonl")) { + decoder = new JsonlDecoder(fileData, decoderOptions); + } else if (fileName.endsWith(".clp.zst")) { + decoder = await ClpIrDecoder.create(fileData); + } else { + throw new Error(`No decoder supports ${fileName}`); + } + + if (fileData.length > MAX_V8_STRING_LENGTH) { + throw new Error(`Cannot handle files larger than ${ + formatSizeInBytes(MAX_V8_STRING_LENGTH) + } due to a limitation in Chromium-based browsers.`); + } + + return decoder; } /** @@ -154,33 +194,6 @@ class LogFileManager { }; } - /** - * Constructs a decoder instance based on the file extension. - * - * @param decoderOptions Initial decoder options. - * @return The constructed decoder. - * @throws {Error} if a decoder cannot be found. - */ - #initDecoder = (decoderOptions: DecoderOptionsType): Decoder => { - let decoder: Decoder; - if (this.#fileName.endsWith(".jsonl")) { - decoder = new JsonlDecoder(this.#fileData, decoderOptions); - } else { - throw new Error(`No decoder supports ${this.#fileName}`); - } - - if (this.#fileData.length > MAX_V8_STRING_LENGTH) { - throw new Error(`Cannot handle files larger than ${ - formatSizeInBytes(MAX_V8_STRING_LENGTH) - } due to a limitation in Chromium-based browsers.`); - } - - this.#numEvents = decoder.getEstimatedNumEvents(); - console.log(`Found ${this.#numEvents} log events.`); - - return decoder; - }; - /** * Gets the range of log event numbers for the page containing the given cursor. * diff --git a/new-log-viewer/src/services/decoders/ClpIrDecoder.ts b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts new file mode 100644 index 00000000..4cab4590 --- /dev/null +++ b/new-log-viewer/src/services/decoders/ClpIrDecoder.ts @@ -0,0 +1,51 @@ +import clpFfiJsModuleInit, {ClpIrStreamReader} from "clp-ffi-js"; + +import {Nullable} from "../../typings/common"; +import { + Decoder, + DecodeResultType, + LogEventCount, +} from "../../typings/decoders"; + + +class ClpIrDecoder implements Decoder { + #streamReader: ClpIrStreamReader; + + constructor (streamReader: ClpIrStreamReader) { + this.#streamReader = streamReader; + } + + /** + * Creates a new ClpIrDecoder instance. + * + * @param dataArray The input data array to be passed to the decoder. + * @return The created ClpIrDecoder instance. + */ + static async create (dataArray: Uint8Array): Promise { + const module = await clpFfiJsModuleInit(); + const streamReader = new module.ClpIrStreamReader(dataArray); + return new ClpIrDecoder(streamReader); + } + + getEstimatedNumEvents (): number { + return this.#streamReader.getNumEventsBuffered(); + } + + buildIdx (beginIdx: number, endIdx: number): Nullable { + return { + numInvalidEvents: 0, + numValidEvents: this.#streamReader.deserializeRange(beginIdx, endIdx), + }; + } + + // eslint-disable-next-line class-methods-use-this + setDecoderOptions (): boolean { + return true; + } + + decode (beginIdx: number, endIdx: number): Nullable { + return this.#streamReader.decodeRange(beginIdx, endIdx); + } +} + +export default ClpIrDecoder; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index c30a3700..53408184 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -45,7 +45,7 @@ interface Decoder { * When applicable, deserializes log events in the range `[beginIdx, endIdx)`. * * @param beginIdx - * @param endIdx + * @param endIdx End index. To deserialize to the end of the file, use `LOG_EVENT_FILE_END_IDX`. * @return Count of the successfully deserialized ("valid") log events and count of any * un-deserializable ("invalid") log events within the range; or null if any log event in the * range doesn't exist (e.g., the range exceeds the number of log events in the file). @@ -71,7 +71,13 @@ interface Decoder { decode(beginIdx: number, endIdx: number): Nullable; } +/** + * Index for specifying the end of the file when the exact number of log events it contains is + * unknown. + */ +const LOG_EVENT_FILE_END_IDX: number = 0; +export {LOG_EVENT_FILE_END_IDX}; export type { Decoder, DecodeResultType,