diff --git a/javascript/packages/fury/lib/fury.ts b/javascript/packages/fury/lib/fury.ts index 4587f7ff13..d4400f7f66 100644 --- a/javascript/packages/fury/lib/fury.ts +++ b/javascript/packages/fury/lib/fury.ts @@ -21,10 +21,11 @@ import ClassResolver from "./classResolver"; import { BinaryWriter } from "./writer"; import { BinaryReader } from "./reader"; import { ReferenceResolver } from "./referenceResolver"; -import { ConfigFlags, Serializer, Config, Language, MAGIC_NUMBER } from "./type"; +import { ConfigFlags, Serializer, Config, Language, MAGIC_NUMBER, Mode } from "./type"; import { OwnershipError } from "./error"; import { InputType, ResultType, TypeDescription } from "./description"; import { generateSerializer, AnySerializer } from "./gen"; +import { TypeMeta } from "./meta/TypeMeta"; export default class { binaryReader: BinaryReader; @@ -32,12 +33,14 @@ export default class { classResolver = new ClassResolver(); referenceResolver: ReferenceResolver; anySerializer: AnySerializer; + typeMeta = TypeMeta; constructor(public config: Config = { refTracking: false, useSliceString: false, hooks: { }, + mode: Mode.SchemaConsistent, }) { this.binaryReader = new BinaryReader(config); this.binaryWriter = new BinaryWriter(config); diff --git a/javascript/packages/fury/lib/gen/builder.ts b/javascript/packages/fury/lib/gen/builder.ts index d82891535a..f78d5a3a25 100644 --- a/javascript/packages/fury/lib/gen/builder.ts +++ b/javascript/packages/fury/lib/gen/builder.ts @@ -22,7 +22,17 @@ import { getMeta } from "../meta"; import { TypeDescription } from "../description"; import Fury from "../fury"; -class BinaryReaderBuilder { +class TypeMetaBuilder { + constructor(private fury: string) { + + } + + fromBytes(reader: string) { + return `${this.fury}.typeMeta.fromBytes(${reader})`; + } +} + +export class BinaryReaderBuilder { constructor(private holder: string) { } @@ -309,6 +319,7 @@ class ClassResolverBuilder { export class CodecBuilder { reader: BinaryReaderBuilder; writer: BinaryWriterBuilder; + typeMeta: TypeMetaBuilder; // Use the TypeMetaWrapper referenceResolver: ReferenceResolverBuilder; classResolver: ClassResolverBuilder; @@ -321,6 +332,7 @@ export class CodecBuilder { this.writer = new BinaryWriterBuilder(bw); this.classResolver = new ClassResolverBuilder(cr); this.referenceResolver = new ReferenceResolverBuilder(rr); + this.typeMeta = new TypeMetaBuilder("fury"); // Initialize the TypeMetaWrapper } furyName() { diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index d9ddf7c639..3602b6d692 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -17,7 +17,7 @@ * under the License. */ -import { InternalSerializerType, MaxInt32 } from "../type"; +import { InternalSerializerType, MaxInt32, Mode } from "../type"; import { Scope } from "./scope"; import { CodecBuilder } from "./builder"; import { ObjectTypeDescription, TypeDescription } from "../description"; @@ -25,17 +25,7 @@ import { fromString } from "../platformBuffer"; import { CodegenRegistry } from "./router"; import { BaseSerializerGenerator, RefState } from "./serializer"; import SerializerResolver from "../classResolver"; -import { MetaString } from "../meta/MetaString"; - -// Ensure MetaString methods are correctly implemented -const computeMetaInformation = (description: any) => { - const metaInfo = JSON.stringify(description); - return MetaString.encode(metaInfo); -}; - -const decodeMetaInformation = (encodedMetaInfo: Uint8Array) => { - return MetaString.decode(encodedMetaInfo); -}; +import { FieldInfo, TypeMeta } from "../meta/TypeMeta"; function computeFieldHash(hash: number, id: number): number { let newHash = (hash) * 31 + (id); @@ -80,11 +70,22 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { writeStmt(accessor: string): string { const options = this.description.options; const expectHash = computeStructHash(this.description); - const metaInformation = Buffer.from(computeMetaInformation(this.description)); + // const metaInformation = Buffer.from(computeMetaInformation(this.description)); + const fields = Object.entries(this.description).map(([key, value]) => { + return new FieldInfo(key, value.type); + }); + const typeMetaBinary = new Uint8Array(TypeMeta.fromFields(256, fields).toBytes()); + const typeMetaDeclare = this.scope.declare("typeMeta", `new Uint8Array([${typeMetaBinary.toString()}])`); return ` ${this.builder.writer.int32(expectHash)}; - ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; + + ${ + this.builder.fury.config.mode === Mode.Compatible + ? this.builder.writer.buffer(typeMetaDeclare) + : "" + } + ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -99,9 +100,8 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { readStmt(accessor: (expr: string) => string, refState: RefState): string { const options = this.description.options; const expectHash = computeStructHash(this.description); - const encodedMetaInformation = computeMetaInformation(this.description); const result = this.scope.uniqueName("result"); - const pass = this.builder.reader.int32(); + return ` if (${this.builder.reader.int32()} !== ${expectHash}) { throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}"); @@ -111,8 +111,13 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { return `${CodecBuilder.safePropName(key)}: null`; }).join(",\n")} }; + ${this.maybeReference(result, refState)} - ${this.builder.reader.buffer(encodedMetaInformation.byteLength)} + ${ + this.builder.fury.config.mode === Mode.Compatible + ? this.builder.typeMeta.fromBytes(this.builder.reader.ownName()) + : "" + } ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -125,8 +130,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { `; } - // /8 /7 /20 % 2 - // is there a ratio from length / deserializer private safeTag() { return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); } diff --git a/javascript/packages/fury/lib/meta/MetaString.ts b/javascript/packages/fury/lib/meta/MetaString.ts index b4afe153be..1843c87b70 100644 --- a/javascript/packages/fury/lib/meta/MetaString.ts +++ b/javascript/packages/fury/lib/meta/MetaString.ts @@ -59,21 +59,17 @@ export class MetaString { // Decoding function that extracts bits per character from the first byte static decode(bytes: Uint8Array): string { - const bitsPerChar = bytes[0] & 0x0F; + const bitsPerChar = bytes[0]; const totalBits = (bytes.length * 8); // Adjusted for metadata bits const chars: string[] = []; let currentBit = 8; // Start after the first 8 metadata bits - while (currentBit < totalBits) { + while (currentBit + bitsPerChar <= totalBits) { let value = 0; for (let i = 0; i < bitsPerChar; i++) { const bytePos = Math.floor(currentBit / 8); const bitPos = currentBit % 8; - if (bytePos >= bytes.length) { - throw new RangeError("Offset is outside the bounds of the DataView"); - } - if (bytes[bytePos] & (1 << (7 - bitPos))) { value |= (1 << (bitsPerChar - i - 1)); } diff --git a/javascript/packages/fury/lib/meta/TypeMeta.ts b/javascript/packages/fury/lib/meta/TypeMeta.ts new file mode 100644 index 0000000000..9934f261a1 --- /dev/null +++ b/javascript/packages/fury/lib/meta/TypeMeta.ts @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BinaryWriter } from "../writer"; +import { BinaryReader } from "../reader"; +import { MetaString } from "./MetaString"; + +enum Encoding { + Utf8, + AllToLowerSpecial, + LowerUpperDigitSpecial, +} + +export class FieldInfo { + constructor(private fieldName: string, private fieldId: number) { + } + + static u8ToEncoding(value: number) { + switch (value) { + case 0x00: + return Encoding.Utf8; + case 0x01: + return Encoding.AllToLowerSpecial; + case 0x02: + return Encoding.LowerUpperDigitSpecial; + } + } + + static fromBytes(reader: BinaryReader) { + const header = reader.uint8(); + let size = (header & 0b11100000) >> 5; + size = (size === 0b111) ? reader.varInt32() + 7 : size; + const typeId = reader.int16(); + // reader.skip(size); + const fieldName = MetaString.decode(reader.buffer(size)); // now we commentd this line , the code work well + return new FieldInfo(fieldName, typeId); + } + + toBytes() { + const writer = new BinaryWriter({}); + const metaString = MetaString.encode(this.fieldName); + let header = 1 << 2; + const size = metaString.byteLength; + const bigSize = size >= 7; + if (bigSize) { + header |= 0b11100000; + writer.uint8(header); + writer.varInt32(size - 7); + } else { + header |= size << 5; + writer.uint8(header); + } + writer.int16(this.fieldId); + writer.buffer(metaString); + return writer.dump(); + } +} + +// Using classes to emulate struct methods in Rust +class TypeMetaLayer { + constructor(private typeId: number, private fieldInfo: FieldInfo[]) { + } + + getTypeId() { + return this.typeId; + } + + getFieldInfo() { + return this.fieldInfo; + } + + toBytes() { + const writer = new BinaryWriter({}); + writer.varInt32(this.fieldInfo.length); + writer.varInt32(this.typeId); + for (const field of this.fieldInfo) { + writer.buffer(field.toBytes()); + } + return writer.dump(); + } + + static fromBytes(reader: BinaryReader) { + const fieldNum = reader.varInt32(); + const typeId = reader.varInt32(); + const fieldInfo = []; + for (let i = 0; i < fieldNum; i++) { + fieldInfo.push(FieldInfo.fromBytes(reader)); + } + return new TypeMetaLayer(typeId, fieldInfo); + } +} + +export class TypeMeta { + constructor(private hash: bigint, private layers: TypeMetaLayer[]) { + } + + getFieldInfo() { + return this.layers[0].getFieldInfo(); + } + + getTypeId() { + return this.layers[0].getTypeId(); + } + + static fromFields(typeId: number, fieldInfo: FieldInfo[]) { + return new TypeMeta(BigInt(0), [new TypeMetaLayer(typeId, fieldInfo)]); + } + + static fromBytes(reader: BinaryReader) { + const header = reader.uint64(); + const hash = header >> BigInt(8); + const layerCount = header & BigInt(0b1111); + const layers = []; + for (let i = 0; i < layerCount; i++) { + layers.push(TypeMetaLayer.fromBytes(reader)); + } + return new TypeMeta(hash, layers); + } + + toBytes() { + const writer = new BinaryWriter({}); + writer.uint64(BigInt((this.hash << BigInt(8)) | BigInt((this.layers.length & 0b1111)))); + for (const layer of this.layers) { + writer.buffer(layer.toBytes()); + } + return writer.dump(); + } +} diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index 9fac1f5901..23956da896 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -100,6 +100,11 @@ export interface Hps { stringCopy: (str: string, dist: Uint8Array, offset: number) => void; } +export enum Mode { + SchemaConsistent, + Compatible, +} + export interface Config { hps?: Hps; refTracking?: boolean; @@ -107,6 +112,7 @@ export interface Config { hooks?: { afterCodeGenerated?: (code: string) => string; }; + mode?: Mode; } export enum Language {