feat(javascript): Added serialization and serialization for Type Meta Layer

javascript/packages/fury/lib/fury.ts

@@ -21,23 +21,26 @@ import ClassResolver from "./classResolver";
 import { BinaryWriter } from "./writer";
 import { BinaryReader } from "./reader";
 import { ReferenceResolver } from "./referenceResolver";
-import { ConfigFlags, Serializer, Config, Language, MAGIC_NUMBER } from "./type";
+import { ConfigFlags, Serializer, Config, Language, MAGIC_NUMBER, Mode } from "./type";
 import { OwnershipError } from "./error";
 import { InputType, ResultType, TypeDescription } from "./description";
 import { generateSerializer, AnySerializer } from "./gen";
+import { TypeMeta } from "./meta/TypeMeta";
 
 export default class {
   binaryReader: BinaryReader;
   binaryWriter: BinaryWriter;
   classResolver = new ClassResolver();
   referenceResolver: ReferenceResolver;
   anySerializer: AnySerializer;
+  typeMeta = TypeMeta;
 
   constructor(public config: Config = {
     refTracking: false,
     useSliceString: false,
     hooks: {
     },
+    mode: Mode.SchemaConsistent,
   }) {
     this.binaryReader = new BinaryReader(config);
     this.binaryWriter = new BinaryWriter(config);

javascript/packages/fury/lib/gen/builder.ts

@@ -22,7 +22,17 @@ import { getMeta } from "../meta";
 import { TypeDescription } from "../description";
 import Fury from "../fury";
 
-class BinaryReaderBuilder {
+class TypeMetaBuilder {
+  constructor(private fury: string) {
+
+  }
+
+  fromBytes(reader: string) {
+    return `${this.fury}.typeMeta.fromBytes(${reader})`;
+  }
+}
+
+export class BinaryReaderBuilder {
   constructor(private holder: string) {
 
   }
@@ -309,6 +319,7 @@ class ClassResolverBuilder {
 export class CodecBuilder {
   reader: BinaryReaderBuilder;
   writer: BinaryWriterBuilder;
+  typeMeta: TypeMetaBuilder; // Use the TypeMetaWrapper
   referenceResolver: ReferenceResolverBuilder;
   classResolver: ClassResolverBuilder;
 
@@ -321,6 +332,7 @@ export class CodecBuilder {
     this.writer = new BinaryWriterBuilder(bw);
     this.classResolver = new ClassResolverBuilder(cr);
     this.referenceResolver = new ReferenceResolverBuilder(rr);
+    this.typeMeta = new TypeMetaBuilder("fury"); // Initialize the TypeMetaWrapper
   }
 
   furyName() {

javascript/packages/fury/lib/gen/object.ts

@@ -25,17 +25,7 @@ import { fromString } from "../platformBuffer";
 import { CodegenRegistry } from "./router";
 import { BaseSerializerGenerator, RefState } from "./serializer";
 import SerializerResolver from "../classResolver";
-import { MetaString } from "../meta/MetaString";
-
-// Ensure MetaString methods are correctly implemented
-const computeMetaInformation = (description: any) => {
-  const metaInfo = JSON.stringify(description);
-  return MetaString.encode(metaInfo);
-};
-
-const decodeMetaInformation = (encodedMetaInfo: Uint8Array) => {
-  return MetaString.decode(encodedMetaInfo);
-};
+import { FieldInfo, TypeMeta } from "../meta/TypeMeta";
 
 function computeFieldHash(hash: number, id: number): number {
   let newHash = (hash) * 31 + (id);
@@ -80,11 +70,18 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator {
   writeStmt(accessor: string): string {
     const options = this.description.options;
     const expectHash = computeStructHash(this.description);
-    const metaInformation = Buffer.from(computeMetaInformation(this.description));
+    // const metaInformation = Buffer.from(computeMetaInformation(this.description));
+    const fields = Object.entries(this.description).map(([key, value]) => {
+      return new FieldInfo(key, value.type);
+    });
+    const typeMetaBinary = new Uint8Array(TypeMeta.fromFields(256, fields).toBytes());
+    const typeMetaDeclare = this.scope.declare("typeMeta", `new Uint8Array([${typeMetaBinary.toString()}])`);
 
     return `
       ${this.builder.writer.int32(expectHash)};
-      ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)};
+
+      ${this.builder.writer.buffer(typeMetaDeclare)};
+
       ${Object.entries(options.props).sort().map(([key, inner]) => {
         const InnerGeneratorClass = CodegenRegistry.get(inner.type);
         if (!InnerGeneratorClass) {
@@ -99,9 +96,8 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator {
   readStmt(accessor: (expr: string) => string, refState: RefState): string {
     const options = this.description.options;
     const expectHash = computeStructHash(this.description);
-    const encodedMetaInformation = computeMetaInformation(this.description);
     const result = this.scope.uniqueName("result");
-    const pass = this.builder.reader.int32();
+
     return `
       if (${this.builder.reader.int32()} !== ${expectHash}) {
           throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}");
@@ -111,8 +107,9 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator {
           return `${CodecBuilder.safePropName(key)}: null`;
         }).join(",\n")}
       };
+
       ${this.maybeReference(result, refState)}
-      ${this.builder.reader.buffer(encodedMetaInformation.byteLength)}
+      ${this.builder.typeMeta.fromBytes(this.builder.reader.ownName())}
       ${Object.entries(options.props).sort().map(([key, inner]) => {
         const InnerGeneratorClass = CodegenRegistry.get(inner.type);
         if (!InnerGeneratorClass) {
@@ -125,8 +122,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator {
     `;
   }
 
-  // /8 /7 /20 % 2
-  // is there a ratio from length / deserializer
   private safeTag() {
     return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag);
   }

javascript/packages/fury/lib/meta/MetaString.ts

@@ -59,21 +59,17 @@ export class MetaString {
 
   // Decoding function that extracts bits per character from the first byte
   static decode(bytes: Uint8Array): string {
-    const bitsPerChar = bytes[0] & 0x0F;
+    const bitsPerChar = bytes[0];
     const totalBits = (bytes.length * 8); // Adjusted for metadata bits
     const chars: string[] = [];
     let currentBit = 8; // Start after the first 8 metadata bits
 
-    while (currentBit < totalBits) {
+    while (currentBit + bitsPerChar <= totalBits) {
       let value = 0;
       for (let i = 0; i < bitsPerChar; i++) {
         const bytePos = Math.floor(currentBit / 8);
         const bitPos = currentBit % 8;
 
-        if (bytePos >= bytes.length) {
-          throw new RangeError("Offset is outside the bounds of the DataView");
-        }
-
         if (bytes[bytePos] & (1 << (7 - bitPos))) {
           value |= (1 << (bitsPerChar - i - 1));
         }

javascript/packages/fury/lib/meta/TypeMeta.ts

@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { BinaryWriter } from "../writer";
+import { BinaryReader } from "../reader";
+import { MetaString } from "./MetaString";
+
+enum Encoding {
+  Utf8,
+  AllToLowerSpecial,
+  LowerUpperDigitSpecial,
+}
+
+export class FieldInfo {
+  constructor(private fieldName: string, private fieldId: number) {
+  }
+
+  static u8ToEncoding(value: number) {
+    switch (value) {
+      case 0x00:
+        return Encoding.Utf8;
+      case 0x01:
+        return Encoding.AllToLowerSpecial;
+      case 0x02:
+        return Encoding.LowerUpperDigitSpecial;
+    }
+  }
+
+  static fromBytes(reader: BinaryReader) {
+    const header = reader.uint8();
+    let size = (header & 0b11100000) >> 5;
+    size = (size === 0b111) ? reader.varInt32() + 7 : size;
+    const typeId = reader.int16();
+    // reader.skip(size);
+    const fieldName = MetaString.decode(reader.buffer(size)); // now we commentd this line , the code work well
+    return new FieldInfo(fieldName, typeId);
+  }
+
+  toBytes() {
+    const writer = new BinaryWriter({});
+    const metaString = MetaString.encode(this.fieldName);
+    let header = 1 << 2;
+    const size = metaString.byteLength;
+    const bigSize = size >= 7;
+    if (bigSize) {
+      header |= 0b11100000;
+      writer.uint8(header);
+      writer.varInt32(size - 7);
+    } else {
+      header |= size << 5;
+      writer.uint8(header);
+    }
+    writer.int16(this.fieldId);
+    writer.buffer(metaString);
+    return writer.dump();
+  }
+}
+
+// Using classes to emulate struct methods in Rust
+class TypeMetaLayer {
+  constructor(private typeId: number, private fieldInfo: FieldInfo[]) {
+  }
+
+  getTypeId() {
+    return this.typeId;
+  }
+
+  getFieldInfo() {
+    return this.fieldInfo;
+  }
+
+  toBytes() {
+    const writer = new BinaryWriter({});
+    writer.varInt32(this.fieldInfo.length);
+    writer.varInt32(this.typeId);
+    for (const field of this.fieldInfo) {
+      writer.buffer(field.toBytes());
+    }
+    return writer.dump();
+  }
+
+  static fromBytes(reader: BinaryReader) {
+    const fieldNum = reader.varInt32();
+    const typeId = reader.varInt32();
+    const fieldInfo = [];
+    for (let i = 0; i < fieldNum; i++) {
+      fieldInfo.push(FieldInfo.fromBytes(reader));
+    }
+    return new TypeMetaLayer(typeId, fieldInfo);
+  }
+}
+
+export class TypeMeta {
+  constructor(private hash: bigint, private layers: TypeMetaLayer[]) {
+  }
+
+  getFieldInfo() {
+    return this.layers[0].getFieldInfo();
+  }
+
+  getTypeId() {
+    return this.layers[0].getTypeId();
+  }
+
+  static fromFields(typeId: number, fieldInfo: FieldInfo[]) {
+    return new TypeMeta(BigInt(0), [new TypeMetaLayer(typeId, fieldInfo)]);
+  }
+
+  static fromBytes(reader: BinaryReader) {
+    const header = reader.uint64();
+    const hash = header >> BigInt(8);
+    const layerCount = header & BigInt(0b1111);
+    const layers = [];
+    for (let i = 0; i < layerCount; i++) {
+      layers.push(TypeMetaLayer.fromBytes(reader));
+    }
+    return new TypeMeta(hash, layers);
+  }
+
+  toBytes() {
+    const writer = new BinaryWriter({});
+    writer.uint64(BigInt((this.hash << BigInt(8)) | BigInt((this.layers.length & 0b1111))));
+    for (const layer of this.layers) {
+      writer.buffer(layer.toBytes());
+    }
+    return writer.dump();
+  }
+}

javascript/packages/fury/lib/type.ts

@@ -100,13 +100,19 @@ export interface Hps {
   stringCopy: (str: string, dist: Uint8Array, offset: number) => void;
 }
 
+export enum Mode {
+  SchemaConsistent,
+  Compatible,
+}
+
 export interface Config {
   hps?: Hps;
   refTracking?: boolean;
   useSliceString?: boolean;
   hooks?: {
     afterCodeGenerated?: (code: string) => string;
   };
+  mode?: Mode;
 }
 
 export enum Language {