feat: Added MetaString Class for Unicode Encoding/Decoding in Type.Ob…

…ject Code Generation (#1774)  ## What does this PR do? The `MetaString` class is introduced to handle Unicode encoding and decoding for strings used in the code generation process for `Type.Object`. This ensures that strings are properly encoded and decoded during storage and transmission. - **Encoding**: Converts Unicode strings into an encoded format. - **Decoding**: Converts encoded strings back to their original Unicode format. The `MetaString` class is used in `object.ts` to handle string encoding and decoding when generating code for `Type.Object`. ## Related issues Is there any related issue? Please attach here. - #1670 - #1671 - #1674 ## Does this PR introduce any user-facing change?  - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? @theweipeng Please have a look at my PR ## Benchmark
apache · Jul 29, 2024 · ba4ac8e · ba4ac8e
1 parent 7517c23
commit ba4ac8e
Show file tree

Hide file tree

Showing 4 changed files with 240 additions and 33 deletions.
diff --git a/javascript/packages/fury/lib/gen/builder.ts b/javascript/packages/fury/lib/gen/builder.ts
@@ -205,7 +205,7 @@ class BinaryWriterBuilder {
     return `${this.holder}.uint64(${v})`;
   }
 
-  buffer(v: string) {
+  buffer(v: string) { // Accepting Uint8Array as a parameter
     return `${this.holder}.buffer(${v})`;
   }
 

diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts
@@ -25,6 +25,17 @@ import { fromString } from "../platformBuffer";
 import { CodegenRegistry } from "./router";
 import { BaseSerializerGenerator, RefState } from "./serializer";
 import SerializerResolver from "../classResolver";
+import { MetaString } from "../meta/MetaString";
+
+// Ensure MetaString methods are correctly implemented
+const computeMetaInformation = (description: any) => {
+  const metaInfo = JSON.stringify(description);
+  return MetaString.encode(metaInfo);
+};
+
+const decodeMetaInformation = (encodedMetaInfo: Uint8Array) => {
+  return MetaString.decode(encodedMetaInfo);
+};
 
 function computeFieldHash(hash: number, id: number): number {
   let newHash = (hash) * 31 + (id);
@@ -69,56 +80,61 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator {
   writeStmt(accessor: string): string {
     const options = this.description.options;
     const expectHash = computeStructHash(this.description);
+    const metaInformation = Buffer.from(computeMetaInformation(this.description));
 
     return `
-            ${this.builder.writer.int32(expectHash)};
-            ${Object.entries(options.props).sort().map(([key, inner]) => {
-            const InnerGeneratorClass = CodegenRegistry.get(inner.type);
-            if (!InnerGeneratorClass) {
-                throw new Error(`${inner.type} generator not exists`);
-            }
-            const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope);
-            return innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`);
-        }).join(";\n")
-            }
-        `;
+      ${this.builder.writer.int32(expectHash)};
+      ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)};
+      ${Object.entries(options.props).sort().map(([key, inner]) => {
+        const InnerGeneratorClass = CodegenRegistry.get(inner.type);
+        if (!InnerGeneratorClass) {
+            throw new Error(`${inner.type} generator not exists`);
+        }
+        const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope);
+        return innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`);
+      }).join(";\n")}
+    `;
   }
 
   readStmt(accessor: (expr: string) => string, refState: RefState): string {
     const options = this.description.options;
     const expectHash = computeStructHash(this.description);
+    const encodedMetaInformation = computeMetaInformation(this.description);
     const result = this.scope.uniqueName("result");
+    const pass = this.builder.reader.int32();
     return `
-        if (${this.builder.reader.int32()} !== ${expectHash}) {
-            throw new Error("validate hash failed: ${this.safeTag()}. expect ${expectHash}");
-        }
-        const ${result} = {
-            ${Object.entries(options.props).sort().map(([key]) => {
-            return `${CodecBuilder.safePropName(key)}: null`;
+      if (${this.builder.reader.int32()} !== ${expectHash}) {
+          throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}");
+      }
+      const ${result} = {
+        ${Object.entries(options.props).sort().map(([key]) => {
+          return `${CodecBuilder.safePropName(key)}: null`;
         }).join(",\n")}
-        };
-        ${this.maybeReference(result, refState)}
-        ${Object.entries(options.props).sort().map(([key, inner]) => {
-            const InnerGeneratorClass = CodegenRegistry.get(inner.type);
-            if (!InnerGeneratorClass) {
-                throw new Error(`${inner.type} generator not exists`);
-            }
-            const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope);
-            return innerGenerator.toReadEmbed(expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`);
-        }).join(";\n")
-            }
-        ${accessor(result)}
-         `;
+      };
+      ${this.maybeReference(result, refState)}
+      ${this.builder.reader.buffer(encodedMetaInformation.byteLength)}
+      ${Object.entries(options.props).sort().map(([key, inner]) => {
+        const InnerGeneratorClass = CodegenRegistry.get(inner.type);
+        if (!InnerGeneratorClass) {
+          throw new Error(`${inner.type} generator not exists`);
+        }
+        const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope);
+        return innerGenerator.toReadEmbed(expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`);
+      }).join(";\n")}
+      ${accessor(result)}
+    `;
   }
 
+  // /8 /7 /20 % 2
+  // is there a ratio from length / deserializer
   private safeTag() {
     return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag);
   }
 
   toReadEmbed(accessor: (expr: string) => string, excludeHead?: boolean, refState?: RefState): string {
     const name = this.scope.declare(
       "tag_ser",
-            `fury.classResolver.getSerializerByTag("${this.safeTag()}")`
+      `fury.classResolver.getSerializerByTag("${this.safeTag()}")`
     );
     if (!excludeHead) {
       return accessor(`${name}.read()`);
@@ -129,7 +145,7 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator {
   toWriteEmbed(accessor: string, excludeHead?: boolean): string {
     const name = this.scope.declare(
       "tag_ser",
-            `fury.classResolver.getSerializerByTag("${this.safeTag()}")`
+      `fury.classResolver.getSerializerByTag("${this.safeTag()}")`
     );
     if (!excludeHead) {
       return `${name}.write(${accessor})`;

diff --git a/javascript/packages/fury/lib/meta/MetaString.ts b/javascript/packages/fury/lib/meta/MetaString.ts
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export class MetaString {
+  static LOWER_SPECIAL = 5;
+  static LOWER_UPPER_DIGIT_SPECIAL = 6;
+  static UTF_8 = 8;
+
+  // Encode function that infers the bits per character
+  static encode(str: string): Uint8Array {
+    const bitsPerChar = MetaString.inferBitsPerChar(str);
+    const totalBits = str.length * bitsPerChar + 8; // Adjusted for metadata bits
+    const byteLength = Math.ceil(totalBits / 8);
+    const bytes = new Uint8Array(byteLength);
+    let currentBit = 8; // Start after the first 8 metadata bits
+
+    for (const char of str) {
+      const value = bitsPerChar === MetaString.LOWER_SPECIAL
+        ? MetaString.charToValueLowerSpecial(char)
+        : bitsPerChar === MetaString.LOWER_UPPER_DIGIT_SPECIAL
+          ? MetaString.charToValueLowerUpperDigitSpecial(char)
+          : MetaString.charToValueUTF8(char);
+
+      for (let i = bitsPerChar - 1; i >= 0; i--) {
+        if ((value & (1 << i)) !== 0) {
+          const bytePos = Math.floor(currentBit / 8);
+          const bitPos = currentBit % 8;
+
+          if (bytePos >= byteLength) {
+            throw new RangeError("Offset is outside the bounds of the DataView");
+          }
+          bytes[bytePos] |= (1 << (7 - bitPos));
+        }
+        currentBit++;
+      }
+    }
+
+    // Store bitsPerChar in the first byte
+    bytes[0] = bitsPerChar;
+
+    return bytes;
+  }
+
+  // Decoding function that extracts bits per character from the first byte
+  static decode(bytes: Uint8Array): string {
+    const bitsPerChar = bytes[0] & 0x0F;
+    const totalBits = (bytes.length * 8); // Adjusted for metadata bits
+    const chars: string[] = [];
+    let currentBit = 8; // Start after the first 8 metadata bits
+
+    while (currentBit < totalBits) {
+      let value = 0;
+      for (let i = 0; i < bitsPerChar; i++) {
+        const bytePos = Math.floor(currentBit / 8);
+        const bitPos = currentBit % 8;
+
+        if (bytePos >= bytes.length) {
+          throw new RangeError("Offset is outside the bounds of the DataView");
+        }
+
+        if (bytes[bytePos] & (1 << (7 - bitPos))) {
+          value |= (1 << (bitsPerChar - i - 1));
+        }
+        currentBit++;
+      }
+
+      chars.push(bitsPerChar === MetaString.LOWER_SPECIAL
+        ? MetaString.valueToCharLowerSpecial(value)
+        : bitsPerChar === MetaString.LOWER_UPPER_DIGIT_SPECIAL
+          ? MetaString.valueToCharLowerUpperDigitSpecial(value)
+          : MetaString.valueToCharUTF8(value));
+    }
+
+    return chars.join("");
+  }
+
+  // Infer bits per character based on the content of the string
+  static inferBitsPerChar(str: string): number {
+    if (/^[a-z._$|]+$/.test(str)) {
+      return MetaString.LOWER_SPECIAL;
+    } else if (/^[a-zA-Z0-9._]+$/.test(str)) {
+      return MetaString.LOWER_UPPER_DIGIT_SPECIAL;
+    }
+    return MetaString.UTF_8; // Default to UTF-8
+  }
+
+  // Convert a character to its value for LOWER_SPECIAL encoding
+  static charToValueLowerSpecial(char: string): number {
+    if (char >= "a" && char <= "z") {
+      return char.charCodeAt(0) - "a".charCodeAt(0);
+    } else if (char === ".") {
+      return 26;
+    } else if (char === "_") {
+      return 27;
+    } else if (char === "$") {
+      return 28;
+    } else if (char === "|") {
+      return 29;
+    }
+    throw new Error(`Invalid character for LOWER_SPECIAL: ${char}`);
+  }
+
+  static valueToCharLowerSpecial(value: number): string {
+    if (value >= 0 && value <= 25) {
+      return String.fromCharCode("a".charCodeAt(0) + value);
+    } else if (value === 26) {
+      return ".";
+    } else if (value === 27) {
+      return "_";
+    } else if (value === 28) {
+      return "$";
+    } else if (value === 29) {
+      return "|";
+    }
+    throw new Error(`Invalid value for LOWER_SPECIAL: ${value}`);
+  }
+
+  static charToValueLowerUpperDigitSpecial(char: string): number {
+    if (char >= "a" && char <= "z") {
+      return char.charCodeAt(0) - "a".charCodeAt(0);
+    } else if (char >= "A" && char <= "Z") {
+      return char.charCodeAt(0) - "A".charCodeAt(0) + 26;
+    } else if (char >= "0" && char <= "9") {
+      return char.charCodeAt(0) - "0".charCodeAt(0) + 52;
+    } else if (char === ".") {
+      return 62;
+    } else if (char === "_") {
+      return 63;
+    }
+    throw new Error(`Invalid character for LOWER_UPPER_DIGIT_SPECIAL: ${char}`);
+  }
+
+  static valueToCharLowerUpperDigitSpecial(value: number): string {
+    if (value >= 0 && value <= 25) {
+      return String.fromCharCode("a".charCodeAt(0) + value);
+    } else if (value >= 26 && value <= 51) {
+      return String.fromCharCode("A".charCodeAt(0) + value - 26);
+    } else if (value >= 52 && value <= 61) {
+      return String.fromCharCode("0".charCodeAt(0) + value - 52);
+    } else if (value === 62) {
+      return ".";
+    } else if (value === 63) {
+      return "_";
+    }
+    throw new Error(`Invalid value for LOWER_UPPER_DIGIT_SPECIAL: ${value}`);
+  }
+
+  static charToValueUTF8(char: string): number {
+    return char.charCodeAt(0);
+  }
+
+  static valueToCharUTF8(value: number): string {
+    return String.fromCharCode(value);
+  }
+}
diff --git a/javascript/test/object.test.ts b/javascript/test/object.test.ts
@@ -233,6 +233,26 @@ describe('object', () => {
     const obj = deserialize(bin);
     expect({kind: "123", path: null}).toEqual(obj)
 })
+
+  test('should handle emojis', () => {
+    const description = {
+      type: InternalSerializerType.OBJECT as const,
+      options: {
+        props: {
+          a: {
+            type: InternalSerializerType.STRING as const,
+          },
+        },
+        tag: "example.emoji"
+      }
+    };
+
+    const fury = new Fury({ refTracking: true });
+    const { serialize, deserialize } = fury.registerSerializer(description);
+    const input = serialize({ a: "Hello, world! 🌍😊" });
+    const result = deserialize(input);
+    expect(result).toEqual({ a: "Hello, world! 🌍😊" });
+  });
 });