From 2f2e60f8d1ee4f60cd93c26d88bdd6d90d5dc12e Mon Sep 17 00:00:00 2001 From: weipeng Date: Fri, 26 Jan 2024 17:23:48 +0800 Subject: [PATCH] feat(JavaScript): Support oneof (#1348) ### 1. Fixed a performance bug that caused writeInt32 to slow down Before: ```JavaScript // reader/index.ts function writeVarInt32() { buffer.byteLength } ``` After: ```JavaScript // reader/index.ts const byteLength = buffer.byteLength; function writeVarInt32() { byteLength } ``` The byteLength property in a Buffer is slow to access. It appears that when we access byteLength, the V8 engine processes it using a hash lookup. so we store it in closure. ### 2. Support Oneof Sometimes, the data we want to serialize does not have a confirmed type; instead, it could be one of several confirmed types. If we use an object to handle this situation, the size of the resulting binary will be too large, as it will contain much unused information. usage: ```JavaScript const oneOfThree = Type.oneof({ option1: Type.string(), option2: Type.object("foo", { a: Type.int32() }), option3: Type.int32(), }); const fury = new Fury({ refTracking: true }); const { serialize, deserialize } = fury.registerSerializer(oneOfThree); const obj = { option1: "hello" } const input = serialize(obj); const result = deserialize( input ); expect(result).toEqual(obj.option1) ``` --- javascript/package.json | 3 +- javascript/packages/fury/lib/description.ts | 205 +++++++++++++----- javascript/packages/fury/lib/fury.ts | 8 +- javascript/packages/fury/lib/gen/any.ts | 43 +++- javascript/packages/fury/lib/gen/builder.ts | 7 + javascript/packages/fury/lib/gen/index.ts | 11 +- javascript/packages/fury/lib/gen/object.ts | 2 +- javascript/packages/fury/lib/gen/oneof.ts | 115 ++++++++++ .../packages/fury/lib/gen/serializer.ts | 17 +- javascript/packages/fury/lib/meta.ts | 7 +- .../packages/fury/lib/platformBuffer.ts | 10 +- javascript/packages/fury/lib/reader/index.ts | 8 +- .../packages/fury/lib/referenceResolver.ts | 2 +- javascript/packages/fury/lib/type.ts | 25 ++- javascript/packages/hps/index.ts | 4 +- javascript/test/any.test.ts | 10 +- javascript/test/oneof.test.ts | 90 ++++++++ 17 files changed, 467 insertions(+), 100 deletions(-) create mode 100644 javascript/packages/fury/lib/gen/oneof.ts create mode 100644 javascript/test/oneof.test.ts diff --git a/javascript/package.json b/javascript/package.json index a0345c208e..653adfdde4 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -1,7 +1,8 @@ { "scripts": { "test": "npm run build && jest", - "build": "npm run build -w packages/fury -w packages/hps", + "clear": "rm -rf ./packages/fury/dist && rm -rf ./packages/hps/dist", + "build": "npm run clear && npm run build -w packages/fury -w packages/hps", "lint": "eslint .", "lint-fix": "eslint . --fix" }, diff --git a/javascript/packages/fury/lib/description.ts b/javascript/packages/fury/lib/description.ts index 42855e4f57..d332e74028 100644 --- a/javascript/packages/fury/lib/description.ts +++ b/javascript/packages/fury/lib/description.ts @@ -20,112 +20,136 @@ import { InternalSerializerType } from "./type"; export interface TypeDescription { - type: InternalSerializerType - label?: string + type: InternalSerializerType; + label?: string; } export interface ObjectTypeDescription extends TypeDescription { options: { - props: { [key: string]: TypeDescription } - tag: string - } + props: { [key: string]: TypeDescription }; + tag: string; + }; } export interface EnumTypeDescription extends TypeDescription { options: { - inner: { [key: string]: any } - } + inner: { [key: string]: any }; + }; +} + +export interface OneofTypeDescription extends TypeDescription { + options: { + inner: { [key: string]: TypeDescription }; + }; } export interface ArrayTypeDescription extends TypeDescription { options: { - inner: TypeDescription - } + inner: TypeDescription; + }; } export interface TupleTypeDescription extends TypeDescription { options: { - inner: TypeDescription[] - } + inner: TypeDescription[]; + }; } export interface SetTypeDescription extends TypeDescription { options: { - key: TypeDescription - } + key: TypeDescription; + }; } export interface MapTypeDescription extends TypeDescription { options: { - key: TypeDescription - value: TypeDescription - } + key: TypeDescription; + value: TypeDescription; + }; } type Props = T extends { options: { - props?: infer T2 extends { [key: string]: any } - tag: string - } + props?: infer T2 extends { [key: string]: any }; + tag: string; + }; } ? { - [P in keyof T2]?: (ToRecordType | null); + [P in keyof T2]?: (InputType | null); } : unknown; type InnerProps = T extends { options: { - inner: infer T2 extends TypeDescription - } + inner: infer T2 extends TypeDescription; + }; } - ? (ToRecordType | null)[] + ? (InputType | null)[] : unknown; type MapProps = T extends { options: { - key: infer T2 extends TypeDescription - value: infer T3 extends TypeDescription - } + key: infer T2 extends TypeDescription; + value: infer T3 extends TypeDescription; + }; } - ? Map, ToRecordType | null> + ? Map, InputType | null> : unknown; type TupleProps = T extends { options: { - inner: infer T2 extends readonly [...TypeDescription[]] - } + inner: infer T2 extends readonly [...TypeDescription[]]; + }; } - ? { [K in keyof T2]: ToRecordType } + ? { [K in keyof T2]: InputType } : unknown; type Value = T extends { [s: string]: infer T2 } ? T2 : unknown; type EnumProps = T extends { options: { - inner: infer T2 - } + inner: infer T2; + }; } ? Value : unknown; +type OneofProps = T extends { + options: { + inner?: infer T2 extends { [key: string]: any }; + }; +} + ? { + [P in keyof T2]?: (InputType | null); + } + : unknown; + +type OneofResult = T extends { + options: { + inner?: infer T2; + }; +} + ? ResultType> + : unknown; + type SetProps = T extends { options: { - key: infer T2 extends TypeDescription - } + key: infer T2 extends TypeDescription; + }; } - ? Set<(ToRecordType | null)> + ? Set<(InputType | null)> : unknown; -export type ToRecordType = T extends { - type: InternalSerializerType.FURY_TYPE_TAG +export type InputType = T extends { + type: InternalSerializerType.FURY_TYPE_TAG; } ? Props : T extends { - type: InternalSerializerType.STRING + type: InternalSerializerType.STRING; } ? string : T extends { - type: InternalSerializerType.TUPLE + type: InternalSerializerType.TUPLE; } ? TupleProps : T extends { @@ -137,51 +161,122 @@ export type ToRecordType = T extends { | InternalSerializerType.INT16 | InternalSerializerType.INT32 | InternalSerializerType.FLOAT - | InternalSerializerType.DOUBLE + | InternalSerializerType.DOUBLE; } ? number : T extends { type: InternalSerializerType.UINT64 - | InternalSerializerType.INT64 + | InternalSerializerType.INT64; } ? bigint : T extends { - type: InternalSerializerType.MAP + type: InternalSerializerType.MAP; } ? MapProps : T extends { - type: InternalSerializerType.FURY_SET + type: InternalSerializerType.FURY_SET; } ? SetProps : T extends { - type: InternalSerializerType.ARRAY + type: InternalSerializerType.ARRAY; } ? InnerProps : T extends { - type: InternalSerializerType.BOOL + type: InternalSerializerType.BOOL; } ? boolean : T extends { - type: InternalSerializerType.DATE + type: InternalSerializerType.DATE; } ? Date : T extends { - type: InternalSerializerType.TIMESTAMP + type: InternalSerializerType.TIMESTAMP; } ? number : T extends { - type: InternalSerializerType.BINARY + type: InternalSerializerType.BINARY; } ? Uint8Array : T extends { - type: InternalSerializerType.ANY + type: InternalSerializerType.ANY; } ? any : T extends { - type: InternalSerializerType.ENUM + type: InternalSerializerType.ENUM; } - ? EnumProps : unknown; + ? EnumProps : T extends { + type: InternalSerializerType.ONEOF; + } ? OneofProps : unknown; + +export type ResultType = T extends { + type: InternalSerializerType.FURY_TYPE_TAG; +} + ? Props + : T extends { + type: InternalSerializerType.STRING; + } + ? string + : T extends { + type: InternalSerializerType.TUPLE; + } + ? TupleProps + : T extends { + type: + | InternalSerializerType.UINT8 + | InternalSerializerType.UINT16 + | InternalSerializerType.UINT32 + | InternalSerializerType.INT8 + | InternalSerializerType.INT16 + | InternalSerializerType.INT32 + | InternalSerializerType.FLOAT + | InternalSerializerType.DOUBLE; + } + ? number + + : T extends { + type: InternalSerializerType.UINT64 + | InternalSerializerType.INT64; + } + ? bigint + : T extends { + type: InternalSerializerType.MAP; + } + ? MapProps + : T extends { + type: InternalSerializerType.FURY_SET; + } + ? SetProps + : T extends { + type: InternalSerializerType.ARRAY; + } + ? InnerProps + : T extends { + type: InternalSerializerType.BOOL; + } + ? boolean + : T extends { + type: InternalSerializerType.DATE; + } + ? Date + : T extends { + type: InternalSerializerType.TIMESTAMP; + } + ? number + : T extends { + type: InternalSerializerType.BINARY; + } + ? Uint8Array + : T extends { + type: InternalSerializerType.ANY; + } + ? any + : T extends { + type: InternalSerializerType.ENUM; + } + ? EnumProps : T extends { + type: InternalSerializerType.ONEOF; + } ? OneofResult : unknown; export const Type = { any() { @@ -197,6 +292,14 @@ export const Type = { }, }; }, + oneof(inner?: T) { + return { + type: InternalSerializerType.ONEOF as const, + options: { + inner, + }, + }; + }, string() { return { type: InternalSerializerType.STRING as const, diff --git a/javascript/packages/fury/lib/fury.ts b/javascript/packages/fury/lib/fury.ts index ffc99563e3..1ea7d26c54 100644 --- a/javascript/packages/fury/lib/fury.ts +++ b/javascript/packages/fury/lib/fury.ts @@ -23,7 +23,7 @@ import { BinaryReader } from "./reader"; import { ReferenceResolver } from "./referenceResolver"; import { ConfigFlags, Serializer, Config, Language, BinaryReader as BinaryReaderType, BinaryWriter as BinaryWriterType } from "./type"; import { OwnershipError } from "./error"; -import { ToRecordType, TypeDescription } from "./description"; +import { InputType, ResultType, TypeDescription } from "./description"; import { generateSerializer, AnySerializer } from "./gen"; export default class { @@ -50,14 +50,14 @@ export default class { const serializer = generateSerializer(this, description); return { serializer, - serialize: (data: ToRecordType) => { + serialize: (data: InputType) => { return this.serialize(data, serializer); }, - serializeVolatile: (data: ToRecordType) => { + serializeVolatile: (data: InputType) => { return this.serializeVolatile(data, serializer); }, deserialize: (bytes: Uint8Array) => { - return this.deserialize(bytes, serializer) as ToRecordType; + return this.deserialize(bytes, serializer) as ResultType; }, }; } diff --git a/javascript/packages/fury/lib/gen/any.ts b/javascript/packages/fury/lib/gen/any.ts index 1117e46a9d..e0335f9b22 100644 --- a/javascript/packages/fury/lib/gen/any.ts +++ b/javascript/packages/fury/lib/gen/any.ts @@ -94,12 +94,12 @@ class AnySerializerGenerator extends BaseSerializerGenerator { this.description = description; } - writeStmt(accessor: string): string { - return `${this.builder.furyName()}.anySerializer.writeInner(${accessor})`; + writeStmt(): string { + throw new Error("Type Any writeStmt can't inline"); } - readStmt(accessor: (expr: string) => string): string { - return `${this.builder.furyName()}.anySerializer.readInner(${accessor})`; + readStmt(): string { + throw new Error("Type Any readStmt can't inline"); } toReadEmbed(accessor: (expr: string) => string, excludeHead = false): string { @@ -115,6 +115,41 @@ class AnySerializerGenerator extends BaseSerializerGenerator { } return `${this.builder.furyName()}.anySerializer.write(${accessor})`; } + + toSerializer() { + this.scope.assertNameNotDuplicate("read"); + this.scope.assertNameNotDuplicate("readInner"); + this.scope.assertNameNotDuplicate("write"); + this.scope.assertNameNotDuplicate("writeInner"); + + const declare = ` + const readInner = (fromRef) => { + throw new Error("Type Any readInner can't call directly"); + }; + const read = () => { + ${this.toReadEmbed(expr => `return ${expr}`)} + }; + const writeInner = (v) => { + throw new Error("Type Any writeInner can't call directly"); + }; + const write = (v) => { + ${this.toWriteEmbed("v")} + }; + `; + return ` + return function (fury, external) { + ${this.scope.generate()} + ${declare} + return { + read, + readInner, + write, + writeInner, + meta: ${JSON.stringify(this.builder.meta(this.description))} + }; + } + `; + } } CodegenRegistry.register(InternalSerializerType.ANY, AnySerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/builder.ts b/javascript/packages/fury/lib/gen/builder.ts index 8e34451a1d..1811852e5b 100644 --- a/javascript/packages/fury/lib/gen/builder.ts +++ b/javascript/packages/fury/lib/gen/builder.ts @@ -331,6 +331,13 @@ export class CodecBuilder { return v.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); } + static safeString(target: string) { + if (!CodecBuilder.isDotPropAccessor(target) || CodecBuilder.isReserved(target)) { + return `"${CodecBuilder.replaceBackslashAndQuote(target)}"`; + } + return `"${target}"`; + } + static safePropAccessor(prop: string) { if (!CodecBuilder.isDotPropAccessor(prop) || CodecBuilder.isReserved(prop)) { return `["${CodecBuilder.replaceBackslashAndQuote(prop)}"]`; diff --git a/javascript/packages/fury/lib/gen/index.ts b/javascript/packages/fury/lib/gen/index.ts index 3971b7a697..c482c1790b 100644 --- a/javascript/packages/fury/lib/gen/index.ts +++ b/javascript/packages/fury/lib/gen/index.ts @@ -18,7 +18,7 @@ */ import { InternalSerializerType } from "../type"; -import { ArrayTypeDescription, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from "../description"; +import { ArrayTypeDescription, MapTypeDescription, ObjectTypeDescription, OneofTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from "../description"; import { CodegenRegistry } from "./router"; import { CodecBuilder } from "./builder"; import { Scope } from "./scope"; @@ -35,6 +35,7 @@ import "./tuple"; import "./typedArray"; import Fury from "../fury"; import "./enum"; +import "./oneof"; export { AnySerializer } from "./any"; @@ -86,6 +87,14 @@ function regDependencies(fury: Fury, description: TypeDescription) { regDependencies(fury, x); }); } + if (description.type === InternalSerializerType.ONEOF) { + const options = (description).options; + if (options.inner) { + Object.values(options.inner).forEach((x) => { + regDependencies(fury, x); + }); + } + } } export const generateSerializer = (fury: Fury, description: TypeDescription) => { diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index 1b86e028f1..2258dee71a 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -112,7 +112,7 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { `; } - safeTag() { + private safeTag() { return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); } diff --git a/javascript/packages/fury/lib/gen/oneof.ts b/javascript/packages/fury/lib/gen/oneof.ts new file mode 100644 index 0000000000..4cb8b93b44 --- /dev/null +++ b/javascript/packages/fury/lib/gen/oneof.ts @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { OneofTypeDescription, Type, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType, RefFlags } from "../type"; +import { Scope } from "./scope"; + +class OneofSerializerGenerator extends BaseSerializerGenerator { + description: OneofTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(): string { + throw new Error("Type oneof writeStmt can't inline"); + } + + readStmt(): string { + throw new Error("Type oneof readStmt can't inline"); + } + + toWriteEmbed(accessor: string, excludeHead = false) { + if (excludeHead) { + throw new Error("Oneof can't excludeHead"); + } + if (Object.values(this.description.options.inner).length < 1) { + throw new Error("Type oneof must contain at least one field"); + } + const stmts = [ + `if (${accessor} === null || ${accessor} === undefined) { + ${this.builder.writer.int8(RefFlags.NullFlag)}; + }`, + ]; + Object.entries(this.description.options.inner).forEach(([key, value]) => { + const InnerGeneratorClass = CodegenRegistry.get(value.type); + if (!InnerGeneratorClass) { + throw new Error(`${value.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(value, this.builder, this.scope); + stmts.push(` if (${CodecBuilder.safeString(key)} in ${accessor}) { + ${innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`)} + }`); + }); + stmts.push(` + { + ${this.builder.writer.int8(RefFlags.NullFlag)}; + } + `); + return stmts.join("else"); + } + + toReadEmbed(accessor: (expr: string) => string, excludeHead = false): string { + const AnyGeneratorClass = CodegenRegistry.get(InternalSerializerType.ANY)!; + const anyGenerator = new AnyGeneratorClass(Type.any(), this.builder, this.scope); + return anyGenerator.toReadEmbed(accessor, excludeHead); + } + + toSerializer() { + this.scope.assertNameNotDuplicate("read"); + this.scope.assertNameNotDuplicate("readInner"); + this.scope.assertNameNotDuplicate("write"); + this.scope.assertNameNotDuplicate("writeInner"); + + const declare = ` + const readInner = (fromRef) => { + throw new Error("Type oneof readInner can't call directly"); + }; + const read = () => { + ${this.toReadEmbed(expr => `return ${expr}`)} + }; + const writeInner = (v) => { + throw new Error("Type oneof writeInner can't call directly"); + }; + const write = (v) => { + ${this.toWriteEmbed("v")} + }; + `; + return ` + return function (fury, external) { + ${this.scope.generate()} + ${declare} + return { + read, + readInner, + write, + writeInner, + meta: ${JSON.stringify(this.builder.meta(this.description))} + }; + } + `; + } +} + +CodegenRegistry.register(InternalSerializerType.ONEOF, OneofSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/serializer.ts b/javascript/packages/fury/lib/gen/serializer.ts index f9adc56420..16d7020efb 100644 --- a/javascript/packages/fury/lib/gen/serializer.ts +++ b/javascript/packages/fury/lib/gen/serializer.ts @@ -25,11 +25,9 @@ import { Scope } from "./scope"; import { TypeDescription, ObjectTypeDescription } from "../description"; export interface SerializerGenerator { - writeStmt(accessor: string): string - readStmt(accessor: (expr: string) => string, refState: RefState): string - toSerializer(): string - toWriteEmbed(accessor: string, excludeHead?: boolean): string - toReadEmbed(accessor: (expr: string) => string, excludeHead?: boolean, refState?: RefState): string + toSerializer(): string; + toWriteEmbed(accessor: string, excludeHead?: boolean): string; + toReadEmbed(accessor: (expr: string) => string, excludeHead?: boolean, refState?: RefState): string; } export enum RefStateType { @@ -124,10 +122,6 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { } } - safeTag() { - return CodecBuilder.replaceBackslashAndQuote(( this.description).options.tag); - } - protected wrapWriteHead(accessor: string, stmt: (accessor: string) => string) { const meta = this.builder.meta(this.description); @@ -135,7 +129,8 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { if (this.description.type !== InternalSerializerType.FURY_TYPE_TAG) { return ""; } - const tagWriter = this.scope.declare("tagWriter", `${this.builder.classResolver.createTagWriter(this.safeTag())}`); + const safeTag = CodecBuilder.replaceBackslashAndQuote(( this.description).options.tag); + const tagWriter = this.scope.declare("tagWriter", `${this.builder.classResolver.createTagWriter(safeTag)}`); return `${tagWriter}.write(${this.builder.writer.ownName()})`; }; @@ -171,7 +166,7 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { } } - protected wrapReadHead(accessor: (expr: string) => string, stmt: (accessor: (expr: string) => string, refState: RefState) => string) { + private wrapReadHead(accessor: (expr: string) => string, stmt: (accessor: (expr: string) => string, refState: RefState) => string) { const refFlag = this.scope.uniqueName("refFlag"); return ` diff --git a/javascript/packages/fury/lib/meta.ts b/javascript/packages/fury/lib/meta.ts index d46dafb65a..4c8234aea6 100644 --- a/javascript/packages/fury/lib/meta.ts +++ b/javascript/packages/fury/lib/meta.ts @@ -23,9 +23,9 @@ import { ObjectTypeDescription, TypeDescription } from "./description"; import { InternalSerializerType } from "./type"; export type Meta = { - fixedSize: number - needToWriteRef: boolean - type: InternalSerializerType + fixedSize: number; + needToWriteRef: boolean; + type: InternalSerializerType; }; export const getMeta = (description: TypeDescription, fury: Fury): Meta => { @@ -138,6 +138,7 @@ export const getMeta = (description: TypeDescription, fury: Fury): Meta => { needToWriteRef: Boolean(fury.config.refTracking) && true, type, }; + case InternalSerializerType.ONEOF: case InternalSerializerType.ANY: return { fixedSize: 11, diff --git a/javascript/packages/fury/lib/platformBuffer.ts b/javascript/packages/fury/lib/platformBuffer.ts index fee395acc2..9c82a02352 100644 --- a/javascript/packages/fury/lib/platformBuffer.ts +++ b/javascript/packages/fury/lib/platformBuffer.ts @@ -20,11 +20,11 @@ import { isNodeEnv } from "./util"; export interface PlatformBuffer extends Uint8Array { - latin1Slice(start: number, end: number): string - utf8Slice(start: number, end: number): string - latin1Write(v: string, offset: number): void - utf8Write(v: string, offset: number): void - copy(target: Uint8Array, targetStart?: number, sourceStart?: number, sourceEnd?: number): void + latin1Slice(start: number, end: number): string; + utf8Slice(start: number, end: number): string; + latin1Write(v: string, offset: number): void; + utf8Write(v: string, offset: number): void; + copy(target: Uint8Array, targetStart?: number, sourceStart?: number, sourceEnd?: number): void; } export class BrowserBuffer extends Uint8Array implements PlatformBuffer { diff --git a/javascript/packages/fury/lib/reader/index.ts b/javascript/packages/fury/lib/reader/index.ts index e32b02f7e8..b12bf12133 100644 --- a/javascript/packages/fury/lib/reader/index.ts +++ b/javascript/packages/fury/lib/reader/index.ts @@ -28,14 +28,16 @@ export const BinaryReader = (config: Config) => { let dataView!: DataView; let buffer!: PlatformBuffer; let bigString: string; + let byteLength: number; const stringLatin1 = sliceStringEnable ? stringLatin1Fast : stringLatin1Slow; function reset(ab: Uint8Array) { buffer = fromUint8Array(ab); + byteLength = buffer.byteLength; dataView = new DataView(buffer.buffer, buffer.byteOffset); if (sliceStringEnable) { - bigString = buffer.latin1Slice(0, buffer.byteLength); + bigString = buffer.latin1Slice(0, byteLength); } cursor = 0; } @@ -153,7 +155,7 @@ export const BinaryReader = (config: Config) => { function varUInt32() { // Reduce memory reads as much as possible. Reading a uint32 at once is far faster than reading four uint8s separately. - if (buffer.byteLength - cursor >= 5) { + if (byteLength - cursor >= 5) { const u32 = dataView.getUint32(cursor++, true); let result = u32 & 0x7f; if ((u32 & 0x80) != 0) { @@ -208,7 +210,7 @@ export const BinaryReader = (config: Config) => { function varUInt64() { // Creating BigInts is too performance-intensive; we'll use uint32 instead. - if (buffer.byteLength - cursor < 8) { + if (byteLength - cursor < 8) { let byte = bigUInt8(); let result = byte & 0x7fn; if ((byte & 0x80n) != 0n) { diff --git a/javascript/packages/fury/lib/referenceResolver.ts b/javascript/packages/fury/lib/referenceResolver.ts index a1fbb5af86..d826d724c4 100644 --- a/javascript/packages/fury/lib/referenceResolver.ts +++ b/javascript/packages/fury/lib/referenceResolver.ts @@ -30,7 +30,7 @@ export const makeHead = (flag: RefFlags, type: InternalSerializerType) => { export const ReferenceResolver = ( config: { - refTracking?: boolean + refTracking?: boolean; }, binaryWriter: BinaryWriter, binaryReader: BinaryReader, diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index fe5036329b..fd893af066 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -54,6 +54,7 @@ export enum InternalSerializerType { FURY_PRIMITIVE_DOUBLE_ARRAY = 263, FURY_STRING_ARRAY = 264, ANY = -1, + ONEOF = -2, } export enum ConfigFlags { @@ -65,11 +66,11 @@ export enum ConfigFlags { // read, write export type Serializer = { - read: () => T2 - write: (v: T2) => T - readInner: (refValue?: boolean) => T2 - writeInner: (v: T2) => T - meta: Meta + read: () => T2; + write: (v: T2) => T; + readInner: (refValue?: boolean) => T2; + writeInner: (v: T2) => T; + meta: Meta; }; export enum RefFlags { @@ -94,17 +95,17 @@ export const LATIN1 = 0; export const UTF8 = 1; export interface Hps { - isLatin1: (str: string) => boolean - stringCopy: (str: string, dist: Uint8Array, offset: number) => void + isLatin1: (str: string) => boolean; + stringCopy: (str: string, dist: Uint8Array, offset: number) => void; } export interface Config { - hps?: Hps - refTracking?: boolean - useSliceString?: boolean + hps?: Hps; + refTracking?: boolean; + useSliceString?: boolean; hooks?: { - afterCodeGenerated?: (code: string) => string - } + afterCodeGenerated?: (code: string) => string; + }; } export enum Language { diff --git a/javascript/packages/hps/index.ts b/javascript/packages/hps/index.ts index cddf256502..f991eb4b39 100644 --- a/javascript/packages/hps/index.ts +++ b/javascript/packages/hps/index.ts @@ -20,8 +20,8 @@ const hps: Hps = require("bindings")("hps.node"); interface Hps { - isLatin1: (str: string) => boolean - stringCopy: (str: string, dist: Uint8Array, offset: number) => void + isLatin1: (str: string) => boolean; + stringCopy: (str: string, dist: Uint8Array, offset: number) => void; } const { isLatin1, stringCopy } = hps; diff --git a/javascript/test/any.test.ts b/javascript/test/any.test.ts index a41b8548ce..0cddde767f 100644 --- a/javascript/test/any.test.ts +++ b/javascript/test/any.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import Fury, { TypeDescription, InternalSerializerType } from '../packages/fury/index'; +import Fury, { TypeDescription, InternalSerializerType, Type } from '../packages/fury/index'; import { describe, expect, test } from '@jest/globals'; describe('bool', () => { @@ -122,4 +122,12 @@ describe('bool', () => { expect(ret instanceof Array).toBe(true) expect(ret).toEqual(obj) }); + + test('should root any work', () => { + const fury = new Fury(); + const { serialize, deserialize } = fury.registerSerializer(Type.any()); + const bin = serialize("hello"); + const result = deserialize(bin); + expect(result).toEqual("hello") + }); }); diff --git a/javascript/test/oneof.test.ts b/javascript/test/oneof.test.ts new file mode 100644 index 0000000000..0cabf19030 --- /dev/null +++ b/javascript/test/oneof.test.ts @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fury, { TypeDescription, InternalSerializerType, Type } from '../packages/fury/index'; +import { describe, expect, test } from '@jest/globals'; + +const oneOfThree = Type.oneof({ + option1: Type.string(), + option2: Type.object("foo", { + a: Type.int32() + }), + option3: Type.int32(), +}); + +describe('oneof', () => { + test('option1: should oneof work1', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(oneOfThree); + const obj = { + option1: "hello" + } + const input = serialize(obj); + const result = deserialize( + input + ); + expect(result).toEqual(obj.option1) + }); + test('option2: should oneof work', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(oneOfThree); + const obj = { + option2: { + a: 123 + } + } + const input = serialize(obj); + const result = deserialize( + input + ); + expect(result).toEqual(obj.option2) + }); + test('option3: should oneof work', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(oneOfThree); + const obj = { + option3: 123 + } + const input = serialize(obj); + const result = deserialize( + input + ); + expect(result).toEqual(obj.option3) + }); + test('should nested oneof work1', () => { + const fury = new Fury({ refTracking: true }); + const { serialize, deserialize } = fury.registerSerializer(Type.object("foo2", { + f: oneOfThree + })); + const obj = { + option1: "hello" + } + const input = serialize({ + f: obj + }); + const result = deserialize( + input + ); + expect(result).toEqual({ + f: obj.option1 + }) + }); +}); + +