diff --git a/javascript/benchmark/index.js b/javascript/benchmark/index.js index 5dd96f0fb1..016b7710ce 100644 --- a/javascript/benchmark/index.js +++ b/javascript/benchmark/index.js @@ -28,6 +28,7 @@ const Type = Fury.Type; const assert = require('assert'); const { spawn } = require("child_process"); + const sample = { id: 123456, name: "John Doe", @@ -106,8 +107,9 @@ const sample = { `, }; + const description = utils.mockData2Description(sample, "fury.test.foo"); -const { serialize, deserialize } = fury.registerSerializer(description); +const { serialize, deserialize, serializeVolatile } = fury.registerSerializer(description); const furyAb = serialize(sample); const sampleJson = JSON.stringify(sample); @@ -164,7 +166,7 @@ async function start() { var suite = new Benchmark.Suite(); suite .add("fury", function () { - serialize(sample); + serializeVolatile(sample).dispose(); }) .add("json", function () { JSON.stringify(sample); diff --git a/javascript/benchmark/sample.jpg b/javascript/benchmark/sample.jpg index 5c98c2d23e..4d74cefa83 100644 Binary files a/javascript/benchmark/sample.jpg and b/javascript/benchmark/sample.jpg differ diff --git a/javascript/jest.config.js b/javascript/jest.config.js index 7015870259..7059d3bbf1 100644 --- a/javascript/jest.config.js +++ b/javascript/jest.config.js @@ -30,6 +30,9 @@ module.exports = { ], transform: { '\\.ts$': ['ts-jest', { + tsconfig: { + target: "ES2021" + }, diagnostics: { ignoreCodes: [151001] } diff --git a/javascript/package.json b/javascript/package.json index 500922c3ac..e8c6b3df74 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -12,6 +12,8 @@ ], "devDependencies": { "@stylistic/eslint-plugin": "^1.5.1", - "eslint": "^8.55.0" + "@types/js-beautify": "^1.14.3", + "eslint": "^8.55.0", + "js-beautify": "^1.14.11" } } diff --git a/javascript/packages/fury/index.ts b/javascript/packages/fury/index.ts index b305c4574e..5334f007fb 100644 --- a/javascript/packages/fury/index.ts +++ b/javascript/packages/fury/index.ts @@ -18,10 +18,9 @@ */ import { - genSerializer, -} from "./lib/codeGen"; + generateSerializer, +} from "./lib/gen"; import { - Cast, ObjectTypeDescription, TypeDescription, ArrayTypeDescription, @@ -45,13 +44,7 @@ export default class { private fury: Fury = FuryInternal(this.config || {}); registerSerializer(description: T) { - if ( - description.type !== InternalSerializerType.FURY_TYPE_TAG - || !Cast(description)?.options.tag - ) { - throw new Error("root type should be object"); - } - const serializer = genSerializer(this.fury, description); + const serializer = generateSerializer(this.fury, description); return { serializer, serialize: (data: ToRecordType) => { diff --git a/javascript/packages/fury/lib/internalSerializer/any.ts b/javascript/packages/fury/lib/any.ts similarity index 57% rename from javascript/packages/fury/lib/internalSerializer/any.ts rename to javascript/packages/fury/lib/any.ts index 00fa993808..7fc0d6f1f0 100644 --- a/javascript/packages/fury/lib/internalSerializer/any.ts +++ b/javascript/packages/fury/lib/any.ts @@ -17,53 +17,59 @@ * under the License. */ -import { Fury, MaxInt32, MinInt32, Serializer } from "../type"; -import { InternalSerializerType, RefFlags } from "../type"; +import { Type } from "./description"; +import { getMeta } from "./meta"; +import { Fury, MaxInt32, MinInt32, Serializer } from "./type"; +import { InternalSerializerType, RefFlags } from "./type"; export default (fury: Fury) => { const { binaryReader, binaryWriter, referenceResolver, classResolver } = fury; - function detectSerializer(cursor: number) { + function detectSerializer() { const typeId = binaryReader.int16(); let serializer: Serializer; if (typeId === InternalSerializerType.FURY_TYPE_TAG) { - serializer = classResolver.getSerializerByTag(classResolver.detectTag(binaryReader)); + const tag = classResolver.readTag(binaryReader)(); + serializer = classResolver.getSerializerByTag(tag); } else { serializer = classResolver.getSerializerById(typeId); } if (!serializer) { throw new Error(`cant find implements of typeId: ${typeId}`); } - binaryReader.setCursor(cursor); - return serializer; } return { + readInner: () => { + throw new Error("any can not call readInner"); + }, + writeInner: () => { + throw new Error("any can not call writeInner"); + }, read: () => { - const cursor = binaryReader.getCursor(); const flag = referenceResolver.readRefFlag(); switch (flag) { case RefFlags.RefValueFlag: - return detectSerializer(cursor).read(); + return detectSerializer().readInner(); case RefFlags.RefFlag: return referenceResolver.getReadObjectByRefId(binaryReader.varUInt32()); case RefFlags.NullFlag: return null; case RefFlags.NotNullValueFlag: - return detectSerializer(cursor).read(); + return detectSerializer().readInner(); } }, write: (v: any) => { - const { write: writeInt64, config: int64Config } = classResolver.getSerializerById(InternalSerializerType.INT64); - const { write: writeDouble, config: doubleConfig } = classResolver.getSerializerById(InternalSerializerType.DOUBLE); - const { write: writeInt32, config: int32Config } = classResolver.getSerializerById(InternalSerializerType.INT32); - const { write: writeBool, config: boolConfig } = classResolver.getSerializerById(InternalSerializerType.BOOL); - const { write: stringWrite, config: stringConfig } = classResolver.getSerializerById(InternalSerializerType.STRING); - const { write: arrayWrite, config: arrayConfig } = classResolver.getSerializerById(InternalSerializerType.ARRAY); - const { write: mapWrite, config: mapConfig } = classResolver.getSerializerById(InternalSerializerType.MAP); - const { write: setWrite, config: setConfig } = classResolver.getSerializerById(InternalSerializerType.FURY_SET); - const { write: timestampWrite, config: timestampConfig } = classResolver.getSerializerById(InternalSerializerType.TIMESTAMP); + const { write: writeInt64, meta: int64Meta } = classResolver.getSerializerById(InternalSerializerType.INT64); + const { write: writeDouble, meta: doubleMeta } = classResolver.getSerializerById(InternalSerializerType.DOUBLE); + const { write: writeInt32, meta: int32Meta } = classResolver.getSerializerById(InternalSerializerType.INT32); + const { write: writeBool, meta: boolMeta } = classResolver.getSerializerById(InternalSerializerType.BOOL); + const { write: stringWrite, meta: stringMeta } = classResolver.getSerializerById(InternalSerializerType.STRING); + const { write: arrayWrite, meta: arrayMeta } = classResolver.getSerializerById(InternalSerializerType.ARRAY); + const { write: mapWrite, meta: mapMeta } = classResolver.getSerializerById(InternalSerializerType.MAP); + const { write: setWrite, meta: setMeta } = classResolver.getSerializerById(InternalSerializerType.FURY_SET); + const { write: timestampWrite, meta: timestampMeta } = classResolver.getSerializerById(InternalSerializerType.TIMESTAMP); // NullFlag if (v === null || v === undefined) { @@ -81,68 +87,64 @@ export default (fury: Fury) => { } if (Number.isInteger(v)) { if (v > MaxInt32 || v < MinInt32) { - binaryWriter.reserve(int64Config().reserve); + binaryWriter.reserve(int64Meta.fixedSize); writeInt64(BigInt(v)); return; } else { - binaryWriter.reserve(int32Config().reserve); + binaryWriter.reserve(int32Meta.fixedSize); writeInt32(v); return; } } else { - binaryWriter.reserve(doubleConfig().reserve); + binaryWriter.reserve(doubleMeta.fixedSize); writeDouble(v); return; } } if (typeof v === "bigint") { - binaryWriter.reserve(int64Config().reserve); + binaryWriter.reserve(int64Meta.fixedSize); writeInt64(v); return; } if (typeof v === "boolean") { - binaryWriter.reserve(boolConfig().reserve); + binaryWriter.reserve(boolMeta.fixedSize); writeBool(v); return; } if (v instanceof Date) { - binaryWriter.reserve(timestampConfig().reserve); + binaryWriter.reserve(timestampMeta.fixedSize); timestampWrite(v); return; } if (typeof v === "string") { - binaryWriter.reserve(stringConfig().reserve); + binaryWriter.reserve(stringMeta.fixedSize); stringWrite(v); return; } if (v instanceof Map) { binaryWriter.reserve(5); - binaryWriter.reserve(mapConfig().reserve); + binaryWriter.reserve(mapMeta.fixedSize); mapWrite(v); return; } if (v instanceof Set) { - binaryWriter.reserve(setConfig().reserve); + binaryWriter.reserve(setMeta.fixedSize); setWrite(v); return; } if (Array.isArray(v)) { - binaryWriter.reserve(arrayConfig().reserve); + binaryWriter.reserve(arrayMeta.fixedSize); arrayWrite(v); return; } throw new Error(`serializer not support ${typeof v} yet`); }, - config: () => { - return { - reserve: 11, - }; - }, + meta: getMeta(Type.any(), fury), }; }; diff --git a/javascript/packages/fury/lib/classResolver.ts b/javascript/packages/fury/lib/classResolver.ts index d344e511b4..751b36c917 100644 --- a/javascript/packages/fury/lib/classResolver.ts +++ b/javascript/packages/fury/lib/classResolver.ts @@ -17,87 +17,102 @@ * under the License. */ -import { arraySerializer, stringArraySerializer, boolArraySerializer, shortArraySerializer, intArraySerializer, longArraySerializer, floatArraySerializer, doubleArraySerializer } from "./internalSerializer/array"; -import stringSerializer from "./internalSerializer/string"; -import binarySerializer from "./internalSerializer/binary"; -import { dateSerializer, timestampSerializer } from "./internalSerializer/datetime"; -import mapSerializer from "./internalSerializer/map"; -import setSerializer from "./internalSerializer/set"; -import boolSerializer from "./internalSerializer/bool"; -import { uInt16Serializer, int16Serializer, int32Serializer, uInt32Serializer, uInt64Serializer, floatSerializer, doubleSerializer, uInt8Serializer, int64Serializer, int8Serializer } from "./internalSerializer/number"; import { InternalSerializerType, Serializer, Fury, BinaryReader, BinaryWriter as TBinaryWriter } from "./type"; -import anySerializer from "./internalSerializer/any"; +import anySerializer from "./any"; import { fromString } from "./platformBuffer"; import { x64hash128 } from "./murmurHash3"; import { BinaryWriter } from "./writer"; +import { generateSerializer } from "./gen"; +import { Type, TypeDescription } from "./description"; const USESTRINGVALUE = 0; const USESTRINGID = 1; -class Lazystring { +class LazyString { private string: string | null = null; private start: number | null = null; private len: number | null = null; static fromPair(start: number, len: number) { - const result = new Lazystring(); + const result = new LazyString(); result.start = start; result.len = len; return result; } static fromString(str: string) { - const result = new Lazystring(); + const result = new LazyString(); result.string = str; return result; } toString(binaryReader: BinaryReader) { if (this.string == null) { - const str = binaryReader.stringUtf8At(this.start!, this.len!); - return str; + this.string = binaryReader.stringUtf8At(this.start!, this.len!); } return this.string; } } +const uninitSerialize = { + read: () => { + throw new Error("uninitSerialize"); + }, + write: () => { + throw new Error("uninitSerialize"); + }, + readInner: () => { + throw new Error("uninitSerialize"); + }, + writeInner: () => { + throw new Error("uninitSerialize"); + }, + meta: { + fixedSize: 0, + noneable: false, + }, +}; + export default class SerializerResolver { private internalSerializer: Serializer[] = new Array(300); private customSerializer: { [key: string]: Serializer } = { }; - private readStringPool: Lazystring[] = []; + private readStringPool: LazyString[] = []; private writeStringCount = 0; private writeStringIndex: number[] = []; + private registerSerializer(fury: Fury, description: TypeDescription) { + return fury.classResolver.registerSerializerById(description.type, generateSerializer(fury, description)); + } + private initInternalSerializer(fury: Fury) { - const _anySerializer = anySerializer(fury); - this.internalSerializer[InternalSerializerType.ANY] = _anySerializer; - this.internalSerializer[InternalSerializerType.STRING] = stringSerializer(fury); - this.internalSerializer[InternalSerializerType.ARRAY] = arraySerializer(fury, _anySerializer); - this.internalSerializer[InternalSerializerType.MAP] = mapSerializer(fury, _anySerializer, _anySerializer); - this.internalSerializer[InternalSerializerType.BOOL] = boolSerializer(fury); - this.internalSerializer[InternalSerializerType.UINT8] = uInt8Serializer(fury); - this.internalSerializer[InternalSerializerType.INT8] = int8Serializer(fury); - this.internalSerializer[InternalSerializerType.UINT16] = uInt16Serializer(fury); - this.internalSerializer[InternalSerializerType.INT16] = int16Serializer(fury); - this.internalSerializer[InternalSerializerType.UINT32] = uInt32Serializer(fury); - this.internalSerializer[InternalSerializerType.INT32] = int32Serializer(fury); - this.internalSerializer[InternalSerializerType.UINT64] = uInt64Serializer(fury); - this.internalSerializer[InternalSerializerType.INT64] = int64Serializer(fury); - this.internalSerializer[InternalSerializerType.FLOAT] = floatSerializer(fury); - this.internalSerializer[InternalSerializerType.DOUBLE] = doubleSerializer(fury); - this.internalSerializer[InternalSerializerType.TIMESTAMP] = timestampSerializer(fury); - this.internalSerializer[InternalSerializerType.DATE] = dateSerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_SET] = setSerializer(fury, anySerializer(fury)); - this.internalSerializer[InternalSerializerType.FURY_STRING_ARRAY] = stringArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY] = boolArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY] = shortArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY] = intArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY] = longArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY] = floatArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY] = doubleArraySerializer(fury); - this.internalSerializer[InternalSerializerType.BINARY] = binarySerializer(fury); + this.internalSerializer[InternalSerializerType.ANY] = anySerializer(fury); + this.registerSerializer(fury, Type.string()); + this.registerSerializer(fury, Type.array(Type.any())); + this.registerSerializer(fury, Type.map(Type.any(), Type.any())); + this.registerSerializer(fury, Type.bool()); + this.registerSerializer(fury, Type.uint8()); + this.registerSerializer(fury, Type.int8()); + this.registerSerializer(fury, Type.uint16()); + this.registerSerializer(fury, Type.int16()); + this.registerSerializer(fury, Type.uint32()); + this.registerSerializer(fury, Type.int32()); + this.registerSerializer(fury, Type.uint64()); + this.registerSerializer(fury, Type.int64()); + this.registerSerializer(fury, Type.float()); + this.registerSerializer(fury, Type.double()); + this.registerSerializer(fury, Type.timestamp()); + this.registerSerializer(fury, Type.date()); + this.registerSerializer(fury, Type.set(Type.any())); + this.registerSerializer(fury, Type.binary()); + this.registerSerializer(fury, Type.stringTypedArray()); + this.registerSerializer(fury, Type.boolTypedArray()); + this.registerSerializer(fury, Type.shortTypedArray()); + this.registerSerializer(fury, Type.intTypedArray()); + this.registerSerializer(fury, Type.longTypedArray()); + this.registerSerializer(fury, Type.floatTypedArray()); + this.registerSerializer(fury, Type.doubleTypedArray()); } init(fury: Fury) { @@ -113,7 +128,16 @@ export default class SerializerResolver { return this.internalSerializer[id]; } - registerSerializerByTag(tag: string, serializer: Serializer) { + registerSerializerById(id: number, serializer: Serializer) { + if (this.internalSerializer[id]) { + Object.assign(this.internalSerializer[id], serializer); + } else { + this.internalSerializer[id] = { ...serializer }; + } + return this.internalSerializer[id]; + } + + registerSerializerByTag(tag: string, serializer: Serializer = uninitSerialize) { if (this.customSerializer[tag]) { Object.assign(this.customSerializer[tag], serializer); } else { @@ -126,25 +150,27 @@ export default class SerializerResolver { return this.customSerializer[tag]; } - createTagWriter(tag: string) { - this.writeStringIndex.push(-1); - const idx = this.writeStringIndex.length - 1; + static tagBuffer(tag: string) { const tagBuffer = fromString(tag); const bufferLen = tagBuffer.byteLength; - const writer = BinaryWriter({}); let tagHash = x64hash128(tagBuffer, 47).getBigUint64(0); - if (tagHash === BigInt(0)) { - tagHash = BigInt(1); + if (tagHash === 0n) { + tagHash = 1n; } writer.uint8(USESTRINGVALUE); writer.uint64(tagHash); writer.int16(bufferLen); writer.bufferWithoutMemCheck(tagBuffer, bufferLen); + return writer.dump(); + } - const fullBuffer = writer.dump(); + createTagWriter(tag: string) { + this.writeStringIndex.push(-1); + const idx = this.writeStringIndex.length - 1; + const fullBuffer = SerializerResolver.tagBuffer(tag); return { write: (binaryWriter: TBinaryWriter) => { @@ -158,28 +184,17 @@ export default class SerializerResolver { this.writeStringIndex[idx] = this.writeStringCount++; binaryWriter.buffer(fullBuffer); }, - bufferLen, }; } - detectTag(binaryReader: BinaryReader) { - const flag = binaryReader.uint8(); - if (flag === USESTRINGVALUE) { - binaryReader.skip(8); // The tag hash is not needed at the moment. - return binaryReader.stringUtf8(binaryReader.int16()); - } else { - return this.readStringPool[binaryReader.int16()].toString(binaryReader); - } - } - readTag(binaryReader: BinaryReader) { const flag = binaryReader.uint8(); if (flag === USESTRINGVALUE) { binaryReader.skip(8); // The tag hash is not needed at the moment. - const start = binaryReader.getCursor(); const len = binaryReader.int16(); + const start = binaryReader.getCursor(); binaryReader.skip(len); - this.readStringPool.push(Lazystring.fromPair(start, len)); + this.readStringPool.push(LazyString.fromPair(start, len)); const idx = this.readStringPool.length; return () => { return this.readStringPool[idx - 1].toString(binaryReader); diff --git a/javascript/packages/fury/lib/codeGen.ts b/javascript/packages/fury/lib/codeGen.ts deleted file mode 100644 index da3ab1e6f7..0000000000 --- a/javascript/packages/fury/lib/codeGen.ts +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, MaxInt32, RefFlags, Fury } from "./type"; -import { replaceBackslashAndQuote, safePropAccessor, safePropName } from "./util"; -import mapSerializer from "./internalSerializer/map"; -import setSerializer from "./internalSerializer/set"; -import { arraySerializer } from "./internalSerializer/array"; -import { tupleSerializer } from "./internalSerializer/tuple"; -import { ArrayTypeDescription, Cast, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from "./description"; -import { fromString } from "./platformBuffer"; - -function computeFieldHash(hash: number, id: number): number { - let newHash = (hash) * 31 + (id); - while (newHash >= MaxInt32) { - newHash = Math.floor(newHash / 7); - } - return newHash; -} - -const computeStringHash = (str: string) => { - const bytes = fromString(str); - let hash = 17; - bytes.forEach((b) => { - hash = hash * 31 + b; - while (hash >= MaxInt32) { - hash = Math.floor(hash / 7); - } - }); - return hash; -}; - -const computeStructHash = (description: TypeDescription) => { - let hash = 17; - for (const [, value] of Object.entries(Cast(description).options.props).sort()) { - let id = value.type; - if (value.type === InternalSerializerType.ARRAY || value.type === InternalSerializerType.MAP) { - id = value.type; // TODO add map key&value type into schema hash - } else if (value.type === InternalSerializerType.FURY_TYPE_TAG) { - id = computeStringHash(Cast(value).options.tag); - } - hash = computeFieldHash(hash, id); - } - return hash; -}; - -function typeHandlerDeclaration(fury: Fury) { - let declarations: string[] = []; - let count = 0; - const exists = new Map(); - function addDeclar(name: string, declar: string, uniqueKey?: string) { - const unique = uniqueKey || name; - if (exists.has(unique)) { - return exists.get(unique)!; - } - declarations.push(declar); - exists.set(unique, name); - return name; - } - - const genBuiltinDeclaration = (type: number) => { - const name = `type_${type}`.replace("-", "_"); - return addDeclar(name, ` - const ${name} = classResolver.getSerializerById(${type})`); - }; - - const genTagDeclaration = (tag: string) => { - const name = `tag_${count++}`; - return addDeclar(name, ` - const ${name} = classResolver.getSerializerByTag("${replaceBackslashAndQuote(tag)}")`, tag); - }; - - const genDeclaration = (description: TypeDescription): string => { - if (description.type === InternalSerializerType.FURY_TYPE_TAG) { - genSerializer(fury, description); - return genTagDeclaration(Cast(description).options.tag); - } - if (description.type === InternalSerializerType.ARRAY) { - const tupleOptions = Cast(description).options; - if (tupleOptions && tupleOptions.isTuple) { - const names = [] as string[]; - Cast(description).options.inner.forEach((v) => { - names.push(genDeclaration(v)); - }); - - const name = `tuple_${names.join("_")}`; - return addDeclar(name, ` - const ${name} = tupleSerializer(fury, [${names.join(", ")}])` - ); - } - - const inner = genDeclaration(Cast(description).options.inner); - const name = `array_${inner}`; - return addDeclar(name, ` - const ${name} = arraySerializer(fury, ${inner})` - ); - } - if (description.type === InternalSerializerType.FURY_SET) { - const inner = genDeclaration(Cast(description).options.key); - const name = `set_${inner}`; - return addDeclar(name, ` - const ${name} = setSerializer(fury, ${inner})` - ); - } - if (description.type === InternalSerializerType.MAP) { - const key = genDeclaration(Cast(description).options.key); - const value = genDeclaration(Cast(description).options.value); - - const name = `map_${key}_${value}`; - return addDeclar(name, ` - const ${name} = mapSerializer(fury, ${key}, ${value})` - ); - } - return genBuiltinDeclaration(description.type); - }; - return { - genDeclaration, - finish() { - const result = { - declarations, - names: [...exists.values()], - }; - declarations = []; - exists.clear(); - return result; - }, - }; -} - -export const generateInlineCode = (fury: Fury, description: TypeDescription) => { - const options = Cast(description).options; - const tag = options?.tag; - const { genDeclaration, finish } = typeHandlerDeclaration(fury); - const expectHash = computeStructHash(description); - const read = ` - // relation tag: ${tag} - const result = { - ${Object.entries(options.props).sort().map(([key]) => { - return `${safePropName(key)}: null`; - }).join(",\n")} - }; - pushReadObject(result); - ${Object.entries(options.props).sort().map(([key, value]) => { - return `result${safePropAccessor(key)} = ${genDeclaration(value)}.read()`; - }).join(";\n") - } - return result; -`; - const write = Object.entries(options.props).sort().map(([key, value]) => { - return `${genDeclaration(value)}.write(v${safePropAccessor(key)})`; - }).join(";\n"); - const { names, declarations } = finish(); - const validTag = replaceBackslashAndQuote(tag); - return new Function( - ` -return function (fury, scope) { - const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; - const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; - ${declarations.join("")} - const tagWriter = classResolver.createTagWriter("${validTag}"); - - const reserves = ${names.map(x => `${x}.config().reserve`).join(" + ")}; - return { - ...referenceResolver.deref(() => { - const hash = binaryReader.int32(); - if (hash !== ${expectHash}) { - throw new Error("validate hash failed: ${validTag}. expect ${expectHash}, but got" + hash); - } - { - ${read} - } - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { - tagWriter.write(binaryWriter); - binaryWriter.int32(${expectHash}); - binaryWriter.reserve(reserves); - ${write} - }), - config() { - return { - reserve: tagWriter.bufferLen + 8, - } - } - } -} -` - ); -}; - -export const genSerializer = (fury: Fury, description: TypeDescription) => { - const tag = Cast(description).options?.tag; - if (fury.classResolver.getSerializerByTag(tag)) { - return fury.classResolver.getSerializerByTag(tag); - } - - fury.classResolver.registerSerializerByTag(tag, fury.classResolver.getSerializerById(InternalSerializerType.ANY)); - - const func = generateInlineCode(fury, description); - return fury.classResolver.registerSerializerByTag(tag, func()(fury, { - InternalSerializerType, - RefFlags, - arraySerializer, - tupleSerializer, - mapSerializer, - setSerializer, - })); -}; diff --git a/javascript/packages/fury/lib/description.ts b/javascript/packages/fury/lib/description.ts index e3723bc73f..832a121420 100644 --- a/javascript/packages/fury/lib/description.ts +++ b/javascript/packages/fury/lib/description.ts @@ -39,7 +39,6 @@ export interface ArrayTypeDescription extends TypeDescription { export interface TupleTypeDescription extends TypeDescription { options: { - isTuple: true inner: TypeDescription[] } } @@ -57,10 +56,6 @@ export interface MapTypeDescription extends TypeDescription { } } -export function Cast(p: TypeDescription) { - return p as unknown as T1; -} - type Props = T extends { options: { props?: infer T2 extends { [key: string]: any } @@ -91,7 +86,7 @@ type MapProps = T extends { type TupleProps = T extends { options: { - inner: infer T2 extends readonly[...TypeDescription[]] + inner: infer T2 extends readonly [...TypeDescription[]] } } ? { [K in keyof T2]: ToRecordType } @@ -122,48 +117,52 @@ export type ToRecordType = T extends { | InternalSerializerType.UINT8 | InternalSerializerType.UINT16 | InternalSerializerType.UINT32 - | InternalSerializerType.UINT64 | InternalSerializerType.INT8 | InternalSerializerType.INT16 | InternalSerializerType.INT32 - | InternalSerializerType.INT64 | InternalSerializerType.FLOAT | InternalSerializerType.DOUBLE } ? number + : T extends { - type: InternalSerializerType.MAP + type: InternalSerializerType.UINT64 + | InternalSerializerType.INT64 } - ? MapProps + ? bigint : T extends { - type: InternalSerializerType.FURY_SET + type: InternalSerializerType.MAP } - ? SetProps + ? MapProps : T extends { - type: InternalSerializerType.ARRAY + type: InternalSerializerType.FURY_SET } - ? InnerProps + ? SetProps : T extends { - type: InternalSerializerType.BOOL + type: InternalSerializerType.ARRAY } - ? boolean + ? InnerProps : T extends { - type: InternalSerializerType.DATE + type: InternalSerializerType.BOOL } - ? Date + ? boolean : T extends { - type: InternalSerializerType.TIMESTAMP + type: InternalSerializerType.DATE } - ? number + ? Date : T extends { - type: InternalSerializerType.BINARY + type: InternalSerializerType.TIMESTAMP } - ? Uint8Array + ? number : T extends { - type: InternalSerializerType.ANY + type: InternalSerializerType.BINARY } - ? any - : unknown; + ? Uint8Array + : T extends { + type: InternalSerializerType.ANY + } + ? any + : unknown; export const Type = { any() { @@ -188,7 +187,6 @@ export const Type = { return { type: InternalSerializerType.TUPLE as const, options: { - isTuple: true, inner: t1, }, }; @@ -292,4 +290,39 @@ export const Type = { type: InternalSerializerType.TIMESTAMP as const, }; }, + stringTypedArray() { + return { + type: InternalSerializerType.FURY_STRING_ARRAY as const, + }; + }, + boolTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY as const, + }; + }, + shortTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY as const, + }; + }, + intTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY as const, + }; + }, + longTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY as const, + }; + }, + floatTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY as const, + }; + }, + doubleTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY as const, + }; + }, }; diff --git a/javascript/packages/fury/lib/fury.ts b/javascript/packages/fury/lib/fury.ts index 5c4c3cef4b..b2c104b3b0 100644 --- a/javascript/packages/fury/lib/fury.ts +++ b/javascript/packages/fury/lib/fury.ts @@ -29,7 +29,7 @@ export default (config: Config) => { const binaryWriter = BinaryWriter(config); const classResolver = new ClassResolver(); - const referenceResolver = ReferenceResolver(config, binaryWriter, binaryReader, classResolver); + const referenceResolver = ReferenceResolver(config, binaryWriter, binaryReader); const fury = { config, @@ -95,11 +95,14 @@ export default (config: Config) => { const cursor = binaryWriter.getCursor(); binaryWriter.skip(4); // preserve 4-byte for nativeObjects start offsets. binaryWriter.uint32(0); // nativeObjects length. - if (serializer) { - serializer.write(data); - } else { - classResolver.getSerializerById(InternalSerializerType.ANY).write(data); + if (!serializer) { + serializer = classResolver.getSerializerById(InternalSerializerType.ANY); } + // reserve fixed size + binaryWriter.reserve(serializer.meta.fixedSize); + // start write + serializer.write(data); + binaryWriter.setUint32Position(cursor, binaryWriter.getCursor()); // nativeObjects start offsets; return binaryWriter; } diff --git a/javascript/packages/fury/lib/gen/any.ts b/javascript/packages/fury/lib/gen/any.ts new file mode 100644 index 0000000000..47994c0501 --- /dev/null +++ b/javascript/packages/fury/lib/gen/any.ts @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class AnySerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private addDep() { + return this.scope.declare( + "any_ser", + this.builder.classResolver.getSerializerById(InternalSerializerType.ANY) + ); + } + + writeStmt(accessor: string): string { + const name = this.addDep(); + return `${name}.writeInner(${accessor})`; + } + + readStmt(accessor: (expr: string) => string): string { + const name = this.addDep(); + return `${name}.readInner(${accessor})`; + } + + toReadEmbed(accessor: (expr: string) => string): string { + const name = this.addDep(); + return accessor(`${name}.read()`); + } + + toWriteEmbed(accessor: string): string { + const name = this.addDep(); + return `${name}.write(${accessor})`; + } +} + +CodegenRegistry.register(InternalSerializerType.ANY, AnySerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/array.ts b/javascript/packages/fury/lib/gen/array.ts new file mode 100644 index 0000000000..002928f24a --- /dev/null +++ b/javascript/packages/fury/lib/gen/array.ts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ArrayTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class ArraySerializerGenerator extends BaseSerializerGenerator { + description: ArrayTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const inner = this.description.options.inner; + return this.builder.meta(inner); + } + + private innerGenerator() { + const inner = this.description.options.inner; + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + return new InnerGeneratorClass(inner, this.builder, this.scope); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const item = this.scope.uniqueName("item"); + return ` + ${this.builder.writer.varUInt32(`${accessor}.length`)} + ${this.builder.writer.reserve(`${innerMeta.fixedSize} * ${accessor}.length`)}; + for (const ${item} of ${accessor}) { + ${innerGenerator.toWriteEmbed(item)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const len = this.scope.uniqueName("len"); + const idx = this.scope.uniqueName("idx"); + + return ` + const ${len} = ${this.builder.reader.varUInt32()}; + const ${result} = new Array(${len}); + ${this.pushReadRefStmt(result)} + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + ${innerGenerator.toReadEmbed(x => `${result}[${idx}] = ${x};`)} + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.ARRAY, ArraySerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/binary.ts b/javascript/packages/fury/lib/gen/binary.ts new file mode 100644 index 0000000000..e18329e199 --- /dev/null +++ b/javascript/packages/fury/lib/gen/binary.ts @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class BinarySerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + return ` + ${this.builder.writer.uint8(1)} + ${this.builder.writer.uint32(`${accessor}.byteLength`)} + ${this.builder.writer.buffer(accessor)} + `; + } + + readStmt(accessor: (expr: string) => string): string { + const result = this.scope.uniqueName("result"); + return ` + ${this.builder.reader.uint8()} + ${result} = ${this.builder.reader.buffer(this.builder.reader.int32())}; + ${this.pushReadRefStmt(result)}; + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.BINARY, BinarySerializerGenerator); diff --git a/javascript/packages/fury/lib/internalSerializer/binary.ts b/javascript/packages/fury/lib/gen/bool.ts similarity index 50% rename from javascript/packages/fury/lib/internalSerializer/binary.ts rename to javascript/packages/fury/lib/gen/bool.ts index fc4694f747..0c4fb5be00 100644 --- a/javascript/packages/fury/lib/internalSerializer/binary.ts +++ b/javascript/packages/fury/lib/gen/bool.ts @@ -17,33 +17,28 @@ * under the License. */ -import { Fury } from "../type"; +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; -export default (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; +class BoolSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; - const { uint8: writeUInt8, int32: writeInt32, buffer: writeBuffer } = binaryWriter; - const { uint8: readUInt8, int32: readInt32, buffer: readBuffer } = binaryReader; - const { pushReadObject } = referenceResolver; + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } - return { - ...referenceResolver.deref(() => { - readUInt8(); // isInBand - const len = readInt32(); - const result = readBuffer(len); - pushReadObject(result); - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.BINARY, (v: Uint8Array) => { - writeUInt8(1); // is inBand - writeInt32(v.byteLength); - writeBuffer(v); - }), - config: () => { - return { - reserve: 8, - }; - }, - }; -}; + writeStmt(accessor: string): string { + return this.builder.writer.uint8(`${accessor} ? 1 : 0`); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(`${this.builder.reader.uint8()} === 1`); + } +} + +CodegenRegistry.register(InternalSerializerType.BOOL, BoolSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/builder.ts b/javascript/packages/fury/lib/gen/builder.ts new file mode 100644 index 0000000000..4fb4142442 --- /dev/null +++ b/javascript/packages/fury/lib/gen/builder.ts @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Fury } from "../type"; +import { Scope } from "./scope"; +import { getMeta } from "../meta"; +import { TypeDescription } from "../description"; + +class BinaryReaderBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + getCursor() { + return `${this.holder}.getCursor()`; + } + + setCursor(v: number | string) { + return `${this.holder}.setCursor(${v})`; + } + + varInt32() { + return `${this.holder}.varInt32()`; + } + + varInt64() { + return `${this.holder}.varInt64()`; + } + + varUInt32() { + return `${this.holder}.varUInt32()`; + } + + varUInt64() { + return `${this.holder}.varUInt64()`; + } + + int8() { + return `${this.holder}.int8()`; + } + + buffer(len: string | number) { + return `${this.holder}.buffer(${len})`; + } + + bufferRef() { + return `${this.holder}.bufferRef()`; + } + + uint8() { + return `${this.holder}.uint8()`; + } + + stringUtf8At() { + return `${this.holder}.stringUtf8At()`; + } + + stringUtf8() { + return `${this.holder}.stringUtf8()`; + } + + stringLatin1() { + return `${this.holder}.stringLatin1()`; + } + + stringOfVarUInt32() { + return `${this.holder}.stringOfVarUInt32()`; + } + + double() { + return `${this.holder}.double()`; + } + + float() { + return `${this.holder}.float()`; + } + + uint16() { + return `${this.holder}.uint16()`; + } + + int16() { + return `${this.holder}.int16()`; + } + + uint64() { + return `${this.holder}.uint64()`; + } + + skip() { + return `${this.holder}.skip()`; + } + + int64() { + return `${this.holder}.int64()`; + } + + sliLong() { + return `${this.holder}.sliLong()`; + } + + uint32() { + return `${this.holder}.uint32()`; + } + + int32() { + return `${this.holder}.int32()`; + } +} + +class BinaryWriterBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + skip(v: number | string) { + return `${this.holder}.skip(${v})`; + } + + getByteLen() { + return `${this.holder}.getByteLen()`; + } + + getReserved() { + return `${this.holder}.getReserved()`; + } + + reserve(v: number | string) { + return `${this.holder}.reserve(${v})`; + } + + uint16(v: number | string) { + return `${this.holder}.uint16(${v})`; + } + + int8(v: number | string) { + return `${this.holder}.int8(${v})`; + } + + int24(v: number | string) { + return `${this.holder}.int24(${v})`; + } + + uint8(v: number | string) { + return `${this.holder}.uint8(${v})`; + } + + int16(v: number | string) { + return `${this.holder}.int16(${v})`; + } + + varInt32(v: number | string) { + return `${this.holder}.varInt32(${v})`; + } + + varUInt32(v: number | string) { + return `${this.holder}.varUInt32(${v})`; + } + + varUInt64(v: number | string) { + return `${this.holder}.varUInt64(${v})`; + } + + varInt64(v: number | string) { + return `${this.holder}.varInt64(${v})`; + } + + stringOfVarUInt32(str: string) { + return `${this.holder}.stringOfVarUInt32(${str})`; + } + + bufferWithoutMemCheck(v: string) { + return `${this.holder}.bufferWithoutMemCheck(${v})`; + } + + uint64(v: number | string) { + return `${this.holder}.uint64(${v})`; + } + + buffer(v: string) { + return `${this.holder}.buffer(${v})`; + } + + double(v: number | string) { + return `${this.holder}.double(${v})`; + } + + float(v: number | string) { + return `${this.holder}.float(${v})`; + } + + int64(v: number | string) { + return `${this.holder}.int64(${v})`; + } + + sliLong(v: number | string) { + return `${this.holder}.sliLong(${v})`; + } + + uint32(v: number | string) { + return `${this.holder}.uint32(${v})`; + } + + int32(v: number | string) { + return `${this.holder}.int32(${v})`; + } + + getCursor() { + return `${this.holder}.getCursor()`; + } +} + +class ReferenceResolverBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + getReadObjectByRefId(id: string | number) { + return `${this.holder}.getReadObjectByRefId(${id})`; + } + + pushReadObject(obj: string) { + return `${this.holder}.pushReadObject(${obj})`; + } + + pushWriteObject(obj: string) { + return `${this.holder}.pushWriteObject(${obj})`; + } + + existsWriteObject(obj: string) { + return `${this.holder}.existsWriteObject(${obj})`; + } +} + +class ClassResolverBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + getSerializerById(id: string | number) { + return `${this.holder}.getSerializerById(${id})`; + } + + getSerializerByTag(tag: string) { + return `${this.holder}.getSerializerByTag(${tag})`; + } + + createTagWriter(tag: string) { + return `${this.holder}.createTagWriter("${tag}")`; + } + + readTag(binaryReader: string) { + return `${this.holder}.readTag(${binaryReader})`; + } +} + +export class CodecBuilder { + reader: BinaryReaderBuilder; + writer: BinaryWriterBuilder; + referenceResolver: ReferenceResolverBuilder; + classResolver: ClassResolverBuilder; + + constructor(scope: Scope, private fury: Fury) { + const br = scope.declareByName("br", "fury.binaryReader"); + const bw = scope.declareByName("bw", "fury.binaryWriter"); + const cr = scope.declareByName("cr", "fury.classResolver"); + const rr = scope.declareByName("rr", "fury.referenceResolver"); + this.reader = new BinaryReaderBuilder(br); + this.writer = new BinaryWriterBuilder(bw); + this.classResolver = new ClassResolverBuilder(cr); + this.referenceResolver = new ReferenceResolverBuilder(rr); + } + + meta(description: TypeDescription) { + return getMeta(description, this.fury); + } + + config() { + return this.fury.config; + } + + static isReserved(key: string) { + return /^(?:do|if|in|for|let|new|try|var|case|else|enum|eval|false|null|this|true|void|with|break|catch|class|const|super|throw|while|yield|delete|export|import|public|return|static|switch|typeof|default|extends|finally|package|private|continue|debugger|function|arguments|interface|protected|implements|instanceof)$/.test(key); + } + + static isDotPropAccessor(prop: string) { + return /^[a-zA-Z_$][0-9a-zA-Z_$]*$/.test(prop); + } + + static replaceBackslashAndQuote(v: string) { + return v.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); + } + + static safePropAccessor(prop: string) { + if (!CodecBuilder.isDotPropAccessor(prop) || CodecBuilder.isReserved(prop)) { + return `["${CodecBuilder.replaceBackslashAndQuote(prop)}"]`; + } + return `.${prop}`; + } + + static safePropName(prop: string) { + if (!CodecBuilder.isDotPropAccessor(prop) || CodecBuilder.isReserved(prop)) { + return `["${CodecBuilder.replaceBackslashAndQuote(prop)}"]`; + } + return prop; + } +} diff --git a/javascript/packages/fury/lib/gen/datetime.ts b/javascript/packages/fury/lib/gen/datetime.ts new file mode 100644 index 0000000000..bce8239ca5 --- /dev/null +++ b/javascript/packages/fury/lib/gen/datetime.ts @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class TimestampSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + if (/^-?[0-9]+$/.test(accessor)) { + return this.builder.writer.int64(`BigInt(${accessor})`); + } + return this.builder.writer.int64(`BigInt(${accessor}.getTime())`); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(`new Date(Number(${this.builder.reader.int64()}))`); + } +} + +class DateSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + const epoch = this.scope.declareByName("epoch", `new Date("1970/01/01 00:00").getTime()`); + if (/^-?[0-9]+$/.test(accessor)) { + return ` + ${this.builder.writer.int32(`Math.floor((${accessor} - ${epoch}) / 1000 / (24 * 60 * 60))`)} + `; + } + return ` + ${this.builder.writer.int32(`Math.floor((${accessor}.getTime() - ${epoch}) / 1000 / (24 * 60 * 60))`)} + `; + } + + readStmt(accessor: (expr: string) => string): string { + const epoch = this.scope.declareByName("epoch", `new Date("1970/01/01 00:00").getTime()`); + return accessor(` + new Date(${epoch} + (${this.builder.reader.int32()} * (24 * 60 * 60) * 1000)) + `); + } +} + +CodegenRegistry.register(InternalSerializerType.DATE, DateSerializerGenerator); +CodegenRegistry.register(InternalSerializerType.TIMESTAMP, TimestampSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/index.ts b/javascript/packages/fury/lib/gen/index.ts new file mode 100644 index 0000000000..84f60aa015 --- /dev/null +++ b/javascript/packages/fury/lib/gen/index.ts @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { InternalSerializerType, Fury } from "../type"; +import { ArrayTypeDescription, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from "../description"; +import { CodegenRegistry } from "./router"; +import { CodecBuilder } from "./builder"; +import { Scope } from "./scope"; +import "./array"; +import "./object"; +import "./string"; +import "./binary"; +import "./bool"; +import "./datetime"; +import "./map"; +import "./number"; +import "./set"; +import "./any"; +import "./tuple"; +import "./typedArray"; + +export const generate = (fury: Fury, description: TypeDescription) => { + const InnerGeneratorClass = CodegenRegistry.get(description.type); + if (!InnerGeneratorClass) { + throw new Error(`${description.type} generator not exists`); + } + const scope = new Scope(); + const generator = new InnerGeneratorClass(description, new CodecBuilder(scope, fury), scope); + + const funcString = generator.toSerializer(); + const afterCodeGenerated = fury.config?.hooks?.afterCodeGenerated; + if (typeof afterCodeGenerated === "function") { + return new Function(afterCodeGenerated(funcString)); + } + return new Function(funcString); +}; + +function regDependencies(fury: Fury, description: TypeDescription) { + if (description.type === InternalSerializerType.FURY_TYPE_TAG) { + const options = (description).options; + if (options.props) { + fury.classResolver.registerSerializerByTag(options.tag); + Object.values(options.props).forEach((x) => { + regDependencies(fury, x); + }); + const func = generate(fury, description); + fury.classResolver.registerSerializerByTag(options.tag, func()(fury, {})); + } + } + if (description.type === InternalSerializerType.ARRAY) { + regDependencies(fury, (description).options.inner); + } + if (description.type === InternalSerializerType.FURY_SET) { + regDependencies(fury, (description).options.key); + } + if (description.type === InternalSerializerType.MAP) { + regDependencies(fury, (description).options.key); + regDependencies(fury, (description).options.value); + } + if (description.type === InternalSerializerType.TUPLE) { + (description).options.inner.forEach((x) => { + regDependencies(fury, x); + }); + } +} + +export const generateSerializer = (fury: Fury, description: TypeDescription) => { + regDependencies(fury, description); + if (description.type === InternalSerializerType.FURY_TYPE_TAG) { + return fury.classResolver.getSerializerByTag((description).options.tag); + } + const func = generate(fury, description); + return func()(fury, {}); +}; diff --git a/javascript/packages/fury/lib/gen/map.ts b/javascript/packages/fury/lib/gen/map.ts new file mode 100644 index 0000000000..6ee9f1b7a5 --- /dev/null +++ b/javascript/packages/fury/lib/gen/map.ts @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { MapTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class MapSerializerGenerator extends BaseSerializerGenerator { + description: MapTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const key = this.description.options.key; + const value = this.description.options.value; + return [this.builder.meta(key), this.builder.meta(value)]; + } + + private innerGenerator() { + const key = this.description.options.key; + const value = this.description.options.value; + + const KeyGeneratorClass = CodegenRegistry.get(key.type); + const ValueGeneratorClass = CodegenRegistry.get(value.type); + if (!KeyGeneratorClass) { + throw new Error(`${key.type} generator not exists`); + } + if (!ValueGeneratorClass) { + throw new Error(`${value.type} generator not exists`); + } + return [new KeyGeneratorClass(key, this.builder, this.scope), new ValueGeneratorClass(value, this.builder, this.scope)]; + } + + writeStmt(accessor: string): string { + const [keyMeta, valueMeta] = this.innerMeta(); + const [keyGenerator, valueGenerator] = this.innerGenerator(); + const key = this.scope.uniqueName("key"); + const value = this.scope.uniqueName("value"); + + return ` + ${this.builder.writer.varUInt32(`${accessor}.size`)} + ${this.builder.writer.reserve(`${keyMeta.fixedSize + valueMeta.fixedSize} * ${accessor}.size`)}; + for (const [${key}, ${value}] of ${accessor}.entries()) { + ${keyGenerator.toWriteEmbed(key)} + ${valueGenerator.toWriteEmbed(value)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const [keyGenerator, valueGenerator] = this.innerGenerator(); + const key = this.scope.uniqueName("key"); + const value = this.scope.uniqueName("value"); + + const result = this.scope.uniqueName("result"); + const idx = this.scope.uniqueName("idx"); + const len = this.scope.uniqueName("len"); + + return ` + const ${result} = new Map(); + ${this.pushReadRefStmt(result)}; + const ${len} = ${this.builder.reader.varUInt32()}; + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + let ${key}; + let ${value}; + ${keyGenerator.toReadEmbed(x => `${key} = ${x};`)} + ${valueGenerator.toReadEmbed(x => `${value} = ${x};`)} + ${result}.set(${key}, ${value}); + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.MAP, MapSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/number.ts b/javascript/packages/fury/lib/gen/number.ts new file mode 100644 index 0000000000..1ca410a562 --- /dev/null +++ b/javascript/packages/fury/lib/gen/number.ts @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +function buildNumberSerializer(writeFun: (builder: CodecBuilder, accessor: string) => string, read: (builder: CodecBuilder) => string) { + return class NumberSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + return writeFun(this.builder, accessor); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(read(this.builder)); + } + }; +} + +CodegenRegistry.register(InternalSerializerType.UINT8, + buildNumberSerializer( + (builder, accessor) => builder.writer.uint8(accessor), + builder => builder.reader.uint8() + ) +); +CodegenRegistry.register(InternalSerializerType.INT8, + buildNumberSerializer( + (builder, accessor) => builder.writer.int8(accessor), + builder => builder.reader.int8() + ) +); +CodegenRegistry.register(InternalSerializerType.UINT16, + buildNumberSerializer( + (builder, accessor) => builder.writer.uint16(accessor), + builder => builder.reader.uint16() + ) +); +CodegenRegistry.register(InternalSerializerType.INT16, + buildNumberSerializer( + (builder, accessor) => builder.writer.int16(accessor), + builder => builder.reader.int16() + ) +); +CodegenRegistry.register(InternalSerializerType.UINT32, + buildNumberSerializer( + (builder, accessor) => builder.writer.uint32(accessor), + builder => builder.reader.uint32() + ) +); +CodegenRegistry.register(InternalSerializerType.INT32, + buildNumberSerializer( + (builder, accessor) => builder.writer.int32(accessor), + builder => builder.reader.int32() + ) +); +CodegenRegistry.register(InternalSerializerType.UINT64, + buildNumberSerializer( + (builder, accessor) => builder.writer.varUInt64(accessor), + builder => builder.reader.varUInt64() + ) +); +CodegenRegistry.register(InternalSerializerType.INT64, + buildNumberSerializer( + (builder, accessor) => builder.writer.sliLong(accessor), + builder => builder.reader.sliLong() + ) +); +CodegenRegistry.register(InternalSerializerType.FLOAT, + buildNumberSerializer( + (builder, accessor) => builder.writer.float(accessor), + builder => builder.reader.float() + ) +); +CodegenRegistry.register(InternalSerializerType.DOUBLE, + buildNumberSerializer( + (builder, accessor) => builder.writer.double(accessor), + builder => builder.reader.double() + ) +); diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts new file mode 100644 index 0000000000..0d97ce49db --- /dev/null +++ b/javascript/packages/fury/lib/gen/object.ts @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { InternalSerializerType, MaxInt32 } from "../type"; +import { Scope } from "./scope"; +import { CodecBuilder } from "./builder"; +import { ObjectTypeDescription, TypeDescription } from "../description"; +import { fromString } from "../platformBuffer"; +import { CodegenRegistry } from "./router"; +import { BaseSerializerGenerator } from "./serializer"; + +function computeFieldHash(hash: number, id: number): number { + let newHash = (hash) * 31 + (id); + while (newHash >= MaxInt32) { + newHash = Math.floor(newHash / 7); + } + return newHash; +} + +const computeStringHash = (str: string) => { + const bytes = fromString(str); + let hash = 17; + bytes.forEach((b) => { + hash = hash * 31 + b; + while (hash >= MaxInt32) { + hash = Math.floor(hash / 7); + } + }); + return hash; +}; + +const computeStructHash = (description: TypeDescription) => { + let hash = 17; + for (const [, value] of Object.entries((description).options.props).sort()) { + let id = value.type; + if (value.type === InternalSerializerType.ARRAY || value.type === InternalSerializerType.TUPLE || value.type === InternalSerializerType.MAP) { + id = Math.floor(value.type); // TODO add map key&value type into schema hash + } else if (value.type === InternalSerializerType.FURY_TYPE_TAG) { + id = computeStringHash((value).options.tag); + } + hash = computeFieldHash(hash, id); + } + return hash; +}; + +class ObjectSerializerGenerator extends BaseSerializerGenerator { + description: ObjectTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + const options = this.description.options; + const expectHash = computeStructHash(this.description); + const tagWriter = this.scope.declare("tagWriter", `${this.builder.classResolver.createTagWriter(this.safeTag())}`); + + return ` + ${tagWriter}.write(${this.builder.writer.ownName()}); + ${this.builder.writer.int32(expectHash)}; + ${Object.entries(options.props).sort().map(([key, inner]) => { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); + return innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`); + }).join(";\n") + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const options = this.description.options; + const expectHash = computeStructHash(this.description); + const result = this.scope.uniqueName("result"); + return ` + if (${this.builder.reader.int32()} !== ${expectHash}) { + throw new Error("validate hash failed: ${this.safeTag()}. expect ${expectHash}"); + } + const ${result} = { + ${Object.entries(options.props).sort().map(([key]) => { + return `${CodecBuilder.safePropName(key)}: null`; + }).join(",\n")} + }; + ${this.pushReadRefStmt(result)} + ${Object.entries(options.props).sort().map(([key, inner]) => { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); + return innerGenerator.toReadEmbed(expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`); + }).join(";\n") + } + ${accessor(result)} + `; + } + + safeTag() { + return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); + } + + toReadEmbed(accessor: (expr: string) => string): string { + const name = this.scope.declare( + "tag_ser", + `fury.classResolver.getSerializerByTag("${this.safeTag()}")` + ); + return accessor(`${name}.read()`); + } + + toWriteEmbed(accessor: string): string { + const name = this.scope.declare( + "tag_ser", + `fury.classResolver.getSerializerByTag("${this.safeTag()}")` + ); + return `${name}.write(${accessor})`; + } +} + +CodegenRegistry.register(InternalSerializerType.FURY_TYPE_TAG, ObjectSerializerGenerator); diff --git a/javascript/packages/fury/lib/internalSerializer/bool.ts b/javascript/packages/fury/lib/gen/router.ts similarity index 52% rename from javascript/packages/fury/lib/internalSerializer/bool.ts rename to javascript/packages/fury/lib/gen/router.ts index 66e3d04cfc..a8cd043a1f 100644 --- a/javascript/packages/fury/lib/internalSerializer/bool.ts +++ b/javascript/packages/fury/lib/gen/router.ts @@ -17,28 +17,22 @@ * under the License. */ -import { Fury } from "../type"; -import { InternalSerializerType, RefFlags } from "../type"; +import { TypeDescription } from "../description"; +import { SerializerGenerator } from "./serializer"; +import { InternalSerializerType } from "../type"; +import { CodecBuilder } from "./builder"; +import { Scope } from "./scope"; -export default (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { uint8: readUInt8 } = binaryReader; - const { int8: writeInt8, uint8: writeUInt8 } = binaryWriter; - return { - ...referenceResolver.deref(() => { - return readUInt8() === 0 ? false : true; - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.BOOL, false, (v: boolean) => { - writeUInt8(v ? 1 : 0); - }), - writeWithoutType: (v: boolean) => { - writeInt8(RefFlags.NotNullValueFlag); - writeUInt8(v ? 1 : 0); - }, - config: () => { - return { - reserve: 4, - }; - }, - }; -}; +type SerializerGeneratorConstructor = new (description: TypeDescription, builder: CodecBuilder, scope: Scope) => SerializerGenerator; + +export class CodegenRegistry { + static map = new Map(); + + static register(type: InternalSerializerType, generator: SerializerGeneratorConstructor) { + this.map.set(InternalSerializerType[type], generator); + } + + static get(type: InternalSerializerType) { + return this.map.get(InternalSerializerType[type]); + } +} diff --git a/javascript/packages/fury/lib/gen/scope.ts b/javascript/packages/fury/lib/gen/scope.ts new file mode 100644 index 0000000000..a5f51e099f --- /dev/null +++ b/javascript/packages/fury/lib/gen/scope.ts @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export class Scope { + private declares: Map = new Map(); + private idx = 0; + + private addDeclar(stmt: string, name: string) { + if (this.declares.has(stmt)) { + return this.declares.get(stmt)!; + } + this.declares.set(stmt, name); + return name; + } + + uniqueName(prefix: string) { + return `${prefix}_${this.idx++}`; + } + + declareByName(name: string, stmt: string) { + return this.addDeclar(stmt, name); + } + + assertNameNotDuplicate(name: string) { + for (const item of this.declares.values()) { + if (item === name) { + throw new Error(`const ${name} declare duplicate`); + } + } + } + + declare(prefix: string, stmt: string) { + return this.addDeclar(stmt, this.uniqueName(prefix)); + } + + generate() { + return Array.from(this.declares.entries()).map(x => `const ${x[1]} = ${x[0]};`).join("\n"); + } +} diff --git a/javascript/packages/fury/lib/gen/serializer.ts b/javascript/packages/fury/lib/gen/serializer.ts new file mode 100644 index 0000000000..34eaa17aa6 --- /dev/null +++ b/javascript/packages/fury/lib/gen/serializer.ts @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { InternalSerializerType } from "../type"; +import { CodecBuilder } from "./builder"; +import { makeHead } from "../referenceResolver"; +import { RefFlags } from "../type"; +import { Scope } from "./scope"; +import { TypeDescription } from "../description"; + +export interface SerializerGenerator { + writeStmt(accessor: string): string + readStmt(accessor: (expr: string) => string): string + toSerializer(): string + toWriteEmbed(accessor: string, withHead?: boolean): string + toReadEmbed(accessor: (expr: string) => string, withHead?: boolean): string +} + +export abstract class BaseSerializerGenerator implements SerializerGenerator { + constructor( + protected description: TypeDescription, + protected builder: CodecBuilder, + protected scope: Scope, + ) { + + } + + abstract writeStmt(accessor: string): string; + + abstract readStmt(accessor: (expr: string) => string): string; + + protected pushReadRefStmt(accessor: string) { + if (!this.builder.config().refTracking) { + return ""; + } + return this.builder.referenceResolver.pushReadObject(accessor); + } + + protected wrapWriteHead(accessor: string, stmt: (accessor: string) => string) { + const meta = this.builder.meta(this.description); + const noneable = meta.noneable; + if (noneable) { + const head = makeHead(RefFlags.RefValueFlag, this.description.type); + + const normaStmt = ` + if (${accessor} !== null && ${accessor} !== undefined) { + ${this.builder.writer.int24(head)}; + ${stmt(accessor)}; + } else { + ${this.builder.writer.int8(RefFlags.NullFlag)}; + } + `; + if (this.builder.config().refTracking) { + const existsId = this.scope.uniqueName("existsId"); + return ` + const ${existsId} = ${this.builder.referenceResolver.existsWriteObject(accessor)}; + if (typeof ${existsId} === "number") { + ${this.builder.writer.int8(RefFlags.RefFlag)} + ${this.builder.writer.varUInt32(existsId)} + } else { + ${this.builder.referenceResolver.pushWriteObject(accessor)} + ${normaStmt} + } + `; + } else { + return normaStmt; + } + } else { + const head = makeHead(RefFlags.NotNullValueFlag, this.description.type); + return ` + ${this.builder.writer.int24(head)}; + if (${accessor} !== null && ${accessor} !== undefined) { + ${stmt(accessor)}; + } else { + ${typeof meta.default === "string" ? stmt(`"${meta.default}"`) : stmt(meta.default)}; + }`; + } + } + + protected wrapReadHead(accessor: (expr: string) => string, stmt: (accessor: (expr: string) => string) => string) { + return ` + switch (${this.builder.reader.int8()}) { + case ${RefFlags.NotNullValueFlag}: + case ${RefFlags.RefValueFlag}: + if (${this.builder.reader.int16()} === ${InternalSerializerType.FURY_TYPE_TAG}) { + ${this.builder.classResolver.readTag(this.builder.reader.ownName())}; + } + ${stmt(accessor)} + break; + case ${RefFlags.RefFlag}: + ${accessor(this.builder.referenceResolver.getReadObjectByRefId(this.builder.reader.varUInt32()))} + break; + case ${RefFlags.NullFlag}: + ${accessor("null")} + break; + } + `; + } + + toWriteEmbed(accessor: string, withHead = true) { + if (!withHead) { + return this.writeStmt(accessor); + } + return this.wrapWriteHead(accessor, accessor => this.writeStmt(accessor)); + } + + toReadEmbed(accessor: (expr: string) => string, withHead = true) { + if (!withHead) { + return this.readStmt(accessor); + } + return this.wrapReadHead(accessor, accessor => this.readStmt(accessor)); + } + + toSerializer() { + this.scope.assertNameNotDuplicate("read"); + this.scope.assertNameNotDuplicate("readInner"); + this.scope.assertNameNotDuplicate("write"); + this.scope.assertNameNotDuplicate("writeInner"); + + const declare = ` + const readInner = () => { + ${this.readStmt(x => `return ${x}`)} + }; + const read = () => { + ${this.wrapReadHead(x => `return ${x}`, accessor => accessor(`readInner()`))} + }; + const writeInner = (v) => { + ${this.writeStmt("v")} + }; + const write = (v) => { + ${this.wrapWriteHead("v", accessor => `writeInner(${accessor})`)} + }; + `; + return ` + return function (fury, external) { + ${this.scope.generate()} + ${declare} + return { + read, + readInner, + write, + writeInner, + meta: ${JSON.stringify(this.builder.meta(this.description))} + }; + } + `; + } +} diff --git a/javascript/packages/fury/lib/gen/set.ts b/javascript/packages/fury/lib/gen/set.ts new file mode 100644 index 0000000000..3a595b163a --- /dev/null +++ b/javascript/packages/fury/lib/gen/set.ts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SetTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class SetSerializerGenerator extends BaseSerializerGenerator { + description: SetTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const inner = this.description.options.key; + return this.builder.meta(inner); + } + + private innerGenerator() { + const inner = this.description.options.key; + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + return new InnerGeneratorClass(inner, this.builder, this.scope); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const item = this.scope.uniqueName("item"); + return ` + ${this.builder.writer.varUInt32(`${accessor}.size`)} + ${this.builder.writer.reserve(`${innerMeta.fixedSize} * ${accessor}.size`)}; + for (const ${item} of ${accessor}.values()) { + ${innerGenerator.toWriteEmbed(item)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const idx = this.scope.uniqueName("idx"); + const len = this.scope.uniqueName("len"); + + return ` + const ${result} = new Set(); + ${this.pushReadRefStmt(result)} + const ${len} = ${this.builder.reader.varUInt32()}; + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + ${innerGenerator.toReadEmbed(x => `${result}.add(${x});`)} + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.FURY_SET, SetSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/string.ts b/javascript/packages/fury/lib/gen/string.ts new file mode 100644 index 0000000000..475b7a41a3 --- /dev/null +++ b/javascript/packages/fury/lib/gen/string.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class StringSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + return this.builder.writer.stringOfVarUInt32(accessor); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(this.builder.reader.stringOfVarUInt32()); + } +} + +CodegenRegistry.register(InternalSerializerType.STRING, StringSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/tuple.ts b/javascript/packages/fury/lib/gen/tuple.ts new file mode 100644 index 0000000000..ee3d8ce172 --- /dev/null +++ b/javascript/packages/fury/lib/gen/tuple.ts @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TupleTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class TupleSerializerGenerator extends BaseSerializerGenerator { + description: TupleTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const inner = this.description.options.inner; + return inner.map(x => this.builder.meta(x)); + } + + private innerGenerator() { + const inner = this.description.options.inner; + return inner.map((x) => { + const InnerGeneratorClass = CodegenRegistry.get(x.type); + if (!InnerGeneratorClass) { + throw new Error(`${x.type} generator not exists`); + } + return new InnerGeneratorClass(x, this.builder, this.scope); + }); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const fixedSize = innerMeta.reduce((x, y) => x + y.fixedSize, 0); + + return ` + ${this.builder.writer.varUInt32(innerMeta.length)} + ${this.builder.writer.reserve(fixedSize)}; + ${ + innerGenerator.map((generator, index) => { + return generator.toWriteEmbed(`${accessor}[${index}]`); + }).join("\n") + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const len = this.scope.uniqueName("len"); + + return ` + const ${len} = ${this.builder.reader.varUInt32()}; + const ${result} = new Array(${len}); + ${this.pushReadRefStmt(result)} + ${ + innerGenerator.map((generator, index) => { + return ` + if (${len} > ${index}) { + ${generator.toReadEmbed(expr => `${result}[${index}] = ${expr}`)} + } + `; + }).join("\n") + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.TUPLE, TupleSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/typedArray.ts b/javascript/packages/fury/lib/gen/typedArray.ts new file mode 100644 index 0000000000..4d83019071 --- /dev/null +++ b/javascript/packages/fury/lib/gen/typedArray.ts @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Type, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +function build(inner: TypeDescription) { + return class TypedArraySerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + return this.builder.meta(inner); + } + + private innerGenerator() { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + return new InnerGeneratorClass(inner, this.builder, this.scope); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const item = this.scope.uniqueName("item"); + return ` + ${this.builder.writer.varUInt32(`${accessor}.length`)} + ${this.builder.writer.reserve(`${innerMeta.fixedSize} * ${accessor}.length`)}; + for (const ${item} of ${accessor}) { + ${innerGenerator.toWriteEmbed(item, false)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const len = this.scope.uniqueName("len"); + const idx = this.scope.uniqueName("idx"); + + return ` + const ${len} = ${this.builder.reader.varUInt32()}; + const ${result} = new Array(${len}); + ${this.pushReadRefStmt(result)} + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + ${innerGenerator.toReadEmbed(x => `${result}[${idx}] = ${x};`, false)} + } + ${accessor(result)} + `; + } + }; +} +CodegenRegistry.register(InternalSerializerType.FURY_STRING_ARRAY, build(Type.string())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY, build(Type.bool())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY, build(Type.int64())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY, build(Type.int32())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY, build(Type.float())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY, build(Type.double())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY, build(Type.int16())); diff --git a/javascript/packages/fury/lib/internalSerializer/array.ts b/javascript/packages/fury/lib/internalSerializer/array.ts deleted file mode 100644 index dae37a3384..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/array.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, Serializer } from "../type"; -import { Fury } from "../type"; - -export const buildArray = (fury: Fury, item: Serializer, type: InternalSerializerType) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - - const { pushReadObject } = referenceResolver; - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { varUInt32: readVarUInt32 } = binaryReader; - const { write, read } = item; - const innerHeadSize = (item.config().reserve); - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Array(len); - pushReadObject(result); - for (let i = 0; i < result.length; i++) { - result[i] = read(); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(type, (v: any[]) => { - writeVarUInt32(v.length); - - reserves(innerHeadSize * v.length); - - for (const x of v) { - write(x); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -const buildTypedArray = (fury: Fury, serializeType: InternalSerializerType, typeArrayType: InternalSerializerType) => { - const serializer = fury.classResolver.getSerializerById(serializeType); - return buildArray(fury, { - read: serializer.readWithoutType!, - write: serializer.writeWithoutType!, - config: serializer.config, - }, typeArrayType); -}; - -export const stringArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.STRING, InternalSerializerType.FURY_STRING_ARRAY); -export const boolArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.BOOL, InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY); -export const longArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.INT64, InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY); -export const intArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.INT32, InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY); -export const floatArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.FLOAT, InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY); -export const doubleArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.DOUBLE, InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY); -export const shortArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.INT16, InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY); - -export const arraySerializer = (fury: Fury, item: Serializer) => buildArray(fury, item, InternalSerializerType.ARRAY); diff --git a/javascript/packages/fury/lib/internalSerializer/datetime.ts b/javascript/packages/fury/lib/internalSerializer/datetime.ts deleted file mode 100644 index ddcb8a6958..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/datetime.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Fury } from "../type"; -import { InternalSerializerType } from "../type"; - -const epochDate = new Date("1970/01/01 00:00"); -const epoch = epochDate.getTime(); - -export const timestampSerializer = (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { int64: writeInt64 } = binaryWriter; - const { int64: readInt64 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return new Date(Number(readInt64())); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.TIMESTAMP, epochDate, (v: Date) => { - writeInt64(BigInt(v.getTime())); - }), - config: () => { - return { - reserve: 11, - }; - }, - }; -}; - -export const dateSerializer = (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { int32: writeInt32 } = binaryWriter; - const { int32: readInt32 } = binaryReader; - return { - ...referenceResolver.deref(() => { - const day = readInt32(); - return new Date(epoch + (day * (24 * 60 * 60) * 1000)); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.DATE, epochDate, (v: Date) => { - const diff = v.getTime() - epoch; - const day = Math.floor(diff / 1000 / (24 * 60 * 60)); - writeInt32(day); - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/map.ts b/javascript/packages/fury/lib/internalSerializer/map.ts deleted file mode 100644 index 53a280a7ab..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/map.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, Fury, Serializer } from "../type"; - -export default (fury: Fury, keySerializer: Serializer, valueSerializer: Serializer) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { varUInt32: readVarUInt32 } = binaryReader; - - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { pushReadObject } = referenceResolver; - const innerHeadSize = keySerializer.config().reserve + valueSerializer.config().reserve; - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Map(); - pushReadObject(result); - for (let index = 0; index < len; index++) { - const key = keySerializer.read(); - const value = valueSerializer.read(); - result.set(key, value); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.MAP, (v: Map) => { - const len = v.size; - writeVarUInt32(len); - reserves(innerHeadSize * v.size); - for (const [key, value] of v.entries()) { - keySerializer.write(key); - valueSerializer.write(value); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/number.ts b/javascript/packages/fury/lib/internalSerializer/number.ts deleted file mode 100644 index 87cd6b148e..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/number.ts +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, RefFlags, Fury } from "../type"; - -export const uInt8Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint8: writeUInt8 } = binaryWriter; - const { uint8: readUInt8 } = binaryReader; - return { - ...referenceResolver.deref(() => { - return readUInt8(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT8, 0, (v: number) => { - writeUInt8(v); - }), - config: () => { - return { - reserve: 4, - }; - }, - }; -}; - -export const floatSerializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, float: writeFloat } = binaryWriter; - const { float: readFloat } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readFloat(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.FLOAT, 0, (v: number) => { - writeFloat(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeFloat(v); - }, - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -export const doubleSerializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, double: writeDouble } = binaryWriter; - const { double: readDouble } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readDouble(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.DOUBLE, 0, (v: number) => { - writeDouble(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeDouble(v); - }, - config: () => { - return { - reserve: 11, - }; - }, - }; -}; - -export const int8Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8 } = binaryWriter; - const { int8: readInt8 } = binaryReader; - return { - ...referenceResolver.deref(() => { - return readInt8(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT8, 0, (v: number) => { - writeInt8(v); - }), - config: () => { - return { - reserve: 4, - }; - }, - }; -}; - -export const uInt16Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint16: writeUInt16 } = binaryWriter; - const { uint16: readUInt16 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readUInt16(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT16, 0, (v: number) => { - writeUInt16(v); - }), - config: () => { - return { - reserve: 5, - }; - }, - }; -}; - -export const int16Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int16: writeInt16, int8: writeInt8 } = binaryWriter; - const { int16: readInt16 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readInt16(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT16, 0, (v: number) => { - writeInt16(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeInt16(v); - }, - config: () => { - return { - reserve: 5, - }; - }, - }; -}; - -export const uInt32Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint32: writeUInt32 } = binaryWriter; - const { uint32: readUInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readUInt32(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT32, 0, (v: number) => { - writeUInt32(v); - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -export const int32Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, int32: writeInt32 } = binaryWriter; - const { int32: readInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readInt32(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT32, 0, (v: number) => { - writeInt32(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeInt32(v); - }, - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -export const uInt64Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint64: writeUInt64 } = binaryWriter; - const { uint64: readUInt64 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readUInt64(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT64, BigInt(0), (v: bigint) => { - writeUInt64(v); - }), - config: () => { - return { - reserve: 11, - }; - }, - }; -}; - -export const int64Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, int64: writeInt64 } = binaryWriter; - const { int64: readInt64 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readInt64(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT64, BigInt(0), (v: bigint) => { - writeInt64(v); - }), - writeWithoutType: (v: bigint) => { - writeInt8(RefFlags.NotNullValueFlag); - writeInt64(v); - }, - config: () => { - return { - reserve: 11, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/set.ts b/javascript/packages/fury/lib/internalSerializer/set.ts deleted file mode 100644 index adae269305..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/set.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Fury, Serializer } from "../type"; -import { InternalSerializerType } from "../type"; - -export default (fury: Fury, nestedSerializer: Serializer) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { varUInt32: readVarUInt32 } = binaryReader; - const { pushReadObject } = referenceResolver; - const innerHeadSize = nestedSerializer.config().reserve; - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Set(); - pushReadObject(result); - for (let index = 0; index < len; index++) { - result.add(nestedSerializer.read()); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_SET, (v: Set) => { - const len = v.size; - writeVarUInt32(len); - reserves(innerHeadSize * v.size); - for (const value of v.values()) { - nestedSerializer.write(value); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/string.ts b/javascript/packages/fury/lib/internalSerializer/string.ts deleted file mode 100644 index ebba64b0a6..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/string.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Fury } from "../type"; -import { InternalSerializerType, RefFlags } from "../type"; - -export default (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { stringOfVarUInt32: writeStringOfVarUInt32, int8 } = binaryWriter; - const { stringOfVarUInt32: readStringOfVarUInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readStringOfVarUInt32(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.STRING, "", (v: string) => { - writeStringOfVarUInt32(v); - }), - writeWithoutType: (v: string) => { - if (v === null) { - binaryWriter.int8(RefFlags.NullFlag); - return; - } - int8(RefFlags.NotNullValueFlag); - writeStringOfVarUInt32(v); - }, - config: () => { - return { - reserve: 8, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/tuple.ts b/javascript/packages/fury/lib/internalSerializer/tuple.ts deleted file mode 100644 index 046679292e..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/tuple.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, Serializer } from "../type"; -import { Fury } from "../type"; - -export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - - const { pushReadObject } = referenceResolver; - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { varUInt32: readVarUInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Array(len); - pushReadObject(result); - for (let i = 0; i < len; i++) { - const item = serializers[i]; - result[i] = item.read(); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.TUPLE, (v: any[]) => { - writeVarUInt32(serializers.length); - - for (let i = 0; i < serializers.length; i++) { - const item = serializers[i]; - reserves(item.config().reserve); - item.write(v[i]); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/meta.ts b/javascript/packages/fury/lib/meta.ts new file mode 100644 index 0000000000..69a7ac88d6 --- /dev/null +++ b/javascript/packages/fury/lib/meta.ts @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ClassResolver from "./classResolver"; +import { ObjectTypeDescription, TypeDescription } from "./description"; +import { Fury, InternalSerializerType } from "./type"; + +export type Meta = { + fixedSize: number + noneable: boolean + default?: T +}; + +const epochDate = new Date("1970/01/01 00:00"); + +export const getMeta = (description: TypeDescription, fury: Fury): Meta => { + const type = description.type; + switch (type) { + case InternalSerializerType.STRING: + return { + fixedSize: 8, + noneable: false, + default: "", + }; + case InternalSerializerType.ARRAY: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.TUPLE: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.MAP: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.BOOL: + case InternalSerializerType.UINT8: + case InternalSerializerType.INT8: + return { + fixedSize: 4, + noneable: false, + default: 0, + }; + case InternalSerializerType.UINT16: + case InternalSerializerType.INT16: + return { + fixedSize: 5, + noneable: false, + default: 0, + }; + case InternalSerializerType.UINT32: + case InternalSerializerType.INT32: + case InternalSerializerType.FLOAT: + return { + fixedSize: 7, + noneable: false, + default: 0, + }; + case InternalSerializerType.UINT64: + case InternalSerializerType.INT64: + case InternalSerializerType.DOUBLE: + return { + fixedSize: 11, + noneable: false, + default: 0, + }; + case InternalSerializerType.BINARY: + return { + fixedSize: 8, + noneable: true, + }; + case InternalSerializerType.DATE: + return { + fixedSize: 7, + noneable: false, + default: epochDate.getTime(), + }; + case InternalSerializerType.TIMESTAMP: + return { + fixedSize: 11, + noneable: false, + default: epochDate.getTime(), + }; + case InternalSerializerType.FURY_TYPE_TAG: + { + const options = (description).options; + let fixedSize = ClassResolver.tagBuffer(options.tag).byteLength + 8; + if (options.props) { + Object.values(options.props).forEach(x => fixedSize += getMeta(x, fury).fixedSize); + } else { + fixedSize += fury.classResolver.getSerializerByTag(options.tag).meta.fixedSize; + } + return { + fixedSize, + noneable: true, + }; + } + + case InternalSerializerType.FURY_SET: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY: + case InternalSerializerType.FURY_STRING_ARRAY: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.ANY: + return { + fixedSize: 11, + noneable: true, + }; + default: + throw new Error(`Meta of ${description.type} not exists`); + } +}; diff --git a/javascript/packages/fury/lib/reader.ts b/javascript/packages/fury/lib/reader.ts index 9513d34396..46b4e5fe96 100644 --- a/javascript/packages/fury/lib/reader.ts +++ b/javascript/packages/fury/lib/reader.ts @@ -20,6 +20,7 @@ import { Config, LATIN1 } from "./type"; import { isNodeEnv } from "./util"; import { PlatformBuffer, alloc, fromUint8Array } from "./platformBuffer"; +import { read1, read10, read11, read12, read13, read14, read15, read2, read3, read4, read5, read6, read7, read8, read9 } from "./string"; export const BinaryReader = (config: Config) => { const sliceStringEnable = isNodeEnv && config.useSliceString; @@ -87,6 +88,16 @@ export const BinaryReader = (config: Config) => { return result; } + function sliLong() { + const i = dataView.getUint32(cursor, true); + if ((i & 0b1) != 0b1) { + cursor += 4; + return BigInt(i >> 1); + } + cursor += 1; + return varInt64(); + } + function float() { const result = dataView.getFloat32(cursor, true); cursor += 4; @@ -122,9 +133,44 @@ export const BinaryReader = (config: Config) => { } function stringLatin1Slow(len: number) { - const result = buffer.latin1Slice(cursor, cursor + len); + const rawCursor = cursor; cursor += len; - return result; + switch (len) { + case 0: + return ""; + case 1: + return read1(buffer, rawCursor); + case 2: + return read2(buffer, rawCursor); + case 3: + return read3(buffer, rawCursor); + case 4: + return read4(buffer, rawCursor); + case 5: + return read5(buffer, rawCursor); + case 6: + return read6(buffer, rawCursor); + case 7: + return read7(buffer, rawCursor); + case 8: + return read8(buffer, rawCursor); + case 9: + return read9(buffer, rawCursor); + case 10: + return read10(buffer, rawCursor); + case 11: + return read11(buffer, rawCursor); + case 12: + return read12(buffer, rawCursor); + case 13: + return read13(buffer, rawCursor); + case 14: + return read14(buffer, rawCursor); + case 15: + return read15(buffer, rawCursor); + default: + return buffer.latin1Slice(rawCursor, cursor); + } } function binary(len: number) { @@ -144,21 +190,67 @@ export const BinaryReader = (config: Config) => { return (v >> 1) ^ -(v & 1); } + function zigZagBigInt(v: bigint) { + return (v >> 1n) ^ -(v & 1n); + } + function varUInt32() { - let byte_ = int8(); + let byte_ = uint8(); let result = byte_ & 0x7f; if ((byte_ & 0x80) != 0) { - byte_ = int8(); + byte_ = uint8(); result |= (byte_ & 0x7f) << 7; if ((byte_ & 0x80) != 0) { - byte_ = int8(); + byte_ = uint8(); result |= (byte_ & 0x7f) << 14; if ((byte_ & 0x80) != 0) { - byte_ = int8(); + byte_ = uint8(); result |= (byte_ & 0x7f) << 21; if ((byte_ & 0x80) != 0) { - byte_ = int8(); - result |= (byte_ & 0x7f) << 28; + byte_ = uint8(); + result |= (byte_) << 28; + } + } + } + } + return result; + } + + function bigUInt8() { + return BigInt(uint8() >>> 0); + } + + function varUInt64() { + let byte_ = bigUInt8(); + let result = byte_ & 0x7fn; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 7n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 14n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 21n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 28n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 35n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 42n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 49n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_) << 56n; + } + } + } + } } } } @@ -170,11 +262,17 @@ export const BinaryReader = (config: Config) => { return zigZag(varUInt32()); } + function varInt64() { + return zigZagBigInt(varUInt64()); + } + return { getCursor: () => cursor, setCursor: (v: number) => (cursor = v), varInt32, + varInt64, varUInt32, + varUInt64, int8, buffer: binary, bufferRef, @@ -191,6 +289,7 @@ export const BinaryReader = (config: Config) => { uint64, skip, int64, + sliLong, uint32, int32, }; diff --git a/javascript/packages/fury/lib/referenceResolver.ts b/javascript/packages/fury/lib/referenceResolver.ts index c79f5400e1..702bfd2f18 100644 --- a/javascript/packages/fury/lib/referenceResolver.ts +++ b/javascript/packages/fury/lib/referenceResolver.ts @@ -21,14 +21,11 @@ import { RefFlags, BinaryReader, BinaryWriter, - SerializerRead, InternalSerializerType, - SerializerWrite, } from "./type"; -import type ClassResolver from "./classResolver"; export const makeHead = (flag: RefFlags, type: InternalSerializerType) => { - return (((type << 16) >>> 16) << 8) | ((flag << 24) >>> 24); + return (((Math.floor(type) << 16) >>> 16) << 8) | ((flag << 24) >>> 24); }; export const ReferenceResolver = ( @@ -37,7 +34,6 @@ export const ReferenceResolver = ( }, binaryWriter: BinaryWriter, binaryReader: BinaryReader, - classResolver: ClassResolver ) => { let readObjects: any[] = []; let writeObjects: any[] = []; @@ -71,95 +67,6 @@ export const ReferenceResolver = ( } } - function skipType() { - const typeId = binaryReader.int16(); - if (typeId === InternalSerializerType.FURY_TYPE_TAG) { - classResolver.readTag(binaryReader); - } - } - - function withNullableOrRefWriter( - type: InternalSerializerType, - fn: SerializerWrite - ) { - const int24 = binaryWriter.int24; - const head = makeHead(RefFlags.RefValueFlag, type); - if (config.refTracking) { - return (v: T) => { - if (v !== null && v !== undefined) { - const existsId = existsWriteObject(v); - if (typeof existsId === "number") { - binaryWriter.int8(RefFlags.RefFlag); - binaryWriter.varUInt32(existsId); - } else { - int24(head); - pushWriteObject(v); - fn(v); - } - } else { - binaryWriter.int8(RefFlags.NullFlag); - } - }; - } else { - return (v: T) => { - if (v !== null && v !== undefined) { - int24(head); - fn(v); - } else { - binaryWriter.int8(RefFlags.NullFlag); - } - }; - } - } - - function withNotNullableWriter( - type: InternalSerializerType, - defaultValue: T, - fn: SerializerWrite - ) { - const head = makeHead(RefFlags.NotNullValueFlag, type); - const int24 = binaryWriter.int24; - return (v: T) => { - int24(head); - if (v == null || v == undefined) { - fn(defaultValue); - } else { - fn(v); - } - }; - } - - function deref(fn: SerializerRead) { - return { - read: () => { - switch (readRefFlag()) { - case RefFlags.RefValueFlag: - skipType(); - return fn(); - case RefFlags.RefFlag: - return getReadObjectByRefId(binaryReader.varUInt32()); - case RefFlags.NullFlag: - return null; - case RefFlags.NotNullValueFlag: - skipType(); - return fn(); - } - }, - readWithoutType: () => { - switch (readRefFlag()) { - case RefFlags.RefValueFlag: - return fn(); - case RefFlags.RefFlag: - return getReadObjectByRefId(binaryReader.varUInt32()); - case RefFlags.NullFlag: - return null; - case RefFlags.NotNullValueFlag: - return fn(); - } - }, - }; - } - return { existsWriteObject, pushWriteObject, @@ -167,8 +74,5 @@ export const ReferenceResolver = ( readRefFlag, getReadObjectByRefId, reset, - withNotNullableWriter, - withNullableOrRefWriter, - deref, }; }; diff --git a/javascript/packages/fury/lib/string.ts b/javascript/packages/fury/lib/string.ts new file mode 100644 index 0000000000..3905254e77 --- /dev/null +++ b/javascript/packages/fury/lib/string.ts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const read1 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor]); +}; +export const read2 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1]); +}; +export const read3 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2]); +}; +export const read4 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3]); +}; +export const read5 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4]); +}; +export const read6 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5]); +}; +export const read7 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6]); +}; +export const read8 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7]); +}; +export const read9 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8]); +}; +export const read10 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9]); +}; +export const read11 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10]); +}; +export const read12 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11]); +}; +export const read13 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11], buffer[cursor + 12]); +}; +export const read14 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11], buffer[cursor + 12], buffer[cursor + 13]); +}; +export const read15 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11], buffer[cursor + 12], buffer[cursor + 13], buffer[cursor + 14]); +}; diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index 39cd50669f..2a1def72cc 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -20,6 +20,7 @@ import type { BinaryWriter } from "./writer"; import type { BinaryReader } from "./reader"; import type FuryFunc from "./fury"; +import { Meta } from "./meta"; export type Fury = ReturnType; export type BinaryWriter = ReturnType; @@ -28,7 +29,7 @@ export type BinaryReader = ReturnType; export enum InternalSerializerType { STRING = 13, ARRAY = 25, - TUPLE = 25, + TUPLE = 25.1, MAP = 30, BOOL = 1, UINT8 = 2, @@ -63,25 +64,13 @@ export enum ConfigFlags { isOutOfBandFlag = 8, } -export type SerializerRead = ( -) => T; - -export type SerializerWrite = ( - v: T, -) => void; - -export type SerializerConfig = ( -) => { - reserve: number -}; - // read, write export type Serializer = { - read: SerializerRead - write: SerializerWrite - readWithoutType?: SerializerRead - writeWithoutType?: SerializerWrite - config: SerializerConfig + read: () => T2 + write: (v: T2) => T + readInner: () => T2 + writeInner: (v: T2) => T + meta: Meta }; export enum RefFlags { @@ -98,6 +87,9 @@ export enum RefFlags { export const MaxInt32 = 2147483647; export const MinInt32 = -2147483648; +export const HalfMaxInt32 = MaxInt32 / 2; +export const HalfMinInt32 = MinInt32 / 2; + export const LATIN1 = 0; export const UTF8 = 1; @@ -110,6 +102,9 @@ export interface Config { hps?: Hps refTracking?: boolean useSliceString?: boolean + hooks?: { + afterCodeGenerated?: (code: string) => string + } } export enum Language { diff --git a/javascript/packages/fury/lib/util.ts b/javascript/packages/fury/lib/util.ts index f7d33f0962..d78445e2dd 100644 --- a/javascript/packages/fury/lib/util.ts +++ b/javascript/packages/fury/lib/util.ts @@ -17,32 +17,6 @@ * under the License. */ -const isReserved = (key: string) => { - return /^(?:do|if|in|for|let|new|try|var|case|else|enum|eval|false|null|this|true|void|with|break|catch|class|const|super|throw|while|yield|delete|export|import|public|return|static|switch|typeof|default|extends|finally|package|private|continue|debugger|function|arguments|interface|protected|implements|instanceof)$/.test(key); -}; - -const isDotPropAccessor = (prop: string) => { - return /^[a-zA-Z_$][0-9a-zA-Z_$]*$/.test(prop); -}; - -export const replaceBackslashAndQuote = (v: string) => { - return v.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); -}; - -export const safePropAccessor = (prop: string) => { - if (!isDotPropAccessor(prop) || isReserved(prop)) { - return `["${replaceBackslashAndQuote(prop)}"]`; - } - return `.${prop}`; -}; - -export const safePropName = (prop: string) => { - if (!isDotPropAccessor(prop) || isReserved(prop)) { - return `["${replaceBackslashAndQuote(prop)}"]`; - } - return prop; -}; - export const isNodeEnv: boolean = typeof process !== "undefined" && process.versions != null diff --git a/javascript/packages/fury/lib/writer.ts b/javascript/packages/fury/lib/writer.ts index 4f14212312..d32017003e 100644 --- a/javascript/packages/fury/lib/writer.ts +++ b/javascript/packages/fury/lib/writer.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Config, LATIN1, UTF8 } from "./type"; +import { Config, HalfMaxInt32, HalfMinInt32, LATIN1, UTF8 } from "./type"; import { PlatformBuffer, alloc, strByteLength } from "./platformBuffer"; import { OwnershipError } from "./error"; @@ -97,13 +97,27 @@ export const BinaryWriter = (config: Config) => { cursor += 4; } - function int64(v: bigint | number) { - if (typeof v === "number") { - dataView.setBigInt64(cursor, BigInt(v), true); + function int64(v: bigint) { + dataView.setBigInt64(cursor, v, true); + cursor += 8; + } + + function sliLong(v: bigint | number) { + if (v <= HalfMaxInt32 && v >= HalfMinInt32) { + // write: + // 00xxx -> 0xxx + // 11xxx -> 1xxx + // read: + // 0xxx -> 00xxx + // 1xxx -> 11xxx + dataView.setUint32(cursor, Number(v) << 1, true); + cursor += 4; } else { - dataView.setBigInt64(cursor, v, true); + const BIG_LONG_FLAG = 0b1; // bit 0 set, means big long. + dataView.setUint8(cursor, BIG_LONG_FLAG); + cursor += 1; + varInt64(BigInt(v)); } - cursor += 8; } function float(v: number) { @@ -122,12 +136,8 @@ export const BinaryWriter = (config: Config) => { cursor += v.byteLength; } - function uint64(v: bigint | number) { - if (typeof v === "number") { - dataView.setBigUint64(cursor, BigInt(v), true); - } else { - dataView.setBigUint64(cursor, v, true); - } + function uint64(v: bigint) { + dataView.setBigUint64(cursor, v, true); cursor += 8; } @@ -215,16 +225,12 @@ export const BinaryWriter = (config: Config) => { cursor += len; } - function zigZag(v: number) { - return (v << 1) ^ (v >> 31); - } - - function varInt32(val: number) { - return varUInt32(zigZag(val)); + function varInt32(v: number) { + return varUInt32((v << 1) ^ (v >> 31)); } function varUInt32(val: number) { - val = val >>> 0; + val = (val >>> 0) & 0xFFFFFFFF; // keep only the lower 32 bits while (val > 127) { arrayBuffer[cursor++] = val & 127 | 128; val >>>= 7; @@ -232,6 +238,27 @@ export const BinaryWriter = (config: Config) => { arrayBuffer[cursor++] = val; } + function varInt64(v: bigint) { + if (typeof v !== "bigint") { + v = BigInt(v); + } + return varUInt64((v << 1n) ^ (v >> 63n)); + } + + function varUInt64(val: bigint | number) { + if (typeof val !== "bigint") { + val = BigInt(val); + } + val = val & 0xFFFFFFFFFFFFFFFFn; // keep only the lower 64 bits + + while (val > 127) { + arrayBuffer[cursor++] = Number(val & 127n | 128n); + val >>= 7n; + } + arrayBuffer[cursor++] = Number(val); + return; + } + function tryFreePool() { if (byteLength > MAX_POOL_SIZE) { initPoll(); @@ -287,6 +314,8 @@ export const BinaryWriter = (config: Config) => { int16, varInt32, varUInt32, + varUInt64, + varInt64, stringOfVarUInt32: config?.hps ? stringOfVarUInt32Fast() : stringOfVarUInt32Slow, @@ -296,6 +325,7 @@ export const BinaryWriter = (config: Config) => { double, float, int64, + sliLong, uint32, int32, getCursor, diff --git a/javascript/packages/fury/tsconfig.json b/javascript/packages/fury/tsconfig.json index 72b76112ad..176ef55b31 100644 --- a/javascript/packages/fury/tsconfig.json +++ b/javascript/packages/fury/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ diff --git a/javascript/packages/hps/tsconfig.json b/javascript/packages/hps/tsconfig.json index 72b76112ad..176ef55b31 100644 --- a/javascript/packages/hps/tsconfig.json +++ b/javascript/packages/hps/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ diff --git a/javascript/test/__snapshots__/codeGen.test.ts.snap b/javascript/test/__snapshots__/codeGen.test.ts.snap index 0fb0461e6e..fdcc52af8b 100644 --- a/javascript/test/__snapshots__/codeGen.test.ts.snap +++ b/javascript/test/__snapshots__/codeGen.test.ts.snap @@ -3,114 +3,604 @@ exports[`codeGen can generate tuple declaration code 1`] = ` "function anonymous( ) { - -return function (fury, scope) { - const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; - const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; +return function(fury, external) { + const br = fury.binaryReader; + const bw = fury.binaryWriter; + const cr = fury.classResolver; + const rr = fury.referenceResolver; + const tag_ser_3 = fury.classResolver.getSerializerByTag("example.foo.1"); + const tag_ser_4 = fury.classResolver.getSerializerByTag("example.foo.2"); + const tag_ser_12 = fury.classResolver.getSerializerByTag("example.bar.1"); + const tag_ser_13 = fury.classResolver.getSerializerByTag("example.bar.2"); + const tagWriter_19 = cr.createTagWriter("tuple-object-wrapper"); - const tag_0 = classResolver.getSerializerByTag("example.foo.1") - const tag_1 = classResolver.getSerializerByTag("example.foo.2") - const tuple_tag_0_tag_1 = tupleSerializer(fury, [tag_0, tag_1]) - const tag_5 = classResolver.getSerializerByTag("example.bar.1") - const tag_6 = classResolver.getSerializerByTag("example.bar.2") - const tuple_tag_0_tag_5_tag_6 = tupleSerializer(fury, [tag_0, tag_5, tag_6]) - const tagWriter = classResolver.createTagWriter("tuple-object-wrapper"); - - const reserves = tag_0.config().reserve + tag_1.config().reserve + tuple_tag_0_tag_1.config().reserve + tag_5.config().reserve + tag_6.config().reserve + tuple_tag_0_tag_5_tag_6.config().reserve; - return { - ...referenceResolver.deref(() => { - const hash = binaryReader.int32(); - if (hash !== 16469457) { - throw new Error("validate hash failed: tuple-object-wrapper. expect 16469457, but got" + hash); + const readInner = () => { + + if (br.int32() !== 16469457) { + throw new Error("validate hash failed: tuple-object-wrapper. expect 16469457"); + } + const result_0 = { + tuple1: null, + tuple1_: null, + tuple2: null, + tuple2_: null + }; + rr.pushReadObject(result_0) + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_2 = br.varUInt32(); + const result_1 = new Array(len_2); + rr.pushReadObject(result_1) + + if (len_2 > 0) { + result_1[0] = tag_ser_3.read() + } + + + if (len_2 > 1) { + result_1[1] = tag_ser_4.read() + } + + result_0.tuple1 = result_1 + + break; + case -2: + result_0.tuple1 = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple1 = null + break; + }; + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_6 = br.varUInt32(); + const result_5 = new Array(len_6); + rr.pushReadObject(result_5) + + if (len_6 > 0) { + result_5[0] = tag_ser_3.read() + } + + + if (len_6 > 1) { + result_5[1] = tag_ser_4.read() + } + + result_0.tuple1_ = result_5 + + break; + case -2: + result_0.tuple1_ = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple1_ = null + break; + }; + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_10 = br.varUInt32(); + const result_9 = new Array(len_10); + rr.pushReadObject(result_9) + + if (len_10 > 0) { + result_9[0] = tag_ser_3.read() + } + + + if (len_10 > 1) { + result_9[1] = tag_ser_12.read() + } + + + if (len_10 > 2) { + result_9[2] = tag_ser_13.read() + } + + result_0.tuple2 = result_9 + + break; + case -2: + result_0.tuple2 = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple2 = null + break; + }; + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_15 = br.varUInt32(); + const result_14 = new Array(len_15); + rr.pushReadObject(result_14) + + if (len_15 > 0) { + result_14[0] = tag_ser_3.read() + } + + + if (len_15 > 1) { + result_14[1] = tag_ser_12.read() + } + + + if (len_15 > 2) { + result_14[2] = tag_ser_13.read() + } + + result_0.tuple2_ = result_14 + + break; + case -2: + result_0.tuple2_ = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple2_ = null + break; + } + + return result_0 + + }; + const read = () => { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + return readInner() + break; + case -2: + return rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + return null + break; + } + + }; + const writeInner = (v) => { + + tagWriter_19.write(bw); + bw.int32(16469457); + + const existsId_22 = rr.existsWriteObject(v.tuple1); + if (typeof existsId_22 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_22) + } else { + rr.pushWriteObject(v.tuple1) + + if (v.tuple1 !== null && v.tuple1 !== undefined) { + bw.int24(6400); + + bw.varUInt32(2) + bw.reserve(148); + tag_ser_3.write(v.tuple1[0]) + tag_ser_4.write(v.tuple1[1]); + } else { + bw.int8(-3); } - { - - // relation tag: tuple-object-wrapper - const result = { - tuple1: null, -tuple1_: null, -tuple2: null, -tuple2_: null + }; - pushReadObject(result); - result.tuple1 = tuple_tag_0_tag_1.read(); -result.tuple1_ = tuple_tag_0_tag_1.read(); -result.tuple2 = tuple_tag_0_tag_5_tag_6.read(); -result.tuple2_ = tuple_tag_0_tag_5_tag_6.read() - return result; - + + const existsId_25 = rr.existsWriteObject(v.tuple1_); + if (typeof existsId_25 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_25) + } else { + rr.pushWriteObject(v.tuple1_) + + if (v.tuple1_ !== null && v.tuple1_ !== undefined) { + bw.int24(6400); + + bw.varUInt32(2) + bw.reserve(148); + tag_ser_3.write(v.tuple1_[0]) + tag_ser_4.write(v.tuple1_[1]); + } else { + bw.int8(-3); } - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { - tagWriter.write(binaryWriter); - binaryWriter.int32(16469457); - binaryWriter.reserve(reserves); - tuple_tag_0_tag_1.write(v.tuple1); -tuple_tag_0_tag_1.write(v.tuple1_); -tuple_tag_0_tag_5_tag_6.write(v.tuple2); -tuple_tag_0_tag_5_tag_6.write(v.tuple2_) - }), - config() { - return { - reserve: tagWriter.bufferLen + 8, + + }; + + const existsId_29 = rr.existsWriteObject(v.tuple2); + if (typeof existsId_29 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_29) + } else { + rr.pushWriteObject(v.tuple2) + + if (v.tuple2 !== null && v.tuple2 !== undefined) { + bw.int24(6400); + + bw.varUInt32(3) + bw.reserve(222); + tag_ser_3.write(v.tuple2[0]) + tag_ser_12.write(v.tuple2[1]) + tag_ser_13.write(v.tuple2[2]); + } else { + bw.int8(-3); + } + + }; + + const existsId_33 = rr.existsWriteObject(v.tuple2_); + if (typeof existsId_33 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_33) + } else { + rr.pushWriteObject(v.tuple2_) + + if (v.tuple2_ !== null && v.tuple2_ !== undefined) { + bw.int24(6400); + + bw.varUInt32(3) + bw.reserve(222); + tag_ser_3.write(v.tuple2_[0]) + tag_ser_12.write(v.tuple2_[1]) + tag_ser_13.write(v.tuple2_[2]); + } else { + bw.int8(-3); } + } - } + + + }; + const write = (v) => { + + const existsId_34 = rr.existsWriteObject(v); + if (typeof existsId_34 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_34) + } else { + rr.pushWriteObject(v) + + if (v !== null && v !== undefined) { + bw.int24(65536); + writeInner(v); + } else { + bw.int8(-3); + } + + } + + }; + + return { + read, + readInner, + write, + writeInner, + meta: { + "fixedSize": 67, + "noneable": true + } + }; } - }" `; exports[`codeGen can generate tuple declaration code 2`] = ` "function anonymous( ) { - -return function (fury, scope) { - const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; - const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; +return function(fury, external) { + const br = fury.binaryReader; + const bw = fury.binaryWriter; + const cr = fury.classResolver; + const rr = fury.referenceResolver; + const tagWriter_6 = cr.createTagWriter("tuple-object-type3-tag"); - const type_13 = classResolver.getSerializerById(13) - const type_1 = classResolver.getSerializerById(1) - const type_6 = classResolver.getSerializerById(6) - const type_14 = classResolver.getSerializerById(14) - const tuple_type_14 = tupleSerializer(fury, [type_14]) - const tuple_type_13_type_1_type_6_tuple_type_14 = tupleSerializer(fury, [type_13, type_1, type_6, tuple_type_14]) - const tagWriter = classResolver.createTagWriter("tuple-object-type3-tag"); - - const reserves = type_13.config().reserve + type_1.config().reserve + type_6.config().reserve + type_14.config().reserve + tuple_type_14.config().reserve + tuple_type_13_type_1_type_6_tuple_type_14.config().reserve; - return { - ...referenceResolver.deref(() => { - const hash = binaryReader.int32(); - if (hash !== 552) { - throw new Error("validate hash failed: tuple-object-type3-tag. expect 552, but got" + hash); - } - { - - // relation tag: tuple-object-type3-tag - const result = { - tuple: null + const readInner = () => { + + if (br.int32() !== 552) { + throw new Error("validate hash failed: tuple-object-type3-tag. expect 552"); + } + const result_0 = { + tuple: null }; - pushReadObject(result); - result.tuple = tuple_type_13_type_1_type_6_tuple_type_14.read() - return result; - + rr.pushReadObject(result_0) + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_2 = br.varUInt32(); + const result_1 = new Array(len_2); + rr.pushReadObject(result_1) + + if (len_2 > 0) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + result_1[0] = br.stringOfVarUInt32() + break; + case -2: + result_1[0] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[0] = null + break; + } + + } + + + if (len_2 > 1) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + result_1[1] = br.uint8() === 1 + break; + case -2: + result_1[1] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[1] = null + break; + } + + } + + + if (len_2 > 2) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + result_1[2] = br.uint32() + break; + case -2: + result_1[2] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[2] = null + break; + } + + } + + + if (len_2 > 3) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_4 = br.varUInt32(); + const result_3 = new Array(len_4); + rr.pushReadObject(result_3) + + if (len_4 > 0) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + br.uint8() + result_5 = br.buffer(br.int32()); + rr.pushReadObject(result_5); + result_3[0] = result_5 + + break; + case -2: + result_3[0] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_3[0] = null + break; + } + + } + + result_1[3] = result_3 + + break; + case -2: + result_1[3] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[3] = null + break; + } + + } + + result_0.tuple = result_1 + + break; + case -2: + result_0.tuple = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple = null + break; + } + + return result_0 + + }; + const read = () => { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + return readInner() + break; + case -2: + return rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + return null + break; + } + + }; + const writeInner = (v) => { + + tagWriter_6.write(bw); + bw.int32(552); + + const existsId_9 = rr.existsWriteObject(v.tuple); + if (typeof existsId_9 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_9) + } else { + rr.pushWriteObject(v.tuple) + + if (v.tuple !== null && v.tuple !== undefined) { + bw.int24(6400); + + bw.varUInt32(4) + bw.reserve(26); + + bw.int24(3583); + if (v.tuple[0] !== null && v.tuple[0] !== undefined) { + bw.stringOfVarUInt32(v.tuple[0]); + } else { + bw.stringOfVarUInt32(""); + } + + bw.int24(511); + if (v.tuple[1] !== null && v.tuple[1] !== undefined) { + bw.uint8(v.tuple[1] ? 1 : 0); + } else { + bw.uint8(0 ? 1 : 0); + } + + bw.int24(1791); + if (v.tuple[2] !== null && v.tuple[2] !== undefined) { + bw.uint32(v.tuple[2]); + } else { + bw.uint32(0); + } + + const existsId_8 = rr.existsWriteObject(v.tuple[3]); + if (typeof existsId_8 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_8) + } else { + rr.pushWriteObject(v.tuple[3]) + + if (v.tuple[3] !== null && v.tuple[3] !== undefined) { + bw.int24(6400); + + bw.varUInt32(1) + bw.reserve(8); + + const existsId_7 = rr.existsWriteObject(v.tuple[3][0]); + if (typeof existsId_7 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_7) + } else { + rr.pushWriteObject(v.tuple[3][0]) + + if (v.tuple[3][0] !== null && v.tuple[3][0] !== undefined) { + bw.int24(3584); + + bw.uint8(1) + bw.uint32(v.tuple[3][0].byteLength) + bw.buffer(v.tuple[3][0]); + } else { + bw.int8(-3); + } + + } + + ; + } else { + bw.int8(-3); + } + + } + + ; + } else { + bw.int8(-3); } - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { - tagWriter.write(binaryWriter); - binaryWriter.int32(552); - binaryWriter.reserve(reserves); - tuple_type_13_type_1_type_6_tuple_type_14.write(v.tuple) - }), - config() { - return { - reserve: tagWriter.bufferLen + 8, + + } + + + }; + const write = (v) => { + + const existsId_10 = rr.existsWriteObject(v); + if (typeof existsId_10 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_10) + } else { + rr.pushWriteObject(v) + + if (v !== null && v !== undefined) { + bw.int24(65536); + writeInner(v); + } else { + bw.int8(-3); } + } - } + + }; + + return { + read, + readInner, + write, + writeInner, + meta: { + "fixedSize": 48, + "noneable": true + } + }; } - }" `; diff --git a/javascript/test/any.test.ts b/javascript/test/any.test.ts index e0c2d94fa5..05f791b710 100644 --- a/javascript/test/any.test.ts +++ b/javascript/test/any.test.ts @@ -47,7 +47,7 @@ describe('bool', () => { test('should write big number work', () => { const fury = new Fury(); const bin = fury.serialize(3000000000); - expect(fury.deserialize(bin)).toBe(BigInt(3000000000)) + expect(fury.deserialize(bin)).toBe(3000000000n); }); test('should write INFINITY work', () => { diff --git a/javascript/test/array.test.ts b/javascript/test/array.test.ts index ea9c7d2217..b862226e17 100644 --- a/javascript/test/array.test.ts +++ b/javascript/test/array.test.ts @@ -69,7 +69,7 @@ describe('array', () => { a3: [3, 5, 76], a4: [634, 564, 76], a6: [234243.555, 55654.6786], - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }, serializer); const result = fury.deserialize( input @@ -81,7 +81,7 @@ describe('array', () => { a3: [3, 5, 76], a4: [634, 564, 76], a6: [234243.555, 55654.6786], - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }) }); test('should string array work', () => { @@ -100,13 +100,13 @@ describe('array', () => { const fury = new Fury({ refTracking: true }); const serializer = fury.registerSerializer(description).serializer; const input = fury.serialize({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }, serializer); const result = fury.deserialize( input ); expect(result).toEqual({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }) }); test('should string array work when latin1 enable', () => { @@ -125,13 +125,13 @@ describe('array', () => { const fury = new Fury({ refTracking: true }); const serializer = fury.registerSerializer(description).serializer; const input = fury.serialize({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }, serializer); const result = fury.deserialize( input ); expect(result).toEqual({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }) }); test('should floatarray work', () => { diff --git a/javascript/test/codeGen.test.ts b/javascript/test/codeGen.test.ts index fae12a1f05..5432396532 100644 --- a/javascript/test/codeGen.test.ts +++ b/javascript/test/codeGen.test.ts @@ -19,16 +19,23 @@ import { describe, expect, test } from '@jest/globals'; import { tupleObjectDescription, tupleObjectType3Description } from './fixtures/tuple'; -import { generateInlineCode } from '../packages/fury/lib/codeGen'; +import { generate } from '../packages/fury/lib/gen/index'; import FuryInternal from '../packages/fury/lib/fury'; +import * as beautify from 'js-beautify'; describe('codeGen', () => { test('can generate tuple declaration code', () => { - const fury = FuryInternal({ refTracking: true }); - const fn = generateInlineCode(fury, tupleObjectDescription); + const fury = FuryInternal({ + refTracking: true, hooks: { + afterCodeGenerated: (code: string) => { + return beautify.js(code, { indent_size: 2, space_in_empty_paren: true, indent_empty_lines: true }); + } + } + }); + const fn = generate(fury, tupleObjectDescription); expect(fn.toString()).toMatchSnapshot(); - const fn2 = generateInlineCode(fury, tupleObjectType3Description); + const fn2 = generate(fury, tupleObjectType3Description); expect(fn2.toString()).toMatchSnapshot(); }) }) diff --git a/javascript/test/fury.test.ts b/javascript/test/fury.test.ts index 55e306a2c2..c3723aa9c2 100644 --- a/javascript/test/fury.test.ts +++ b/javascript/test/fury.test.ts @@ -17,8 +17,9 @@ * under the License. */ -import Fury, { TypeDescription, InternalSerializerType } from '../packages/fury/index'; +import Fury, { TypeDescription, Type } from '../packages/fury/index'; import { describe, expect, test } from '@jest/globals'; +import { fromUint8Array } from '../packages/fury/lib/platformBuffer'; describe('fury', () => { test('should deserialize null work', () => { @@ -55,4 +56,64 @@ describe('fury', () => { expect(error.message).toBe('outofband mode is not supported now'); } }); + + test('should register work', () => { + const fury = new Fury(); + const { serialize, deserialize } = fury.registerSerializer(Type.array(Type.string())); + const bin = serialize(["hello", "world"]); + expect(deserialize(bin)).toEqual(["hello", "world"]); + }); + + describe('serializer description should work', () => { + test('can serialize and deserialize primitive types', () => { + const description = Type.int8() + testDescription(description, 123) + + const description2 = Type.int16() + testDescription(description2, 123) + + const description3 = Type.int32() + testDescription(description3, 123) + + const description4 = Type.bool() + testDescription(description4, true) + + // has precision problem + // const description5 = Type.float() + // testDescription(description5, 123.456) + + const description6 = Type.double() + testDescription(description6, 123.456789) + + const description7 = Type.binary() + testDescription(description7, new Uint8Array([1, 2, 3]), fromUint8Array(new Uint8Array([1, 2, 3]))); + + const description8 = Type.string() + testDescription(description8, '123') + + const description9 = Type.set(Type.string()) + testDescription(description9, new Set(['123'])) + }) + + test('can serialize and deserialize array', () => { + const description = Type.array(Type.int8()) + testDescription(description, [1, 2, 3]) + testDescription(description, []) + }) + + test('can serialize and deserialize tuple', () => { + const description = Type.tuple([Type.int8(), Type.int16(), Type.timestamp()]) + testDescription(description, [1, 2, new Date()]) + }) + + + function testDescription(description: TypeDescription, input: any, expected?: any) { + const fury = new Fury(); + const serialize = fury.registerSerializer(description); + const result = serialize.deserialize( + serialize.serialize(input) + ); + expect(result).toEqual(expected ?? input) + } + }) }); diff --git a/javascript/test/io.test.ts b/javascript/test/io.test.ts index 38f1b3f2c6..8d5edf9f7c 100644 --- a/javascript/test/io.test.ts +++ b/javascript/test/io.test.ts @@ -41,7 +41,6 @@ function num2Bin(num: number) { 8, 16, 32, - 64 ].forEach((x, y) => { { writer[`uint${x}`](10); @@ -93,7 +92,6 @@ function num2Bin(num: number) { 8, 16, 32, - 64 ].forEach((x, y) => { { writer[`int${x}`](10); @@ -354,5 +352,59 @@ function num2Bin(num: number) { expect(reader.int8()).toBe(10); expect(reader.int16()).toBe(20); }); + + test('should varUInt64 work', () => { + const writer = BinaryWriter(config); + writer.varUInt64(2n ** 2n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varUInt64()).toBe(2n ** 2n); + }); + + test('should varUInt64 work', () => { + const writer = BinaryWriter(config); + writer.varUInt64(2n ** 63n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varUInt64()).toBe(2n ** 63n); + }); + + test('should varInt64 work', () => { + const writer = BinaryWriter(config); + writer.varInt64(2n ** 2n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varInt64()).toBe(2n ** 2n); + }); + + test('should varInt64 work', () => { + const writer = BinaryWriter(config); + writer.varInt64(2n ** 62n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varInt64()).toBe(2n ** 62n); + }); + + test('should silong work', () => { + const writer = BinaryWriter(config); + writer.sliLong(2n ** 2n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.sliLong()).toBe(2n ** 2n); + }); + + test('should silong work', () => { + const writer = BinaryWriter(config); + writer.sliLong(2n ** 62n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.sliLong()).toBe(2n ** 62n); + }); }); }) diff --git a/javascript/test/object.test.ts b/javascript/test/object.test.ts index 88692b5c51..8e8676acc6 100644 --- a/javascript/test/object.test.ts +++ b/javascript/test/object.test.ts @@ -218,17 +218,6 @@ describe('object', () => { expect(result).toEqual({ a: { b: "hel", c: [{ d: "hello" }] } }) }); - test('should register work', () => { - const description = Type.string(); - const fury = new Fury({ refTracking: true }); - try { - fury.registerSerializer(description); - throw new Error('unreachable code') - } catch (error) { - expect(error.message).toBe("root type should be object"); - } - }); - test("should partial record work", () => { const hps = undefined; const description = Type.object('ws-channel-protocol', {