Skip to content

Commit

Permalink
feat(javascript): Refactor & Compress Long (#1313)
Browse files Browse the repository at this point in the history
## Refactor the code generator
1. Currently, JavaScript can only register a Object by Type.object,
because the generator is too simple
2. Primitive generator of types like string、number can't be inlined in
the holder collection, which cause necessary polymorphic that affect the
performance

## Enhancement performance
1. Currently, Buffer.latin1Slice would transfer from native code
generated by JIT to v8 runtime, This has a significant impact on
performance, so we use String.fromCharCode to create little string when
the string length is less than 15. It is a magic number, but I have
tested it and it works fine.

## Improve
When useSliceString is disabled, the performance of deserialization
improves by 100%. It is 3 times faster than native JSON and twice as
fast as protobuf.

When useSliceString is enabled, the performance of deserialization
improves by 30%. It is 5 times faster than native JSON and 3 times
faster than protobuf.

### Before:

#### Disable useSliceString

![image](https://github.com/apache/incubator-fury/assets/16490211/83278ece-0eba-4aa5-81a5-e2806bfd1997)

#### Enable useSliceString

![image](https://github.com/apache/incubator-fury/assets/16490211/abaeb420-274a-4aae-b432-750e96061668)

### After:
#### Disable useSliceString

![image](https://github.com/apache/incubator-fury/assets/16490211/2d0e1d3f-e3e4-432e-8b20-8d934b848a76)

#### Enable useSliceString

![image](https://github.com/apache/incubator-fury/assets/16490211/a9939cbf-cede-4a1d-8720-ac0f04a3abd6)
  • Loading branch information
theweipeng authored Jan 6, 2024
1 parent f49e662 commit 44cfbb7
Show file tree
Hide file tree
Showing 50 changed files with 2,885 additions and 1,292 deletions.
6 changes: 4 additions & 2 deletions javascript/benchmark/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const Type = Fury.Type;
const assert = require('assert');
const { spawn } = require("child_process");


const sample = {
id: 123456,
name: "John Doe",
Expand Down Expand Up @@ -106,8 +107,9 @@ const sample = {
`,
};


const description = utils.mockData2Description(sample, "fury.test.foo");
const { serialize, deserialize } = fury.registerSerializer(description);
const { serialize, deserialize, serializeVolatile } = fury.registerSerializer(description);

const furyAb = serialize(sample);
const sampleJson = JSON.stringify(sample);
Expand Down Expand Up @@ -164,7 +166,7 @@ async function start() {
var suite = new Benchmark.Suite();
suite
.add("fury", function () {
serialize(sample);
serializeVolatile(sample).dispose();
})
.add("json", function () {
JSON.stringify(sample);
Expand Down
Binary file modified javascript/benchmark/sample.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions javascript/jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ module.exports = {
],
transform: {
'\\.ts$': ['ts-jest', {
tsconfig: {
target: "ES2021"
},
diagnostics: {
ignoreCodes: [151001]
}
Expand Down
4 changes: 3 additions & 1 deletion javascript/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
],
"devDependencies": {
"@stylistic/eslint-plugin": "^1.5.1",
"eslint": "^8.55.0"
"@types/js-beautify": "^1.14.3",
"eslint": "^8.55.0",
"js-beautify": "^1.14.11"
}
}
13 changes: 3 additions & 10 deletions javascript/packages/fury/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,9 @@
*/

import {
genSerializer,
} from "./lib/codeGen";
generateSerializer,
} from "./lib/gen";
import {
Cast,
ObjectTypeDescription,
TypeDescription,
ArrayTypeDescription,
Expand All @@ -45,13 +44,7 @@ export default class {
private fury: Fury = FuryInternal(this.config || {});

registerSerializer<T extends TypeDescription>(description: T) {
if (
description.type !== InternalSerializerType.FURY_TYPE_TAG
|| !Cast<ObjectTypeDescription>(description)?.options.tag
) {
throw new Error("root type should be object");
}
const serializer = genSerializer(this.fury, description);
const serializer = generateSerializer(this.fury, description);
return {
serializer,
serialize: (data: ToRecordType<T>) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,53 +17,59 @@
* under the License.
*/

import { Fury, MaxInt32, MinInt32, Serializer } from "../type";
import { InternalSerializerType, RefFlags } from "../type";
import { Type } from "./description";
import { getMeta } from "./meta";
import { Fury, MaxInt32, MinInt32, Serializer } from "./type";
import { InternalSerializerType, RefFlags } from "./type";

export default (fury: Fury) => {
const { binaryReader, binaryWriter, referenceResolver, classResolver } = fury;

function detectSerializer(cursor: number) {
function detectSerializer() {
const typeId = binaryReader.int16();
let serializer: Serializer;
if (typeId === InternalSerializerType.FURY_TYPE_TAG) {
serializer = classResolver.getSerializerByTag(classResolver.detectTag(binaryReader));
const tag = classResolver.readTag(binaryReader)();
serializer = classResolver.getSerializerByTag(tag);
} else {
serializer = classResolver.getSerializerById(typeId);
}
if (!serializer) {
throw new Error(`cant find implements of typeId: ${typeId}`);
}
binaryReader.setCursor(cursor);

return serializer;
}

return {
readInner: () => {
throw new Error("any can not call readInner");
},
writeInner: () => {
throw new Error("any can not call writeInner");
},
read: () => {
const cursor = binaryReader.getCursor();
const flag = referenceResolver.readRefFlag();
switch (flag) {
case RefFlags.RefValueFlag:
return detectSerializer(cursor).read();
return detectSerializer().readInner();
case RefFlags.RefFlag:
return referenceResolver.getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
return detectSerializer(cursor).read();
return detectSerializer().readInner();
}
},
write: (v: any) => {
const { write: writeInt64, config: int64Config } = classResolver.getSerializerById(InternalSerializerType.INT64);
const { write: writeDouble, config: doubleConfig } = classResolver.getSerializerById(InternalSerializerType.DOUBLE);
const { write: writeInt32, config: int32Config } = classResolver.getSerializerById(InternalSerializerType.INT32);
const { write: writeBool, config: boolConfig } = classResolver.getSerializerById(InternalSerializerType.BOOL);
const { write: stringWrite, config: stringConfig } = classResolver.getSerializerById(InternalSerializerType.STRING);
const { write: arrayWrite, config: arrayConfig } = classResolver.getSerializerById(InternalSerializerType.ARRAY);
const { write: mapWrite, config: mapConfig } = classResolver.getSerializerById(InternalSerializerType.MAP);
const { write: setWrite, config: setConfig } = classResolver.getSerializerById(InternalSerializerType.FURY_SET);
const { write: timestampWrite, config: timestampConfig } = classResolver.getSerializerById(InternalSerializerType.TIMESTAMP);
const { write: writeInt64, meta: int64Meta } = classResolver.getSerializerById(InternalSerializerType.INT64);
const { write: writeDouble, meta: doubleMeta } = classResolver.getSerializerById(InternalSerializerType.DOUBLE);
const { write: writeInt32, meta: int32Meta } = classResolver.getSerializerById(InternalSerializerType.INT32);
const { write: writeBool, meta: boolMeta } = classResolver.getSerializerById(InternalSerializerType.BOOL);
const { write: stringWrite, meta: stringMeta } = classResolver.getSerializerById(InternalSerializerType.STRING);
const { write: arrayWrite, meta: arrayMeta } = classResolver.getSerializerById(InternalSerializerType.ARRAY);
const { write: mapWrite, meta: mapMeta } = classResolver.getSerializerById(InternalSerializerType.MAP);
const { write: setWrite, meta: setMeta } = classResolver.getSerializerById(InternalSerializerType.FURY_SET);
const { write: timestampWrite, meta: timestampMeta } = classResolver.getSerializerById(InternalSerializerType.TIMESTAMP);

// NullFlag
if (v === null || v === undefined) {
Expand All @@ -81,68 +87,64 @@ export default (fury: Fury) => {
}
if (Number.isInteger(v)) {
if (v > MaxInt32 || v < MinInt32) {
binaryWriter.reserve(int64Config().reserve);
binaryWriter.reserve(int64Meta.fixedSize);
writeInt64(BigInt(v));
return;
} else {
binaryWriter.reserve(int32Config().reserve);
binaryWriter.reserve(int32Meta.fixedSize);
writeInt32(v);
return;
}
} else {
binaryWriter.reserve(doubleConfig().reserve);
binaryWriter.reserve(doubleMeta.fixedSize);
writeDouble(v);
return;
}
}

if (typeof v === "bigint") {
binaryWriter.reserve(int64Config().reserve);
binaryWriter.reserve(int64Meta.fixedSize);
writeInt64(v);
return;
}

if (typeof v === "boolean") {
binaryWriter.reserve(boolConfig().reserve);
binaryWriter.reserve(boolMeta.fixedSize);
writeBool(v);
return;
}

if (v instanceof Date) {
binaryWriter.reserve(timestampConfig().reserve);
binaryWriter.reserve(timestampMeta.fixedSize);
timestampWrite(v);
return;
}

if (typeof v === "string") {
binaryWriter.reserve(stringConfig().reserve);
binaryWriter.reserve(stringMeta.fixedSize);
stringWrite(v);
return;
}

if (v instanceof Map) {
binaryWriter.reserve(5);
binaryWriter.reserve(mapConfig().reserve);
binaryWriter.reserve(mapMeta.fixedSize);
mapWrite(v);
return;
}
if (v instanceof Set) {
binaryWriter.reserve(setConfig().reserve);
binaryWriter.reserve(setMeta.fixedSize);
setWrite(v);
return;
}
if (Array.isArray(v)) {
binaryWriter.reserve(arrayConfig().reserve);
binaryWriter.reserve(arrayMeta.fixedSize);
arrayWrite(v);
return;
}

throw new Error(`serializer not support ${typeof v} yet`);
},
config: () => {
return {
reserve: 11,
};
},
meta: getMeta(Type.any(), fury),
};
};
Loading

0 comments on commit 44cfbb7

Please sign in to comment.