Skip to content

Commit

Permalink
feature: compress the numbers so that they correspond to standard pro…
Browse files Browse the repository at this point in the history 
…tocol
  • Loading branch information
@theweipeng
theweipeng committed Jan 2, 2024
1 parent 1134cce commit 77fce72
Show file tree
Hide file tree
Showing 11 changed files with 83 additions and 53 deletions.
2 changes: 1 addition & 1 deletion javascript/packages/fury/lib/internalSerializer/any.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export default (fury: Fury) => {
case RefFlags.RefValueFlag:
return detectSerializer(cursor).read();
case RefFlags.RefFlag:
return referenceResolver.getReadObjectByRefId(binaryReader.varInt32());
return referenceResolver.getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/array.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ export const buildArray = (fury: Fury, item: Serializer, type: InternalSerialize
const { binaryReader, binaryWriter, referenceResolver } = fury;

const { pushReadObject } = referenceResolver;
const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { varUInt32: readVarUInt32 } = binaryReader;
const { write, read } = item;
const innerHeadSize = (item.config().reserve);
return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Array(len);
pushReadObject(result);
for (let i = 0; i < result.length; i++) {
Expand All @@ -39,7 +39,7 @@ export const buildArray = (fury: Fury, item: Serializer, type: InternalSerialize
return result;
}),
write: referenceResolver.withNullableOrRefWriter(type, (v: any[]) => {
writeVarInt32(v.length);
writeVarUInt32(v.length);

reserves(innerHeadSize * v.length);

Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/map.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ import { InternalSerializerType, Fury, Serializer } from "../type";

export default (fury: Fury, keySerializer: Serializer, valueSerializer: Serializer) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: readVarUInt32 } = binaryReader;

const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { pushReadObject } = referenceResolver;
const innerHeadSize = keySerializer.config().reserve + valueSerializer.config().reserve;
return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Map();
pushReadObject(result);
for (let index = 0; index < len; index++) {
Expand All @@ -40,7 +40,7 @@ export default (fury: Fury, keySerializer: Serializer, valueSerializer: Serializ
}),
write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.MAP, (v: Map<any, any>) => {
const len = v.size;
writeVarInt32(len);
writeVarUInt32(len);
reserves(innerHeadSize * v.size);
for (const [key, value] of v.entries()) {
keySerializer.write(key);
Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/set.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ import { InternalSerializerType } from "../type";

export default (fury: Fury, nestedSerializer: Serializer) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;
const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { varUInt32: readVarUInt32 } = binaryReader;
const { pushReadObject } = referenceResolver;
const innerHeadSize = nestedSerializer.config().reserve;
return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Set();
pushReadObject(result);
for (let index = 0; index < len; index++) {
Expand All @@ -38,7 +38,7 @@ export default (fury: Fury, nestedSerializer: Serializer) => {
}),
write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_SET, (v: Set<any>) => {
const len = v.size;
writeVarInt32(len);
writeVarUInt32(len);
reserves(innerHeadSize * v.size);
for (const value of v.values()) {
nestedSerializer.write(value);
Expand Down
10 changes: 5 additions & 5 deletions javascript/packages/fury/lib/internalSerializer/string.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,23 @@ import { InternalSerializerType, RefFlags } from "../type";

export default (fury: Fury) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;
const { stringOfVarInt32: writeStringOfVarInt32, int8 } = binaryWriter;
const { stringOfVarInt32: readStringOfVarInt32 } = binaryReader;
const { stringOfVarUInt32: writeStringOfVarUInt32, int8 } = binaryWriter;
const { stringOfVarUInt32: readStringOfVarUInt32 } = binaryReader;

return {
...referenceResolver.deref(() => {
return readStringOfVarInt32();
return readStringOfVarUInt32();
}),
write: referenceResolver.withNotNullableWriter(InternalSerializerType.STRING, "", (v: string) => {
writeStringOfVarInt32(v);
writeStringOfVarUInt32(v);
}),
writeWithoutType: (v: string) => {
if (v === null) {
binaryWriter.int8(RefFlags.NullFlag);
return;
}
int8(RefFlags.NotNullValueFlag);
writeStringOfVarInt32(v);
writeStringOfVarUInt32(v);
},
config: () => {
return {
Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/tuple.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;

const { pushReadObject } = referenceResolver;
const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { varUInt32: readVarUInt32 } = binaryReader;

return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Array(len);
pushReadObject(result);
for (let i = 0; i < len; i++) {
Expand All @@ -39,7 +39,7 @@ export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => {
return result;
}),
write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.TUPLE, (v: any[]) => {
writeVarInt32(serializers.length);
writeVarUInt32(serializers.length);

for (let i = 0; i < serializers.length; i++) {
const item = serializers[i];
Expand Down
17 changes: 13 additions & 4 deletions javascript/packages/fury/lib/reader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,9 @@ export const BinaryReader = (config: Config) => {
return result;
}

function stringOfVarInt32() {
function stringOfVarUInt32() {
const isLatin1 = uint8() === LATIN1;
const len = varInt32();
const len = varUInt32();
return isLatin1 ? stringLatin1(len) : stringUtf8(len);
}

Expand Down Expand Up @@ -140,7 +140,11 @@ export const BinaryReader = (config: Config) => {
return result;
}

function varInt32() {
function zigZag(v: number) {
return (v >> 1) ^ -(v & 1);
}

function varUInt32() {
let byte_ = int8();
let result = byte_ & 0x7f;
if ((byte_ & 0x80) != 0) {
Expand All @@ -162,10 +166,15 @@ export const BinaryReader = (config: Config) => {
return result;
}

function varInt32() {
return zigZag(varUInt32());
}

return {
getCursor: () => cursor,
setCursor: (v: number) => (cursor = v),
varInt32,
varUInt32,
int8,
buffer: binary,
bufferRef,
Expand All @@ -174,7 +183,7 @@ export const BinaryReader = (config: Config) => {
stringUtf8At,
stringUtf8,
stringLatin1,
stringOfVarInt32,
stringOfVarUInt32,
double,
float,
uint16,
Expand Down
6 changes: 3 additions & 3 deletions javascript/packages/fury/lib/referenceResolver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ export const ReferenceResolver = (
const existsId = existsWriteObject(v);
if (typeof existsId === "number") {
binaryWriter.int8(RefFlags.RefFlag);
binaryWriter.varInt32(existsId);
binaryWriter.varUInt32(existsId);
} else {
int24(head);
pushWriteObject(v);
Expand Down Expand Up @@ -137,7 +137,7 @@ export const ReferenceResolver = (
skipType();
return fn();
case RefFlags.RefFlag:
return getReadObjectByRefId(binaryReader.varInt32());
return getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
Expand All @@ -150,7 +150,7 @@ export const ReferenceResolver = (
case RefFlags.RefValueFlag:
return fn();
case RefFlags.RefFlag:
return getReadObjectByRefId(binaryReader.varInt32());
return getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
Expand Down
23 changes: 16 additions & 7 deletions javascript/packages/fury/lib/writer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,13 @@ export const BinaryWriter = (config: Config) => {
}
}

function stringOfVarInt32Fast() {
function stringOfVarUInt32Fast() {
const { isLatin1: detectIsLatin1, stringCopy } = config!.hps!;
return function (v: string) {
const isLatin1 = detectIsLatin1(v);
const len = isLatin1 ? v.length : strByteLength(v);
dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8);
varInt32(len);
varUInt32(len);
reserve(len);
if (isLatin1) {
stringCopy(v, arrayBuffer, cursor);
Expand All @@ -191,11 +191,11 @@ export const BinaryWriter = (config: Config) => {
};
}

function stringOfVarInt32Slow(v: string) {
function stringOfVarUInt32Slow(v: string) {
const len = strByteLength(v);
const isLatin1 = len === v.length;
dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8);
varInt32(len);
varUInt32(len);
reserve(len);
if (isLatin1) {
if (len < 40) {
Expand All @@ -215,7 +215,15 @@ export const BinaryWriter = (config: Config) => {
cursor += len;
}

function zigZag(v: number) {
return (v << 1) ^ (v >> 31);
}

function varInt32(val: number) {
return varUInt32(zigZag(val));
}

function varUInt32(val: number) {
val = val >>> 0;
while (val > 127) {
arrayBuffer[cursor++] = val & 127 | 128;
Expand Down Expand Up @@ -278,9 +286,10 @@ export const BinaryWriter = (config: Config) => {
uint8,
int16,
varInt32,
stringOfVarInt32: config?.hps
? stringOfVarInt32Fast()
: stringOfVarInt32Slow,
varUInt32,
stringOfVarUInt32: config?.hps
? stringOfVarUInt32Fast()
: stringOfVarUInt32Slow,
bufferWithoutMemCheck,
uint64,
buffer,
Expand Down
Loading

0 comments on commit 77fce72

Please sign in to comment.