From 44cfbb7e22aff615ac46389a3d1d4fd1f805b14a Mon Sep 17 00:00:00 2001 From: weipeng Date: Sun, 7 Jan 2024 00:43:36 +0800 Subject: [PATCH] feat(javascript): Refactor & Compress Long (#1313) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Refactor the code generator 1. Currently, JavaScript can only register a Object by Type.object, because the generator is too simple 2. Primitive generator of types like string、number can't be inlined in the holder collection, which cause necessary polymorphic that affect the performance ## Enhancement performance 1. Currently, Buffer.latin1Slice would transfer from native code generated by JIT to v8 runtime, This has a significant impact on performance, so we use String.fromCharCode to create little string when the string length is less than 15. It is a magic number, but I have tested it and it works fine. ## Improve When useSliceString is disabled, the performance of deserialization improves by 100%. It is 3 times faster than native JSON and twice as fast as protobuf. When useSliceString is enabled, the performance of deserialization improves by 30%. It is 5 times faster than native JSON and 3 times faster than protobuf. ### Before: #### Disable useSliceString ![image](https://github.com/apache/incubator-fury/assets/16490211/83278ece-0eba-4aa5-81a5-e2806bfd1997) #### Enable useSliceString ![image](https://github.com/apache/incubator-fury/assets/16490211/abaeb420-274a-4aae-b432-750e96061668) ### After: #### Disable useSliceString ![image](https://github.com/apache/incubator-fury/assets/16490211/2d0e1d3f-e3e4-432e-8b20-8d934b848a76) #### Enable useSliceString ![image](https://github.com/apache/incubator-fury/assets/16490211/a9939cbf-cede-4a1d-8720-ac0f04a3abd6) --- javascript/benchmark/index.js | 6 +- javascript/benchmark/sample.jpg | Bin 27259 -> 28373 bytes javascript/jest.config.js | 3 + javascript/package.json | 4 +- javascript/packages/fury/index.ts | 13 +- .../fury/lib/{internalSerializer => }/any.ts | 68 +- javascript/packages/fury/lib/classResolver.ts | 141 ++-- javascript/packages/fury/lib/codeGen.ts | 224 ------ javascript/packages/fury/lib/description.ts | 85 ++- javascript/packages/fury/lib/fury.ts | 13 +- javascript/packages/fury/lib/gen/any.ts | 63 ++ javascript/packages/fury/lib/gen/array.ts | 80 +++ javascript/packages/fury/lib/gen/binary.ts | 54 ++ .../binary.ts => gen/bool.ts} | 47 +- javascript/packages/fury/lib/gen/builder.ts | 339 +++++++++ javascript/packages/fury/lib/gen/datetime.ts | 76 ++ javascript/packages/fury/lib/gen/index.ts | 90 +++ javascript/packages/fury/lib/gen/map.ts | 97 +++ javascript/packages/fury/lib/gen/number.ts | 105 +++ javascript/packages/fury/lib/gen/object.ts | 138 ++++ .../bool.ts => gen/router.ts} | 42 +- javascript/packages/fury/lib/gen/scope.ts | 55 ++ .../packages/fury/lib/gen/serializer.ts | 164 +++++ javascript/packages/fury/lib/gen/set.ts | 80 +++ javascript/packages/fury/lib/gen/string.ts | 44 ++ javascript/packages/fury/lib/gen/tuple.ts | 90 +++ .../packages/fury/lib/gen/typedArray.ts | 85 +++ .../fury/lib/internalSerializer/array.ts | 75 -- .../fury/lib/internalSerializer/datetime.ts | 66 -- .../fury/lib/internalSerializer/map.ts | 56 -- .../fury/lib/internalSerializer/number.ts | 238 ------- .../fury/lib/internalSerializer/set.ts | 53 -- .../fury/lib/internalSerializer/string.ts | 49 -- .../fury/lib/internalSerializer/tuple.ts | 56 -- javascript/packages/fury/lib/meta.ts | 143 ++++ javascript/packages/fury/lib/reader.ts | 115 ++- .../packages/fury/lib/referenceResolver.ts | 98 +-- javascript/packages/fury/lib/string.ts | 64 ++ javascript/packages/fury/lib/type.ts | 31 +- javascript/packages/fury/lib/util.ts | 26 - javascript/packages/fury/lib/writer.ts | 68 +- javascript/packages/fury/tsconfig.json | 2 +- javascript/packages/hps/tsconfig.json | 2 +- .../test/__snapshots__/codeGen.test.ts.snap | 670 +++++++++++++++--- javascript/test/any.test.ts | 2 +- javascript/test/array.test.ts | 12 +- javascript/test/codeGen.test.ts | 15 +- javascript/test/fury.test.ts | 63 +- javascript/test/io.test.ts | 56 +- javascript/test/object.test.ts | 11 - 50 files changed, 2885 insertions(+), 1292 deletions(-) rename javascript/packages/fury/lib/{internalSerializer => }/any.ts (57%) delete mode 100644 javascript/packages/fury/lib/codeGen.ts create mode 100644 javascript/packages/fury/lib/gen/any.ts create mode 100644 javascript/packages/fury/lib/gen/array.ts create mode 100644 javascript/packages/fury/lib/gen/binary.ts rename javascript/packages/fury/lib/{internalSerializer/binary.ts => gen/bool.ts} (50%) create mode 100644 javascript/packages/fury/lib/gen/builder.ts create mode 100644 javascript/packages/fury/lib/gen/datetime.ts create mode 100644 javascript/packages/fury/lib/gen/index.ts create mode 100644 javascript/packages/fury/lib/gen/map.ts create mode 100644 javascript/packages/fury/lib/gen/number.ts create mode 100644 javascript/packages/fury/lib/gen/object.ts rename javascript/packages/fury/lib/{internalSerializer/bool.ts => gen/router.ts} (52%) create mode 100644 javascript/packages/fury/lib/gen/scope.ts create mode 100644 javascript/packages/fury/lib/gen/serializer.ts create mode 100644 javascript/packages/fury/lib/gen/set.ts create mode 100644 javascript/packages/fury/lib/gen/string.ts create mode 100644 javascript/packages/fury/lib/gen/tuple.ts create mode 100644 javascript/packages/fury/lib/gen/typedArray.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/array.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/datetime.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/map.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/number.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/set.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/string.ts delete mode 100644 javascript/packages/fury/lib/internalSerializer/tuple.ts create mode 100644 javascript/packages/fury/lib/meta.ts create mode 100644 javascript/packages/fury/lib/string.ts diff --git a/javascript/benchmark/index.js b/javascript/benchmark/index.js index 5dd96f0fb1..016b7710ce 100644 --- a/javascript/benchmark/index.js +++ b/javascript/benchmark/index.js @@ -28,6 +28,7 @@ const Type = Fury.Type; const assert = require('assert'); const { spawn } = require("child_process"); + const sample = { id: 123456, name: "John Doe", @@ -106,8 +107,9 @@ const sample = { `, }; + const description = utils.mockData2Description(sample, "fury.test.foo"); -const { serialize, deserialize } = fury.registerSerializer(description); +const { serialize, deserialize, serializeVolatile } = fury.registerSerializer(description); const furyAb = serialize(sample); const sampleJson = JSON.stringify(sample); @@ -164,7 +166,7 @@ async function start() { var suite = new Benchmark.Suite(); suite .add("fury", function () { - serialize(sample); + serializeVolatile(sample).dispose(); }) .add("json", function () { JSON.stringify(sample); diff --git a/javascript/benchmark/sample.jpg b/javascript/benchmark/sample.jpg index 5c98c2d23edf0a529fe9686434c344fe6fbae9dd..4d74cefa83d1b6eb236ca99768afea5517732877 100644 GIT binary patch delta 7935 zcmch6cT^PHvUej0g5)S!38F-i90ydANDw4v4=5RijD$uIR3r!^3Q}$2eKtZgIz{whjtbPbwlQ}a z>-7%uJzsh8C?`WDS~+|7KFUy!&32?0SN5HP{QP)&qwz4-Vm~HmDNJP0$aZ8yw2}a9 zUJP1yDy~jGyW>@)@pNR1>=L8XDJrVO4TF+Do!FyH+IL$tW8tlSkdO+Lw>u}2dP-PE zC6pc4m<_u6YJ)T@-RBqUENRvGMi(fVMs9R4FQ;=GT6rUEC76f+#C--egg%4%|1hqA zso2-EoUAv1Ym{u9RZk~v%#xpZ8k~}fK2M=1R&xic*0gAZ3Xt5Hf=}XC0*+X zK*mRKh79XSO!I$ULtLO`0)PJSB>+kf2|y#w@nHEn{z)qV2IvMY3~Zyyj>m4C~EhsiF)jJaAXhUkCKjm+gAK}ugeX`T9B zV>%$+{E1>R<;G>qrS*GbNmV9WW0i^oX$BlQI+fp3a=Eb7<9tN=DC9D%z3hFS3jwI zg`%?07pyqJ^Y1k@ZHR5+WH!^_i=b_{7)*Aad+jJxZrjQ@9s?emr7mOqkrOT7CqvS7KUP;n zz&!*f_+!0-eyOl#QluhK!dDUOd?E<$TO8nZ_APXd2ye*nK_<2Tx6|0Mq}i9-t~q<9 zyqeX@1K+#z6=^kYxpX)EvJ1$=hoeu#SO@@vbxn~I_vAE7h(!OM;)33^>r~7Q_qSXb zRS=3aL%@{3jK2$Fd{^w8^<&tITxP2J6uIJsw)C6&~V}->S3BZ6M0pOnl3Ba5q z0f>y8L!VNB3k^0{9UlU)lMk-R5r9NA4#oYKd!I8`ZigfAi(E*YO#K;ze<;jLxX5%w zo@L+JQtH@sK3Ae-L2_BHL)3iG%*Hm^%v4D7#CkXRS=yT!c83wKn-b%&`{omFXiO;5 z$zdrF{WucjnTc_q$b8IMXDk!DAyx%ab>@6Gf16#OEc@fAu=pB>xL;={Kpjoq$>Cqd zW4z?2lrocK**DZ+m_K#=Ui;!%tnwF;Z-cpLrOE<0pHsa{c22mm3RR?G*ppp-r)kl# z<(fgMqI_Y}*y5krH(KbMCN7&vEJ}#Gm84UKWwFBir8vJIT8k7}M&=LIAWr@aa? zTe!_=K>LO68Kvi?FxD4L=i{GNuCsTW(eL54735Nbbd=2=Oiq647d*Em`_q&41?wzD zVvK39&D<5oiH4n3^bh#DNN$5>Vqg#2X^&ZD^1}r_2T$*CFNvosQenYY@=(+|0)3zM zxS$-kC{`o7EwRZJDpAAk4{1!@5JG6OFuYfxzMz@MN z4SeaAl_+Tul$pN4k0(%v@Z>Jjrf!Sh(-@dRr_`mj%A~;fBNvYf1D{(=Xcq z8+qG#+ajc8PH`w0QUi<=h z$~K;E5})$OyLnVU#={rvIox4&g+KVYDCFYNaagt)D}fJ}Mjzq8w)lVCgMSJQ$qJFEqqHJb_4UkH-Tv|f)_A)vp~0YeN+oxPRZprGQ&%0Oa z)0aPs)?>IOzJX3t1c1B@^Az9F7mRyr!`npwas$r3&u_bfnKXvwqnq1RI(m99`RP}O zG&pj?CRzt%C;Js|tjD?BYVfxr0M8!d>H9edfI9(bpZ>1=wcc-AVxv2rmwRtq;Ysk6 zg7H*x5$@?6SQ!v!cz$a6dGTvzlDAuQIs0_*BeZDKwdGV`!#5;UXkg{{d-jNiudetV_&h=-~x3Z6S1A_HAehCQ3P?!NSL8>(0RuDHY$z?C6fXK zT9giaAHA>l3fw0n0M#>R{64cg1i-9>&7^Ak#1d}erzDzhw(>+530AMhe z+Sg$Oi)Ie?q1D5=`g=~zqn8@F2|z~zmg$8pTrowyG(Nyo?Gl@VuEkpX<4(@Krg%x-46Y^7=HNm@Ss!% z&pd-uh3$C|fDffT%EWXp;+3vVxz*v!?uu+Yv0_?6?^F|jHF5%g0CABZ_g^J9Tzo8E zuN6%3(mbf^A3Jyz5~PLHtqU|@HDgWXAr3CS^vC76LWKvXzB8C8q*G8u_t~B6qCLy* zi)5_ZV;>uQOQr+-zt$z^*}3mqw$kl&Me^JXp#J)zgDXN{qG45ld6;h{ztm8D_r`Pe z(Z0;OqMw#4`SuNGu6}cmNMVw|6$P3ub@`yvk^XrFo6VpQ(aC!wrCWLB(Z|B#xy;vD z-!MP&Bsw3+HP5WSUQ1TvyE5fUu7 zBv~_fp{nE$V**fiz3G71M%iQ_^!J%jPbdN8t-0yhzK<4Kp*n}KHS@lg7>(q5~9VmIMG@qJYQ4 z2mqaR9?9G#gX{Dv9XC9m#SdklS%OnYJg(;0#E(V1EGi;W!TjjDR!RId$TLk!!TvE6 z1ZR*}yvV{cS$`q`Fmrz#7R2*>wn1FsW(4cN>fMm>pD@Rc$}}@!Nf&mxyKgVo#9b8( z;D10I?=Bess`6DV*!(0p$$6LWNIzvThwqvkrzQzLDGyUi0P4kXXC6CQn{Wc)XLE*` z`DJ}Cd*LVAfEs4G)7B)_|B?kV^_gZDEKaMpMmG04RLyDY0j+RUjjXI%jP;e%w8dd2!$ zqf6wVkls?}b`=BqwvfrY5pI|GvvLA3134ju@;LXA0|97q!!r^oF_Hbb#&tkXA3~h} zLW2MMoQb0xm&pguS`Oe#x(rYU4V76&7GE!QcMCl;;>ylAuVV>*vnt7Px=$J_fHbPH2oz{Xa6|}G zW*s9Jc{%9s?#U03X*(6Jj$=fE9)m7$$4@;wJ!O~LZBjVk>gYnJ(@N41Y5W+5$t_pg zDOrtdSWr)DQ9dkSBiMrKY@%o0U^sguufi&MQ^MW)Mt22Y!G*j~vvO+hI5N9jTpFG^ z>K!g^F_8NQZAei^+l)jHC?-4a*E@aO?y65_jd&-+#AY?RJZ>i_iBiz3i+Q)twstrx z7!v;N1lwv*kuT#ygs>GL(Pp9=&8a%9n6#Q~g_A_kFm`a721R@=$j_wkOEpH>*^e3f zF;|bpq6ThLWIt94)fev-Mdz%Z-Xo=wZXWAmc572`G?o$=ry!GZN5mURB9N-J|MHjq z6N10}+YwsPBct)>-Y5~^Z5U$qYULk~0^;#%u@?A-T!^bv1)tcc9{)CB)B>@6bU~F8 z1hi$Dk`!CYOb{IRLqyOTN+IuZJP&^c3qiP%v5ap>BLKJW{_1I(5`4lS5^oJb#B858 z0wdeXpr_##k}?Kp{A6uYKx;e%U-H7 zN*cNOx=US^>~b_4 zw+_XoS`PUZ#hC4I{lpup9hc)&NzPqZU<3BGYup4>$FPdYe)2(TL3W(IZEs>pJiznc z3S%a{?n)1I1uD{dTXDUzbIQctY8diKF`m>zYDceWJJ!bX@0$^yU%7Z~nbXgBswuM4 z!tLg^t~OKR#6HZ4q5keLy*SdJKV_EEem73gio0dvQRFg8c`9M~a<_1WV!1LYBa4^m z(>|!L)C$O$jEYI_6}Y`~W{a+)x4t+!tDn){lc2XiWl-TP7efS+rL9({BA-jH9LNQ8 z=dBq~7Syx6&aCG{PivOSxWigoywe}mZIiD`tc0>w4sYGFb`AQT&6hGyxzQ}yukhf; z{f+mv3QFv{T^cyCTNP+^11R^*gb#7}T#IAuA2f zS``TKoPt~ESsqd*An`o;M&2`SfZHHKL{Apcc)1a>6dF`a9#bN*cskeZ!2o-)7mT^n z=r;5iZ<-RSf}Wa;6VLGellrK?Z)7QFD??)%VAx<%Ow5EFC2tq|P;U<55|SBMc71 zR#xhR*rvD-`OXtXr60 z&66&0bV$=l?ucQU;`8FmjM>sqzk*zk*i>Qj`3KjKLcG7a_rDZCav+V=F3)P6|0|Rf z%3CW!yP^L1{fsfMdmDpyc#VWQ_KtXz^*l8``%cq+Wb==5%QP#=jm>b%axtD_?&SB^ ze_j`9OE&a@2eMMMlDFYluu`rQTh!a4Y=#!N(!ToUizL-AG1wFi(GcBG*Nf8^D#j1e zNQR|!1e+%&#QDNqT$qNh+6P`tqDvWf#`ebP0>l}WL}P#Rk=u$#&DNK8B}1tP?>Cqg znBM%}OqFG7u|)Ga__Jgxg%ipegj+e9DC{O7Pqp5Y;p?I8&GB{IpabLaUb@Fm|SDG1*}6R@FDG zsB1MC=Eon6D)iJN;m{|82+!K&iVJ#&28(E-0ug00Wb9U>huXc0g%*zHQSVEiA*%uy z1wQWc8J-Y7Xh1}$_Wa8@4%d|$w6`jKha89*&d;O{UH)39GwHl!gv4Ks;uRgcxL9fH z>e#_*KoPYT$n=N$&IX%O@93npwattbiI#gAr&-rkzs!lkcj`-u)Y*vC?-jOkX+(d? z>&5m==n@@&w_GiyoT=C>jr~q|aL)T!G6NL{ysbWIBvb2ZD9^YQipDUtS%=4|c#9J5 zd?8V)WWi=rPMajcd!t-Xg$2?Tc7`MDDGY1y)um2+xhBe zBo{P2pH*sif|KoT>=U|s`p^PgBq=JR0Y5qJia z1U>*lqnV2Yp!Ggf%=Nv6loKTY+mTQcssMES1)^xqU`*JUk>3iR&(FHwJ#24Kh2qddPqJ_G4 zY}7UZSYOh^T^JRtF9tiH8W7)T59QTZgR=f2klwoZYy#F2{~r2nh2)ib0uT>%zt+=0 z!NsD2ze$JGL=g2X!SkPRk-~S|2d=%3KHG2qXPKAzpQl~~x46nZ%jAzm$Ioxqg|E%7 z8anOIv^sGX-|rBm=)A0EyZFJBxVCigi+ez?ShN&KdmK8pZ+bgND|eDf$a%1HxrdyX z=*W#)>kGSX092x`BjDGwK{fB?S7wb}yfW@Qm70_jZ$U5v(EC^TO&h zul*ldeu-x4yC({)Ce+2fuP2XfHKr8x$qd!J;LiBV!&t5C)w+5ofAWHpcg9_EbhFpe z_WZ@C8{GO>lC9-R%fl3rrch5|iU8n(J0ZHkmSf@@e?m1w0LCx%pik)vjZO|d=CnuA zK^n*K164If<44aB2;CvbDCD4Ea!Eeuu!^t_wDv&OEKJqN*~-tJjzb42#K_ zaEaBhU&>Ig^)Vjmscg8!?k+XqKH<>#*o8pl$JolL^i2#`(z?A_99y-WE=ta-{s8R4 zMhZ94X?auiNo)6lZ`<#yF`Zt2jYtsp0EJR^!B+&}7tC>@3GYfxymdIqq6 z)%T9mRFd~LDPm|fm%=x_M=7@$vB48nmo@Ef1}1pdDRW++YSWZ!(+D0Q6U`79Hl8uXS9kZmNyecHPqcqlt&lH9eE^MhFe4I$)(tF_79k-CMF{CW6W!XFK(lyc5G& zzCBF@c(OAU2qjKpf8V)ui z06e|RSYoIQG2a(Fxp#zHg8i~dQ_kZf0REv(e?8Q^#&-N}AOvXb* z;4*luB{g$O?JnYK$h`X#I*jy%Dzm0H!nYm;e9( delta 7167 zcmch6XH*p1wr;hEsDK2?QKbnIB?rlDvPjM}K}3?!A0A2q7hs8wsu`K`IW8ESY6c0L5Fn5Rq;B@>7X zD8AuLBKh5w1p;FI-!XKW)a(-CDr~I_zCF;hRc=M7QABzMw;oB3DxthZ=s@Kw$1}_j z&_|{Q0j=M#xOmNpC(aO{%uWRXWV8^lFX`uJKqmYDY7+SWS>8Xz2@&mkIQwvvy)bCr zy|Tb|()HGa=I}E0FGN_8MARGFtc7u@Xz~u`3a#&SuuxPhH3xUzoCwS0$;a8pRRa&6 zBd3$ynmxKhEsCb7UW=xX?fEHeRKCCF`>&a#tE>N`C$-g8d5XUOGsYeoqv9HHvv~SC zpqQ8Q-Cd_q+)*syL}@{p2}JE(V+YNayU?c_jQCx!BR<{kU_|7676Re{2YgKjcnIr* zfL(792;egO$7Wh*31e4E?S8jH0E-j??6tt|JR?ZJUi@R)|K=E;kLw{IyAqrs!@2zO zhk&aJ4ZI=yCu;gSCsW<$4MIpe1U%u~%loIUCjS?7b(h~wb!BEJCzUa}+)D%j zi`lzo1w&C&x%2p#6IFY4m|v3(0crzg;RA@qe% zXdN=t9p*nkj3h|y8p4Re7x*s*Pc&kuFwRW9Q)EM?sy^Ytr?{S&Km- zD>|5pX2#Nkb&kh>6vG9nUpVP?tMIznUl9H2m$O?TPlQ=T374KuR3eYSKo$i7tGp19 zvJU}!lGve2rGsrdUrvF|N34|kmZOIPi69oF3{~Af1W@DJ1F`3WGy_dPtec^Sy>mb z*255b2et$od&M_*^R4&dCrl_7YIMMky{4#R^jw`p1G%w8x=$yo;*n_-Nwn}U*&DEa z6)0zpo`Hnn?442%!YPo z$9=t#?(#1m#Hcb|xD%N`LQNui;eF+<)H}yu_gl7hKb)P#^<4|Q__IO-WtWr@ovaBY zq9pP#Bg0s#Slk~%tmfJ5;^)E&PG3e}(dx^hwiH#rvj)H}#+6_5y077-aH)doxx~ST zANi-FifS|X#_FZ1<*>Zoj(`PE(rr0s_5&0nrz$FmpBl!7iW52kRDH|!=@obeYuly=UD)WrGsL9h^gE56D2lx7P5LjUyFX3++C zOCax(22gVQJJ*o$r?f-}xS>mRc1*mh(DytsA|5F3PY_TF0clF>%@A<5c-~A%J*-HV zFloquzTMbaq;m3x1rA%*dHv%O39E$TW|Z z0O_o(-2>9nn%Sg;kJtF;C#TXu)~*0Cx{0U3=VQx7sJ=%-WkN>+2t~_)DMV=^MX2`V zb=JC$OVZbxtK1qr1iz#-lQ-{qR8@Z77x{2-WYKy-VRk|=4o{Un7%1Btf(YDHbbd(~ zY#>~>+akdyzdbz{Yt~GKfO~ttD?6vBlsO8WN+xF9GP84Nb+`xf6J&KkSXJzM@0#o& zhf%hXsG#iz2)Nvf$}y34l~b`%lehP@K7vGp^YmEKl-5~iW4u}Ne}rbyo=B@PdF^ez%Tce{fTT zLa|^mn4rVE{d=p&Ld1sCPGk9wpmf8mj?s!BNOQ1*SXW31^u?f zMaYqa_c^E15HM!=_kr2W*vZTy&iwQRJW;n4kJPFe{Q_qAZ4{>7Oij+ny&?T)YqZ$q z^}H(mLIe>b*MRf}dF&U}l+B)iZ3FLYygO^B)zr$1C#=P80Sr5m-G-ckMMhk_5t~eL zk*i{1B|RzL9%A=nVwNP3*Dw2d*sPO(YU+uJ*6pqC^yIO3JW#%Hy#kl-ScomVM-Z5WvF1{G8E046l|MF9DlOaC$Z*@! z^vQ5mu!b3Hh1AxA;MM%N5j;R=s1DGR&YTflB@BF1yiWc7vruf5|H`*J`9|=_B>TB5oCVs)wGhxC zf+EJ2S4XD~SLRoVV5xds9%urWjXN1b9mTX@}Ee7JiKtuRXX-EmmJ7(CH$n}dud*oM@I ziJGyPNmfQdKwoS3XgX*9W>FWMlDdDZ^v?6N1Fm4^ME!%8~YELwjZxBoR{auxPH9?Pmp>n8c#J79wB^X zB_hOtOj|v;3>#Zz+UK8EgOx5%KOSLVl&=2D{OsQM=Uvp~5e55VCAHVyEFQ{9P8T}P zcfBJq9I`f2dzfBf^Mh9N7H)~1>BdzJ)g&(}I9ngteF#{eSar|YrNVKeu$17K@B{>? z3)v1gs^_{CJQ((mC1-{keXo(e!tAh4LX(+?y3Y(Jp6<`68_d)&GL!|s3g~$9Le0Z- zyiYC2LwIvLZF?FisP$=`Iot{Ec~P9=q5p<8odMac=_{dKdN%7Jd3r@|dL*n_K^Jvl z&$nOZ($4tTExtWmvtp5o^mi_?SwAb!uQ_|(S5V&=K={M^TYmL0qT5H8R$G2=kiJPB z5K|{*y}?0cXu$y^#(}^&G%8(f234WM03*X#@GHRnGm2s2gzm!r8<77-P=A^5w~vUS zPZ^}QHW(6;C}9>s7HSDCkoYA8G0TulrYHkG+9i_hPjsu(Jh-&p=qof8l$Tu`MXCJG zdqGi9&u0Jm?Qut_!6h6A_dwymw!gT1g_YIQM=9&mx?vF%E42J6iqA5`6X)5*5eFSP zUOl_i+zA%8x36>Y>&Q^RR>+uS<{Q`7UC|nTLVm8l14FPcHqoWR4w;(n(aE$Xq0cpC z#rUJhidVR*1q%9+Tar63%`>Y}Vv}oouHM$#zKo)c&F*&$rmmREFMK=$Hc_I?bQenM zQ9|@QC>pu%s0uz((42mLB8R#aZHY^wm24(j;hPi`G(XnrEqAY!mho@1Wu> z)z`4t*{y5b^k7}_=Ok&5yL&6sQmjC zZXqM@oWvo`MO+ifpMwP92Cv$yWBE(^A9RE^L!`M)Pqg z-D5IM9>>S?`wN`u-tFz))EhfHRbELbxm4Nhc-!7DV*=J7M$vhxW~oF2KJHy zjGIU@3^adZ%ab4(rghRo8KR2`ig#6~g_C`R8Soy8BM@MZP_C?H zII$s{Itbf3nd+IUD^IIKU$mXFb7d5f9ByA?E)7Jo$1i;gh1W)qwpFzBb|cj6AMz<| z{98SaY@uParwDD^SH{4kaYUELO4#R%6#bP(%8xEOGjnt_TwN|N!>JS^Y<@bFtc~8m zNGuN;JpaQ`drL5k&aaD$LzX9k@heMDLFqfM^?)*0?($rZh3ZEdqP)rEv!5SwAQF6M z*h}(D^{F;c8#I`%kN9_5lQqW7#TaFQctg03} zIm_xs-0nN{tHUkTO*c8I3|7h}{9?NAPml1QODU1c7Tsz}-adgox}0vCt0)2|2DK$P zp+(>763)Iy-U`BPxrZcb^M^(9bq$?JNX_b!qDlKIDK6Xl_U*L?elfTdp6nFn;Ie18 zKIrC?F36L{w7_4>DNLi>d2xk#`3q)F;XTaCT+a8D&J@O$Z`^|2W9sAPlR^8RQkel)-lpAUnTl0 z>s=l*lL#izcnEFpjo?5X=)*zFwmHvN6laYihANKSYmsHyBfktw$8VKh&7Q3npox4N zE?Jkm%4lX)FkSY^O#m#KRA*ydsM&W+fD5XA+IIZ$2%h5GdC!3Ko%J^>o)|MBbBl_~ z6m`K^U$5jej`yO>pA`R!Ea#*X;A%nHDIMM8h#&=Z`yeuc_DnQJT;3gqinu_5Dlrwq zu&7#J04>L--Zu^`sU06+erF@1TB9vRaW|g{ixYOwM&gZzB~B$-onnME_FISBUX?ME z_;Hbid`|9Fy?YgyZ&Y9&+Vn38Gd39!6cQ8(j17%l*3AlPeD5vTvRHig4fRUglg^8j zSQ71Esd!(B1BBkg-#$`V+`o9ts=sat>UaADCTUr}EATbl3FokwvCu^dmwh^|VZ80@ z@4Kwg>+}6i%v{n3mB^^8Wb6VT1~z#I%f@dX>HbuY^`xD8`HZjSR5R_RXh(>1_7@l< zMhqSY0}q;Xzn!%=7Y9<9XE`P~t8*B!0pG3-TN7x1A_*Y4PYXsFjVLWj6DxN+F>qC* z*u2fOeV$&M+i?2~v*qf*C+F=+?$pm}x0~h;9-Ko!5(>vh%=a+XMCP2qJCC6HEl_qsUo@8`Ice83)d#x35BVX+ z)7(XMx_tw7LkjR7gPjI5gKPgQXRT=spB$xNfFn7kYGnDuyeA5N-I$__+2c{)OWbO zKTp^y1tWLc!QIqmXu;2^7vE46)Yd!vL>v^(a`cXuy;)l=N_$OlkjII_x_o$IKf%jl zk1AH4vUu8hU3|@I_*^e6%w0aMKisGWWPA^)0-x+mqR2FKaX#w_)(H z9H)-7-Lp)Mdma$?7zW!&lfNJa8?e}@*C_PnZi+5(43xL|!QS*$>&FqWXcpxSvV>R# z#kN}oe8_J_UP{i*FcNq)V0^XAknf<}R+g~mb z8gD7m|F|qxa9^v8`PB__(!>EkDxTO_1uvJOG{_u}I6B3fwr z7Ragq7nGNX7-*`eW;*!g<;Am9eA#RGthTRMNB)`9%M$-?oNCOtJ`fbSlEC9PC>*Du ze8p@(YkTFxnZc#u?UF**>exWA$ZC8-3djsjrCurQzR%w+uI_t5?_#l_5#K{U`#)O_ zZzSZ_PHA+^yW$keETl4gxl{9#S*_Q)wOU8y>ub&0gsr>72HQAZV=2P>o=Au6zqv>2 zmagw{?DZr^@Vox@0gkn1_BUJKrO{ZWMN49V8Ue1l+dA21&G()%AeCl0&sat?%>Q)8 z?uZg=H{7oyS>E%8BmBQ0pU-YVsaxcNd}y(gCJ4BLJSXO9UpA~Hno5U6+%wEeHjC@V zj}>@=Zr0bNP0#Z!KQ*}x_u+H$=ekH8bR$93vRaX0_c-0ldGL^3c;G<+McT1B;hekrch8N_a}seRt9M1N1R?br61F?r#`Vg&gMUdd80OtC*SN++ zk#L2+(oW1!^BDOXOLcM+UySeVB*0eXI&slE5U{bVkH0+3AKgokCpJ1ZG9iFJ6D#md zz%=Cm%s5M6pMGtZg8LVR#J?m`$UCr9pFSwJ?W^4FA>^1tK6M+jSN7wwcO|vEc~7do?URt`$~I-8P-@Ty0Jl1*T6GJ#wDy0e%LT%+Qw_M2+*;<-$rGsNPW2nCZ(p zZkm~NF1n?os6}ss1;s9Bk1$u&KDmwr_H!Xk>LZ;@Ju_gDt;@~cxlQ_i0mb`opw#B{ zud&iq%3XTx!R32Pk< z%XvD{FWWI=a6Fx%?cM<~ z(description: T) { - if ( - description.type !== InternalSerializerType.FURY_TYPE_TAG - || !Cast(description)?.options.tag - ) { - throw new Error("root type should be object"); - } - const serializer = genSerializer(this.fury, description); + const serializer = generateSerializer(this.fury, description); return { serializer, serialize: (data: ToRecordType) => { diff --git a/javascript/packages/fury/lib/internalSerializer/any.ts b/javascript/packages/fury/lib/any.ts similarity index 57% rename from javascript/packages/fury/lib/internalSerializer/any.ts rename to javascript/packages/fury/lib/any.ts index 00fa993808..7fc0d6f1f0 100644 --- a/javascript/packages/fury/lib/internalSerializer/any.ts +++ b/javascript/packages/fury/lib/any.ts @@ -17,53 +17,59 @@ * under the License. */ -import { Fury, MaxInt32, MinInt32, Serializer } from "../type"; -import { InternalSerializerType, RefFlags } from "../type"; +import { Type } from "./description"; +import { getMeta } from "./meta"; +import { Fury, MaxInt32, MinInt32, Serializer } from "./type"; +import { InternalSerializerType, RefFlags } from "./type"; export default (fury: Fury) => { const { binaryReader, binaryWriter, referenceResolver, classResolver } = fury; - function detectSerializer(cursor: number) { + function detectSerializer() { const typeId = binaryReader.int16(); let serializer: Serializer; if (typeId === InternalSerializerType.FURY_TYPE_TAG) { - serializer = classResolver.getSerializerByTag(classResolver.detectTag(binaryReader)); + const tag = classResolver.readTag(binaryReader)(); + serializer = classResolver.getSerializerByTag(tag); } else { serializer = classResolver.getSerializerById(typeId); } if (!serializer) { throw new Error(`cant find implements of typeId: ${typeId}`); } - binaryReader.setCursor(cursor); - return serializer; } return { + readInner: () => { + throw new Error("any can not call readInner"); + }, + writeInner: () => { + throw new Error("any can not call writeInner"); + }, read: () => { - const cursor = binaryReader.getCursor(); const flag = referenceResolver.readRefFlag(); switch (flag) { case RefFlags.RefValueFlag: - return detectSerializer(cursor).read(); + return detectSerializer().readInner(); case RefFlags.RefFlag: return referenceResolver.getReadObjectByRefId(binaryReader.varUInt32()); case RefFlags.NullFlag: return null; case RefFlags.NotNullValueFlag: - return detectSerializer(cursor).read(); + return detectSerializer().readInner(); } }, write: (v: any) => { - const { write: writeInt64, config: int64Config } = classResolver.getSerializerById(InternalSerializerType.INT64); - const { write: writeDouble, config: doubleConfig } = classResolver.getSerializerById(InternalSerializerType.DOUBLE); - const { write: writeInt32, config: int32Config } = classResolver.getSerializerById(InternalSerializerType.INT32); - const { write: writeBool, config: boolConfig } = classResolver.getSerializerById(InternalSerializerType.BOOL); - const { write: stringWrite, config: stringConfig } = classResolver.getSerializerById(InternalSerializerType.STRING); - const { write: arrayWrite, config: arrayConfig } = classResolver.getSerializerById(InternalSerializerType.ARRAY); - const { write: mapWrite, config: mapConfig } = classResolver.getSerializerById(InternalSerializerType.MAP); - const { write: setWrite, config: setConfig } = classResolver.getSerializerById(InternalSerializerType.FURY_SET); - const { write: timestampWrite, config: timestampConfig } = classResolver.getSerializerById(InternalSerializerType.TIMESTAMP); + const { write: writeInt64, meta: int64Meta } = classResolver.getSerializerById(InternalSerializerType.INT64); + const { write: writeDouble, meta: doubleMeta } = classResolver.getSerializerById(InternalSerializerType.DOUBLE); + const { write: writeInt32, meta: int32Meta } = classResolver.getSerializerById(InternalSerializerType.INT32); + const { write: writeBool, meta: boolMeta } = classResolver.getSerializerById(InternalSerializerType.BOOL); + const { write: stringWrite, meta: stringMeta } = classResolver.getSerializerById(InternalSerializerType.STRING); + const { write: arrayWrite, meta: arrayMeta } = classResolver.getSerializerById(InternalSerializerType.ARRAY); + const { write: mapWrite, meta: mapMeta } = classResolver.getSerializerById(InternalSerializerType.MAP); + const { write: setWrite, meta: setMeta } = classResolver.getSerializerById(InternalSerializerType.FURY_SET); + const { write: timestampWrite, meta: timestampMeta } = classResolver.getSerializerById(InternalSerializerType.TIMESTAMP); // NullFlag if (v === null || v === undefined) { @@ -81,68 +87,64 @@ export default (fury: Fury) => { } if (Number.isInteger(v)) { if (v > MaxInt32 || v < MinInt32) { - binaryWriter.reserve(int64Config().reserve); + binaryWriter.reserve(int64Meta.fixedSize); writeInt64(BigInt(v)); return; } else { - binaryWriter.reserve(int32Config().reserve); + binaryWriter.reserve(int32Meta.fixedSize); writeInt32(v); return; } } else { - binaryWriter.reserve(doubleConfig().reserve); + binaryWriter.reserve(doubleMeta.fixedSize); writeDouble(v); return; } } if (typeof v === "bigint") { - binaryWriter.reserve(int64Config().reserve); + binaryWriter.reserve(int64Meta.fixedSize); writeInt64(v); return; } if (typeof v === "boolean") { - binaryWriter.reserve(boolConfig().reserve); + binaryWriter.reserve(boolMeta.fixedSize); writeBool(v); return; } if (v instanceof Date) { - binaryWriter.reserve(timestampConfig().reserve); + binaryWriter.reserve(timestampMeta.fixedSize); timestampWrite(v); return; } if (typeof v === "string") { - binaryWriter.reserve(stringConfig().reserve); + binaryWriter.reserve(stringMeta.fixedSize); stringWrite(v); return; } if (v instanceof Map) { binaryWriter.reserve(5); - binaryWriter.reserve(mapConfig().reserve); + binaryWriter.reserve(mapMeta.fixedSize); mapWrite(v); return; } if (v instanceof Set) { - binaryWriter.reserve(setConfig().reserve); + binaryWriter.reserve(setMeta.fixedSize); setWrite(v); return; } if (Array.isArray(v)) { - binaryWriter.reserve(arrayConfig().reserve); + binaryWriter.reserve(arrayMeta.fixedSize); arrayWrite(v); return; } throw new Error(`serializer not support ${typeof v} yet`); }, - config: () => { - return { - reserve: 11, - }; - }, + meta: getMeta(Type.any(), fury), }; }; diff --git a/javascript/packages/fury/lib/classResolver.ts b/javascript/packages/fury/lib/classResolver.ts index d344e511b4..751b36c917 100644 --- a/javascript/packages/fury/lib/classResolver.ts +++ b/javascript/packages/fury/lib/classResolver.ts @@ -17,87 +17,102 @@ * under the License. */ -import { arraySerializer, stringArraySerializer, boolArraySerializer, shortArraySerializer, intArraySerializer, longArraySerializer, floatArraySerializer, doubleArraySerializer } from "./internalSerializer/array"; -import stringSerializer from "./internalSerializer/string"; -import binarySerializer from "./internalSerializer/binary"; -import { dateSerializer, timestampSerializer } from "./internalSerializer/datetime"; -import mapSerializer from "./internalSerializer/map"; -import setSerializer from "./internalSerializer/set"; -import boolSerializer from "./internalSerializer/bool"; -import { uInt16Serializer, int16Serializer, int32Serializer, uInt32Serializer, uInt64Serializer, floatSerializer, doubleSerializer, uInt8Serializer, int64Serializer, int8Serializer } from "./internalSerializer/number"; import { InternalSerializerType, Serializer, Fury, BinaryReader, BinaryWriter as TBinaryWriter } from "./type"; -import anySerializer from "./internalSerializer/any"; +import anySerializer from "./any"; import { fromString } from "./platformBuffer"; import { x64hash128 } from "./murmurHash3"; import { BinaryWriter } from "./writer"; +import { generateSerializer } from "./gen"; +import { Type, TypeDescription } from "./description"; const USESTRINGVALUE = 0; const USESTRINGID = 1; -class Lazystring { +class LazyString { private string: string | null = null; private start: number | null = null; private len: number | null = null; static fromPair(start: number, len: number) { - const result = new Lazystring(); + const result = new LazyString(); result.start = start; result.len = len; return result; } static fromString(str: string) { - const result = new Lazystring(); + const result = new LazyString(); result.string = str; return result; } toString(binaryReader: BinaryReader) { if (this.string == null) { - const str = binaryReader.stringUtf8At(this.start!, this.len!); - return str; + this.string = binaryReader.stringUtf8At(this.start!, this.len!); } return this.string; } } +const uninitSerialize = { + read: () => { + throw new Error("uninitSerialize"); + }, + write: () => { + throw new Error("uninitSerialize"); + }, + readInner: () => { + throw new Error("uninitSerialize"); + }, + writeInner: () => { + throw new Error("uninitSerialize"); + }, + meta: { + fixedSize: 0, + noneable: false, + }, +}; + export default class SerializerResolver { private internalSerializer: Serializer[] = new Array(300); private customSerializer: { [key: string]: Serializer } = { }; - private readStringPool: Lazystring[] = []; + private readStringPool: LazyString[] = []; private writeStringCount = 0; private writeStringIndex: number[] = []; + private registerSerializer(fury: Fury, description: TypeDescription) { + return fury.classResolver.registerSerializerById(description.type, generateSerializer(fury, description)); + } + private initInternalSerializer(fury: Fury) { - const _anySerializer = anySerializer(fury); - this.internalSerializer[InternalSerializerType.ANY] = _anySerializer; - this.internalSerializer[InternalSerializerType.STRING] = stringSerializer(fury); - this.internalSerializer[InternalSerializerType.ARRAY] = arraySerializer(fury, _anySerializer); - this.internalSerializer[InternalSerializerType.MAP] = mapSerializer(fury, _anySerializer, _anySerializer); - this.internalSerializer[InternalSerializerType.BOOL] = boolSerializer(fury); - this.internalSerializer[InternalSerializerType.UINT8] = uInt8Serializer(fury); - this.internalSerializer[InternalSerializerType.INT8] = int8Serializer(fury); - this.internalSerializer[InternalSerializerType.UINT16] = uInt16Serializer(fury); - this.internalSerializer[InternalSerializerType.INT16] = int16Serializer(fury); - this.internalSerializer[InternalSerializerType.UINT32] = uInt32Serializer(fury); - this.internalSerializer[InternalSerializerType.INT32] = int32Serializer(fury); - this.internalSerializer[InternalSerializerType.UINT64] = uInt64Serializer(fury); - this.internalSerializer[InternalSerializerType.INT64] = int64Serializer(fury); - this.internalSerializer[InternalSerializerType.FLOAT] = floatSerializer(fury); - this.internalSerializer[InternalSerializerType.DOUBLE] = doubleSerializer(fury); - this.internalSerializer[InternalSerializerType.TIMESTAMP] = timestampSerializer(fury); - this.internalSerializer[InternalSerializerType.DATE] = dateSerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_SET] = setSerializer(fury, anySerializer(fury)); - this.internalSerializer[InternalSerializerType.FURY_STRING_ARRAY] = stringArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY] = boolArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY] = shortArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY] = intArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY] = longArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY] = floatArraySerializer(fury); - this.internalSerializer[InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY] = doubleArraySerializer(fury); - this.internalSerializer[InternalSerializerType.BINARY] = binarySerializer(fury); + this.internalSerializer[InternalSerializerType.ANY] = anySerializer(fury); + this.registerSerializer(fury, Type.string()); + this.registerSerializer(fury, Type.array(Type.any())); + this.registerSerializer(fury, Type.map(Type.any(), Type.any())); + this.registerSerializer(fury, Type.bool()); + this.registerSerializer(fury, Type.uint8()); + this.registerSerializer(fury, Type.int8()); + this.registerSerializer(fury, Type.uint16()); + this.registerSerializer(fury, Type.int16()); + this.registerSerializer(fury, Type.uint32()); + this.registerSerializer(fury, Type.int32()); + this.registerSerializer(fury, Type.uint64()); + this.registerSerializer(fury, Type.int64()); + this.registerSerializer(fury, Type.float()); + this.registerSerializer(fury, Type.double()); + this.registerSerializer(fury, Type.timestamp()); + this.registerSerializer(fury, Type.date()); + this.registerSerializer(fury, Type.set(Type.any())); + this.registerSerializer(fury, Type.binary()); + this.registerSerializer(fury, Type.stringTypedArray()); + this.registerSerializer(fury, Type.boolTypedArray()); + this.registerSerializer(fury, Type.shortTypedArray()); + this.registerSerializer(fury, Type.intTypedArray()); + this.registerSerializer(fury, Type.longTypedArray()); + this.registerSerializer(fury, Type.floatTypedArray()); + this.registerSerializer(fury, Type.doubleTypedArray()); } init(fury: Fury) { @@ -113,7 +128,16 @@ export default class SerializerResolver { return this.internalSerializer[id]; } - registerSerializerByTag(tag: string, serializer: Serializer) { + registerSerializerById(id: number, serializer: Serializer) { + if (this.internalSerializer[id]) { + Object.assign(this.internalSerializer[id], serializer); + } else { + this.internalSerializer[id] = { ...serializer }; + } + return this.internalSerializer[id]; + } + + registerSerializerByTag(tag: string, serializer: Serializer = uninitSerialize) { if (this.customSerializer[tag]) { Object.assign(this.customSerializer[tag], serializer); } else { @@ -126,25 +150,27 @@ export default class SerializerResolver { return this.customSerializer[tag]; } - createTagWriter(tag: string) { - this.writeStringIndex.push(-1); - const idx = this.writeStringIndex.length - 1; + static tagBuffer(tag: string) { const tagBuffer = fromString(tag); const bufferLen = tagBuffer.byteLength; - const writer = BinaryWriter({}); let tagHash = x64hash128(tagBuffer, 47).getBigUint64(0); - if (tagHash === BigInt(0)) { - tagHash = BigInt(1); + if (tagHash === 0n) { + tagHash = 1n; } writer.uint8(USESTRINGVALUE); writer.uint64(tagHash); writer.int16(bufferLen); writer.bufferWithoutMemCheck(tagBuffer, bufferLen); + return writer.dump(); + } - const fullBuffer = writer.dump(); + createTagWriter(tag: string) { + this.writeStringIndex.push(-1); + const idx = this.writeStringIndex.length - 1; + const fullBuffer = SerializerResolver.tagBuffer(tag); return { write: (binaryWriter: TBinaryWriter) => { @@ -158,28 +184,17 @@ export default class SerializerResolver { this.writeStringIndex[idx] = this.writeStringCount++; binaryWriter.buffer(fullBuffer); }, - bufferLen, }; } - detectTag(binaryReader: BinaryReader) { - const flag = binaryReader.uint8(); - if (flag === USESTRINGVALUE) { - binaryReader.skip(8); // The tag hash is not needed at the moment. - return binaryReader.stringUtf8(binaryReader.int16()); - } else { - return this.readStringPool[binaryReader.int16()].toString(binaryReader); - } - } - readTag(binaryReader: BinaryReader) { const flag = binaryReader.uint8(); if (flag === USESTRINGVALUE) { binaryReader.skip(8); // The tag hash is not needed at the moment. - const start = binaryReader.getCursor(); const len = binaryReader.int16(); + const start = binaryReader.getCursor(); binaryReader.skip(len); - this.readStringPool.push(Lazystring.fromPair(start, len)); + this.readStringPool.push(LazyString.fromPair(start, len)); const idx = this.readStringPool.length; return () => { return this.readStringPool[idx - 1].toString(binaryReader); diff --git a/javascript/packages/fury/lib/codeGen.ts b/javascript/packages/fury/lib/codeGen.ts deleted file mode 100644 index da3ab1e6f7..0000000000 --- a/javascript/packages/fury/lib/codeGen.ts +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, MaxInt32, RefFlags, Fury } from "./type"; -import { replaceBackslashAndQuote, safePropAccessor, safePropName } from "./util"; -import mapSerializer from "./internalSerializer/map"; -import setSerializer from "./internalSerializer/set"; -import { arraySerializer } from "./internalSerializer/array"; -import { tupleSerializer } from "./internalSerializer/tuple"; -import { ArrayTypeDescription, Cast, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from "./description"; -import { fromString } from "./platformBuffer"; - -function computeFieldHash(hash: number, id: number): number { - let newHash = (hash) * 31 + (id); - while (newHash >= MaxInt32) { - newHash = Math.floor(newHash / 7); - } - return newHash; -} - -const computeStringHash = (str: string) => { - const bytes = fromString(str); - let hash = 17; - bytes.forEach((b) => { - hash = hash * 31 + b; - while (hash >= MaxInt32) { - hash = Math.floor(hash / 7); - } - }); - return hash; -}; - -const computeStructHash = (description: TypeDescription) => { - let hash = 17; - for (const [, value] of Object.entries(Cast(description).options.props).sort()) { - let id = value.type; - if (value.type === InternalSerializerType.ARRAY || value.type === InternalSerializerType.MAP) { - id = value.type; // TODO add map key&value type into schema hash - } else if (value.type === InternalSerializerType.FURY_TYPE_TAG) { - id = computeStringHash(Cast(value).options.tag); - } - hash = computeFieldHash(hash, id); - } - return hash; -}; - -function typeHandlerDeclaration(fury: Fury) { - let declarations: string[] = []; - let count = 0; - const exists = new Map(); - function addDeclar(name: string, declar: string, uniqueKey?: string) { - const unique = uniqueKey || name; - if (exists.has(unique)) { - return exists.get(unique)!; - } - declarations.push(declar); - exists.set(unique, name); - return name; - } - - const genBuiltinDeclaration = (type: number) => { - const name = `type_${type}`.replace("-", "_"); - return addDeclar(name, ` - const ${name} = classResolver.getSerializerById(${type})`); - }; - - const genTagDeclaration = (tag: string) => { - const name = `tag_${count++}`; - return addDeclar(name, ` - const ${name} = classResolver.getSerializerByTag("${replaceBackslashAndQuote(tag)}")`, tag); - }; - - const genDeclaration = (description: TypeDescription): string => { - if (description.type === InternalSerializerType.FURY_TYPE_TAG) { - genSerializer(fury, description); - return genTagDeclaration(Cast(description).options.tag); - } - if (description.type === InternalSerializerType.ARRAY) { - const tupleOptions = Cast(description).options; - if (tupleOptions && tupleOptions.isTuple) { - const names = [] as string[]; - Cast(description).options.inner.forEach((v) => { - names.push(genDeclaration(v)); - }); - - const name = `tuple_${names.join("_")}`; - return addDeclar(name, ` - const ${name} = tupleSerializer(fury, [${names.join(", ")}])` - ); - } - - const inner = genDeclaration(Cast(description).options.inner); - const name = `array_${inner}`; - return addDeclar(name, ` - const ${name} = arraySerializer(fury, ${inner})` - ); - } - if (description.type === InternalSerializerType.FURY_SET) { - const inner = genDeclaration(Cast(description).options.key); - const name = `set_${inner}`; - return addDeclar(name, ` - const ${name} = setSerializer(fury, ${inner})` - ); - } - if (description.type === InternalSerializerType.MAP) { - const key = genDeclaration(Cast(description).options.key); - const value = genDeclaration(Cast(description).options.value); - - const name = `map_${key}_${value}`; - return addDeclar(name, ` - const ${name} = mapSerializer(fury, ${key}, ${value})` - ); - } - return genBuiltinDeclaration(description.type); - }; - return { - genDeclaration, - finish() { - const result = { - declarations, - names: [...exists.values()], - }; - declarations = []; - exists.clear(); - return result; - }, - }; -} - -export const generateInlineCode = (fury: Fury, description: TypeDescription) => { - const options = Cast(description).options; - const tag = options?.tag; - const { genDeclaration, finish } = typeHandlerDeclaration(fury); - const expectHash = computeStructHash(description); - const read = ` - // relation tag: ${tag} - const result = { - ${Object.entries(options.props).sort().map(([key]) => { - return `${safePropName(key)}: null`; - }).join(",\n")} - }; - pushReadObject(result); - ${Object.entries(options.props).sort().map(([key, value]) => { - return `result${safePropAccessor(key)} = ${genDeclaration(value)}.read()`; - }).join(";\n") - } - return result; -`; - const write = Object.entries(options.props).sort().map(([key, value]) => { - return `${genDeclaration(value)}.write(v${safePropAccessor(key)})`; - }).join(";\n"); - const { names, declarations } = finish(); - const validTag = replaceBackslashAndQuote(tag); - return new Function( - ` -return function (fury, scope) { - const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; - const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; - ${declarations.join("")} - const tagWriter = classResolver.createTagWriter("${validTag}"); - - const reserves = ${names.map(x => `${x}.config().reserve`).join(" + ")}; - return { - ...referenceResolver.deref(() => { - const hash = binaryReader.int32(); - if (hash !== ${expectHash}) { - throw new Error("validate hash failed: ${validTag}. expect ${expectHash}, but got" + hash); - } - { - ${read} - } - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { - tagWriter.write(binaryWriter); - binaryWriter.int32(${expectHash}); - binaryWriter.reserve(reserves); - ${write} - }), - config() { - return { - reserve: tagWriter.bufferLen + 8, - } - } - } -} -` - ); -}; - -export const genSerializer = (fury: Fury, description: TypeDescription) => { - const tag = Cast(description).options?.tag; - if (fury.classResolver.getSerializerByTag(tag)) { - return fury.classResolver.getSerializerByTag(tag); - } - - fury.classResolver.registerSerializerByTag(tag, fury.classResolver.getSerializerById(InternalSerializerType.ANY)); - - const func = generateInlineCode(fury, description); - return fury.classResolver.registerSerializerByTag(tag, func()(fury, { - InternalSerializerType, - RefFlags, - arraySerializer, - tupleSerializer, - mapSerializer, - setSerializer, - })); -}; diff --git a/javascript/packages/fury/lib/description.ts b/javascript/packages/fury/lib/description.ts index e3723bc73f..832a121420 100644 --- a/javascript/packages/fury/lib/description.ts +++ b/javascript/packages/fury/lib/description.ts @@ -39,7 +39,6 @@ export interface ArrayTypeDescription extends TypeDescription { export interface TupleTypeDescription extends TypeDescription { options: { - isTuple: true inner: TypeDescription[] } } @@ -57,10 +56,6 @@ export interface MapTypeDescription extends TypeDescription { } } -export function Cast(p: TypeDescription) { - return p as unknown as T1; -} - type Props = T extends { options: { props?: infer T2 extends { [key: string]: any } @@ -91,7 +86,7 @@ type MapProps = T extends { type TupleProps = T extends { options: { - inner: infer T2 extends readonly[...TypeDescription[]] + inner: infer T2 extends readonly [...TypeDescription[]] } } ? { [K in keyof T2]: ToRecordType } @@ -122,48 +117,52 @@ export type ToRecordType = T extends { | InternalSerializerType.UINT8 | InternalSerializerType.UINT16 | InternalSerializerType.UINT32 - | InternalSerializerType.UINT64 | InternalSerializerType.INT8 | InternalSerializerType.INT16 | InternalSerializerType.INT32 - | InternalSerializerType.INT64 | InternalSerializerType.FLOAT | InternalSerializerType.DOUBLE } ? number + : T extends { - type: InternalSerializerType.MAP + type: InternalSerializerType.UINT64 + | InternalSerializerType.INT64 } - ? MapProps + ? bigint : T extends { - type: InternalSerializerType.FURY_SET + type: InternalSerializerType.MAP } - ? SetProps + ? MapProps : T extends { - type: InternalSerializerType.ARRAY + type: InternalSerializerType.FURY_SET } - ? InnerProps + ? SetProps : T extends { - type: InternalSerializerType.BOOL + type: InternalSerializerType.ARRAY } - ? boolean + ? InnerProps : T extends { - type: InternalSerializerType.DATE + type: InternalSerializerType.BOOL } - ? Date + ? boolean : T extends { - type: InternalSerializerType.TIMESTAMP + type: InternalSerializerType.DATE } - ? number + ? Date : T extends { - type: InternalSerializerType.BINARY + type: InternalSerializerType.TIMESTAMP } - ? Uint8Array + ? number : T extends { - type: InternalSerializerType.ANY + type: InternalSerializerType.BINARY } - ? any - : unknown; + ? Uint8Array + : T extends { + type: InternalSerializerType.ANY + } + ? any + : unknown; export const Type = { any() { @@ -188,7 +187,6 @@ export const Type = { return { type: InternalSerializerType.TUPLE as const, options: { - isTuple: true, inner: t1, }, }; @@ -292,4 +290,39 @@ export const Type = { type: InternalSerializerType.TIMESTAMP as const, }; }, + stringTypedArray() { + return { + type: InternalSerializerType.FURY_STRING_ARRAY as const, + }; + }, + boolTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY as const, + }; + }, + shortTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY as const, + }; + }, + intTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY as const, + }; + }, + longTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY as const, + }; + }, + floatTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY as const, + }; + }, + doubleTypedArray() { + return { + type: InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY as const, + }; + }, }; diff --git a/javascript/packages/fury/lib/fury.ts b/javascript/packages/fury/lib/fury.ts index 5c4c3cef4b..b2c104b3b0 100644 --- a/javascript/packages/fury/lib/fury.ts +++ b/javascript/packages/fury/lib/fury.ts @@ -29,7 +29,7 @@ export default (config: Config) => { const binaryWriter = BinaryWriter(config); const classResolver = new ClassResolver(); - const referenceResolver = ReferenceResolver(config, binaryWriter, binaryReader, classResolver); + const referenceResolver = ReferenceResolver(config, binaryWriter, binaryReader); const fury = { config, @@ -95,11 +95,14 @@ export default (config: Config) => { const cursor = binaryWriter.getCursor(); binaryWriter.skip(4); // preserve 4-byte for nativeObjects start offsets. binaryWriter.uint32(0); // nativeObjects length. - if (serializer) { - serializer.write(data); - } else { - classResolver.getSerializerById(InternalSerializerType.ANY).write(data); + if (!serializer) { + serializer = classResolver.getSerializerById(InternalSerializerType.ANY); } + // reserve fixed size + binaryWriter.reserve(serializer.meta.fixedSize); + // start write + serializer.write(data); + binaryWriter.setUint32Position(cursor, binaryWriter.getCursor()); // nativeObjects start offsets; return binaryWriter; } diff --git a/javascript/packages/fury/lib/gen/any.ts b/javascript/packages/fury/lib/gen/any.ts new file mode 100644 index 0000000000..47994c0501 --- /dev/null +++ b/javascript/packages/fury/lib/gen/any.ts @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class AnySerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private addDep() { + return this.scope.declare( + "any_ser", + this.builder.classResolver.getSerializerById(InternalSerializerType.ANY) + ); + } + + writeStmt(accessor: string): string { + const name = this.addDep(); + return `${name}.writeInner(${accessor})`; + } + + readStmt(accessor: (expr: string) => string): string { + const name = this.addDep(); + return `${name}.readInner(${accessor})`; + } + + toReadEmbed(accessor: (expr: string) => string): string { + const name = this.addDep(); + return accessor(`${name}.read()`); + } + + toWriteEmbed(accessor: string): string { + const name = this.addDep(); + return `${name}.write(${accessor})`; + } +} + +CodegenRegistry.register(InternalSerializerType.ANY, AnySerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/array.ts b/javascript/packages/fury/lib/gen/array.ts new file mode 100644 index 0000000000..002928f24a --- /dev/null +++ b/javascript/packages/fury/lib/gen/array.ts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ArrayTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class ArraySerializerGenerator extends BaseSerializerGenerator { + description: ArrayTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const inner = this.description.options.inner; + return this.builder.meta(inner); + } + + private innerGenerator() { + const inner = this.description.options.inner; + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + return new InnerGeneratorClass(inner, this.builder, this.scope); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const item = this.scope.uniqueName("item"); + return ` + ${this.builder.writer.varUInt32(`${accessor}.length`)} + ${this.builder.writer.reserve(`${innerMeta.fixedSize} * ${accessor}.length`)}; + for (const ${item} of ${accessor}) { + ${innerGenerator.toWriteEmbed(item)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const len = this.scope.uniqueName("len"); + const idx = this.scope.uniqueName("idx"); + + return ` + const ${len} = ${this.builder.reader.varUInt32()}; + const ${result} = new Array(${len}); + ${this.pushReadRefStmt(result)} + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + ${innerGenerator.toReadEmbed(x => `${result}[${idx}] = ${x};`)} + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.ARRAY, ArraySerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/binary.ts b/javascript/packages/fury/lib/gen/binary.ts new file mode 100644 index 0000000000..e18329e199 --- /dev/null +++ b/javascript/packages/fury/lib/gen/binary.ts @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class BinarySerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + return ` + ${this.builder.writer.uint8(1)} + ${this.builder.writer.uint32(`${accessor}.byteLength`)} + ${this.builder.writer.buffer(accessor)} + `; + } + + readStmt(accessor: (expr: string) => string): string { + const result = this.scope.uniqueName("result"); + return ` + ${this.builder.reader.uint8()} + ${result} = ${this.builder.reader.buffer(this.builder.reader.int32())}; + ${this.pushReadRefStmt(result)}; + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.BINARY, BinarySerializerGenerator); diff --git a/javascript/packages/fury/lib/internalSerializer/binary.ts b/javascript/packages/fury/lib/gen/bool.ts similarity index 50% rename from javascript/packages/fury/lib/internalSerializer/binary.ts rename to javascript/packages/fury/lib/gen/bool.ts index fc4694f747..0c4fb5be00 100644 --- a/javascript/packages/fury/lib/internalSerializer/binary.ts +++ b/javascript/packages/fury/lib/gen/bool.ts @@ -17,33 +17,28 @@ * under the License. */ -import { Fury } from "../type"; +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; -export default (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; +class BoolSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; - const { uint8: writeUInt8, int32: writeInt32, buffer: writeBuffer } = binaryWriter; - const { uint8: readUInt8, int32: readInt32, buffer: readBuffer } = binaryReader; - const { pushReadObject } = referenceResolver; + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } - return { - ...referenceResolver.deref(() => { - readUInt8(); // isInBand - const len = readInt32(); - const result = readBuffer(len); - pushReadObject(result); - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.BINARY, (v: Uint8Array) => { - writeUInt8(1); // is inBand - writeInt32(v.byteLength); - writeBuffer(v); - }), - config: () => { - return { - reserve: 8, - }; - }, - }; -}; + writeStmt(accessor: string): string { + return this.builder.writer.uint8(`${accessor} ? 1 : 0`); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(`${this.builder.reader.uint8()} === 1`); + } +} + +CodegenRegistry.register(InternalSerializerType.BOOL, BoolSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/builder.ts b/javascript/packages/fury/lib/gen/builder.ts new file mode 100644 index 0000000000..4fb4142442 --- /dev/null +++ b/javascript/packages/fury/lib/gen/builder.ts @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Fury } from "../type"; +import { Scope } from "./scope"; +import { getMeta } from "../meta"; +import { TypeDescription } from "../description"; + +class BinaryReaderBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + getCursor() { + return `${this.holder}.getCursor()`; + } + + setCursor(v: number | string) { + return `${this.holder}.setCursor(${v})`; + } + + varInt32() { + return `${this.holder}.varInt32()`; + } + + varInt64() { + return `${this.holder}.varInt64()`; + } + + varUInt32() { + return `${this.holder}.varUInt32()`; + } + + varUInt64() { + return `${this.holder}.varUInt64()`; + } + + int8() { + return `${this.holder}.int8()`; + } + + buffer(len: string | number) { + return `${this.holder}.buffer(${len})`; + } + + bufferRef() { + return `${this.holder}.bufferRef()`; + } + + uint8() { + return `${this.holder}.uint8()`; + } + + stringUtf8At() { + return `${this.holder}.stringUtf8At()`; + } + + stringUtf8() { + return `${this.holder}.stringUtf8()`; + } + + stringLatin1() { + return `${this.holder}.stringLatin1()`; + } + + stringOfVarUInt32() { + return `${this.holder}.stringOfVarUInt32()`; + } + + double() { + return `${this.holder}.double()`; + } + + float() { + return `${this.holder}.float()`; + } + + uint16() { + return `${this.holder}.uint16()`; + } + + int16() { + return `${this.holder}.int16()`; + } + + uint64() { + return `${this.holder}.uint64()`; + } + + skip() { + return `${this.holder}.skip()`; + } + + int64() { + return `${this.holder}.int64()`; + } + + sliLong() { + return `${this.holder}.sliLong()`; + } + + uint32() { + return `${this.holder}.uint32()`; + } + + int32() { + return `${this.holder}.int32()`; + } +} + +class BinaryWriterBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + skip(v: number | string) { + return `${this.holder}.skip(${v})`; + } + + getByteLen() { + return `${this.holder}.getByteLen()`; + } + + getReserved() { + return `${this.holder}.getReserved()`; + } + + reserve(v: number | string) { + return `${this.holder}.reserve(${v})`; + } + + uint16(v: number | string) { + return `${this.holder}.uint16(${v})`; + } + + int8(v: number | string) { + return `${this.holder}.int8(${v})`; + } + + int24(v: number | string) { + return `${this.holder}.int24(${v})`; + } + + uint8(v: number | string) { + return `${this.holder}.uint8(${v})`; + } + + int16(v: number | string) { + return `${this.holder}.int16(${v})`; + } + + varInt32(v: number | string) { + return `${this.holder}.varInt32(${v})`; + } + + varUInt32(v: number | string) { + return `${this.holder}.varUInt32(${v})`; + } + + varUInt64(v: number | string) { + return `${this.holder}.varUInt64(${v})`; + } + + varInt64(v: number | string) { + return `${this.holder}.varInt64(${v})`; + } + + stringOfVarUInt32(str: string) { + return `${this.holder}.stringOfVarUInt32(${str})`; + } + + bufferWithoutMemCheck(v: string) { + return `${this.holder}.bufferWithoutMemCheck(${v})`; + } + + uint64(v: number | string) { + return `${this.holder}.uint64(${v})`; + } + + buffer(v: string) { + return `${this.holder}.buffer(${v})`; + } + + double(v: number | string) { + return `${this.holder}.double(${v})`; + } + + float(v: number | string) { + return `${this.holder}.float(${v})`; + } + + int64(v: number | string) { + return `${this.holder}.int64(${v})`; + } + + sliLong(v: number | string) { + return `${this.holder}.sliLong(${v})`; + } + + uint32(v: number | string) { + return `${this.holder}.uint32(${v})`; + } + + int32(v: number | string) { + return `${this.holder}.int32(${v})`; + } + + getCursor() { + return `${this.holder}.getCursor()`; + } +} + +class ReferenceResolverBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + getReadObjectByRefId(id: string | number) { + return `${this.holder}.getReadObjectByRefId(${id})`; + } + + pushReadObject(obj: string) { + return `${this.holder}.pushReadObject(${obj})`; + } + + pushWriteObject(obj: string) { + return `${this.holder}.pushWriteObject(${obj})`; + } + + existsWriteObject(obj: string) { + return `${this.holder}.existsWriteObject(${obj})`; + } +} + +class ClassResolverBuilder { + constructor(private holder: string) { + + } + + ownName() { + return this.holder; + } + + getSerializerById(id: string | number) { + return `${this.holder}.getSerializerById(${id})`; + } + + getSerializerByTag(tag: string) { + return `${this.holder}.getSerializerByTag(${tag})`; + } + + createTagWriter(tag: string) { + return `${this.holder}.createTagWriter("${tag}")`; + } + + readTag(binaryReader: string) { + return `${this.holder}.readTag(${binaryReader})`; + } +} + +export class CodecBuilder { + reader: BinaryReaderBuilder; + writer: BinaryWriterBuilder; + referenceResolver: ReferenceResolverBuilder; + classResolver: ClassResolverBuilder; + + constructor(scope: Scope, private fury: Fury) { + const br = scope.declareByName("br", "fury.binaryReader"); + const bw = scope.declareByName("bw", "fury.binaryWriter"); + const cr = scope.declareByName("cr", "fury.classResolver"); + const rr = scope.declareByName("rr", "fury.referenceResolver"); + this.reader = new BinaryReaderBuilder(br); + this.writer = new BinaryWriterBuilder(bw); + this.classResolver = new ClassResolverBuilder(cr); + this.referenceResolver = new ReferenceResolverBuilder(rr); + } + + meta(description: TypeDescription) { + return getMeta(description, this.fury); + } + + config() { + return this.fury.config; + } + + static isReserved(key: string) { + return /^(?:do|if|in|for|let|new|try|var|case|else|enum|eval|false|null|this|true|void|with|break|catch|class|const|super|throw|while|yield|delete|export|import|public|return|static|switch|typeof|default|extends|finally|package|private|continue|debugger|function|arguments|interface|protected|implements|instanceof)$/.test(key); + } + + static isDotPropAccessor(prop: string) { + return /^[a-zA-Z_$][0-9a-zA-Z_$]*$/.test(prop); + } + + static replaceBackslashAndQuote(v: string) { + return v.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); + } + + static safePropAccessor(prop: string) { + if (!CodecBuilder.isDotPropAccessor(prop) || CodecBuilder.isReserved(prop)) { + return `["${CodecBuilder.replaceBackslashAndQuote(prop)}"]`; + } + return `.${prop}`; + } + + static safePropName(prop: string) { + if (!CodecBuilder.isDotPropAccessor(prop) || CodecBuilder.isReserved(prop)) { + return `["${CodecBuilder.replaceBackslashAndQuote(prop)}"]`; + } + return prop; + } +} diff --git a/javascript/packages/fury/lib/gen/datetime.ts b/javascript/packages/fury/lib/gen/datetime.ts new file mode 100644 index 0000000000..bce8239ca5 --- /dev/null +++ b/javascript/packages/fury/lib/gen/datetime.ts @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class TimestampSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + if (/^-?[0-9]+$/.test(accessor)) { + return this.builder.writer.int64(`BigInt(${accessor})`); + } + return this.builder.writer.int64(`BigInt(${accessor}.getTime())`); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(`new Date(Number(${this.builder.reader.int64()}))`); + } +} + +class DateSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + const epoch = this.scope.declareByName("epoch", `new Date("1970/01/01 00:00").getTime()`); + if (/^-?[0-9]+$/.test(accessor)) { + return ` + ${this.builder.writer.int32(`Math.floor((${accessor} - ${epoch}) / 1000 / (24 * 60 * 60))`)} + `; + } + return ` + ${this.builder.writer.int32(`Math.floor((${accessor}.getTime() - ${epoch}) / 1000 / (24 * 60 * 60))`)} + `; + } + + readStmt(accessor: (expr: string) => string): string { + const epoch = this.scope.declareByName("epoch", `new Date("1970/01/01 00:00").getTime()`); + return accessor(` + new Date(${epoch} + (${this.builder.reader.int32()} * (24 * 60 * 60) * 1000)) + `); + } +} + +CodegenRegistry.register(InternalSerializerType.DATE, DateSerializerGenerator); +CodegenRegistry.register(InternalSerializerType.TIMESTAMP, TimestampSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/index.ts b/javascript/packages/fury/lib/gen/index.ts new file mode 100644 index 0000000000..84f60aa015 --- /dev/null +++ b/javascript/packages/fury/lib/gen/index.ts @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { InternalSerializerType, Fury } from "../type"; +import { ArrayTypeDescription, MapTypeDescription, ObjectTypeDescription, SetTypeDescription, TupleTypeDescription, TypeDescription } from "../description"; +import { CodegenRegistry } from "./router"; +import { CodecBuilder } from "./builder"; +import { Scope } from "./scope"; +import "./array"; +import "./object"; +import "./string"; +import "./binary"; +import "./bool"; +import "./datetime"; +import "./map"; +import "./number"; +import "./set"; +import "./any"; +import "./tuple"; +import "./typedArray"; + +export const generate = (fury: Fury, description: TypeDescription) => { + const InnerGeneratorClass = CodegenRegistry.get(description.type); + if (!InnerGeneratorClass) { + throw new Error(`${description.type} generator not exists`); + } + const scope = new Scope(); + const generator = new InnerGeneratorClass(description, new CodecBuilder(scope, fury), scope); + + const funcString = generator.toSerializer(); + const afterCodeGenerated = fury.config?.hooks?.afterCodeGenerated; + if (typeof afterCodeGenerated === "function") { + return new Function(afterCodeGenerated(funcString)); + } + return new Function(funcString); +}; + +function regDependencies(fury: Fury, description: TypeDescription) { + if (description.type === InternalSerializerType.FURY_TYPE_TAG) { + const options = (description).options; + if (options.props) { + fury.classResolver.registerSerializerByTag(options.tag); + Object.values(options.props).forEach((x) => { + regDependencies(fury, x); + }); + const func = generate(fury, description); + fury.classResolver.registerSerializerByTag(options.tag, func()(fury, {})); + } + } + if (description.type === InternalSerializerType.ARRAY) { + regDependencies(fury, (description).options.inner); + } + if (description.type === InternalSerializerType.FURY_SET) { + regDependencies(fury, (description).options.key); + } + if (description.type === InternalSerializerType.MAP) { + regDependencies(fury, (description).options.key); + regDependencies(fury, (description).options.value); + } + if (description.type === InternalSerializerType.TUPLE) { + (description).options.inner.forEach((x) => { + regDependencies(fury, x); + }); + } +} + +export const generateSerializer = (fury: Fury, description: TypeDescription) => { + regDependencies(fury, description); + if (description.type === InternalSerializerType.FURY_TYPE_TAG) { + return fury.classResolver.getSerializerByTag((description).options.tag); + } + const func = generate(fury, description); + return func()(fury, {}); +}; diff --git a/javascript/packages/fury/lib/gen/map.ts b/javascript/packages/fury/lib/gen/map.ts new file mode 100644 index 0000000000..6ee9f1b7a5 --- /dev/null +++ b/javascript/packages/fury/lib/gen/map.ts @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { MapTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class MapSerializerGenerator extends BaseSerializerGenerator { + description: MapTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const key = this.description.options.key; + const value = this.description.options.value; + return [this.builder.meta(key), this.builder.meta(value)]; + } + + private innerGenerator() { + const key = this.description.options.key; + const value = this.description.options.value; + + const KeyGeneratorClass = CodegenRegistry.get(key.type); + const ValueGeneratorClass = CodegenRegistry.get(value.type); + if (!KeyGeneratorClass) { + throw new Error(`${key.type} generator not exists`); + } + if (!ValueGeneratorClass) { + throw new Error(`${value.type} generator not exists`); + } + return [new KeyGeneratorClass(key, this.builder, this.scope), new ValueGeneratorClass(value, this.builder, this.scope)]; + } + + writeStmt(accessor: string): string { + const [keyMeta, valueMeta] = this.innerMeta(); + const [keyGenerator, valueGenerator] = this.innerGenerator(); + const key = this.scope.uniqueName("key"); + const value = this.scope.uniqueName("value"); + + return ` + ${this.builder.writer.varUInt32(`${accessor}.size`)} + ${this.builder.writer.reserve(`${keyMeta.fixedSize + valueMeta.fixedSize} * ${accessor}.size`)}; + for (const [${key}, ${value}] of ${accessor}.entries()) { + ${keyGenerator.toWriteEmbed(key)} + ${valueGenerator.toWriteEmbed(value)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const [keyGenerator, valueGenerator] = this.innerGenerator(); + const key = this.scope.uniqueName("key"); + const value = this.scope.uniqueName("value"); + + const result = this.scope.uniqueName("result"); + const idx = this.scope.uniqueName("idx"); + const len = this.scope.uniqueName("len"); + + return ` + const ${result} = new Map(); + ${this.pushReadRefStmt(result)}; + const ${len} = ${this.builder.reader.varUInt32()}; + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + let ${key}; + let ${value}; + ${keyGenerator.toReadEmbed(x => `${key} = ${x};`)} + ${valueGenerator.toReadEmbed(x => `${value} = ${x};`)} + ${result}.set(${key}, ${value}); + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.MAP, MapSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/number.ts b/javascript/packages/fury/lib/gen/number.ts new file mode 100644 index 0000000000..1ca410a562 --- /dev/null +++ b/javascript/packages/fury/lib/gen/number.ts @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +function buildNumberSerializer(writeFun: (builder: CodecBuilder, accessor: string) => string, read: (builder: CodecBuilder) => string) { + return class NumberSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + return writeFun(this.builder, accessor); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(read(this.builder)); + } + }; +} + +CodegenRegistry.register(InternalSerializerType.UINT8, + buildNumberSerializer( + (builder, accessor) => builder.writer.uint8(accessor), + builder => builder.reader.uint8() + ) +); +CodegenRegistry.register(InternalSerializerType.INT8, + buildNumberSerializer( + (builder, accessor) => builder.writer.int8(accessor), + builder => builder.reader.int8() + ) +); +CodegenRegistry.register(InternalSerializerType.UINT16, + buildNumberSerializer( + (builder, accessor) => builder.writer.uint16(accessor), + builder => builder.reader.uint16() + ) +); +CodegenRegistry.register(InternalSerializerType.INT16, + buildNumberSerializer( + (builder, accessor) => builder.writer.int16(accessor), + builder => builder.reader.int16() + ) +); +CodegenRegistry.register(InternalSerializerType.UINT32, + buildNumberSerializer( + (builder, accessor) => builder.writer.uint32(accessor), + builder => builder.reader.uint32() + ) +); +CodegenRegistry.register(InternalSerializerType.INT32, + buildNumberSerializer( + (builder, accessor) => builder.writer.int32(accessor), + builder => builder.reader.int32() + ) +); +CodegenRegistry.register(InternalSerializerType.UINT64, + buildNumberSerializer( + (builder, accessor) => builder.writer.varUInt64(accessor), + builder => builder.reader.varUInt64() + ) +); +CodegenRegistry.register(InternalSerializerType.INT64, + buildNumberSerializer( + (builder, accessor) => builder.writer.sliLong(accessor), + builder => builder.reader.sliLong() + ) +); +CodegenRegistry.register(InternalSerializerType.FLOAT, + buildNumberSerializer( + (builder, accessor) => builder.writer.float(accessor), + builder => builder.reader.float() + ) +); +CodegenRegistry.register(InternalSerializerType.DOUBLE, + buildNumberSerializer( + (builder, accessor) => builder.writer.double(accessor), + builder => builder.reader.double() + ) +); diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts new file mode 100644 index 0000000000..0d97ce49db --- /dev/null +++ b/javascript/packages/fury/lib/gen/object.ts @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { InternalSerializerType, MaxInt32 } from "../type"; +import { Scope } from "./scope"; +import { CodecBuilder } from "./builder"; +import { ObjectTypeDescription, TypeDescription } from "../description"; +import { fromString } from "../platformBuffer"; +import { CodegenRegistry } from "./router"; +import { BaseSerializerGenerator } from "./serializer"; + +function computeFieldHash(hash: number, id: number): number { + let newHash = (hash) * 31 + (id); + while (newHash >= MaxInt32) { + newHash = Math.floor(newHash / 7); + } + return newHash; +} + +const computeStringHash = (str: string) => { + const bytes = fromString(str); + let hash = 17; + bytes.forEach((b) => { + hash = hash * 31 + b; + while (hash >= MaxInt32) { + hash = Math.floor(hash / 7); + } + }); + return hash; +}; + +const computeStructHash = (description: TypeDescription) => { + let hash = 17; + for (const [, value] of Object.entries((description).options.props).sort()) { + let id = value.type; + if (value.type === InternalSerializerType.ARRAY || value.type === InternalSerializerType.TUPLE || value.type === InternalSerializerType.MAP) { + id = Math.floor(value.type); // TODO add map key&value type into schema hash + } else if (value.type === InternalSerializerType.FURY_TYPE_TAG) { + id = computeStringHash((value).options.tag); + } + hash = computeFieldHash(hash, id); + } + return hash; +}; + +class ObjectSerializerGenerator extends BaseSerializerGenerator { + description: ObjectTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + const options = this.description.options; + const expectHash = computeStructHash(this.description); + const tagWriter = this.scope.declare("tagWriter", `${this.builder.classResolver.createTagWriter(this.safeTag())}`); + + return ` + ${tagWriter}.write(${this.builder.writer.ownName()}); + ${this.builder.writer.int32(expectHash)}; + ${Object.entries(options.props).sort().map(([key, inner]) => { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); + return innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`); + }).join(";\n") + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const options = this.description.options; + const expectHash = computeStructHash(this.description); + const result = this.scope.uniqueName("result"); + return ` + if (${this.builder.reader.int32()} !== ${expectHash}) { + throw new Error("validate hash failed: ${this.safeTag()}. expect ${expectHash}"); + } + const ${result} = { + ${Object.entries(options.props).sort().map(([key]) => { + return `${CodecBuilder.safePropName(key)}: null`; + }).join(",\n")} + }; + ${this.pushReadRefStmt(result)} + ${Object.entries(options.props).sort().map(([key, inner]) => { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); + return innerGenerator.toReadEmbed(expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`); + }).join(";\n") + } + ${accessor(result)} + `; + } + + safeTag() { + return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); + } + + toReadEmbed(accessor: (expr: string) => string): string { + const name = this.scope.declare( + "tag_ser", + `fury.classResolver.getSerializerByTag("${this.safeTag()}")` + ); + return accessor(`${name}.read()`); + } + + toWriteEmbed(accessor: string): string { + const name = this.scope.declare( + "tag_ser", + `fury.classResolver.getSerializerByTag("${this.safeTag()}")` + ); + return `${name}.write(${accessor})`; + } +} + +CodegenRegistry.register(InternalSerializerType.FURY_TYPE_TAG, ObjectSerializerGenerator); diff --git a/javascript/packages/fury/lib/internalSerializer/bool.ts b/javascript/packages/fury/lib/gen/router.ts similarity index 52% rename from javascript/packages/fury/lib/internalSerializer/bool.ts rename to javascript/packages/fury/lib/gen/router.ts index 66e3d04cfc..a8cd043a1f 100644 --- a/javascript/packages/fury/lib/internalSerializer/bool.ts +++ b/javascript/packages/fury/lib/gen/router.ts @@ -17,28 +17,22 @@ * under the License. */ -import { Fury } from "../type"; -import { InternalSerializerType, RefFlags } from "../type"; +import { TypeDescription } from "../description"; +import { SerializerGenerator } from "./serializer"; +import { InternalSerializerType } from "../type"; +import { CodecBuilder } from "./builder"; +import { Scope } from "./scope"; -export default (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { uint8: readUInt8 } = binaryReader; - const { int8: writeInt8, uint8: writeUInt8 } = binaryWriter; - return { - ...referenceResolver.deref(() => { - return readUInt8() === 0 ? false : true; - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.BOOL, false, (v: boolean) => { - writeUInt8(v ? 1 : 0); - }), - writeWithoutType: (v: boolean) => { - writeInt8(RefFlags.NotNullValueFlag); - writeUInt8(v ? 1 : 0); - }, - config: () => { - return { - reserve: 4, - }; - }, - }; -}; +type SerializerGeneratorConstructor = new (description: TypeDescription, builder: CodecBuilder, scope: Scope) => SerializerGenerator; + +export class CodegenRegistry { + static map = new Map(); + + static register(type: InternalSerializerType, generator: SerializerGeneratorConstructor) { + this.map.set(InternalSerializerType[type], generator); + } + + static get(type: InternalSerializerType) { + return this.map.get(InternalSerializerType[type]); + } +} diff --git a/javascript/packages/fury/lib/gen/scope.ts b/javascript/packages/fury/lib/gen/scope.ts new file mode 100644 index 0000000000..a5f51e099f --- /dev/null +++ b/javascript/packages/fury/lib/gen/scope.ts @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export class Scope { + private declares: Map = new Map(); + private idx = 0; + + private addDeclar(stmt: string, name: string) { + if (this.declares.has(stmt)) { + return this.declares.get(stmt)!; + } + this.declares.set(stmt, name); + return name; + } + + uniqueName(prefix: string) { + return `${prefix}_${this.idx++}`; + } + + declareByName(name: string, stmt: string) { + return this.addDeclar(stmt, name); + } + + assertNameNotDuplicate(name: string) { + for (const item of this.declares.values()) { + if (item === name) { + throw new Error(`const ${name} declare duplicate`); + } + } + } + + declare(prefix: string, stmt: string) { + return this.addDeclar(stmt, this.uniqueName(prefix)); + } + + generate() { + return Array.from(this.declares.entries()).map(x => `const ${x[1]} = ${x[0]};`).join("\n"); + } +} diff --git a/javascript/packages/fury/lib/gen/serializer.ts b/javascript/packages/fury/lib/gen/serializer.ts new file mode 100644 index 0000000000..34eaa17aa6 --- /dev/null +++ b/javascript/packages/fury/lib/gen/serializer.ts @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { InternalSerializerType } from "../type"; +import { CodecBuilder } from "./builder"; +import { makeHead } from "../referenceResolver"; +import { RefFlags } from "../type"; +import { Scope } from "./scope"; +import { TypeDescription } from "../description"; + +export interface SerializerGenerator { + writeStmt(accessor: string): string + readStmt(accessor: (expr: string) => string): string + toSerializer(): string + toWriteEmbed(accessor: string, withHead?: boolean): string + toReadEmbed(accessor: (expr: string) => string, withHead?: boolean): string +} + +export abstract class BaseSerializerGenerator implements SerializerGenerator { + constructor( + protected description: TypeDescription, + protected builder: CodecBuilder, + protected scope: Scope, + ) { + + } + + abstract writeStmt(accessor: string): string; + + abstract readStmt(accessor: (expr: string) => string): string; + + protected pushReadRefStmt(accessor: string) { + if (!this.builder.config().refTracking) { + return ""; + } + return this.builder.referenceResolver.pushReadObject(accessor); + } + + protected wrapWriteHead(accessor: string, stmt: (accessor: string) => string) { + const meta = this.builder.meta(this.description); + const noneable = meta.noneable; + if (noneable) { + const head = makeHead(RefFlags.RefValueFlag, this.description.type); + + const normaStmt = ` + if (${accessor} !== null && ${accessor} !== undefined) { + ${this.builder.writer.int24(head)}; + ${stmt(accessor)}; + } else { + ${this.builder.writer.int8(RefFlags.NullFlag)}; + } + `; + if (this.builder.config().refTracking) { + const existsId = this.scope.uniqueName("existsId"); + return ` + const ${existsId} = ${this.builder.referenceResolver.existsWriteObject(accessor)}; + if (typeof ${existsId} === "number") { + ${this.builder.writer.int8(RefFlags.RefFlag)} + ${this.builder.writer.varUInt32(existsId)} + } else { + ${this.builder.referenceResolver.pushWriteObject(accessor)} + ${normaStmt} + } + `; + } else { + return normaStmt; + } + } else { + const head = makeHead(RefFlags.NotNullValueFlag, this.description.type); + return ` + ${this.builder.writer.int24(head)}; + if (${accessor} !== null && ${accessor} !== undefined) { + ${stmt(accessor)}; + } else { + ${typeof meta.default === "string" ? stmt(`"${meta.default}"`) : stmt(meta.default)}; + }`; + } + } + + protected wrapReadHead(accessor: (expr: string) => string, stmt: (accessor: (expr: string) => string) => string) { + return ` + switch (${this.builder.reader.int8()}) { + case ${RefFlags.NotNullValueFlag}: + case ${RefFlags.RefValueFlag}: + if (${this.builder.reader.int16()} === ${InternalSerializerType.FURY_TYPE_TAG}) { + ${this.builder.classResolver.readTag(this.builder.reader.ownName())}; + } + ${stmt(accessor)} + break; + case ${RefFlags.RefFlag}: + ${accessor(this.builder.referenceResolver.getReadObjectByRefId(this.builder.reader.varUInt32()))} + break; + case ${RefFlags.NullFlag}: + ${accessor("null")} + break; + } + `; + } + + toWriteEmbed(accessor: string, withHead = true) { + if (!withHead) { + return this.writeStmt(accessor); + } + return this.wrapWriteHead(accessor, accessor => this.writeStmt(accessor)); + } + + toReadEmbed(accessor: (expr: string) => string, withHead = true) { + if (!withHead) { + return this.readStmt(accessor); + } + return this.wrapReadHead(accessor, accessor => this.readStmt(accessor)); + } + + toSerializer() { + this.scope.assertNameNotDuplicate("read"); + this.scope.assertNameNotDuplicate("readInner"); + this.scope.assertNameNotDuplicate("write"); + this.scope.assertNameNotDuplicate("writeInner"); + + const declare = ` + const readInner = () => { + ${this.readStmt(x => `return ${x}`)} + }; + const read = () => { + ${this.wrapReadHead(x => `return ${x}`, accessor => accessor(`readInner()`))} + }; + const writeInner = (v) => { + ${this.writeStmt("v")} + }; + const write = (v) => { + ${this.wrapWriteHead("v", accessor => `writeInner(${accessor})`)} + }; + `; + return ` + return function (fury, external) { + ${this.scope.generate()} + ${declare} + return { + read, + readInner, + write, + writeInner, + meta: ${JSON.stringify(this.builder.meta(this.description))} + }; + } + `; + } +} diff --git a/javascript/packages/fury/lib/gen/set.ts b/javascript/packages/fury/lib/gen/set.ts new file mode 100644 index 0000000000..3a595b163a --- /dev/null +++ b/javascript/packages/fury/lib/gen/set.ts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SetTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class SetSerializerGenerator extends BaseSerializerGenerator { + description: SetTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const inner = this.description.options.key; + return this.builder.meta(inner); + } + + private innerGenerator() { + const inner = this.description.options.key; + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + return new InnerGeneratorClass(inner, this.builder, this.scope); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const item = this.scope.uniqueName("item"); + return ` + ${this.builder.writer.varUInt32(`${accessor}.size`)} + ${this.builder.writer.reserve(`${innerMeta.fixedSize} * ${accessor}.size`)}; + for (const ${item} of ${accessor}.values()) { + ${innerGenerator.toWriteEmbed(item)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const idx = this.scope.uniqueName("idx"); + const len = this.scope.uniqueName("len"); + + return ` + const ${result} = new Set(); + ${this.pushReadRefStmt(result)} + const ${len} = ${this.builder.reader.varUInt32()}; + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + ${innerGenerator.toReadEmbed(x => `${result}.add(${x});`)} + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.FURY_SET, SetSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/string.ts b/javascript/packages/fury/lib/gen/string.ts new file mode 100644 index 0000000000..475b7a41a3 --- /dev/null +++ b/javascript/packages/fury/lib/gen/string.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class StringSerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + writeStmt(accessor: string): string { + return this.builder.writer.stringOfVarUInt32(accessor); + } + + readStmt(accessor: (expr: string) => string): string { + return accessor(this.builder.reader.stringOfVarUInt32()); + } +} + +CodegenRegistry.register(InternalSerializerType.STRING, StringSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/tuple.ts b/javascript/packages/fury/lib/gen/tuple.ts new file mode 100644 index 0000000000..ee3d8ce172 --- /dev/null +++ b/javascript/packages/fury/lib/gen/tuple.ts @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TupleTypeDescription, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +class TupleSerializerGenerator extends BaseSerializerGenerator { + description: TupleTypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + const inner = this.description.options.inner; + return inner.map(x => this.builder.meta(x)); + } + + private innerGenerator() { + const inner = this.description.options.inner; + return inner.map((x) => { + const InnerGeneratorClass = CodegenRegistry.get(x.type); + if (!InnerGeneratorClass) { + throw new Error(`${x.type} generator not exists`); + } + return new InnerGeneratorClass(x, this.builder, this.scope); + }); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const fixedSize = innerMeta.reduce((x, y) => x + y.fixedSize, 0); + + return ` + ${this.builder.writer.varUInt32(innerMeta.length)} + ${this.builder.writer.reserve(fixedSize)}; + ${ + innerGenerator.map((generator, index) => { + return generator.toWriteEmbed(`${accessor}[${index}]`); + }).join("\n") + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const len = this.scope.uniqueName("len"); + + return ` + const ${len} = ${this.builder.reader.varUInt32()}; + const ${result} = new Array(${len}); + ${this.pushReadRefStmt(result)} + ${ + innerGenerator.map((generator, index) => { + return ` + if (${len} > ${index}) { + ${generator.toReadEmbed(expr => `${result}[${index}] = ${expr}`)} + } + `; + }).join("\n") + } + ${accessor(result)} + `; + } +} + +CodegenRegistry.register(InternalSerializerType.TUPLE, TupleSerializerGenerator); diff --git a/javascript/packages/fury/lib/gen/typedArray.ts b/javascript/packages/fury/lib/gen/typedArray.ts new file mode 100644 index 0000000000..4d83019071 --- /dev/null +++ b/javascript/packages/fury/lib/gen/typedArray.ts @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Type, TypeDescription } from "../description"; +import { CodecBuilder } from "./builder"; +import { BaseSerializerGenerator } from "./serializer"; +import { CodegenRegistry } from "./router"; +import { InternalSerializerType } from "../type"; +import { Scope } from "./scope"; + +function build(inner: TypeDescription) { + return class TypedArraySerializerGenerator extends BaseSerializerGenerator { + description: TypeDescription; + + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; + } + + private innerMeta() { + return this.builder.meta(inner); + } + + private innerGenerator() { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + return new InnerGeneratorClass(inner, this.builder, this.scope); + } + + writeStmt(accessor: string): string { + const innerMeta = this.innerMeta(); + const innerGenerator = this.innerGenerator(); + const item = this.scope.uniqueName("item"); + return ` + ${this.builder.writer.varUInt32(`${accessor}.length`)} + ${this.builder.writer.reserve(`${innerMeta.fixedSize} * ${accessor}.length`)}; + for (const ${item} of ${accessor}) { + ${innerGenerator.toWriteEmbed(item, false)} + } + `; + } + + readStmt(accessor: (expr: string) => string): string { + const innerGenerator = this.innerGenerator(); + const result = this.scope.uniqueName("result"); + const len = this.scope.uniqueName("len"); + const idx = this.scope.uniqueName("idx"); + + return ` + const ${len} = ${this.builder.reader.varUInt32()}; + const ${result} = new Array(${len}); + ${this.pushReadRefStmt(result)} + for (let ${idx} = 0; ${idx} < ${len}; ${idx}++) { + ${innerGenerator.toReadEmbed(x => `${result}[${idx}] = ${x};`, false)} + } + ${accessor(result)} + `; + } + }; +} +CodegenRegistry.register(InternalSerializerType.FURY_STRING_ARRAY, build(Type.string())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY, build(Type.bool())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY, build(Type.int64())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY, build(Type.int32())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY, build(Type.float())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY, build(Type.double())); +CodegenRegistry.register(InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY, build(Type.int16())); diff --git a/javascript/packages/fury/lib/internalSerializer/array.ts b/javascript/packages/fury/lib/internalSerializer/array.ts deleted file mode 100644 index dae37a3384..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/array.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, Serializer } from "../type"; -import { Fury } from "../type"; - -export const buildArray = (fury: Fury, item: Serializer, type: InternalSerializerType) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - - const { pushReadObject } = referenceResolver; - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { varUInt32: readVarUInt32 } = binaryReader; - const { write, read } = item; - const innerHeadSize = (item.config().reserve); - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Array(len); - pushReadObject(result); - for (let i = 0; i < result.length; i++) { - result[i] = read(); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(type, (v: any[]) => { - writeVarUInt32(v.length); - - reserves(innerHeadSize * v.length); - - for (const x of v) { - write(x); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -const buildTypedArray = (fury: Fury, serializeType: InternalSerializerType, typeArrayType: InternalSerializerType) => { - const serializer = fury.classResolver.getSerializerById(serializeType); - return buildArray(fury, { - read: serializer.readWithoutType!, - write: serializer.writeWithoutType!, - config: serializer.config, - }, typeArrayType); -}; - -export const stringArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.STRING, InternalSerializerType.FURY_STRING_ARRAY); -export const boolArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.BOOL, InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY); -export const longArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.INT64, InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY); -export const intArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.INT32, InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY); -export const floatArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.FLOAT, InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY); -export const doubleArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.DOUBLE, InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY); -export const shortArraySerializer = (fury: Fury) => buildTypedArray(fury, InternalSerializerType.INT16, InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY); - -export const arraySerializer = (fury: Fury, item: Serializer) => buildArray(fury, item, InternalSerializerType.ARRAY); diff --git a/javascript/packages/fury/lib/internalSerializer/datetime.ts b/javascript/packages/fury/lib/internalSerializer/datetime.ts deleted file mode 100644 index ddcb8a6958..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/datetime.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Fury } from "../type"; -import { InternalSerializerType } from "../type"; - -const epochDate = new Date("1970/01/01 00:00"); -const epoch = epochDate.getTime(); - -export const timestampSerializer = (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { int64: writeInt64 } = binaryWriter; - const { int64: readInt64 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return new Date(Number(readInt64())); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.TIMESTAMP, epochDate, (v: Date) => { - writeInt64(BigInt(v.getTime())); - }), - config: () => { - return { - reserve: 11, - }; - }, - }; -}; - -export const dateSerializer = (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { int32: writeInt32 } = binaryWriter; - const { int32: readInt32 } = binaryReader; - return { - ...referenceResolver.deref(() => { - const day = readInt32(); - return new Date(epoch + (day * (24 * 60 * 60) * 1000)); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.DATE, epochDate, (v: Date) => { - const diff = v.getTime() - epoch; - const day = Math.floor(diff / 1000 / (24 * 60 * 60)); - writeInt32(day); - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/map.ts b/javascript/packages/fury/lib/internalSerializer/map.ts deleted file mode 100644 index 53a280a7ab..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/map.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, Fury, Serializer } from "../type"; - -export default (fury: Fury, keySerializer: Serializer, valueSerializer: Serializer) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { varUInt32: readVarUInt32 } = binaryReader; - - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { pushReadObject } = referenceResolver; - const innerHeadSize = keySerializer.config().reserve + valueSerializer.config().reserve; - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Map(); - pushReadObject(result); - for (let index = 0; index < len; index++) { - const key = keySerializer.read(); - const value = valueSerializer.read(); - result.set(key, value); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.MAP, (v: Map) => { - const len = v.size; - writeVarUInt32(len); - reserves(innerHeadSize * v.size); - for (const [key, value] of v.entries()) { - keySerializer.write(key); - valueSerializer.write(value); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/number.ts b/javascript/packages/fury/lib/internalSerializer/number.ts deleted file mode 100644 index 87cd6b148e..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/number.ts +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, RefFlags, Fury } from "../type"; - -export const uInt8Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint8: writeUInt8 } = binaryWriter; - const { uint8: readUInt8 } = binaryReader; - return { - ...referenceResolver.deref(() => { - return readUInt8(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT8, 0, (v: number) => { - writeUInt8(v); - }), - config: () => { - return { - reserve: 4, - }; - }, - }; -}; - -export const floatSerializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, float: writeFloat } = binaryWriter; - const { float: readFloat } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readFloat(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.FLOAT, 0, (v: number) => { - writeFloat(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeFloat(v); - }, - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -export const doubleSerializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, double: writeDouble } = binaryWriter; - const { double: readDouble } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readDouble(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.DOUBLE, 0, (v: number) => { - writeDouble(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeDouble(v); - }, - config: () => { - return { - reserve: 11, - }; - }, - }; -}; - -export const int8Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8 } = binaryWriter; - const { int8: readInt8 } = binaryReader; - return { - ...referenceResolver.deref(() => { - return readInt8(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT8, 0, (v: number) => { - writeInt8(v); - }), - config: () => { - return { - reserve: 4, - }; - }, - }; -}; - -export const uInt16Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint16: writeUInt16 } = binaryWriter; - const { uint16: readUInt16 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readUInt16(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT16, 0, (v: number) => { - writeUInt16(v); - }), - config: () => { - return { - reserve: 5, - }; - }, - }; -}; - -export const int16Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int16: writeInt16, int8: writeInt8 } = binaryWriter; - const { int16: readInt16 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readInt16(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT16, 0, (v: number) => { - writeInt16(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeInt16(v); - }, - config: () => { - return { - reserve: 5, - }; - }, - }; -}; - -export const uInt32Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint32: writeUInt32 } = binaryWriter; - const { uint32: readUInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readUInt32(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT32, 0, (v: number) => { - writeUInt32(v); - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -export const int32Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, int32: writeInt32 } = binaryWriter; - const { int32: readInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readInt32(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT32, 0, (v: number) => { - writeInt32(v); - }), - writeWithoutType: (v: number) => { - writeInt8(RefFlags.NotNullValueFlag); - writeInt32(v); - }, - config: () => { - return { - reserve: 7, - }; - }, - }; -}; - -export const uInt64Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { uint64: writeUInt64 } = binaryWriter; - const { uint64: readUInt64 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readUInt64(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.UINT64, BigInt(0), (v: bigint) => { - writeUInt64(v); - }), - config: () => { - return { - reserve: 11, - }; - }, - }; -}; - -export const int64Serializer = (fury: Fury) => { - const { binaryWriter, binaryReader, referenceResolver } = fury; - const { int8: writeInt8, int64: writeInt64 } = binaryWriter; - const { int64: readInt64 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readInt64(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.INT64, BigInt(0), (v: bigint) => { - writeInt64(v); - }), - writeWithoutType: (v: bigint) => { - writeInt8(RefFlags.NotNullValueFlag); - writeInt64(v); - }, - config: () => { - return { - reserve: 11, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/set.ts b/javascript/packages/fury/lib/internalSerializer/set.ts deleted file mode 100644 index adae269305..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/set.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Fury, Serializer } from "../type"; -import { InternalSerializerType } from "../type"; - -export default (fury: Fury, nestedSerializer: Serializer) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { varUInt32: readVarUInt32 } = binaryReader; - const { pushReadObject } = referenceResolver; - const innerHeadSize = nestedSerializer.config().reserve; - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Set(); - pushReadObject(result); - for (let index = 0; index < len; index++) { - result.add(nestedSerializer.read()); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_SET, (v: Set) => { - const len = v.size; - writeVarUInt32(len); - reserves(innerHeadSize * v.size); - for (const value of v.values()) { - nestedSerializer.write(value); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/string.ts b/javascript/packages/fury/lib/internalSerializer/string.ts deleted file mode 100644 index ebba64b0a6..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/string.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Fury } from "../type"; -import { InternalSerializerType, RefFlags } from "../type"; - -export default (fury: Fury) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - const { stringOfVarUInt32: writeStringOfVarUInt32, int8 } = binaryWriter; - const { stringOfVarUInt32: readStringOfVarUInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - return readStringOfVarUInt32(); - }), - write: referenceResolver.withNotNullableWriter(InternalSerializerType.STRING, "", (v: string) => { - writeStringOfVarUInt32(v); - }), - writeWithoutType: (v: string) => { - if (v === null) { - binaryWriter.int8(RefFlags.NullFlag); - return; - } - int8(RefFlags.NotNullValueFlag); - writeStringOfVarUInt32(v); - }, - config: () => { - return { - reserve: 8, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/internalSerializer/tuple.ts b/javascript/packages/fury/lib/internalSerializer/tuple.ts deleted file mode 100644 index 046679292e..0000000000 --- a/javascript/packages/fury/lib/internalSerializer/tuple.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { InternalSerializerType, Serializer } from "../type"; -import { Fury } from "../type"; - -export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => { - const { binaryReader, binaryWriter, referenceResolver } = fury; - - const { pushReadObject } = referenceResolver; - const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter; - const { varUInt32: readVarUInt32 } = binaryReader; - - return { - ...referenceResolver.deref(() => { - const len = readVarUInt32(); - const result = new Array(len); - pushReadObject(result); - for (let i = 0; i < len; i++) { - const item = serializers[i]; - result[i] = item.read(); - } - return result; - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.TUPLE, (v: any[]) => { - writeVarUInt32(serializers.length); - - for (let i = 0; i < serializers.length; i++) { - const item = serializers[i]; - reserves(item.config().reserve); - item.write(v[i]); - } - }), - config: () => { - return { - reserve: 7, - }; - }, - }; -}; diff --git a/javascript/packages/fury/lib/meta.ts b/javascript/packages/fury/lib/meta.ts new file mode 100644 index 0000000000..69a7ac88d6 --- /dev/null +++ b/javascript/packages/fury/lib/meta.ts @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ClassResolver from "./classResolver"; +import { ObjectTypeDescription, TypeDescription } from "./description"; +import { Fury, InternalSerializerType } from "./type"; + +export type Meta = { + fixedSize: number + noneable: boolean + default?: T +}; + +const epochDate = new Date("1970/01/01 00:00"); + +export const getMeta = (description: TypeDescription, fury: Fury): Meta => { + const type = description.type; + switch (type) { + case InternalSerializerType.STRING: + return { + fixedSize: 8, + noneable: false, + default: "", + }; + case InternalSerializerType.ARRAY: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.TUPLE: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.MAP: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.BOOL: + case InternalSerializerType.UINT8: + case InternalSerializerType.INT8: + return { + fixedSize: 4, + noneable: false, + default: 0, + }; + case InternalSerializerType.UINT16: + case InternalSerializerType.INT16: + return { + fixedSize: 5, + noneable: false, + default: 0, + }; + case InternalSerializerType.UINT32: + case InternalSerializerType.INT32: + case InternalSerializerType.FLOAT: + return { + fixedSize: 7, + noneable: false, + default: 0, + }; + case InternalSerializerType.UINT64: + case InternalSerializerType.INT64: + case InternalSerializerType.DOUBLE: + return { + fixedSize: 11, + noneable: false, + default: 0, + }; + case InternalSerializerType.BINARY: + return { + fixedSize: 8, + noneable: true, + }; + case InternalSerializerType.DATE: + return { + fixedSize: 7, + noneable: false, + default: epochDate.getTime(), + }; + case InternalSerializerType.TIMESTAMP: + return { + fixedSize: 11, + noneable: false, + default: epochDate.getTime(), + }; + case InternalSerializerType.FURY_TYPE_TAG: + { + const options = (description).options; + let fixedSize = ClassResolver.tagBuffer(options.tag).byteLength + 8; + if (options.props) { + Object.values(options.props).forEach(x => fixedSize += getMeta(x, fury).fixedSize); + } else { + fixedSize += fury.classResolver.getSerializerByTag(options.tag).meta.fixedSize; + } + return { + fixedSize, + noneable: true, + }; + } + + case InternalSerializerType.FURY_SET: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.FURY_PRIMITIVE_BOOL_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_SHORT_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_INT_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_LONG_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_FLOAT_ARRAY: + case InternalSerializerType.FURY_PRIMITIVE_DOUBLE_ARRAY: + case InternalSerializerType.FURY_STRING_ARRAY: + return { + fixedSize: 7, + noneable: true, + }; + case InternalSerializerType.ANY: + return { + fixedSize: 11, + noneable: true, + }; + default: + throw new Error(`Meta of ${description.type} not exists`); + } +}; diff --git a/javascript/packages/fury/lib/reader.ts b/javascript/packages/fury/lib/reader.ts index 9513d34396..46b4e5fe96 100644 --- a/javascript/packages/fury/lib/reader.ts +++ b/javascript/packages/fury/lib/reader.ts @@ -20,6 +20,7 @@ import { Config, LATIN1 } from "./type"; import { isNodeEnv } from "./util"; import { PlatformBuffer, alloc, fromUint8Array } from "./platformBuffer"; +import { read1, read10, read11, read12, read13, read14, read15, read2, read3, read4, read5, read6, read7, read8, read9 } from "./string"; export const BinaryReader = (config: Config) => { const sliceStringEnable = isNodeEnv && config.useSliceString; @@ -87,6 +88,16 @@ export const BinaryReader = (config: Config) => { return result; } + function sliLong() { + const i = dataView.getUint32(cursor, true); + if ((i & 0b1) != 0b1) { + cursor += 4; + return BigInt(i >> 1); + } + cursor += 1; + return varInt64(); + } + function float() { const result = dataView.getFloat32(cursor, true); cursor += 4; @@ -122,9 +133,44 @@ export const BinaryReader = (config: Config) => { } function stringLatin1Slow(len: number) { - const result = buffer.latin1Slice(cursor, cursor + len); + const rawCursor = cursor; cursor += len; - return result; + switch (len) { + case 0: + return ""; + case 1: + return read1(buffer, rawCursor); + case 2: + return read2(buffer, rawCursor); + case 3: + return read3(buffer, rawCursor); + case 4: + return read4(buffer, rawCursor); + case 5: + return read5(buffer, rawCursor); + case 6: + return read6(buffer, rawCursor); + case 7: + return read7(buffer, rawCursor); + case 8: + return read8(buffer, rawCursor); + case 9: + return read9(buffer, rawCursor); + case 10: + return read10(buffer, rawCursor); + case 11: + return read11(buffer, rawCursor); + case 12: + return read12(buffer, rawCursor); + case 13: + return read13(buffer, rawCursor); + case 14: + return read14(buffer, rawCursor); + case 15: + return read15(buffer, rawCursor); + default: + return buffer.latin1Slice(rawCursor, cursor); + } } function binary(len: number) { @@ -144,21 +190,67 @@ export const BinaryReader = (config: Config) => { return (v >> 1) ^ -(v & 1); } + function zigZagBigInt(v: bigint) { + return (v >> 1n) ^ -(v & 1n); + } + function varUInt32() { - let byte_ = int8(); + let byte_ = uint8(); let result = byte_ & 0x7f; if ((byte_ & 0x80) != 0) { - byte_ = int8(); + byte_ = uint8(); result |= (byte_ & 0x7f) << 7; if ((byte_ & 0x80) != 0) { - byte_ = int8(); + byte_ = uint8(); result |= (byte_ & 0x7f) << 14; if ((byte_ & 0x80) != 0) { - byte_ = int8(); + byte_ = uint8(); result |= (byte_ & 0x7f) << 21; if ((byte_ & 0x80) != 0) { - byte_ = int8(); - result |= (byte_ & 0x7f) << 28; + byte_ = uint8(); + result |= (byte_) << 28; + } + } + } + } + return result; + } + + function bigUInt8() { + return BigInt(uint8() >>> 0); + } + + function varUInt64() { + let byte_ = bigUInt8(); + let result = byte_ & 0x7fn; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 7n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 14n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 21n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 28n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 35n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 42n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_ & 0x7fn) << 49n; + if ((byte_ & 0x80n) != 0n) { + byte_ = bigUInt8(); + result |= (byte_) << 56n; + } + } + } + } } } } @@ -170,11 +262,17 @@ export const BinaryReader = (config: Config) => { return zigZag(varUInt32()); } + function varInt64() { + return zigZagBigInt(varUInt64()); + } + return { getCursor: () => cursor, setCursor: (v: number) => (cursor = v), varInt32, + varInt64, varUInt32, + varUInt64, int8, buffer: binary, bufferRef, @@ -191,6 +289,7 @@ export const BinaryReader = (config: Config) => { uint64, skip, int64, + sliLong, uint32, int32, }; diff --git a/javascript/packages/fury/lib/referenceResolver.ts b/javascript/packages/fury/lib/referenceResolver.ts index c79f5400e1..702bfd2f18 100644 --- a/javascript/packages/fury/lib/referenceResolver.ts +++ b/javascript/packages/fury/lib/referenceResolver.ts @@ -21,14 +21,11 @@ import { RefFlags, BinaryReader, BinaryWriter, - SerializerRead, InternalSerializerType, - SerializerWrite, } from "./type"; -import type ClassResolver from "./classResolver"; export const makeHead = (flag: RefFlags, type: InternalSerializerType) => { - return (((type << 16) >>> 16) << 8) | ((flag << 24) >>> 24); + return (((Math.floor(type) << 16) >>> 16) << 8) | ((flag << 24) >>> 24); }; export const ReferenceResolver = ( @@ -37,7 +34,6 @@ export const ReferenceResolver = ( }, binaryWriter: BinaryWriter, binaryReader: BinaryReader, - classResolver: ClassResolver ) => { let readObjects: any[] = []; let writeObjects: any[] = []; @@ -71,95 +67,6 @@ export const ReferenceResolver = ( } } - function skipType() { - const typeId = binaryReader.int16(); - if (typeId === InternalSerializerType.FURY_TYPE_TAG) { - classResolver.readTag(binaryReader); - } - } - - function withNullableOrRefWriter( - type: InternalSerializerType, - fn: SerializerWrite - ) { - const int24 = binaryWriter.int24; - const head = makeHead(RefFlags.RefValueFlag, type); - if (config.refTracking) { - return (v: T) => { - if (v !== null && v !== undefined) { - const existsId = existsWriteObject(v); - if (typeof existsId === "number") { - binaryWriter.int8(RefFlags.RefFlag); - binaryWriter.varUInt32(existsId); - } else { - int24(head); - pushWriteObject(v); - fn(v); - } - } else { - binaryWriter.int8(RefFlags.NullFlag); - } - }; - } else { - return (v: T) => { - if (v !== null && v !== undefined) { - int24(head); - fn(v); - } else { - binaryWriter.int8(RefFlags.NullFlag); - } - }; - } - } - - function withNotNullableWriter( - type: InternalSerializerType, - defaultValue: T, - fn: SerializerWrite - ) { - const head = makeHead(RefFlags.NotNullValueFlag, type); - const int24 = binaryWriter.int24; - return (v: T) => { - int24(head); - if (v == null || v == undefined) { - fn(defaultValue); - } else { - fn(v); - } - }; - } - - function deref(fn: SerializerRead) { - return { - read: () => { - switch (readRefFlag()) { - case RefFlags.RefValueFlag: - skipType(); - return fn(); - case RefFlags.RefFlag: - return getReadObjectByRefId(binaryReader.varUInt32()); - case RefFlags.NullFlag: - return null; - case RefFlags.NotNullValueFlag: - skipType(); - return fn(); - } - }, - readWithoutType: () => { - switch (readRefFlag()) { - case RefFlags.RefValueFlag: - return fn(); - case RefFlags.RefFlag: - return getReadObjectByRefId(binaryReader.varUInt32()); - case RefFlags.NullFlag: - return null; - case RefFlags.NotNullValueFlag: - return fn(); - } - }, - }; - } - return { existsWriteObject, pushWriteObject, @@ -167,8 +74,5 @@ export const ReferenceResolver = ( readRefFlag, getReadObjectByRefId, reset, - withNotNullableWriter, - withNullableOrRefWriter, - deref, }; }; diff --git a/javascript/packages/fury/lib/string.ts b/javascript/packages/fury/lib/string.ts new file mode 100644 index 0000000000..3905254e77 --- /dev/null +++ b/javascript/packages/fury/lib/string.ts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const read1 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor]); +}; +export const read2 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1]); +}; +export const read3 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2]); +}; +export const read4 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3]); +}; +export const read5 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4]); +}; +export const read6 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5]); +}; +export const read7 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6]); +}; +export const read8 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7]); +}; +export const read9 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8]); +}; +export const read10 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9]); +}; +export const read11 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10]); +}; +export const read12 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11]); +}; +export const read13 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11], buffer[cursor + 12]); +}; +export const read14 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11], buffer[cursor + 12], buffer[cursor + 13]); +}; +export const read15 = (buffer: Uint8Array, cursor: number) => { + return String.fromCharCode(buffer[cursor], buffer[cursor + 1], buffer[cursor + 2], buffer[cursor + 3], buffer[cursor + 4], buffer[cursor + 5], buffer[cursor + 6], buffer[cursor + 7], buffer[cursor + 8], buffer[cursor + 9], buffer[cursor + 10], buffer[cursor + 11], buffer[cursor + 12], buffer[cursor + 13], buffer[cursor + 14]); +}; diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index 39cd50669f..2a1def72cc 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -20,6 +20,7 @@ import type { BinaryWriter } from "./writer"; import type { BinaryReader } from "./reader"; import type FuryFunc from "./fury"; +import { Meta } from "./meta"; export type Fury = ReturnType; export type BinaryWriter = ReturnType; @@ -28,7 +29,7 @@ export type BinaryReader = ReturnType; export enum InternalSerializerType { STRING = 13, ARRAY = 25, - TUPLE = 25, + TUPLE = 25.1, MAP = 30, BOOL = 1, UINT8 = 2, @@ -63,25 +64,13 @@ export enum ConfigFlags { isOutOfBandFlag = 8, } -export type SerializerRead = ( -) => T; - -export type SerializerWrite = ( - v: T, -) => void; - -export type SerializerConfig = ( -) => { - reserve: number -}; - // read, write export type Serializer = { - read: SerializerRead - write: SerializerWrite - readWithoutType?: SerializerRead - writeWithoutType?: SerializerWrite - config: SerializerConfig + read: () => T2 + write: (v: T2) => T + readInner: () => T2 + writeInner: (v: T2) => T + meta: Meta }; export enum RefFlags { @@ -98,6 +87,9 @@ export enum RefFlags { export const MaxInt32 = 2147483647; export const MinInt32 = -2147483648; +export const HalfMaxInt32 = MaxInt32 / 2; +export const HalfMinInt32 = MinInt32 / 2; + export const LATIN1 = 0; export const UTF8 = 1; @@ -110,6 +102,9 @@ export interface Config { hps?: Hps refTracking?: boolean useSliceString?: boolean + hooks?: { + afterCodeGenerated?: (code: string) => string + } } export enum Language { diff --git a/javascript/packages/fury/lib/util.ts b/javascript/packages/fury/lib/util.ts index f7d33f0962..d78445e2dd 100644 --- a/javascript/packages/fury/lib/util.ts +++ b/javascript/packages/fury/lib/util.ts @@ -17,32 +17,6 @@ * under the License. */ -const isReserved = (key: string) => { - return /^(?:do|if|in|for|let|new|try|var|case|else|enum|eval|false|null|this|true|void|with|break|catch|class|const|super|throw|while|yield|delete|export|import|public|return|static|switch|typeof|default|extends|finally|package|private|continue|debugger|function|arguments|interface|protected|implements|instanceof)$/.test(key); -}; - -const isDotPropAccessor = (prop: string) => { - return /^[a-zA-Z_$][0-9a-zA-Z_$]*$/.test(prop); -}; - -export const replaceBackslashAndQuote = (v: string) => { - return v.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); -}; - -export const safePropAccessor = (prop: string) => { - if (!isDotPropAccessor(prop) || isReserved(prop)) { - return `["${replaceBackslashAndQuote(prop)}"]`; - } - return `.${prop}`; -}; - -export const safePropName = (prop: string) => { - if (!isDotPropAccessor(prop) || isReserved(prop)) { - return `["${replaceBackslashAndQuote(prop)}"]`; - } - return prop; -}; - export const isNodeEnv: boolean = typeof process !== "undefined" && process.versions != null diff --git a/javascript/packages/fury/lib/writer.ts b/javascript/packages/fury/lib/writer.ts index 4f14212312..d32017003e 100644 --- a/javascript/packages/fury/lib/writer.ts +++ b/javascript/packages/fury/lib/writer.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Config, LATIN1, UTF8 } from "./type"; +import { Config, HalfMaxInt32, HalfMinInt32, LATIN1, UTF8 } from "./type"; import { PlatformBuffer, alloc, strByteLength } from "./platformBuffer"; import { OwnershipError } from "./error"; @@ -97,13 +97,27 @@ export const BinaryWriter = (config: Config) => { cursor += 4; } - function int64(v: bigint | number) { - if (typeof v === "number") { - dataView.setBigInt64(cursor, BigInt(v), true); + function int64(v: bigint) { + dataView.setBigInt64(cursor, v, true); + cursor += 8; + } + + function sliLong(v: bigint | number) { + if (v <= HalfMaxInt32 && v >= HalfMinInt32) { + // write: + // 00xxx -> 0xxx + // 11xxx -> 1xxx + // read: + // 0xxx -> 00xxx + // 1xxx -> 11xxx + dataView.setUint32(cursor, Number(v) << 1, true); + cursor += 4; } else { - dataView.setBigInt64(cursor, v, true); + const BIG_LONG_FLAG = 0b1; // bit 0 set, means big long. + dataView.setUint8(cursor, BIG_LONG_FLAG); + cursor += 1; + varInt64(BigInt(v)); } - cursor += 8; } function float(v: number) { @@ -122,12 +136,8 @@ export const BinaryWriter = (config: Config) => { cursor += v.byteLength; } - function uint64(v: bigint | number) { - if (typeof v === "number") { - dataView.setBigUint64(cursor, BigInt(v), true); - } else { - dataView.setBigUint64(cursor, v, true); - } + function uint64(v: bigint) { + dataView.setBigUint64(cursor, v, true); cursor += 8; } @@ -215,16 +225,12 @@ export const BinaryWriter = (config: Config) => { cursor += len; } - function zigZag(v: number) { - return (v << 1) ^ (v >> 31); - } - - function varInt32(val: number) { - return varUInt32(zigZag(val)); + function varInt32(v: number) { + return varUInt32((v << 1) ^ (v >> 31)); } function varUInt32(val: number) { - val = val >>> 0; + val = (val >>> 0) & 0xFFFFFFFF; // keep only the lower 32 bits while (val > 127) { arrayBuffer[cursor++] = val & 127 | 128; val >>>= 7; @@ -232,6 +238,27 @@ export const BinaryWriter = (config: Config) => { arrayBuffer[cursor++] = val; } + function varInt64(v: bigint) { + if (typeof v !== "bigint") { + v = BigInt(v); + } + return varUInt64((v << 1n) ^ (v >> 63n)); + } + + function varUInt64(val: bigint | number) { + if (typeof val !== "bigint") { + val = BigInt(val); + } + val = val & 0xFFFFFFFFFFFFFFFFn; // keep only the lower 64 bits + + while (val > 127) { + arrayBuffer[cursor++] = Number(val & 127n | 128n); + val >>= 7n; + } + arrayBuffer[cursor++] = Number(val); + return; + } + function tryFreePool() { if (byteLength > MAX_POOL_SIZE) { initPoll(); @@ -287,6 +314,8 @@ export const BinaryWriter = (config: Config) => { int16, varInt32, varUInt32, + varUInt64, + varInt64, stringOfVarUInt32: config?.hps ? stringOfVarUInt32Fast() : stringOfVarUInt32Slow, @@ -296,6 +325,7 @@ export const BinaryWriter = (config: Config) => { double, float, int64, + sliLong, uint32, int32, getCursor, diff --git a/javascript/packages/fury/tsconfig.json b/javascript/packages/fury/tsconfig.json index 72b76112ad..176ef55b31 100644 --- a/javascript/packages/fury/tsconfig.json +++ b/javascript/packages/fury/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ diff --git a/javascript/packages/hps/tsconfig.json b/javascript/packages/hps/tsconfig.json index 72b76112ad..176ef55b31 100644 --- a/javascript/packages/hps/tsconfig.json +++ b/javascript/packages/hps/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ diff --git a/javascript/test/__snapshots__/codeGen.test.ts.snap b/javascript/test/__snapshots__/codeGen.test.ts.snap index 0fb0461e6e..fdcc52af8b 100644 --- a/javascript/test/__snapshots__/codeGen.test.ts.snap +++ b/javascript/test/__snapshots__/codeGen.test.ts.snap @@ -3,114 +3,604 @@ exports[`codeGen can generate tuple declaration code 1`] = ` "function anonymous( ) { - -return function (fury, scope) { - const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; - const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; +return function(fury, external) { + const br = fury.binaryReader; + const bw = fury.binaryWriter; + const cr = fury.classResolver; + const rr = fury.referenceResolver; + const tag_ser_3 = fury.classResolver.getSerializerByTag("example.foo.1"); + const tag_ser_4 = fury.classResolver.getSerializerByTag("example.foo.2"); + const tag_ser_12 = fury.classResolver.getSerializerByTag("example.bar.1"); + const tag_ser_13 = fury.classResolver.getSerializerByTag("example.bar.2"); + const tagWriter_19 = cr.createTagWriter("tuple-object-wrapper"); - const tag_0 = classResolver.getSerializerByTag("example.foo.1") - const tag_1 = classResolver.getSerializerByTag("example.foo.2") - const tuple_tag_0_tag_1 = tupleSerializer(fury, [tag_0, tag_1]) - const tag_5 = classResolver.getSerializerByTag("example.bar.1") - const tag_6 = classResolver.getSerializerByTag("example.bar.2") - const tuple_tag_0_tag_5_tag_6 = tupleSerializer(fury, [tag_0, tag_5, tag_6]) - const tagWriter = classResolver.createTagWriter("tuple-object-wrapper"); - - const reserves = tag_0.config().reserve + tag_1.config().reserve + tuple_tag_0_tag_1.config().reserve + tag_5.config().reserve + tag_6.config().reserve + tuple_tag_0_tag_5_tag_6.config().reserve; - return { - ...referenceResolver.deref(() => { - const hash = binaryReader.int32(); - if (hash !== 16469457) { - throw new Error("validate hash failed: tuple-object-wrapper. expect 16469457, but got" + hash); + const readInner = () => { + + if (br.int32() !== 16469457) { + throw new Error("validate hash failed: tuple-object-wrapper. expect 16469457"); + } + const result_0 = { + tuple1: null, + tuple1_: null, + tuple2: null, + tuple2_: null + }; + rr.pushReadObject(result_0) + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_2 = br.varUInt32(); + const result_1 = new Array(len_2); + rr.pushReadObject(result_1) + + if (len_2 > 0) { + result_1[0] = tag_ser_3.read() + } + + + if (len_2 > 1) { + result_1[1] = tag_ser_4.read() + } + + result_0.tuple1 = result_1 + + break; + case -2: + result_0.tuple1 = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple1 = null + break; + }; + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_6 = br.varUInt32(); + const result_5 = new Array(len_6); + rr.pushReadObject(result_5) + + if (len_6 > 0) { + result_5[0] = tag_ser_3.read() + } + + + if (len_6 > 1) { + result_5[1] = tag_ser_4.read() + } + + result_0.tuple1_ = result_5 + + break; + case -2: + result_0.tuple1_ = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple1_ = null + break; + }; + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_10 = br.varUInt32(); + const result_9 = new Array(len_10); + rr.pushReadObject(result_9) + + if (len_10 > 0) { + result_9[0] = tag_ser_3.read() + } + + + if (len_10 > 1) { + result_9[1] = tag_ser_12.read() + } + + + if (len_10 > 2) { + result_9[2] = tag_ser_13.read() + } + + result_0.tuple2 = result_9 + + break; + case -2: + result_0.tuple2 = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple2 = null + break; + }; + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_15 = br.varUInt32(); + const result_14 = new Array(len_15); + rr.pushReadObject(result_14) + + if (len_15 > 0) { + result_14[0] = tag_ser_3.read() + } + + + if (len_15 > 1) { + result_14[1] = tag_ser_12.read() + } + + + if (len_15 > 2) { + result_14[2] = tag_ser_13.read() + } + + result_0.tuple2_ = result_14 + + break; + case -2: + result_0.tuple2_ = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple2_ = null + break; + } + + return result_0 + + }; + const read = () => { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + return readInner() + break; + case -2: + return rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + return null + break; + } + + }; + const writeInner = (v) => { + + tagWriter_19.write(bw); + bw.int32(16469457); + + const existsId_22 = rr.existsWriteObject(v.tuple1); + if (typeof existsId_22 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_22) + } else { + rr.pushWriteObject(v.tuple1) + + if (v.tuple1 !== null && v.tuple1 !== undefined) { + bw.int24(6400); + + bw.varUInt32(2) + bw.reserve(148); + tag_ser_3.write(v.tuple1[0]) + tag_ser_4.write(v.tuple1[1]); + } else { + bw.int8(-3); } - { - - // relation tag: tuple-object-wrapper - const result = { - tuple1: null, -tuple1_: null, -tuple2: null, -tuple2_: null + }; - pushReadObject(result); - result.tuple1 = tuple_tag_0_tag_1.read(); -result.tuple1_ = tuple_tag_0_tag_1.read(); -result.tuple2 = tuple_tag_0_tag_5_tag_6.read(); -result.tuple2_ = tuple_tag_0_tag_5_tag_6.read() - return result; - + + const existsId_25 = rr.existsWriteObject(v.tuple1_); + if (typeof existsId_25 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_25) + } else { + rr.pushWriteObject(v.tuple1_) + + if (v.tuple1_ !== null && v.tuple1_ !== undefined) { + bw.int24(6400); + + bw.varUInt32(2) + bw.reserve(148); + tag_ser_3.write(v.tuple1_[0]) + tag_ser_4.write(v.tuple1_[1]); + } else { + bw.int8(-3); } - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { - tagWriter.write(binaryWriter); - binaryWriter.int32(16469457); - binaryWriter.reserve(reserves); - tuple_tag_0_tag_1.write(v.tuple1); -tuple_tag_0_tag_1.write(v.tuple1_); -tuple_tag_0_tag_5_tag_6.write(v.tuple2); -tuple_tag_0_tag_5_tag_6.write(v.tuple2_) - }), - config() { - return { - reserve: tagWriter.bufferLen + 8, + + }; + + const existsId_29 = rr.existsWriteObject(v.tuple2); + if (typeof existsId_29 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_29) + } else { + rr.pushWriteObject(v.tuple2) + + if (v.tuple2 !== null && v.tuple2 !== undefined) { + bw.int24(6400); + + bw.varUInt32(3) + bw.reserve(222); + tag_ser_3.write(v.tuple2[0]) + tag_ser_12.write(v.tuple2[1]) + tag_ser_13.write(v.tuple2[2]); + } else { + bw.int8(-3); + } + + }; + + const existsId_33 = rr.existsWriteObject(v.tuple2_); + if (typeof existsId_33 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_33) + } else { + rr.pushWriteObject(v.tuple2_) + + if (v.tuple2_ !== null && v.tuple2_ !== undefined) { + bw.int24(6400); + + bw.varUInt32(3) + bw.reserve(222); + tag_ser_3.write(v.tuple2_[0]) + tag_ser_12.write(v.tuple2_[1]) + tag_ser_13.write(v.tuple2_[2]); + } else { + bw.int8(-3); } + } - } + + + }; + const write = (v) => { + + const existsId_34 = rr.existsWriteObject(v); + if (typeof existsId_34 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_34) + } else { + rr.pushWriteObject(v) + + if (v !== null && v !== undefined) { + bw.int24(65536); + writeInner(v); + } else { + bw.int8(-3); + } + + } + + }; + + return { + read, + readInner, + write, + writeInner, + meta: { + "fixedSize": 67, + "noneable": true + } + }; } - }" `; exports[`codeGen can generate tuple declaration code 2`] = ` "function anonymous( ) { - -return function (fury, scope) { - const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; - const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, tupleSerializer, mapSerializer, setSerializer } = scope; +return function(fury, external) { + const br = fury.binaryReader; + const bw = fury.binaryWriter; + const cr = fury.classResolver; + const rr = fury.referenceResolver; + const tagWriter_6 = cr.createTagWriter("tuple-object-type3-tag"); - const type_13 = classResolver.getSerializerById(13) - const type_1 = classResolver.getSerializerById(1) - const type_6 = classResolver.getSerializerById(6) - const type_14 = classResolver.getSerializerById(14) - const tuple_type_14 = tupleSerializer(fury, [type_14]) - const tuple_type_13_type_1_type_6_tuple_type_14 = tupleSerializer(fury, [type_13, type_1, type_6, tuple_type_14]) - const tagWriter = classResolver.createTagWriter("tuple-object-type3-tag"); - - const reserves = type_13.config().reserve + type_1.config().reserve + type_6.config().reserve + type_14.config().reserve + tuple_type_14.config().reserve + tuple_type_13_type_1_type_6_tuple_type_14.config().reserve; - return { - ...referenceResolver.deref(() => { - const hash = binaryReader.int32(); - if (hash !== 552) { - throw new Error("validate hash failed: tuple-object-type3-tag. expect 552, but got" + hash); - } - { - - // relation tag: tuple-object-type3-tag - const result = { - tuple: null + const readInner = () => { + + if (br.int32() !== 552) { + throw new Error("validate hash failed: tuple-object-type3-tag. expect 552"); + } + const result_0 = { + tuple: null }; - pushReadObject(result); - result.tuple = tuple_type_13_type_1_type_6_tuple_type_14.read() - return result; - + rr.pushReadObject(result_0) + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_2 = br.varUInt32(); + const result_1 = new Array(len_2); + rr.pushReadObject(result_1) + + if (len_2 > 0) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + result_1[0] = br.stringOfVarUInt32() + break; + case -2: + result_1[0] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[0] = null + break; + } + + } + + + if (len_2 > 1) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + result_1[1] = br.uint8() === 1 + break; + case -2: + result_1[1] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[1] = null + break; + } + + } + + + if (len_2 > 2) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + result_1[2] = br.uint32() + break; + case -2: + result_1[2] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[2] = null + break; + } + + } + + + if (len_2 > 3) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + const len_4 = br.varUInt32(); + const result_3 = new Array(len_4); + rr.pushReadObject(result_3) + + if (len_4 > 0) { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + + br.uint8() + result_5 = br.buffer(br.int32()); + rr.pushReadObject(result_5); + result_3[0] = result_5 + + break; + case -2: + result_3[0] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_3[0] = null + break; + } + + } + + result_1[3] = result_3 + + break; + case -2: + result_1[3] = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_1[3] = null + break; + } + + } + + result_0.tuple = result_1 + + break; + case -2: + result_0.tuple = rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + result_0.tuple = null + break; + } + + return result_0 + + }; + const read = () => { + + switch (br.int8()) { + case -1: + case 0: + if (br.int16() === 256) { + cr.readTag(br); + } + return readInner() + break; + case -2: + return rr.getReadObjectByRefId(br.varUInt32()) + break; + case -3: + return null + break; + } + + }; + const writeInner = (v) => { + + tagWriter_6.write(bw); + bw.int32(552); + + const existsId_9 = rr.existsWriteObject(v.tuple); + if (typeof existsId_9 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_9) + } else { + rr.pushWriteObject(v.tuple) + + if (v.tuple !== null && v.tuple !== undefined) { + bw.int24(6400); + + bw.varUInt32(4) + bw.reserve(26); + + bw.int24(3583); + if (v.tuple[0] !== null && v.tuple[0] !== undefined) { + bw.stringOfVarUInt32(v.tuple[0]); + } else { + bw.stringOfVarUInt32(""); + } + + bw.int24(511); + if (v.tuple[1] !== null && v.tuple[1] !== undefined) { + bw.uint8(v.tuple[1] ? 1 : 0); + } else { + bw.uint8(0 ? 1 : 0); + } + + bw.int24(1791); + if (v.tuple[2] !== null && v.tuple[2] !== undefined) { + bw.uint32(v.tuple[2]); + } else { + bw.uint32(0); + } + + const existsId_8 = rr.existsWriteObject(v.tuple[3]); + if (typeof existsId_8 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_8) + } else { + rr.pushWriteObject(v.tuple[3]) + + if (v.tuple[3] !== null && v.tuple[3] !== undefined) { + bw.int24(6400); + + bw.varUInt32(1) + bw.reserve(8); + + const existsId_7 = rr.existsWriteObject(v.tuple[3][0]); + if (typeof existsId_7 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_7) + } else { + rr.pushWriteObject(v.tuple[3][0]) + + if (v.tuple[3][0] !== null && v.tuple[3][0] !== undefined) { + bw.int24(3584); + + bw.uint8(1) + bw.uint32(v.tuple[3][0].byteLength) + bw.buffer(v.tuple[3][0]); + } else { + bw.int8(-3); + } + + } + + ; + } else { + bw.int8(-3); + } + + } + + ; + } else { + bw.int8(-3); } - }), - write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_TYPE_TAG, (v) => { - tagWriter.write(binaryWriter); - binaryWriter.int32(552); - binaryWriter.reserve(reserves); - tuple_type_13_type_1_type_6_tuple_type_14.write(v.tuple) - }), - config() { - return { - reserve: tagWriter.bufferLen + 8, + + } + + + }; + const write = (v) => { + + const existsId_10 = rr.existsWriteObject(v); + if (typeof existsId_10 === "number") { + bw.int8(-2) + bw.varUInt32(existsId_10) + } else { + rr.pushWriteObject(v) + + if (v !== null && v !== undefined) { + bw.int24(65536); + writeInner(v); + } else { + bw.int8(-3); } + } - } + + }; + + return { + read, + readInner, + write, + writeInner, + meta: { + "fixedSize": 48, + "noneable": true + } + }; } - }" `; diff --git a/javascript/test/any.test.ts b/javascript/test/any.test.ts index e0c2d94fa5..05f791b710 100644 --- a/javascript/test/any.test.ts +++ b/javascript/test/any.test.ts @@ -47,7 +47,7 @@ describe('bool', () => { test('should write big number work', () => { const fury = new Fury(); const bin = fury.serialize(3000000000); - expect(fury.deserialize(bin)).toBe(BigInt(3000000000)) + expect(fury.deserialize(bin)).toBe(3000000000n); }); test('should write INFINITY work', () => { diff --git a/javascript/test/array.test.ts b/javascript/test/array.test.ts index ea9c7d2217..b862226e17 100644 --- a/javascript/test/array.test.ts +++ b/javascript/test/array.test.ts @@ -69,7 +69,7 @@ describe('array', () => { a3: [3, 5, 76], a4: [634, 564, 76], a6: [234243.555, 55654.6786], - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }, serializer); const result = fury.deserialize( input @@ -81,7 +81,7 @@ describe('array', () => { a3: [3, 5, 76], a4: [634, 564, 76], a6: [234243.555, 55654.6786], - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }) }); test('should string array work', () => { @@ -100,13 +100,13 @@ describe('array', () => { const fury = new Fury({ refTracking: true }); const serializer = fury.registerSerializer(description).serializer; const input = fury.serialize({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }, serializer); const result = fury.deserialize( input ); expect(result).toEqual({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }) }); test('should string array work when latin1 enable', () => { @@ -125,13 +125,13 @@ describe('array', () => { const fury = new Fury({ refTracking: true }); const serializer = fury.registerSerializer(description).serializer; const input = fury.serialize({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }, serializer); const result = fury.deserialize( input ); expect(result).toEqual({ - a7: ["hello", "world", null] + a7: ["hello", "world", "!"] }) }); test('should floatarray work', () => { diff --git a/javascript/test/codeGen.test.ts b/javascript/test/codeGen.test.ts index fae12a1f05..5432396532 100644 --- a/javascript/test/codeGen.test.ts +++ b/javascript/test/codeGen.test.ts @@ -19,16 +19,23 @@ import { describe, expect, test } from '@jest/globals'; import { tupleObjectDescription, tupleObjectType3Description } from './fixtures/tuple'; -import { generateInlineCode } from '../packages/fury/lib/codeGen'; +import { generate } from '../packages/fury/lib/gen/index'; import FuryInternal from '../packages/fury/lib/fury'; +import * as beautify from 'js-beautify'; describe('codeGen', () => { test('can generate tuple declaration code', () => { - const fury = FuryInternal({ refTracking: true }); - const fn = generateInlineCode(fury, tupleObjectDescription); + const fury = FuryInternal({ + refTracking: true, hooks: { + afterCodeGenerated: (code: string) => { + return beautify.js(code, { indent_size: 2, space_in_empty_paren: true, indent_empty_lines: true }); + } + } + }); + const fn = generate(fury, tupleObjectDescription); expect(fn.toString()).toMatchSnapshot(); - const fn2 = generateInlineCode(fury, tupleObjectType3Description); + const fn2 = generate(fury, tupleObjectType3Description); expect(fn2.toString()).toMatchSnapshot(); }) }) diff --git a/javascript/test/fury.test.ts b/javascript/test/fury.test.ts index 55e306a2c2..c3723aa9c2 100644 --- a/javascript/test/fury.test.ts +++ b/javascript/test/fury.test.ts @@ -17,8 +17,9 @@ * under the License. */ -import Fury, { TypeDescription, InternalSerializerType } from '../packages/fury/index'; +import Fury, { TypeDescription, Type } from '../packages/fury/index'; import { describe, expect, test } from '@jest/globals'; +import { fromUint8Array } from '../packages/fury/lib/platformBuffer'; describe('fury', () => { test('should deserialize null work', () => { @@ -55,4 +56,64 @@ describe('fury', () => { expect(error.message).toBe('outofband mode is not supported now'); } }); + + test('should register work', () => { + const fury = new Fury(); + const { serialize, deserialize } = fury.registerSerializer(Type.array(Type.string())); + const bin = serialize(["hello", "world"]); + expect(deserialize(bin)).toEqual(["hello", "world"]); + }); + + describe('serializer description should work', () => { + test('can serialize and deserialize primitive types', () => { + const description = Type.int8() + testDescription(description, 123) + + const description2 = Type.int16() + testDescription(description2, 123) + + const description3 = Type.int32() + testDescription(description3, 123) + + const description4 = Type.bool() + testDescription(description4, true) + + // has precision problem + // const description5 = Type.float() + // testDescription(description5, 123.456) + + const description6 = Type.double() + testDescription(description6, 123.456789) + + const description7 = Type.binary() + testDescription(description7, new Uint8Array([1, 2, 3]), fromUint8Array(new Uint8Array([1, 2, 3]))); + + const description8 = Type.string() + testDescription(description8, '123') + + const description9 = Type.set(Type.string()) + testDescription(description9, new Set(['123'])) + }) + + test('can serialize and deserialize array', () => { + const description = Type.array(Type.int8()) + testDescription(description, [1, 2, 3]) + testDescription(description, []) + }) + + test('can serialize and deserialize tuple', () => { + const description = Type.tuple([Type.int8(), Type.int16(), Type.timestamp()]) + testDescription(description, [1, 2, new Date()]) + }) + + + function testDescription(description: TypeDescription, input: any, expected?: any) { + const fury = new Fury(); + const serialize = fury.registerSerializer(description); + const result = serialize.deserialize( + serialize.serialize(input) + ); + expect(result).toEqual(expected ?? input) + } + }) }); diff --git a/javascript/test/io.test.ts b/javascript/test/io.test.ts index 38f1b3f2c6..8d5edf9f7c 100644 --- a/javascript/test/io.test.ts +++ b/javascript/test/io.test.ts @@ -41,7 +41,6 @@ function num2Bin(num: number) { 8, 16, 32, - 64 ].forEach((x, y) => { { writer[`uint${x}`](10); @@ -93,7 +92,6 @@ function num2Bin(num: number) { 8, 16, 32, - 64 ].forEach((x, y) => { { writer[`int${x}`](10); @@ -354,5 +352,59 @@ function num2Bin(num: number) { expect(reader.int8()).toBe(10); expect(reader.int16()).toBe(20); }); + + test('should varUInt64 work', () => { + const writer = BinaryWriter(config); + writer.varUInt64(2n ** 2n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varUInt64()).toBe(2n ** 2n); + }); + + test('should varUInt64 work', () => { + const writer = BinaryWriter(config); + writer.varUInt64(2n ** 63n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varUInt64()).toBe(2n ** 63n); + }); + + test('should varInt64 work', () => { + const writer = BinaryWriter(config); + writer.varInt64(2n ** 2n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varInt64()).toBe(2n ** 2n); + }); + + test('should varInt64 work', () => { + const writer = BinaryWriter(config); + writer.varInt64(2n ** 62n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.varInt64()).toBe(2n ** 62n); + }); + + test('should silong work', () => { + const writer = BinaryWriter(config); + writer.sliLong(2n ** 2n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.sliLong()).toBe(2n ** 2n); + }); + + test('should silong work', () => { + const writer = BinaryWriter(config); + writer.sliLong(2n ** 62n); + const ab = writer.dump(); + const reader = BinaryReader({}); + reader.reset(ab) + expect(reader.sliLong()).toBe(2n ** 62n); + }); }); }) diff --git a/javascript/test/object.test.ts b/javascript/test/object.test.ts index 88692b5c51..8e8676acc6 100644 --- a/javascript/test/object.test.ts +++ b/javascript/test/object.test.ts @@ -218,17 +218,6 @@ describe('object', () => { expect(result).toEqual({ a: { b: "hel", c: [{ d: "hello" }] } }) }); - test('should register work', () => { - const description = Type.string(); - const fury = new Fury({ refTracking: true }); - try { - fury.registerSerializer(description); - throw new Error('unreachable code') - } catch (error) { - expect(error.message).toBe("root type should be object"); - } - }); - test("should partial record work", () => { const hps = undefined; const description = Type.object('ws-channel-protocol', {