From c265c4bba34e0dc0e852637041c8531deca22f48 Mon Sep 17 00:00:00 2001 From: ImgBotApp Date: Sun, 2 Dec 2018 18:55:22 +0000 Subject: [PATCH 001/248] [ImgBot] Optimize images /test/webroot/static/mdn.png -- 22.46kb -> 19.91kb (11.37%) --- test/webroot/static/mdn.png | Bin 23002 -> 20386 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/webroot/static/mdn.png b/test/webroot/static/mdn.png index e758c6345fafed63f362eae01e8809a086c76a53..751840e9d36369d9647aa2bf7d35b17d28d699b6 100644 GIT binary patch literal 20386 zcma&O2RzkZ{6DUXknD`Il9jDwuS&=k+3O-?uk6)TC_)jkTZrp&?af8zHOk(EOLq3= ze=ha;et-YR@BeuGA3fd=d7tw>ul+o)b6)3ot*Nd^a_QP792^`HrH2o+aBv{{;GrPA z2;RisYV-y_@NDj>-NV5th$cQV#|OV(w|uCjhJ)kFfrIn>B@WI3c=h}@4vvQ)4$dMJ z2S?%q4i2qLO64Od&~VX0Meza7DfZvTnygsxhREfifjbV)%U0}jA?ia;C=QN#kkW&D zkG)4$#=ZP|cT@LVSA7jGkFq(iUH{_QJD0SYXsXD+@UX?h)Q_=lM%NAge0*DtTKH^shdh{eQPDx_PmnXT_em@JvB?YzurJA2! zeuX#`4ieC$3?iaC{_I=!EF82(*&4a=t4FoJ1wYxZm#!w&KOd^x0*~{$7d~oR1ENwZ z>~3`Lpij-lwHzb70pU_GZVz}S|LhV#h`Xu~BRjtw`w^EZ^vCQjxU4$% z4+L*v^2jQGg)T~-|X32~R2jS$qJhvuJByC#GaT!9YshoORdB65HX_CSVY zcYgJW4=g)LrQ&)Ji^WiT!Rwj)($_OrM~}vF>2_*IOgB~%f8sA_e=fWjR+MkOc9XHK zwvvuqyNvaED3%aimt+6V^km1YE`e0aI9vLz&)2M=Ciy(qGo9q`s*a1DNaKukEi=xy z*p^ZwKGd?8*|SyIU4eN-@?Fo&yK_E9`EgNQb`l~)GRv{6;!cnJ_D@ufP6PC=s-wbP z)fKNjQHaZUENZRe4S(wgiH^~nQG)mzhl^UEt?T)Rk?bRS7^;D8W#{u`*Q+i`S$6J5 z5g*ISo8}{P3>h>)oe&GVNupQT7+vB&?V~w8u1Ss|YpRnmy%~LZN!7PZfJ~2k^@c&O zZi4T9C-Fg|5aNm58$WE55cpV*YQN5Yc5{J_d&9TcPUD}8j^)wHiVI1|S}H%D`^dTz zhYMNKEi>%hD`P{V9OFiCmwd;8%9#R;`d3;_FfUQlx)G>gn@L@2iRIGvCYZ-DLf0Cw z9<DxYB zro&B~OYdCb=?a&J!bZ8h2(iEE;(zZ`pE|^M!Pvu_+k~AKXx7IRNX()iJ*bu3zQ=^o}(($UK{a2AD8KHc#Y^dookk^>2uQh ziDSQ&m(gP;a;%mD64sQUt)n?>mA#H}1b_s8T@YOO#hMa?xU3Gr?Ory@O-6!M*MJKg z-+#8$uCrKe)yoL;>CB2(%M9^oCJsB3t_po2-YMU7>|en}|F(6waQH?XJd-niw5A7a zfBb|>XXjNXKG<%44cIn?`i#|9G9bIlIiczAfU?n5K>5e($D`cqU*$2sU>?%iD4Cg& z=K%^%B~!o#_R^Tr_6W$5uN4?FX!bA>wJ|%$-Ldt3lw078wc74oF{$O!#ltjjBTNR& zV+JvxTH8XzOICs4KAPF$tH8vHW}?56*qWLzW;(1-+H8qfp9VCSoE*NX^`C6y`(o!; zu-1XVx6g8H7x6>qB^9%%Svv~sGzaLcb_0cCGP_T#yWVt7N_uDySUDR$W+uw5QE;}} zoV4#)pR_-^U~|D~BPN8oV<;wJb3$Oqd$KmEq-H32x8Lftrn%jnkj6enI=6h}Dpd{BLzh$| zZy;A);3a8Bd}8~hcE`ov9E1)Rtcyx1^yR)yfp~PAaO{si`jGv|poW(NiG4He9Mqi)0uQ}XS{)@X~(t| zKq6xoc-(y5;GSI9_0Wp^Kru^qQ>HQT2!bK+6b5<_6s|MiD@7yq|9&%{J^5d#_!Rb-wkyxKulLSK9&KoS=}W8y!hR5 z-6r+%R>wFwT>EH>D9g=B%Z}Yi%cEQCzt#^ITZBaBG44i~DU?N)m{$HNlV$EH(^^&k zM&4+ST1DKLt~d|UZi<2CR9cw+c(+fL?V%KVPB>@Rm=^NgD7Vc;nyT+g+@Rqo1hw(T zCa9Wx75USVh6L@zIx1mglLOe z7x4DA9GoBqj`5-*0;yc=*N=tDW);=OKUi65d~3-?$S)IYXKzeuZ;C?D?A<4qY`Vu% z%7PY~J4|WtU?CLacE_|oXNm*1&DO?sGe(EV@Mx;%beNi9feL7uDN~d^KHm)a|T)6P6q5Oqw{D?mqK5Y3l4hoomAN~UC#+;`cNSrM2uzgF}yvfOv0(e zc>kv`S=UgCmrHwSM=ilXDwQ`W*2l??z9G`pCf^&0*#=Zb3auIxA9(fHuI9$t1oULCgl(C(gJNk{anb;s6&6r^ck5fs?7AvZO zz{a6HBDqkeXbn~}kC*1ScoBj|JC%fkkB=HGu^K$F@O|v`^Q(ugNLSH(uL706sVAIz zKG%q^WkT1nE6CIE@s3}a4#N`x;#@oLbf(ks{+}mRaL$Tf_Hd5Qu@qWq4>?^jn<0AK zh|Q&toMSIN*(F`r_Af`Bi&&AXV}bHK3QkuS&Ay$M=1kG{esK%O2IN%9!I76<%l=wk zpk@iI$&s3ncWh4A9-ib>GEs$SpOo;B<0#X)j+{aoA%0HiN=sR4n@V9{K<5zy0w&%@ z+bT}*yU_YBBZN2LiA+)&Y3|ko2wDXZ%UJ z!!FX4>CovyS8^_{exIv@R}KXwY+${^Tp!kC@I7^Ve=R3!chc$<5g$;dLtkXifsp1T z%atY```Jg5$e?g}Q2x}UfAq*pN8@Uf;-G3`1();ady+(2@i1h9NafqjuwOarlOFR% zh*9Z7Kkw$uu|2M(-b1nHK$ck}1SR;m?i!ej^D`AO?^O-Gx|;WKVfZCucH`bg!d@fI z4y;)AKgJj#R>~^-dlYEZ6AQSUCC^?`DuA!uWhwPRCdsa9kRBM)coAd+wfzSr)++V+ z6~K@2sjH^z;1cygdfSbbSjWcq60|YmnkNQs(fom6Acgv_lJ+2#zUIU-f;~mxrg~Kf z3Uq>i_PbCR_P^x%*O7Oj7sBy%sXyL?5JwU7cG139gx24Jl@wjl>|1^cAtt>9-Z6lv z`oDdMM?=Ury*Ke*X1!Bo3A% zLf$hP&X;q&;5hP~6ZfI@_h3!?!MNu<0(P;a&N_3G{!%Vu-1E=Yjv)R%qNe@pFPxJH+WJrEeu4qp3-9k(IZA{PlM?3M!16KtzI*@w zpQ?hr@O1J2N9&?}xpenYkqF4^G@-7wQ#k)O zz5U1de_HxarT3 zyJMFA`6yb*-^aOxSKT@GONnlFf$=-^@r&u4iZR)nHc9;LW*Q=t=iz`Ce@Hl_%MZ)$ zbjw-)3r!-`5m&xdu}Yv({?<;F`6H5BA{5zWrg7gs^Y8bhDOa7UgN%X= zDKcB&K1FWHktfFfJcc9x22x>~w#~!s`_s5CH14!oZ(xO)MSJH)>anmK!9X+CDvD>C z#e5Rg`Pnqy)4yc$NVw0LjY-%L13o;r2o6kGs(Uf%YjKf=gUDyQZ<7zDM!gqmCLjJ~ z5hf2EZ>KuUVgJ;ABf)r2-5PI2c_?c_%)CKx}U%zG;#b*t^It_JC5r5O(K~k~IC58v03w@!MLYps}gch@S zT^9GT3^JGb=MSrQLopmaz8d`VV2UT?_4b9v`DCmy<-V&fMc>yI7XT|2+k*D&Bu}5! z-#&@eS|;-r&5OMo@bdcxPrtq09xkEusx|1M8_g)MFj|!8?6EFY4jblPv4$5X<@Q(< zD!Z4-U#sHl0d!*Goy4^!ZlHVYX7yY{J{!0v1$@zm#$5=<;se*zd|QcIEj$|H<_b0G z_8*U?JPmNK+^jv-%WT)M*u<>3_q%;A)Zrdc?XfBtIW7 zekDn{ZXb)4Y72xSOdeq4T=AmrvqZ&YxN}Tq#POAAdi()IV9JNyA6#os=rBB^LMtaH zClC6>efA$`q==NNz}pskK3-FfXycZ=bnVHaa^QQhCs0K9_Fe$~AQr<&8i})#GB$6U zV~AiLU-%<*Hj*o!sKq}@^WJTw|0v+l`M#`MN5wiItT?%!{;^!hldrYG;}4+q3$PL? z9UOKw(*+;cHhNaV_V+IYjg&aj$u}T2qs2Tw)h%R^v`DH*G@8)^_Z+xs_eLhT+w7}U zhrP|AJ~_S3JG&FIZ%nZ01%Y84(#Lc;l=^9YIpz}bYCI|2X)W`@Ai=s48%HTNR!(8Y zS3iQL?g7VwcD+(7O8s-EjoMCb^yulAyxT;sZ~CRzBCbU=i%S1H4)(*SebzMmcpOQHzxK#NL@A)*RFdtFHlkx3Kf&~lGWjxH< zUN@ZsJM-j_=JHwPlt^XCUEJHg3n3QGUJyhV>r7A^2n}L#e9h_uA`{EUu498yiYMBk z;-iGwq&AnZwT$RnRD))_VU^W}WVR{Tjmbkl!qf&IvLdIbVI{m(CKnyX3@P-HyPXe- zc%j9xk`C+`bzjm@2P0|9*|%No<)P-~@hLf5;Q{BG5>|D7;)Z=Y>pLs!g(25RJ;X&j zC*A|b0L~d;<(2U`>EIg$g!oH9G;gdlMQCJ?EAD+GDLz=4F#|1he0NkirmVt}_M4t*Yz-M2+XG#SP8)U<2&^?rSOn4y89Lt zyx9~7rYVpQXb|UiuiQ-7a58OGhf+x7^*AoPJ0t>AeDFNpfLZjQ$|TONE*@CW+C^NZ z#-oAiL2Dez?<&RW-lW%erM7MlWQl%P)z5j1svH@6 zJ-I%&ZokvUhx2h;2nNb5bH4Y9-jx@`i*HXvNT12g2;q4+*&2;^6@reGDv*xA>PSo) zDUwJn+vrGf;3Tp7OtoKaa_k>{{Fs0XUD~@76N+m`7FyxLxnY%%|T!Dvs<3E zWiN?%Ls1*5y?S4LA409#Sd*16G=XJKU$@0Z272QcB?l5PunAC>si??0uPa9J^LdAk z4flSFf*t=S6jb_vrca^NEdDAslD{1a>!B~ZWo917M|?n8fIxSz{v3ybtTO0JTu@jn zO78jk3GFs2uL=eH+Zd+XZoOg@vGjiE{DhpM7S58q-xUZY4gQp7s) z`(p8`fjs>4uSRt=HJ}&DF7Zx^dO(KkN8Db__IZue_@Bw*e;7eO?wLga;AYCrY%+-j zk@<_y6cPczL+-vU$p65`X`YXwa+819pb-U@`4wiIaenQ1dTde{EWJY`^2YlYCAPsN zsfur-|MxR2?p=V5`b;_2JRt!}#-ovg)?1W>509|I6uYYwp@{aFFwuAjm}6X2$PhAo zC5&^A1Psq;4#Hv-u;06MQW*(lH0E{$vtw0S6<3xt`>8*R2*BGPr@Ck!hMy6`SpFMU zo*Crc-hIf0O;jfq;`%N%(el6mD3)_%)oL>t_!(Nk;<(71c`txez`U=H>mg7i&9W7U zRGw@QTOhnooG)4Td5AeAVuc7X;<8_f?{-uMNL3Hp4ppB@Wis|Kw7asW1~x}nkb<|R z3vZ!+_eej)&PtzanNnF+tXl*$v28A#jvVlp43Zkq`d$c+{FF(c`*pLQlULY)1jNdM zo*yu@w|vyc!?09uPP3a-Xw!0FS*6PhZdTu4jDH5Azg^YGTaM_X7z7=$+@9}s@cHax z)(p5qoZ>(V)~D~KGI^_T+BSP8@IE}_XoR*QJtb7+{cvbgLJD4#<0M8NV3 zZ(xZFvMaj2eMMZmH?Pu45vzIQPnF|r?qqReu3laGi7aH!H!lLi_&hh@(aOl_{Mycb z-Ya|DA#NOhd*kd!+GmkaP8D0CsKsqK>nB-+b(+0zvYc0P=0x$aP7^Fy>An7W%XlFnwaH{-m@nf2 zSid;adVlS6q@lw~q3I4ap-+DG_}O+FF&D{qZ0pu^+wC$#=$UpHAxo^2Hn<|1VL+(Yz zm$|$E%>DU>stoi!B*1u+$lk|;VFspE=yl58gRDy2#vvhL+iK&_VjmfkJu0bJs~3=d z2?#}#0(OqI>Kt0_6ExOYABA*h4MZDp4*0)89V|Q{$S!(s5*b~GfDG;tycw&yaeI&n85;{ zY0==8gJBYzi~@AS=p8|Boctln@iZeeedKK{HT>o=(d+bXqnMv0?07^l{EPtqelFY9(l@@3jQy5g%2(d@ z`IGv%iFv-m%9qQ5;l#d?5O^O1CcJg|^2qGtRGr!)_!~hM!8;=*;U0>7LA z3%ULmX6?1-)h>mmOr}P7qPLpoOJX7OdpfFlADmyoog#F4B3CvV3N~-=tEHluXL=ER$2o#nLA2IoY_26g+1K+AFv0e56b62UbfO=|sE! zY9s#QTSJYFOOO%^4R_LFdXtrBKb6R9azTivH%Ew+w5I;a>^5P{SlkvP)cGBFy>L=d z!*vWI>d>1Bi7NqNt8zIq}Yi_+x!i%IYoB0g?6;(c2f9fCfTs@*A8nM|!IK;O6(Lb4HSGJJczgo;@z&(f-M z?3@p>4;)K6K;#X6vmqA+2@`TXYQ%V!)%F2JxwRZ`)KfE+5`nX+>ZH&bg-Ta3HX_`sBsJ3kG7XP<*{xTs->_PGXaO@MKe2vW8px|Lv8? zSk^$^-U%@-3}4jYb=$wfIe&H=nf;`*?*c5-rkjR%Jj+Kj$4(o4P~dOZ&lMKLK=yCU zFm6@snBv%0yQ{MUUae%4osVY=q)LRxxz?)wM)kIut~MFP1E^1E`f>*)nsDL(1!g!) z+9b@naMl$+-k?Yx&1FZrbMVImYWI(;;8EzWuD!!tyk!tddnc6!{=nv{RWm(TxJ>+@ zHafKub>26l7R-Yh*uME6Ke_8W7yap`qGi|+vmmBVsr|1{#WN30Cc{KB0#N_soz~Ab zA*d;?P&wh`7-WeypN6-v7nP2)6TkV3Gszc{BMEOH$BqAW{XQI@G8x@u_x#ylzS!!_O)NS}cG|`~1@xucFZ72H$ht`*a_%7le`n`e-@ymWA)BBRNQ1 zd_=Q!=c*CzBZEO}$ECiH3;xO0<+JjPl?AYoaM`lT_~~kl4b`R8lM=mDgjWAQw36Wb zl$(qlvE@5^3ue7P%Xp(?Cj8ptvi}AVpA1X{9b7mSyG66z{$WX~?N(=|on*C{*;~Y3^B>liF%&ulq zC{U-Kro#K4WT79M#DGn&=dV?>IYS9~jc3YcKU^W5*(6c?jJAlIvFhCDh!4KbpJZch6pr|&ddqY1BKA(KE2 zksVEU11Y#rGNbi#1hE%mD`a5Rl&D-0)V4kvxH8#1`{sA23O3h;z%nMTB8OkGgy9dM z74a8EJr;6UUCP6fL0eWljZVY zv~G%5R?<#5a`P%RF*{6tG%vF^#QfKs6AXER6c0e1H}Tjy{EjCVg`TuqX{B0i>905A zW`~>h8Y&4PWwuc443{S5NSjXy`g-|zCz^t_;upi~*7nhCS43MI(OZoNBNx!yJ?3+N zV?Qfe?EyrWnp_@?J!V{zq`;(?Y7Kv2)z>Ce~Zp=zRP6%jyQxupZm{j{GBS9(9HfP?<$n!!+{}J(XPM$`Cnzr(?!--43zp2 zQG~`eJu#ME{*s|U$_aHoy(9Y|tK$WVAUx7@=#sLeVaR8Hh22lcDfDO+B^r-2;)#uC z&^}t~3Y_^`oXk-7FFYm|DjK2*z%h&~o?mB62~M9F6mDV8@(4 z7aF~0fuM{0J=GTZf}|3R23gY0+}l5yNg^^kZT|z9l#_eO{{HMBYJEO@+gglf*lHhe z^0D>{ZLC|Z5?9FjZ$5U9()}=7kEw7WhoI%H>s+;Xb>VlsjAor(_tp248hv;O|2o;~ z(1Y0s`b$KBxDf7=&NzNnKu!8uN8u9U`d=5=*Z$%54xpN#A>c?^mH*YhbjvB zG4ngt{L-#1yTgBTpIy~2m`fMH`{H(^N|XQaHjhZ$4m+*CX@)iU{+C;2+vMt9e8rrF zKcJ$Bmo~pfK>R-ll{Wl#SosK>mbd6-RxZo0Fb4_>|D8VW9KR6R-;)&Im~s=tg<;H( zOl7#YQ>_#Da)S@l_t&H=VvI-rf-QCDr$Q$m7X#9im#7WF7J)mG0ZNIvx=OrGZ#D7+ zTlW&8S}l1r0{)~KouiA}@sPs}GkvqWdtyS!=9*^@rmi-G?frSG_;j(bH96ERv3zex zk*Q^PQRHkDcOCioj!!4;Yo>XE0dv^l;o?EBx?k4LXRznG&|!P!9B4!i-l z-$_BIB;;@89S^SKFZE1HT+Z=|dcRi!8O$bK=1TKV|JrTITFqCeKn`pUgE*rmfyZJx zDUrsqxh;a2eoMK)4}YE2SV)h+VMFlKOatm*9tgbutMpk@!{YZ4>-8W>s|V3vz7@nq zx0_?*m^U-*5wSOUtsTdbrO^=bMIO zLtLi?>mA>i8xsc83J=xP&e0ijwBHL;z7d9MaQWSC6k7A(%?ihUQ?#9&Uu*q|qPp@ID6mz< zkfCPc$rlN#8xubgxK6f%kRE&gYe3pT43!wz3)^n_Feb@QTvVc13X+g%v?BHC0U581 z#OF!l^L26jW@ctp0$BbCGRW0g`b9Hv+g|DHp&~sz$d2PL@ntD%VpO!@3iV6$e) z%g^1Z>bmkZNsK#Oi}7H)k(4&t``;={+}#7EF!YTFNlMhmS64fe+4RG_v|V<3gYl$9 zwya%nx-8`SdHT1hVCEPv?rbB+-u%XI{q5ZvCZEk&HCfWJSRnk z-uWc%m1RHWd*ddOvsA%t>W*Pvs~mR8>M{COS2)MYs?gA?Gf_ElZ{wEFZ-j%b&^qgx z6JoyI+TKBP>TOi=fa1)~{Ls(pZ#9ZB;q*^NVk~d3Hpxv_{r<4r>NG8UHjAO}xsN=h zIJNo?PWR{>PA~RRfK*Q1u}Z4;{^>i}?x5i?XZ3R_a>=qL#yA-h^4jBI_2#E#BFPV3 zm)Q4B94b0|c!;^d7F`8b{+ngF6v+JU>R-bEJC!RtuEA#dDdb6oI!Z zAI>r7z5BhatAx0vBHv<_y}BAV*32ZX`;$*{!_dmsuI7lZR*T6fQLY>;v}wx@=+Y#W zxSie-n98`JW!DIP+E%N9A_5Fm>10>whatA#eA1O{@aKkQGoKXOAN7c+I8yto!q1X0 z3u^PBM?y;y9p{kfd#dRBD3R~`MHs&GMk_sM*m@KcJJ*7gdraFa?+c$rB$l!&L5gBRqPnY=25o61o$7WE6CG zGzGr`8c3-}J?ytL^Ey^#dxxB+zqkRnGiB7zoCw@c51@TTY)qW{1h)I3XzS&NRpc|n zO1_QA?W~m(AyLxxiy2rxD-_BwHATjwPBEGMMa9Vzhjo!#dzNTah>NZ9n+o+&QC>&B<#o(7 zs{8b8{OqHD$z;sMp7B^c~o{2=~1iiipg%Uyp zZMil}PJtcbls?uEjqm>Pqtk<+3C@I!%llu>HrU_yu!1bTElcnB;}UI|3SN0rNgvam zoKdI4+0I@Fyn3cksv?A$AQ;u)p`0Y)Y{>skQCZ<=-fFOPwB#(KxmM8^PP4yf#5p!c z`xL7WJNby>XGT)o$V#w5Vn+J7HAS=#r;0+~bgE`FlPn>;%_1&+t(9&)eRq@bs;2<* z9p1q6PL3BM7QyQisP=681cAKU<=Yd9C0peW82EKPT*kSO2L}Y0PHmF!5O7ZYT^ppk zbZa(*j7B-R?MAwZIbDDH;163}DWpT`Zr&tqYA$s^-0K)<^=%}`H854lUB0tSJwV04 znO=XYHW04&s6t&Ue$BwUKz>?QXw^lZhJeb&h(7fEZ2)^mBK zeeOl2I>+P;ILS}OE=@N@xa}*~b;ug4k%J18y}xIO`JQv=kkz~Q;p+Pb2S#RPhjnp$ zK5EjBac6WuRq*R0Yku)&)2kuk`KfQ~m|J7LiXvk`a>_RzZytcBS=PkeX+mS*o~f0| zD?*M7>Rb{B8`f2mMVuv{<f5M8K|xbWeGK3D#`~}n>yMn+^9S`-18La|8}gO(-8yZ!5u`zXPGuiM z`T7SpeR-OBAn=mW8z4Z?q}b`>LfL?CgxM#jh0@mPnT)cd&IjqV(d{XNLe7Ppb*cBE zQZh&o&%@q%c71gv#GDrg_;#oT4K14$IEY*%Cnx=PGEdj_-7_s(Pn+kPSVaTAWb_+; z!)>0yT{6~zOYI%FdWO*_7V2wU2Fl)C=@bt9Bg-UD!npCeOnqX#u0jUMWD?KXylUG1 zeEm1dZndyA`g8BlRZyZcfy%Q163w2m`nkFWZq+!Bf@{WGd9I5`cm%YyRP6^tdPuYrSNMKt9$0%muK=|{eS^W=PZ|rv!eUv z>_AD-EUP;qZ*wdJK57xyUH4@^_pI`$dH;-tic$_iRK4{MPPFzLEX@MB-hRKpto;@YiZCyI>mHCtC z`U181ZcQN-7=R#hO5zP)u#2B_pXullcLu_zK7vMhp5RQin8a8)z^mqKdL3#`PshYhjdl-bKA z$Wp4dM~{2RB-@iQ0c2n?d*tRaK8L_rg*+0n)P0NgYZSt676L)UJ(k?vN z={Zxnf>rZ0YmD*OpL)qMD*ZQ-1iJ1CA?|=PPz7x9GB0#c%ct3NYykV;W4Dljb&?UA z=WUj4>skwiL%XuzsLKpLOlQ!1+TYI}S`&W(TkuA%bcu`{%Yyq1ppDw~+pEo$5?CDR zSuK2apcD{l!PCteZm&2yzmg}q-t@$F@ius^h&M1-%-tn?cEBJ^aZ-Y7oCXr%x`w)K z1erGqr@%VS^ibhlkk^ zg*ce7<4=|@o+leBT-bv`<$BQ0Kw%S830+o^%IA*^pmAl-Ty^h(94T1xqV5QRsf3~2 zRl%G~x&2w~;_z~QWDibLWQD*TF!(r~J{UXJOI=ix;Ef$*>2?3bF7DadA9#sJ8rUcw z6GK|pC2`RACf~$s#=xvfcU;uZ1_l-GWxd`f+gF_m=>{+A>e!3UU%}Qzymhg>s7jx3d4X%co$c0j6BKJWk*g$X;1JG?BDsE1!q>L$QfhSE z$vy61SAA%J{R&stO&NM{8Y@P~WADuoiB?!o;5c{2& zJ4D{na(0!z*+dH94w=)^oc}aOtSR=$VZ91uDXF)gBRu#KFEZv5y;wT1b_Xh)77%ZY zwc3es4=#~Y}A;gg~<_M|6*lX_GSwGGPs%=+eSw-4jo!+q8t3OmoB-bf&= z`6W8`zawx}>I3kNL3L>xCJZ`k$V#ewjh+-_LxLP;g@f1zaEM(Gmu)0kC2Bb@ez{GU zYmNp$<8Y=Ji2cnlWyzZ)5PSWB)V!&`FPGG5*mT-(oR7hUfrTo1b$1c0ejzZcXy<(N zjG7V=ujG~s|M@#%2C_O|^};2-_XIExqh?R4dCGH&tm|Fbz7y>r;m?@;e6foixaM`F z=_?>_9Rw20SrJW;f4H4gB1jg6YM4jonrHmOUG|BTK6(iJV_xbeAvk++IadsvhnBgcWX&f7 zx1;Y8-gqR*#H5dYMk^Lw{zZvc<6EFvy)?L1*N#dm*~^tW@eF-3Xp~I@?saL@2Bu)g z!0JnHf1E$i;rF>a)10feDVpReJFdN+wXh8ZKvxF+%^5t+n5v5dCCv1hx`}(na-w9p z1_I1;ybb)ZH?Zd+ToO=*Z`q1WWtIkN57bL%@BBI7^}3wvU`anu-blZdrbot*-NV#E{~b?)l96rz*ZbqZDXBgJ820b zOVrAW8DLWIf(iZQ+iQDw=yvwPw1SEn00hG(0p9;<+MTVGQW8oE@$zyBfTl`Ke#hPf z*v1=>#>&wnCU7a&UZ&!X+zzHh+3fT>_3UTV!Su$JF6Pjc3+GDNMb3Ih7MoQolZcBW zWsc{Fx|p%-{MkZD%xni9I^A+!pZbm(Nz!fFZzO~)i5*@d#cIwCnO_^r$BC}mTmSR| z7d>P>eFV@i9Kyq8`5#{x_x@xo}Y?gMemprRBHb3HGuLG&cV6%s1EDw zmBqMwfl*Iwpa$%-P^N$F$ZoUUv;-djQU$`^mR0u`g&X8Z3iv&98K%;Y-<91;Ec$D+n}Lng40`jzDwHW{3I0Zq!LC%HS>`?mUE5iaLCwuHRPz79~v4mgJQ3^d3t=O%Qa zNcIH8IfI6^xp_#oK|a6|5-Hu|un?=Fckw}_vxu`ZRHww#oyOb@AiWJOilups%3l+x zKUo9n6RuI0#((N;^SU}ti^ny@F-F@t?xJ@3))q-C0N<;0#jiUTVEwG)H1J{%qwho3 zG1H->;@UrN5bb+VdSu?GVdRf7b5xrFZjUuUxdAjaG+}=^`RYc^cE!eFY7-@;9=}Rd ze`{s$cA;dJ0cqT`*6emCaU(z|Zu~RPTFeZs1V&RSjO|JZ8Wt=voG^MrvMAZgNFmli zYV^9j#vocURKmcwS|Ke`T0jGVerrn{D`pAOh$z>wYgJc%BB7S%p(}w0>K{0!y0JX9}9CJLPi|63Ev1!Dh; z>+YjiaGpl&I+BI~<6jTF9b8(`x*!kE+^Rwuf1SPg%jt4!D3uBjt{=|2_JOr1)#+}nNaait1NVcB1B`}xrE*{g{_4?OwfdUM^rVpB5 zwbHFL^aM6NfFFNfZiHuEIH)7CGMV-qcIG$^AB%S9ip50?SVS z9YAOWFf!Kg0u}5&PVWtMx&P22bn-}UvT8i0a%Cf2H8nd@bsRh^KNT`BLG#-_f%70+ zY{(W)Hrp)FPe>7fURI=P=MRNUQn!n@jKLPL2)|fun^)k5Mcsbe?q#n>jyJ!h_+54zuK*~hG{>Tz0Dht?pk~L9=o1ZdB2`OVLH_n@r~4#$x-wv%6Oq2x z@ac7z>Ekj3(;y%%xUvlH*GFnVS_n9e|0Jg$*~Bg9ydG)o+XjzQ7dNQI`XTPuX*{%8 z{JtMn`zci3V+mvR;cMib2ezyv9FAY_O{Id`4@tPtekhwfkU8z^big)=<5w0PdfVUg zW4>j#yt*T)y%YTARuk6mt(OF{w0)APf6c7Dgq8guUZlrGf!>yG3dheJ>Ka(nRkKNY zS(Q*-YDH9bKiR1iE|VE~=cOGhNlA(sgqX?6VX9t5hDFJbvrG0)ODavjWBj1W$V0iW zB0ZYj_rTRETMdY%mA|Hcjq^jT*}ac@L7UG5gca#+hv1QSjHh}19zn;2MRlhtRujFT zF8(g}(kk*_0i`sN z9>G?yQkn{FzSvZ@|B|f+aO#ti#h)|En}0f>aP3m@yNmP$DX8FsacSo|M?eGS{BR?` z+CeX}SK-QPM}`sao}+>Z30WWPZWQ1s;ItknW~r~lzN0vE=;81x=T$}7Hd1$0&V%IPczAN`xODv<5p^v~Ter`dC;|8A z&Bd}K1&uL3sPtWa#iD!&^3;J+f6>Mq;d}Nv?)JL_8X+PL(3vsNq0OFpnipu*2&>h& zj3>L(2egha&C^60G@E__uqx(+-R8ep+b+uHGhyQHe9D3@w>4AiSD-7kW4%n%B?~Ud z)ju5z071_##doq~|Oe5QI#g3y5W~32AA$)Z} zfzA{>*M^ClDD-bZwkrhw+|Ac;v8V>7#G#4kjUSW-+M#rO`6&_eu^A=N?xdORde619 zmPJX$UoK|VJVDIS4F4(X<#nI~fx~+U*tY+hyNs)8^!D`3SInjXBLpxWR~{OMC)OQ_^X_X~I* zS5%+q0V8y8+(3F$(w&XGNT0IB>G^`&D=uB60jW!G{@cQaTz}$l*?INkgyBB`Mjs+95xQMUa+PuS=(JRhj;A|Rc9d=Kh#QdBo@ zX=Mye`6(*45b~O(?X;cJWge4pnkjNW$dLXM^ACmJ_)&^b@j>Y|FcgM9W{Ww+r))y! zG$-;$-N(ou^uw%O@=)>mfK5``G=Eyr>hWBR=O-q0odD4~6D zGek)PVju7#wt)kW-T#fys_UU$yohm~B7I(6tGcTc1nt^=aX#eiUf*e~?nu4iN1yfr1zlfc4iLbFXJAO=`*WDkwL@*34zgJa5YdS^6);ob5 zmg3({YpMgYNe1Wb8@EMhNMIRVoUj7m>+Acq^-6WK9lqOn*We_Bb)ZXGfekn-cv86$ z2Q;a8d_P9E5!OV4whso(0RfzCeu28BPrYxIhRgKr$*HgFDUZgG`7&Qf2AKivd?eV) z8a%0VgZfuEyYx(oe?YqX1ent2$$lakG_%?#*H;Hl2nT#e9pZBaL=_QA?)FcX{5o)R z`bES$FMXKo^Bd(7td;nlo4dP$S zp?g89S_pPhs>VfkK_F>bq?oxB$`TOq z8=CGuZQRZWPbV)ISj^7MH3{$IvGebc%*n<7Dh$8?@mc^EHOpL(rMc0Zc}vF`+?j~u z;Ml+x7_2>hkhA#L0iI)dZ8Qq}#Rgw~h1FEQk6!mR?+2T?XyMr4agXZN0l$jyF_(g| zTh22sZxAUPOWzVpBr^y-NYMLm5+#9;kG^&~w)K`^`*A*eKPNIB(3;r+# z5`I){e$Wp(C5W!(*f)=DkIAfa?8-2Y%CZQWuF5xLk-f^e~)p4lSnNnr04ZK`v zGEz+ndQk0X773Uc<-BR0s-Zdkpmz6H=HVhdNw5anZ8tL@V?7?_dy2m#Tzl^|OJKc! z>uo7Jp8z?kOpWbO42BQp(Y;x445ZZ#coCB}<;TEO3_sDK$|U>wa={)fTU;i`aA^Y| zRGz?F?DsE*g$d+G+cs>OCkY1d!7zCm5VC;-pve#9blqXRjlH{Zx>T8y2%=syHV`4h zlT&-Ba)!x|U(^3YUS2actrSlZ9G6l1uo}am%9++cg}YQ+aVVfps5-Pn|6VM4&BS}5 z_TV*#YQh(yUJ~D)+MbsSQk=-R{u%G3H?x{pJ7NuP;&9f4&#z-I30&a%uScvX4? zBb}_kNs6;(Pptoo3#2Bd+JHSol3=QQCoE$wz+-KbfR_qF^%s5iCu z<-$dO0nmtJBFuE*l${`lS&>{TcT6JzgYfrhaSs`j$1vpHd8n zne_k6KjBeu&3gvF`=RBTPNn);PNlqAw?eP?1fJi#|L=75Gu{?2#HV~I+*imDsMuTKwWGZyTA`jy>x#kwOmPoD-3wp<4VV)5fHpy%q(@3(zWw)>R3 z@T%yQoL5V$N)qmVj(KzSc{ z;lDq&v)LrY*G3Ajx(xJrxGgB7o4Fl3*Sq=~rw`9JZm&nq(-_ak#`$MBm97MOdOvW4 zi^um8^J^GE9cvWDaDR@Gl^dDUdPa97rLo;B6;?JpQB8YC`{54hy>&yjI` zRGj?fmRE5=kN3P2hyc}L8DBVk=GfQC0h4vb&3&_g?tTjlu`1v;_i9(pa(i)OP^@gr zPk7Y-qn&vH=hd^9T3&qv#)|k<&NYV@UD>zm`_I4P>Swy|zm=Y`;^}>^2DOmy%DyY~ zqEBvLE?JrYDjJTe8ziob4|ufqk22$qub;er&Qvo|^%tIHw1U%j;zWtuguUNYJ|wL3 zPdHTnF`wyy+3r_yz8R;2d^5aGdz4=_Rpwe#J+J?*w7vej3BXK{{HK{g;@Bst<($5i z-ompkJzH~NSN^-kYn+WA^L;paiNAaA?^pBElh^mmmo7h29(U;R zpIGJqU?0GBv%=G+G{>&9^JzE((YGs0y8c{`n%Qn*>$sid z?>~)I^D8+g0v+)GSiBsAxc>U(Cfq@n*6FSXZcO^C-uS%cnX1jn`=Siu?^o4d1{!`v z$7AJxkA%CwW8OUFEPnj`H>;Z6nRpQfQN8uMO}T@_e7Wp$q`v=goP2)I5AoV(NADlK z&&weCZgo3Q@*B|ln1BB}-mt00xe8|j&!M*|#mztsk{}y`^V3So6N^$A%FE03 gGV`*FlM@S4_413-XTP(N0xDwgboFyt=akR{0Jcme3jhEB literal 23002 zcmZs@by$>9w+A|uASFmhr=WC5BOoOpB`D0$gNjHuNRJ>TrF4q4FvLjLkRlS&0}?~1 zGz`+xck`X^+lCsKOuY4>id(@FYd7 z>oxd8=%J$Zlmz?)ki3Wl->Fcabtj}u}4Eo!SPqZ<*xwkIEe;YwSh<{z- zE$-osf&};NLZVSk1eMD55~>il+*>q$Vpw>jQ_Pnou0|j2D#5AsY3vj__`CSE7iKmu zZM~09ec2Q^M{%1&yRTQU^Rgas0xXndqS_s^$z!zM*biw*qdmV5&s0ZU9*OY3s%aL6 z$iNn7jrek1SC1_9J0-8T7P!~<2R>O-C(@s`xb-X|8WD!$4|w5K`S^Q-0K>i&oT1}s z{2k?VMuq7hmV_GEsBYmcx)xir4-k$LIIh&?txAV{2;SvLuIu;ggsh5!f|@#ZY0Zo~ zYbT@8r2JdRn_m!!FkykTX7?s*!eBb`lZc2eYv{~T8!d{`SjaAA)gCYMQ=N0NHDLw$ zH3C`U%S~-Oq%LffNvl5D^66i-f^{`VPI&GUQg#!pzzYT+qSy&%Bma0=opyUwC>dgj zbh+Z>;dhcbYLb(9$_Q25A)7}e!b`NtboC}az1*M2*wRB!4mfL>=_rdhT5i z#1P;|1`&A}e`FDlQ6dbz$XCf;1sj%xdo?Hr%fu#pO!Tx^OX?)OM;W>Lu>G9}k;fxp zsh9NV$EyIuM2#bR)I!3F&;@x;jKgbulY>ULs~; z>&>Uj&w~Ty>vzfUCpxB^Oeevv1?$(<7jy{AWS*(lmnzT`mxW?JZ&9j&#Rp_Qn|b+) zGJLPwtret%Ij!1vbK)DvIBXd+`VdjH@c2auV%|}M(QsQp9_4iyG4jKHq-mFNHntnGQ~C;5 z>dF0L_9ep`?JXdx9?mP>DrZn>e`grxVZ0os^3B5A2RZ$F=l0!uSOdUV%~3S~%vF$v1YsR&Acw{Yl2iHKKa-^5W+IVN9_pQ)4Zn*lJ zkST$^!^*0liZMWaR00CSdwcCk?X~h_K)hNrXw}EgmuD>0fJO4I4v)d;UP47aaw&jN zkT^R0>Zu@{)v?la#eM{XFSq!~8B^E+=jg}z$#M7L8#6bV8Xyo$qJZcFNmHna(vWCK zulM8_odFAKy2h(E$VjgCFGX`SUREjmuy5>e{yGEQmHh46*#+Ih715r={C9JEMsMIc|N5AwD zLzs}hOwF=p_*VX9qTF7eg|JM@$)bD%v~HoA(_TMluYz4{(m@%@eefxxQs-+4pziT+ z1EYkKz2dx~&BoHTq1oe3C{mRkV2h_(av(CtFta z>shdEpo|@$RRx@6X`8WkmbJ|c3!1yOsacHTA#L#yK%guw3I4YwwEUDuy==x-?bBls zHhE_z0u|t-i-bHpyE=a>cpLE3aXIM5a+RHD?E(u?HS}e;$~Sg$ZdbSaUjOajz;ph^ z(R4(xnJkKqu*{i14P6y{m$pTKfjsN^+h3L7G~rJ`-|`4Oae!{nHAgF-@dJ%c8JR9; zVvh$pK!s|}1IijWRg<^neT(J?3CyAf@cD`8D*M3p-{@bSl*SrciTe!&nXC4kHvy94 zL_^*%?#uxmW({_%fA&hieL`X+3Cr~EUabu50*wUzyjTo80ec`rhlf;MweK(_s$Mnr zA_QMDc+hv(Oq)zuHQLt^SJaBfEW~;u<-!pdKp~v8$tWAN)PP9^GmRz2a4K1G2rm+% z>PNdr^Vc=&OB2EN=u`I7O`AT(-B3WAn{0vV>*A4jc9sP^t1)nb>X_ zYP=0?uq-=XKW*(v3$k_+_iW*QST)p!F&A!D&)d7?-lE*I&b%n$#5`?SbTD-!&&x#n6~!_UpB>-IbfoNGl@Yu%CnOK<{d zwc_%$;yBTmc26*m0|m_O%TzT@S*oY))c&sq0P>#g{AospRHTePW99Dz@l%8-*X`-X zFz-}|s*S<%CR#OCr^4%}zf%6bqFXLNF_B+iwD$$SK0YGvYIx(EtUv;*LniMR6j%*o z-0J{JDlt8TGhzAtsZfsfmRD5Yx&$X~fc_pHa$zx{^V|nNC{rG93nSN?jaY(MYYB`` zWBfV*qBMp?Xb+0Tv~S~gv@D% z46nH~c}CqH+5H*E_y1G$cL%!4NE!6!PK8H>)T2WR^6-@orXD7as1bF3DdEj=I<)QDcMFn&r@c9z~B z41XIkIacm2g>O9mIdMO!fO*Y6-66PseGo1xn(Ukmv?1@?H7%cHU6G4aY?x=#Z-R*n zXNU^l>quzw1`+!JR@=)Iapl&-_{w~mxx3MP? zRgjzzWF?jV&!jn8O!Z=b?iSO-ztV5u{hwWBbn5xP72YluNY}#cV8ZCbJi%v9{5keq}R@G5yf&~OJy|LJC#94F9G<{khvJ@ zK<~jNqbG7W91-SyWp}N>>i4{9-z#|>tCxEqSB)1v2nB2M)khr?Dg-jc;HuCYqlN#W ztYT`uoWIB{OoSW0NCKQY>69M~sF}W7{8_bCKzg%wgalw-VSM^Hcp^Z*%hgR9d6ui) z<(sD3aljWN6}Rmz9G#)j&UA96Ug_6B$5r4oce8HTvx>nFf>)~@$hluTiLQ#g#PWL* zZoqTNn63o;`hg9U^6I2ma&|S`gA)pi$3UPuo+MWtz&f|sTag&&E2YkIrfZ5pggKLF zo0d;hOpLMmssOG%Ijl633;}C`D8^aF5xPN!hrCqHIUg4dxx7Vw5)C$pB!P))g%;TI zVE-;v76n~;_cn@X_MGz%Arg*wcbrW1n98y3pBNUk%fC;}B^M0|qQtB%y zKE2Ka*k<{cMm^vg&|&%0!oZMY%rvHH0`q&DtjiM#=f$TlTeu;CtjPU_6Q_Dk0zG8ohSSUT5I|8zvpdx2*_#kq82BZmrw7;-yCVNnpw@t}m`ZtJZj~WH1n5%?K z4f4UlJjm$bkyKZG2nE4_L~0cHt@E%Qj2315pTYmr;M;91D{7L46|#;(y>1Ix10VHO zI35I&7K0#yK;YuW?h%r!ja`mlbb?^^#gs8us z!KV@elN>W}5&nN`S<2;$vaZmUbv6=JgPvTT|93k+)SXNc}xN#ZU`^;|J~;VSUgRJ^nY0K3p+%iy~paj;BTjB%vtxpKb0X%=|BL4{Cr^B2~N76)A?a#@`F(R3)}N z+HFW$b>kbEIcZa@H&cw)zV2MY$1tJTmOGvDzBDtm)LHE5)T_^>y~9mtp;AZk>GK{u zNDn6*Pwe}O+s510UbzaZ((skrrM6Yj^eE@T*Vx@Ai=tNJ^7-M;>{!27HnERNGjtWC zAZh~m^i)gk2`zHfcXc&ZcpP4N=baS1PTq{2-R~pM(Xgd0+=$8Zlq6C$HJvJGL>{f9 zwX`?rfVaUUp)!r(! zUr=DViuWyI2=$-7i@nnU=CG2MkAyabE7KMabr_SD#tOTt!6@1TZSD8H`2i_pVTA|& zX8WDrOn;c5^@H;F+>qV#Wli7E^iLyRr(|SR{Q3H{x&Hw<7revMQa_|>lmv$T@XO1g z@4Zg8NmhlmY}BLOhbDP)F{pwqFFbTzquxEJF|w%fxiymKL3_xtA@H|Q;R2_Y`1Awn z#7Kv4hdQG^8a5lNb9{Ze-h-$djJGt=0j}=axr2Ib9fpYVSVj(p!Url7CJRHv8n!r} zI|ft?vL@wb75uwyM7)u{iAmy2&k(=;q&7&EeamrtAX(D1HB6qt_ zpXNZqNdn#vFPntARz}v;*W>~(S)+1{B^(Abn9%jHSKV&`fpDUqv@|&z-pDpgD@q~} z&ZjC*Ks&Qeim3M-Xd7_QPV|-Xo)_1c-w>Rs;HbZi?%xh% z{7zUjX?pgej&zJ>QH`WV@_Yp<21hy+whQBCwTuOAG*a~I;PtS%A1Qk}9+ zDo(+hIgNqvB7Gm0d5zzGJkc;ZM#k&H*~afH#z;r$NVJiM5*uq({0bJvYoBw!zNj?o zIT{*Y*nDxIy;q`Ns;`iU?t8S)-?d_#Fey}Tdb#wrv;2~oGJ-CFmQwrlEWB=_?$yCZ zULoPiF%9{{63&HBO7N%ALq`IzXi9pW$?1-Fl{QRQ@+*@j6ai?Y3s>fH=}@~WP=+Gbzuu2GiHVqclDj=V(>A*oPsxk``) z6KhR4W$a!%=^jCkOD7u~YOd?GpHw}7O`C5_63egeKea3DE6tSKZ<#B(o*`A1swez? z{C(MNSufq64^=bTY;v4myCj}*hTpuw6P7${O%3eHt$w_XS!KqPZUcqnOdV8JXgc0T ze~7i6L*N5;+-J;_!sgo2(pmxuQ2~=D5r@Oqj~V#~HJ?69Do0V1Xdej+O`hj$)il#; zfK$a^vFQ)UhYC_s$89H<`L}&2zOWOlh2Ie%m`G)`PVFQ$6Zh$(G!o)G-z<+kd1YIU zEtDZueTyPx?|Rdjf+jeG=EusAR#wH>*fREO zFuuMF?1LBoW=GNU-^Ylb5Z|nzdh%_V*KVZyxkQDugA$%G7aY%OH_@%p)O(F8TY*Eh zjQSnyk1=d-kS#A)HsCd+Or;(FP`*^Ziv9;fL5}_^XHLY6u%F9rS|T}0=0uET*lVyi zh_Hqg5ADknV*{%K_m{Nz<3?_U=@Q8IQD zI0f@Nhw$CSNK};COEOb&Zz>l1-S!7NIc7@g#((!M2 zptxlBSb zs+XrHG&pjmi}6mySEG}IMf#=2e4prCBe)JlsN)n$zT<8pKC8B?jPEuU_6+h5yR=Q1 zS<+Jw2`1va)+T%Cr!B5g88L9Zl@$EArr2bgu>8^BlLUJXHA3?r?(@f!{!!Lar|Z)e z?;2sYwDgPXZ$Eor4HP7|v$#6xsFa<B9%1v{rMjYVd|c zfVo+?mAZ;o2Kfw)^LlWG5SEAWw}n~TKU_`r@35U5?Yjs8bP}19Iy}{H8fahqD@Hi6 z`gOIVyys}7W^rl)gWvACLOzJ;!yb62jsy<4MF12-=_X8;+Ya$POMJ;e8oQNj?{giI z4?M$Y>qafVUz;k0Jfe>%)ba_XV8oQsNM<0qcIjMUfR_fr$1jz# zWjTOtfp{j)sCVL1MwGk5TfGNM2Gr_5%Gbk)7+b$j6101=rAGHIwp$N(ge>tMYK?5( z+fDvsqe&G>CxJqAal<50I;+)k*xY)dyKeGdy+KL zL^ph#mVC-0T=qK@MFPy(@qt@dlb@%^jx*8My7F2f?^K8k8y*>U*{gX!XHUakC;m2d z1>z@k?Ag!%$VNDFHSSm03-s#_O`F@Iyd+!mHiGl)AKBn%EEn8i$K!mRouT#;;)rt#I^z>_h3rtZG%9b zJv-rzig`4^nxg|oUQrA_qqqhcog%18dj0LO#kqYtwh$5rOsbsY)1jwGkPPZamQt_@JPHr$s2}*-lV=W2C%i zrN28s01_d_bw?}5LROjEui&QzDO?_(p0antd+o>CbFY_fMPRwen)F2M$>>}!X@52U>zzqbLAbPdB#~)5i1!L#X)ii?`q<&k%a!9wW^W8O<|(Q zyYx)mc#~|2l>o8xC-7cCL?b47u-*Z!!t5BcF-!AuECpV(ww&fn0b{%QJ|9QPfnLXi zD-mRZA!}(Yq5s>)^z&U{T2L%_0)xI-VYHYNxeqCRBuRbe$?;o7Q_thq6a#v7%`ENW zAk5eixFf6VGQ^N?&Rb0oi30dJmn~(gpJcrD%|~k5f)XxQQRZe3ST58Qz_Lh+i0#Z! z6ofhiyCzjI-QwAJiO2eqxV zk0h|rW|H3nL-Bo+wHixN()19$#KhH!fqU2No|IcD`2}VhLnUFCd4B|9Z`mHQ9QW9% z9!3a2Tq}N&=UC>58xC2BK2msMQb0Qrg`gvDaZa_&lzV7Y2H50yMfZ?bC-wEdaO>3#y5qY`uOj`QYV*OK(Lb5FzI^V6CRcMk`01VhR~f-qA;K4w0%;LZ?h zNuExTO1msb$ShYDQ%)Rc9MK)I5=7n-4)b!He=5}QI%e*pm|o9;?S{zD_;UAW;Mhk$ z>3OAdL$L!d(kiq%!0(s5su9L(TF%Z5tg7gt)Lw@WX z%#3)Sk-!z-Cx-~ZEqii%J*_ade=j0boNj!2LliyS@$OkM?aJ!B zt(?EOJgZ<@6eaxtRf4v5|AyC5-j=C>4L~>^kRw~~TvYJd?j`uMA8$zrx*>?}`OZ3r z(cC;Zn>NKZe>ShZ82*q$=t+eqKx!c-rQfEcC`Ug9fKUhY?)~j8AX!`MPnS~D+N6F% zc&7Za4C~-x9eJIw?C#!}*16ZYSm2g$eiv}YXz<8h%E^E`lx~uZP$#dZjW)XJ$6d$a z6+3c)k=JvY*BZFaUzxi!_d^hADXVU4X6lS5#s=a1O!$6vH4iWU)jYc3P>o!bfiAPU zB%t!zS?^GccXQ`GPaJD{0As5+Nr)yM=vobE3lpZ%VXMFZ1 z)pIEzSSc`akTK24hcROO)!W*lQmsno&l69O0{aUlU7n6?K1+~DR_9cSD%*sMAp18N zp`1cLjVt}>#ok7$K4%q-$lS|bl#^8gW~8;9Jz-)(V&jJ{#_xqqqqj8SZ7pBBD8I`D zw?5sx=R_RdJib`DS{c=1dfQh%&mZsg+3z3edn401)mXkJ?+}NRWUo(=X{@Cw9{Pj; zFV~~oY8%8h8jt#FeO<<&Y#xu?;@Iwcpjs+Kf^s-j(xSkp{YyjNQ z&~ulC&8!1WAT7s^IiBoLvodgbGSnCNh_F`TrME_+U&4fqq^=$844Nv`ma4Ej&rIg7 z9M3u{DB09JtnQG2u)GH%`jh9GZJtz7`t1`N-&Fo-Dvq8Yawm*3CG_8{bD-<->jT|d zxud41*k`Zoe9<2*o|H^ZZm;btoxPdr#Z1&p{%QK~2n5MpmB5Fmao zY7T2&Y^F3@VsY>$vW0h3ow%RVkmml$-`$;c>k-18=?5?Ye&+qoX?}(Jm-T146~QS@ zuhc}afx8fm?KdP4y@UntwQ)Q~Ij;xd6GqY-6_+U+*+0CzQ;Eo^Q7jzty~rEF%#}Jh zGd0hpn5@ey>_?FsN2@!@boL|+MJz{^#S|01p&noHYazpup&kkQ3+d|9lh0P8j4&q~ zzxtd5SHtd!slBS6D^WxTugMe!9UnI9*_+8(UwpIwVPR0Apu1f6X)KTpE$0;TPFzk} zI_AG(wcdi>9xpwAZ?J8F2 zl(Nw?#N^w7-q^jBc-nq5JjC&R=TyDBnv*lPZeW%JgRu1tDk9b6s^j^gHi^~>hWqQ| zBXXTAI|TUFc4p;P?AxC)`8;%{ji-OL@1!je)10nBt@@$R2*NdRiak+q-eOPG^cWS1 z-Qh0(xvyIow(wIl2}~;OeQL$ZAUo%fb*B-mfa3 z7_wK_#w2g|SGwsk&@Fv9!(lDLGLln2s;NEmizKOj@;tq_%A&8-t73lx)*(>NMqk0N4fm!lGBCFed;TCri!p`*ZXuuU+Lt z>Sr{<%|@l8s}4Gw6iSbnOP$2_cj5y5Ts&g2g{*(a^7p*D#Ll~A@gRfa@{-Co0-H5+ zHtmOvL|lx~IZLcreaUX80To5V`73ebQP?aO4BLNIm5)gtMojj$vuQ8i{l~MQ6z=r; zDxN$;KwgTuWN!kn`6#`RnL;bpoZ@<6w!AaF3Imc2kcR*H=OW#77aO&n*y;WXLRJ2* z`sTqPkLtniC^T>|h&kciB@S`xm9~S-oHRZwc#xl8w-PF{XU#4Zk;6Dc6npBV%>MUA z?)~UqotMN2n1{PT_#E2v^XR{QNBN6uzcgn8kS>30finYX=OSWM3=HUYjonNyn7A<- zFtwxJ{*I<+|9LAN2F8mGymd)@U+;?j>&+s3F z(An}#TRbxVmB9RaIqRN^bYSeUF@!4q=BzMU)CN^WgZ7S3>&B*nRPMj3WdZpSknYMZ zNF+;hA%g4(N_Lb_+LC`5($(il_~+E~DgVS~H4%A-Pb3*k0x8jW2i3Qd8ODkKBE=K< zcJ|6JSiz@56RZwtg+Hx)VdLr+$q+Oq_F_V|%WLn;Ks`PrAX$6##+`$@=9mWM_IG$7 z#kg%LWjW730HS{gB9t`;#3k3=Lbc=!gr#|uw=Lx?r=niiK-w1%^mI|pZyOeEWcd7I z*OQBG6~(*+O!%ZKepA6al}fthD{|Fy`4gMp!T{&#f~?V^*9pfkMJ719IHy_S!C?#nRhAyWR;hmIhOocSr_?ty$Q3_m})1lLtr?` z>L=HC65NG;8s@`49|%X<3Nw}Cr7QZs1IzLL&}e^0h@TQlzV#t*vN+zTpxE3<;+kae zlh}(duQTf=gWQ~wZdaOZn@cGZt<8GjL%wkEbX<&l0N3{Jkz(kv{a@oF@5Vnbe0-@I zpYn7)C&%$g==`fAG2gW)zf)i*;nk{;NX95j>3+C61g{^D_u$8H3z~o9&QYrYD5BW^pRqtbHYq*Z;^9| z=geBnanHoXQf`o1;2QyitJvxqnTA;`f=(G4w_PmIvya=|o7{F9ahad_d?f7?#PGMD z7i2aQZ^gv?Y%+sDR#v+|hb{f1(D>flE!%YPmCLJ+hdD+25^GS*Ja1YYE2&#FF7EV{ z9eYsz=1OVZ`0u3mw_(752ixOnKQ0Dcy?D)uI!DQD{P#C}q&^S2%w@Al=R7WB*Z;op zR0x9jnUMZ{Y2N-UoO@x4FMr9~OK^Dnnu?$6nlScgUZ`kNY%lABT)E8H{%?Fpp7taB znI#yg#rTrLl3n?0fLot;){-5V3dad;n`Yh04iFU{Xn%8n=_Rc7T!X0PsPQhC#vyES z=SrfQ7Jr_pUfdTe-!CDv{Cf7-XDVY_x(P{)ykcla8?Al!MW)rmv5Q#0ezP zrYPM;*R`6q`TB$&N?haGJ%Q@2P8RZTLLfW&GBbCas+!;ZRBx|p|9vaQ&=+kLY}R5C zFE~#(GTFM78S2@cX31ra2cfjReL7_N4q+o+IaA2qQuD1wU0MG5pfC5q53Uqz2!v&d z3MW_(YGGbWsC!>rNEwi?W_)?su+z3vlG6Ca zgu)8>Mv2nL`wz-0i6CF{+rqdxcjC(3^VCX*^5ZKNA2sqi3&O^3j#}Mv8Vk*FRNDNl zC)hWkG|0f;3G5|(e1D+}t?epWx7IuLrpGF%Ap_8@*mQE&Cf!qetF8$VbTs!6U5ESr z8%MJzzQ20Ry#yrmK&Q!7O&HW75ei21>*@`60{f?s+q~FUn1GFKvc51a|NMD-_+I#- zCdOmzVZUwzJ|a!BL}U+D2A~u*XBTy{&`guO?&WpNVBNSsmx69OjP)x0?6UTE@1*d~ zI#B%*W)TWVB)-BEnm-=RH)M%#S*W(Vf1dqADAZz~m!M#wE%WBRL}ksRHa~~#jF9Y8 zjl|5XPe&D5ciE8zcn=0Vw3yx}ejdesm|qr30p9U=10{jOX^^+w6 zIqHUiY=N$r+l9kT_=tutiI`bkjdu4>^_5wlH}&*i`K-mpN;WMoNoM0b8f(bkc5o?oHVs##3&U3>`!RAlto+isQ7Wp21^yV&_XuisyY z+zVgJaT5>Rb@ay3GqS(!na5${KrQRT3WZvf3;bPMvOwLxbVyTh<;#!9) zm~TO2aN^IH@cJJHn^ma>o1^%+D!ryZv9X$x;zOuTjgzE5mws4cEc;sIAvHBd8wqP* zn&vjTk$uk`WCszqvAcub&DvKk6GLkHDn=kRCNUE@&F^dSCn}egCa{`e#-QnFFgcO3 zagZLuaWQ*3q6Y?$fu~ZZX9nwueG>AJ2S~ZL+{?1MLuap=U2%8XOku_uXfbW|n|qK2 zkNCiEop~Qr{JF1D3}}t87LzAsCr)RQ^QG77;I)f|k*Y@7+jbo(flX^a4yK#x zwbW%xEgq59^*I)Sx9Q=U2ClU5M7=fdfY zECn6F2-mowKIWQ;Fm>*}{xDIgn6<$a6;BPZ+Myn}HdJ-kYkFIbKT8J3+nJT@{Mxj2<7 z165CWm-qeORtaYL)RyHbD+LxlWxhdOHa(E3sJBM_@sbE4)6%RoYYm)dkilExU9nx8 z&23C!@wf904La##a;lKsWqotGSF6IBe5>&o7&LAqh@%Yo$fBp;An3Yhh3#<1+`5XT z+TbE#t++DDeq%09s8&>1L{*yQR{GU|xXk8Fq=yYPBXE(W_F6_BX^T2&4i;^fZ%EMcY% zW=7X++q{S}wFnS2=+=sdx!g+R^tqvQVITp?CTFWA9sg^zXU5?mF?I44)M6zx`L;s9 z#a+6W|C&sQA?=QfD9`?x83uC{HOOya{g*{~{F~<_`AZzfk1S!>MOovmXvTS@_XYsA7s^`N2j8SYa)0+8HBS>XQ)^_{<>H9s+^I zH$o3Vky(1g5Hk<-PN=R|b?(MI3^_SroNm}P_N3<>(DuRc1?l~{A~f(OjzCK8MSx{E z?^2=nza8BUl(Q8|>xPJ=%ZEl{kew`3qo*N)6#F?j#bS9L5SD%3t)$d*yb)d)EP^dD zn+I^Tkps)hceybrc8~^vc%0jHfa8S#lm5;#2pm^ZSVYBN?=M`@M((#&ru4a_`Fl8n z+m(o+EnIdq1R~&6YG-W3oj8+?K+y3xUrw}losAdXsKbrjure>tC_?a5D!F`XaK<*0 zKu~k4+Wecsc&DHwuicm1kKMid(5W`aLdpf){4xpppXcZZ5C5BE%{e84M7AKPCujt` zXDqHA?zu~+?OFJnFG5>aZfLdn+J-I*S9_=K=PDR_jZH>>^~Hw_9xkSs&XusfEq7-T z7Ev%DsJeOMBu=}sY$l%_je6AazBbs`Id%W6`?ow=kPyPLd-45`w}y4(@S?N4&G$E# zbKuz645hrv4(obD?C;qJvpIn!fTE9zy6)j=ti^mb!Ic4+lh_ zW`lh4H-<`3Rj`DEUX>|Z6G)p6Xzs5oZVnf8G)q#}>fs7MtbGH*4HZ7(*vTQo6*ltB zo>+%WaDi#nNyr?~flSqX`cQhsLXvruU1_?!O?bNewg`wb%c=kdlChgT_hEcR%qqy@ zdX(NJk!S0kn!}#3zq(6$zPv@zr`L<=nP;}Qbb}y7e$KFrmsA_Sd!ua21y@dxaMH2* z!GJ+V4|%Vh3;7?MuRu#GcO^%kc5ia6gs!=Dx7uVS7Vc&UkToe)`Z znHb|Svif(?K6^WMBe94YZ73lstra*Hxcl0CdWK;f_YJ|nem-rmF}a*UA6R3PvwrSb zmPZ8cOwexK1$r*O&MpA>x?4tN%rx@+`}&I|cKzFZVMwH8rhSIw(vD=QJxS1#Epur` zZEmT_p5te-iD$?x8E@=lbXyI@&?4qUwQ0T4$h4tw`F(ujr9Ah*T?$B^rHtUYzg8pe zwM!G1R>eo|Czr2g1pZH#mACt7>lb^1hK(%k2er_IODJoMlzJMSyiarSkHt8Vw znc~mjUvG<0l>OV^?EWn$p>OY>ZS^F_JU12-5uC_?k=MQV@Q9vf2JdUL^u)w!&rqf9 z@mm=UVgbtYeTOjq!TY1bpR{iXPUX&{t6Noj0_>+@)D@7g)#x9}7JHq3fi;ET7H$Dz zDg17y5Ne+n;HH+HuO)vSVZlA-F=loarqy`5;>U) z<8s#hbMzKF1Ex+Dc6H6PGURvyac2vw_!NVB_D8nBdp$jpT7RQ%Chs+Hr#Oxp+|riG z1#9k2LLDZGXXIz?F2viYM}Il_*S?;at2`5wmQ}t({(VIHork2X(v$Piz}++yUVz|r zkaBbA@ILoq@nhf~?U}o7OT^8q1H4m$c0}f4Bc)57s?iYZVFD z9=uV%-WrzKigCMmA_{p(o5n zH5zBuu+|jc zCsewAe$wChsG4J8f z``bnmgkHbeZrNGwL8V!?K3jd1c+9Bi@v=Ug$En2b3TZML3(QNdJEL8vG?#py1*>t; zj}xn$U@|uj#^3$iO4_s|Uv*)38ktwLw-D$ndE>Aa=Xd_^_@K1xXvTsx4_wt!k`L%j zRPA01^dG!K>XrXIeSL3LL*r=MTS4C8+%L!|8F#roNZ_-`VwF@9A>2S^IL+QcSFf-8w-so+ zpLL^zi&x>r(UK&o_sSD}@)A4J)tx2zgMYi(0BZk)I0ibvD|q+&hd*V#Si0@6YmtcT zWgoAcpDy*~FX$WEC{yr06th)63^I z83(tZBym?peUo8*YqbSm6dye_<~68Iif9QqR?{8W(4=y=Gx0?`TRkrEGK#Pic`^ux_6tRfNB%@B)AN&S_=VU`8Y34qyAF_X*>$4`52Y`_FXQC&v<)@QO+L3-4(2dt#9eOeVD0_sVO_Fc zWHfq-%k^nev_a;lkl9K=VW7IzPftNHFs;=%&00AO;ZauRIL;sG}4vxn7AoPJY~qk??&+! zlvXdk2U&r_)B8;)O2~spb~eQyGeM+`>5sUJ>ER3iDS1@A%T1n25D7IuchwBMm*BZO zz+-BK87^4?7g=ccDY(rY_ezduiPR}UXPr#whN4bTYa&An-bAgbNbGo&06clce;o$v z9|k^#MCuAXgx$H{(D1~AdeJk6rZd2t$eO8M`{Zv#B1T%M1B}!rR2l8PA;f8pC&LBG zZ}u*3@FGR4omAwzLL?ke=(HRw$-+&Mfbm$10O5y?*`Y+(VJsQzn)jyro{^|bCs5IewwCN zP1WBluh#g}x0RPqWP3W9zzSl)?1$|tUteQKUG(@y_ubG8{Ay2OP;8mmoT-BkBP0*h zxl->0-`Sr$itl}rmwL)i#d^lMwkezQ)8ZQ9?h?g40y19te@Q3DZoW-^?n;Hn_vr1wcf++#esJv<m zsB>|zOJ8myQ1ahQ`$-!hZ=bF4e^yzsiH4-*a>$$96Jf|7K{sF-N1T>Nvr8SWgJv$? z0VArEPoZJXy?T1XZG-IPG^EvPFpZikUE4Ue0ZBxTYgDLV9>sXG;EO|&Y^ADHbJA$n zFqrG_4M6`g2RwJoS<&u&(0y4e%TzU#O4-kuXRA{Q7GR|OU#V+kV6Yv5yCJAP;qbnX z0&kV^v`gIc((4AzwN0>6^9&kEB475EOwCtm3D# znGRq>Ap;uiA$&y&tvWLf=U{)Heaq+{yUep?cxrDZ`OsJDH{#mdGxsBe0ITLB$#XYy&yulopf1}-(V z*?M^9#er4b?KkJ0?sjX3fmgcljYPlr4E*l^rK|r7K#igj^Jzh=2cNn&V{JGFjU=yu z81n%O2b?nx=Np`f3(7&y!*i0CK!G5HgHp7Yb=kqJ_mJx1qQm<*loNaQqcbv@uqM6D zxgKbRU0MVY87OoSVmo_X93&llZ`&cVVdatclm$LQ22|2Jy%d@&v}$eo@IEZkPO7sk zn23W5)be3|=NPY^q)Oe@=~E$eECFsTAq2c`O*Es>++wH{9F(o{RuHz?2Dp7S&*DL`tbY<|DaY?Gt1Za3c7gHbpQAVufyI_(z6uHGK^|n znit)W61TdxY+^JA_qh)F6`A^va*x%zaD=jHe+P4dwd-y=``m&{ecNy}af1DIPHAN` zcuM!zrtA;pUi!bFgPBQbyFjDFpx_foqxmum&5qq1DLq3s=le=HUO36(>-Ye}3T-yC z!W8&-oFP@AX$<=!Q~|AccHo#HY_29m_FV zO(Ir&ZeZw%(B5{LeeOdI`9%(9bNKy{SGURMi{g~AgAbo8YW<~(4B|UCoU@cjbrifS zy534OL;Lb8)fUXe(&dT`RAx&Yyt79aQa^z4=1z@y?EEMMyVS1yQDCZDPD4gwkm8Dm z&%Uj{yQbS>8wDQ!@aH}BSW90L6eZ=@g#k6H_Faj(JROo#_)=qjuQ>Mc!-Sqz6?ii? z7LQ0(@NqQv0uAHUt3qOSy%Tb=r(;|{M35l7`MH|^L-I-$^?>G9t0c5uJ}qfI{mLXs zwEFC8zJ?+eWYZwOfGwlHF2B5y?S*0TndAy=@y)>mzmn0sD;U^*0+r!9ozMUG?gx0) z;+KlHXu+BXbbV9%*^(6f^)c2uGZ+St32W2=#oZ6htTXv)E|QvHExW=-$93 zx8_aZ>-4@3=WII{;c*LsZ|GLxoWj_hiI=)9|KTC1j6{9!iUueJ)D>8l*-|_5^$-&Q z&$Z{|%-=Fza+k&WGl8<>2P}ZaIH8}za*qGnHRNBsWe|)ca;dmF;t)4*M}j!Vy>_T( z`}eB(!hP;>wkU_TaFMNgd8UK4{06d$uzXcyeNuy6&%EVPKOz zk8^h=0Z7~rhW|fZTzNc{T^oMvgkfZFtXa|wqELv+zHc?gK3NK7%f7E!vQ$FZGeW|U z*JQ~uW63Mo*T|OjwPX)v`HtT2pYNN0&;8u@bzSG&_c_n=JNMjRA){YPIdB+~U!D~H z)pyxp?6#gpr&e>aYo^UT@Hl^*){h%6>+9;Y%klXB=^U}T-7jSFg_lfFXWYt#$5#ds z_9kvn_w@V%-7E2&?qctc=VV;EwF?gFe|7z%ChzNIAROStFg~*|GTb^;OXKO@O}uDH z2b~@K9a%Tiw#)tKcYB@7vB;V$?WmRUH>bQnD&kGWNxPy+TVHsrxa&&`u|}bfofTb0 z<10JU8VXfj7G>%S1;yYvF(KT>&~nM1Q~jfIr|!>xQf1iSc-p@1MZIS|ElozRp2FC% z&zWR@3>EK%zO!5Zt?E`*R@h?u{sr{{m}3Y*1RZcTtmSjAk9h(!y@zH?gthJ`Eama> zju_-`_s`u$*B3uZzP_CPec%UmTHg&(&}$d(S8tQ~D1JhX|3eiFE;4QveZFjJ;;pE1 zS??74!-2+y*Wiegnw>>#jN$RB5xAwb6>vSk_vi;LI?8cz`xiJ&34{K)hsIYq>faETKi_p)PtK8;EO-#C;F%!su>#O`Bze{Z=A@9e_W0=IxqP2 zZ1P0==rT)ct27s&Fnh!e+UP79;67Ht@p*Ec@4Qc%*8&{wqzZ{4uifO*;;rS>Q2frE zCp7dSs>*(RlJ-uNBe;TI+N}K}Or>mm;;l_n-3;+W*#(1_;GVKLyFHNlqM2&Ou$#Nj z;sjIE0geH!*Fd+D@Z^Pwd^zns&E$TThB|v=zHBK$)0TjOi(=t--^;cH6rL)|ToLAY zdx~X{g1&ks{PDQ3R%Bq|(pLVq*S9d#ZkJ}l3NdxT6M2o4Ur3yZVId=GQ=<5(OHV;G zY#q4Un&V817(4suA}eL&CGP}ZevyLE$Xkh!3f^7ncRAeyli(F$oTmZm=TkGP>$iw| z#zC&x(pOBUI8(}uf(HhbFKdZ4jxt!?hpnp-$+4%+OQg?#Ck>3U^wT4KRvxqEO17q_ zOb^ph{B;G~ez1{<*&s_uyqSs;?c(**{Fsm8PJ9kGERKwOZ$S*Ifzlk$_ws zlNkPN`RL}F%Nb{bNE^0Ya9MS{REBLgD+YQ9`Zd@OPJaHlu37o6kz#-Eu=`p=CqBNO zFF8;R4!=d$D6A=<&ibG((3JK9n{cZkHO4T={uhCXJxvW;c%S`dA6CebJpA=*0qoyi zdz2({MzTm_MInABvb(N55+BF^+Fm;J2(-Dh`eXImcoE-}0o$EH>480+J0oA+rj+&$ z_p2>L-@0;Y9Bjf<_;MS=TwK-<=vJGEwo_Des|w2D*B8xJWqPnr{i)q@9=yjUMYBKd zD3%xDK1nceJ=lxu5KAx^l9DoOA8j36IBC2)E0n`iHnXXWC!Hvrute<6>g_8R{UBLj zSee*Sf=^nJE3)5Hz0ZYn47Y9lBE(50xMFJsaX)+06)fXoV`CAvBth_d!b#5#hr^9p z8RYhicJ^~{SKTj5!wF22|2yiZOVJiNHD@?@YO#@d{|VhX7CZ)5OQ4De&Mw$yscH7j z20r=TyvGhF(EZ=j`>XFDxP-34=;7@c+1@ICp_~|-8 zAM#Sg=+qTa9K(9{|2#1)*UM@5y&GAnpVfa}k`Ke1vdV<0XnM@>pa2V_08Sx`dP37ADD*$VtS1=H0opYi4Qi?`5x)OL z8leU}vr4g}o@5z22fb4w412=_aXiH_4~A-15QF}hiVG~-VnSZutb@`Wqo<>*?=dDETM8kc>#3@i&)kz7j>6wup&b?E8xB$QqvM&}&*arzaM68?4=meX%ND%G5`vB68y~go zM)eiA|vF@%8tsAGLm*692KB{HNP%j>`p(#+i)t;&U`1( z5;JY-Guv9B^y(y!jWln>Ib(UG*OTA(ol%-hwrL zeP~RDqddUj8)sbmTJpgCa`_rZbPwf*F9f4`o#F4|9E`$LrL&#oba7=Be*7F6KtAFR zfSN&#)YF&TH!ug|X64(G{v{ z&_FhLy(wRG9q})m<+%8p-?Cnxq9SJY>YS#FZ30zECzsS6ZijkZ2MjL{ ziG1?YRpDR}et{kjB>dg~TJjoj*1bi~mbd{stc50*ufnboBZkxxmUh+0a~*|ipEu?d zSuUnCHW|LB-BZe9BzhuZGQi!wzzYz1{C?Ha1Cb&TNgIU<$7b)suN(BKk?ki18<&230JtX#Xt1rbnKZktueo9_$se#57N z#WME-zuiibysb}W2)VQRDe7t1k(Le7<1Z7IR@92=fr0#pe@2_+Z6ykRn(2w@Py0#D z5ku)I#=B7;nufK@-~AMIdz4*dc6iG{saZw02dc;&zrsqxUJAphxmDSIHuq9mTJa}( z^S(R`av%$1gOr%Cd;kggDU#Zb20M(4wXAIj|Fijh@ikACi7qJ~2eEtm&dF-bMtz3nz39^^|EJ{tVa6y`DDJ>ZsfA-En z&n+V3;oYvmG=3b7W$&>BvGey5hZfR4MelN0EjrPJG~$?HS;s74Dv0(xE?bhbL1Ix_ z=>1m9+K`|YIXUZVN6#F_(sIjJpKe6Xh(SKSh!`I-dh|KC@C!PZ(Bf1}&K=Rv1$hI7 z?d?YH6Ouz#RUZV|1| zHwU;)1&F7aSza|*@DT8zmlHiLSJN(^Rx9&*akC zW7-H!uRxPj64u=n*xHeO=KWZ7UAeh#@RT?2A6?0?!eIkstMga&N9;(CRFUZwW5<^9 z)+{@s>mJ!LK3{rFf2!B`1yd7ql4iYl*ipWvvf*&geB;VDW4Va`S|q8}SO-%gM8nqBN{KaY?R&-B|e*T++>sR&L?MP3wW}r-Q5+RLy0!F!q*>ocTr%7F zqmEtt_Oo*#_@Il;+s;Yka-TFJsF3Ek+Y`>epP>Wp5MI@N_2zxl(v!MKhJj7%x-WUX z!BhzLa#mXQ<2-MMZrvR5f9t(pyG0%)dMT8b*yuV?CC6Ush;cb~Y_%+ZEm6vDQeM}s zQU*LcRll0EXgA&akC`25OzlsVo;BaO0YZH8=5-DZ!5=C`#gAx^@&+z{iXg0TtE6?B zYe`%vC*An~_oMx2H?S_9O;$-0LjxLOQJH$mkkZh$ydup?qD+o?qfNZEk$B+`uk0B4%v_`>WKR|Z z2Wux9Q(@f_i#KywXBPhY`c=>uZ{B645WFJI4Kkt`5ULA|OXjU@o9_^`i|-5x#-L2L z4{Em*;JgOgV(v>jvY3hr|Hfgrgr&|1+vOPiHnYq3#n0=vYW%XJB>3dJBG~#tB4_40 z38%k)6)@<{i_-&(#S@Z-j## z))VzKC7@Qg5tcejD2i9C76rz5l`SSIh-%O=ic~@Ll8rl4upu2I43{B zMpd#|vvtDNXDTKfdGonSTJdGSuUj4gY{<5IWZ?4Y(V&F52+E&hebEE_vvNpU3`31cKp_jB_Bb_Bj#539G{UQrmEGd+ z86-7Cr7llrXWI3sKphTTWQ@MACCOP4f2eGp9uJ6)0~XG*k{0ruGv8Bh(J*G?^%A@- zNz%9xuJ_7p1(ieLI|BfOOy)~*amor-Vqovx_5m+lQsr2qW3xaf2UQzu4C*Y|MrDBs zGw*>r>sZj{a!Q1-ww{>_-=MaQF^}fS-(V)t(+)b@5K85$S3qdB6RvNVszl2EK$);) zX{dWNc4{yW4B$D$AnGx%C9jVhsFF1u^&RDlZ`l9>uxG}ar@j<#I5!g&|1>`_M!(vM zWUM_@Hrk%BWb4g;WCaYy5`h8DjA^Xs>%?gNYCBTDwp-O$imDb~{#i&b2;np9pkqV1 zaR*vg&Ezo{kfrL3+!d=5oP^Dr5Hn9(uXY)U4>L zPPBHWD22N>4b$}9IyY;vDdcM;?L?@xK{f<=T?&4=qWH%G)!e5TNF!K{ zwSlelQ9gli2?fO*0@@^ZBOg_@aY$X0(1N3(=B;%aQ{nUU#qUn}Bv@ ze&j#xSyTeJwJqs_0kSy=K^GIKlg$by42j-hQtPP1@(*$IX~j7f`K9-BK>=ila1jY> zi8H6G)~#x(Q-fwe*>Slo&Jhhj`bGs3lxVQ)_iUP+sC1BAZVYDKrZmQTGP4<~tbCFt zV`pa&tzd6s+uO}Bt0;X1&+k%3zAXS4oJ}OORo`z)7ch}w9)m({l2jB&IGqt?Csi9^ z?J^j(F*VSU?)$`;Ht8Vu% z!<t|Z`CEx zsXEaNl)P0_?FA2$Q#<Bqrg1A|ydsY=Jr!N3OHYs%%I5kIb zv}&r^ZJi9+3|P8*QeFWcd@Gun!WV9q?X=}b(z{K}a|q&+<{hFJ1X1RpjMHIX>xU)S zP~rilQufy@cr=w2Ga=Q%>-ArHgP=~bL$HRl~+ux^--<;{)sU-o8<3FZuWRxw%@W3z@fH z`_a%4Jzs{EvEZ@~82m({<>;0EVO*Z^FI zSzN5_c0i2&>47p}C1Plw*)R8knQUFa%(Koc)xnASNr6cBy0))SyExmM|8+9;Yj@0t*%-nQ|`OO3`%a`F`wMm XN1rBB-R;5uNI_a^`l|R_*oglC?9p2= From c9634ff7c17fb69dcc6de5ed0c1673977668a99b Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 16:02:56 +0100 Subject: [PATCH 002/248] Dependencies cleanup --- package-lock.json | 1135 ++++++++++----------------------------------- package.json | 33 +- 2 files changed, 253 insertions(+), 915 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1ae9ad25f..8b53128b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,21 @@ { "name": "phantomas", - "version": "1.20.1", + "version": "2.0.0-beta", "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/node": { + "version": "10.12.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", + "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==", + "dev": true + }, + "@types/semver": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", + "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==", + "dev": true + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -11,129 +23,29 @@ "dev": true }, "agent-base": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.0.tgz", - "integrity": "sha512-c+R/U5X+2zz2+UCrCFv6odQzJdoqI+YecuhnAJLa1zYaMc13zPfwMwZrr91Pd1DYNo/yPRbiM4WVf9whgwFsIg==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", + "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", "requires": { "es6-promisify": "^5.0.0" } }, - "ajv": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", - "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", - "requires": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" - } - }, - "amdefine": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=" - }, - "analyze-css": { - "version": "0.12.6", - "resolved": "https://registry.npmjs.org/analyze-css/-/analyze-css-0.12.6.tgz", - "integrity": "sha512-fGXG/VxD62+2BGDxyzoKaPepw8YT7yvqbpIpwm5N2Ko5Y9TIAxxeGYKNzlmNjsQG5QoNvJY/xCwicFVnGsfVMw==", - "requires": { - "cli": "^1.0.1", - "css": "^2.2.3", - "css-shorthand-properties": "^1.1.0", - "debug": "^3.1.0", - "fast-stats": "0.0.x", - "glob": "^7.1.2", - "http-proxy-agent": "^2.1.0", - "node-fetch": "^2.1.2", - "onecolor": "^3.0.4", - "optimist": "0.6.x", - "slick": "~1.12.1", - "specificity": "^0.3.2" - } - }, - "ansicolors": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.3.2.tgz", - "integrity": "sha1-ZlWX3oap/+Oqm/vmyuXG6kJrSXk=" - }, - "ansistyles": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/ansistyles/-/ansistyles-0.1.3.tgz", - "integrity": "sha1-XeYEFb2gcbs3EnhUyGT0GyMlRTk=" - }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "ascii-table": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/ascii-table/-/ascii-table-0.0.9.tgz", - "integrity": "sha1-BqZgTWpV1L9BqaR9mHLXp42jHnM=" - }, - "asn1": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", - "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, "async": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", - "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", - "requires": { - "lodash": "^4.17.10" - } - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "atob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.1.tgz", - "integrity": "sha1-ri1acpR38onWDdf5amMUoi3Wwio=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + "version": "1.5.2", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", + "dev": true }, - "aws4": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz", - "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, - "bcrypt-pbkdf": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", - "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", - "optional": true, - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "bluebird": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", - "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==", - "dev": true - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -144,47 +56,30 @@ } }, "buffer-from": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.0.tgz", - "integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ==" - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "cli": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cli/-/cli-1.0.1.tgz", "integrity": "sha1-IoF1NPJL+klQw01TLUjsvGIbjBQ=", + "dev": true, "requires": { "exit": "0.1.2", "glob": "^7.1.1" } }, - "co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" - }, "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, - "combined-stream": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", - "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", - "requires": { - "delayed-stream": "~1.0.0" - } - }, "commander": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", - "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", + "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==", "dev": true }, "concat-map": { @@ -194,7 +89,7 @@ }, "concat-stream": { "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "resolved": "http://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "requires": { "buffer-from": "^1.0.0", @@ -204,9 +99,9 @@ } }, "config-chain": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.11.tgz", - "integrity": "sha1-q6CXR9++TD5w52am5BWG4YWfxvI=", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz", + "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==", "dev": true, "requires": { "ini": "^1.3.4", @@ -229,46 +124,10 @@ }, "corser": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/corser/-/corser-2.0.1.tgz", "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=", "dev": true }, - "css": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/css/-/css-2.2.3.tgz", - "integrity": "sha512-0W171WccAjQGGTKLhw4m2nnl0zPHUlTO/I8td4XzJgIB8Hg3ZZx71qT4G4eX8OVsSiaAKiUMy73E3nsbPlg2DQ==", - "requires": { - "inherits": "^2.0.1", - "source-map": "^0.1.38", - "source-map-resolve": "^0.5.1", - "urix": "^0.1.0" - } - }, - "css-shorthand-properties": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/css-shorthand-properties/-/css-shorthand-properties-1.1.0.tgz", - "integrity": "sha512-0Pg/e2U0NgT6Chw1TUe3D2ZTOOO3JFzlDV+bN3hxR8T0RDjnwzmx5WhuYUA0GHYWlPYAYi2/Ac0e8oqgerHDQw==" - }, - "csv-string": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/csv-string/-/csv-string-3.1.3.tgz", - "integrity": "sha512-Tn2bzQQ8xEWPALtKz4ysdqr72hk/m14VuHG54FYm/jF+VUuL7xkNXWQdTYSlarB9P9ecnNy4rgj4dfgJINs/ZQ==" - }, - "cycle": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", - "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=", - "dev": true, - "optional": true - }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, "date-now": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", @@ -276,23 +135,13 @@ "dev": true }, "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "requires": { - "ms": "2.0.0" + "ms": "^2.1.1" } }, - "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, "diff": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.8.tgz", @@ -311,22 +160,22 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, "entities": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", - "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", "dev": true } } }, "domelementtype": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", - "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", "dev": true }, "domhandler": { @@ -348,72 +197,65 @@ "domelementtype": "1" } }, - "ecc-jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", - "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", - "optional": true, - "requires": { - "jsbn": "~0.1.0" - } - }, "ecstatic": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-3.2.0.tgz", - "integrity": "sha512-Goilx/2cfU9vvfQjgtNgc2VmJAD8CasQ6rZDqCd2u4Hsyd/qFET6nBf60jiHodevR3nl3IGzNKtrzPXWP88utQ==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-3.3.0.tgz", + "integrity": "sha512-EblWYTd+wPIAMQ0U4oYJZ7QBypT9ZUIwpqli0bKDjeIIQnXDBK2dXtZ9yzRCOlkW1HkO8gn7/FxLK1yPIW17pw==", "dev": true, "requires": { "he": "^1.1.1", - "mime": "^1.4.1", + "mime": "^1.6.0", "minimist": "^1.1.0", - "url-join": "^2.0.2" + "url-join": "^2.0.5" }, "dependencies": { + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true + }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true } } }, "editorconfig": { - "version": "0.13.3", - "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.13.3.tgz", - "integrity": "sha512-WkjsUNVCu+ITKDj73QDvi0trvpdDWdkDyHybDGSXPfekLCqwmpD7CP7iPbvBgosNuLcI96XTDwNa75JyFl7tEQ==", + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.2.tgz", + "integrity": "sha512-GWjSI19PVJAM9IZRGOS+YKI8LN+/sjkSjNyvxL5ucqP9/IqtYNXBaQ/6c/hkPNYQHyOHra2KoXZI/JVpuqwmcQ==", "dev": true, "requires": { - "bluebird": "^3.0.5", - "commander": "^2.9.0", - "lru-cache": "^3.2.0", - "semver": "^5.1.0", + "@types/node": "^10.11.7", + "@types/semver": "^5.5.0", + "commander": "^2.19.0", + "lru-cache": "^4.1.3", + "semver": "^5.6.0", "sigmund": "^1.0.1" } }, "entities": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, "es6-promise": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz", - "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ==" + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", + "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==" }, "es6-promisify": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", "requires": { "es6-promise": "^4.0.3" } }, - "esprima": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", - "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==" - }, "eventemitter3": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", @@ -423,12 +265,8 @@ "exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" - }, - "extend": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", - "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=", + "dev": true }, "extract-zip": { "version": "1.6.7", @@ -448,35 +286,20 @@ "requires": { "ms": "2.0.0" } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" } } }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, "eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=", "dev": true }, - "fast-deep-equal": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", - "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "fast-stats": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/fast-stats/-/fast-stats-0.0.3.tgz", - "integrity": "sha1-ZQr5Y8P/hcSWo2EPINQM1MFkWU0=" - }, "fd-slicer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", @@ -486,37 +309,29 @@ } }, "follow-redirects": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.0.tgz", - "integrity": "sha512-fdrt472/9qQ6Kgjvb935ig6vJCuofpBUD14f9Vb+SLlm7xIe4Qva5gey8EKtv8lp7ahE1wilg3xL1znpVGtZIA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.0.tgz", + "integrity": "sha512-4Oh4eI3S9OueVV41AgJ1oLjpaJUhbJ7JDGOMhe0AFqoSejl5Q2nn3eGglAzRUKVKZE8jG5MNn66TjCJMAnpsWA==", "dev": true, "requires": { - "debug": "^3.1.0" - } - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", - "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "1.0.6", - "mime-types": "^2.1.12" - } - }, - "fs-extra": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", - "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", - "requires": { - "graceful-fs": "^4.1.2", - "jsonfile": "^2.1.0", - "klaw": "^1.0.0" + "debug": "=3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } } }, "fs.realpath": { @@ -524,18 +339,10 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, "glob": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -545,43 +352,15 @@ "path-is-absolute": "^1.0.0" } }, - "graceful-fs": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", - "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", - "requires": { - "ajv": "^5.1.0", - "har-schema": "^2.0.0" - } - }, - "hasha": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/hasha/-/hasha-2.2.0.tgz", - "integrity": "sha1-eNfL/B5tZjA/55g3NlmEUXsvbuE=", - "requires": { - "is-stream": "^1.0.1", - "pinkie-promise": "^2.0.0" - } - }, "he": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, "htmlparser2": { "version": "3.8.3", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -600,7 +379,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "dev": true, "requires": { @@ -612,7 +391,7 @@ }, "string_decoder": { "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", "dev": true } @@ -629,15 +408,6 @@ "requires-port": "^1.0.0" } }, - "http-proxy-agent": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", - "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", - "requires": { - "agent-base": "4", - "debug": "3.1.0" - } - }, "http-server": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.11.1.tgz", @@ -654,14 +424,23 @@ "union": "~0.4.3" } }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "https-proxy-agent": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", + "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" + "agent-base": "^4.1.0", + "debug": "^3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + } } }, "inflight": { @@ -684,181 +463,60 @@ "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true }, - "is-stream": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, "js-beautify": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.7.5.tgz", - "integrity": "sha512-9OhfAqGOrD7hoQBLJMTA+BKuKmoEtTJXzZ7WDF/9gvjtey1koVLuZqIY6c51aPDjbNdNtIXAkiWKVhziawE9Og==", + "version": "1.8.9", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.8.9.tgz", + "integrity": "sha512-MwPmLywK9RSX0SPsUJjN7i+RQY9w/yC17Lbrq9ViEefpLRgqAR2BgrMN2AbifkUuhDV8tRauLhLda/9+bE0YQA==", "dev": true, "requires": { - "config-chain": "~1.1.5", - "editorconfig": "^0.13.2", + "config-chain": "^1.1.12", + "editorconfig": "^0.15.2", + "glob": "^7.1.3", "mkdirp": "~0.5.0", - "nopt": "~3.0.1" + "nopt": "~4.0.1" } }, - "js-yaml": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz", - "integrity": "sha512-saJstZWv7oNeOyBh3+Dx1qWzhW0+e6/8eDzo7p5rDFqxntSztloLtuKu+Ejhtq82jsilwOIZYsCz+lIjthg1Hw==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true - }, "jshint": { - "version": "github:macbre/jshint#e22e19bd4af05a666cbd5ad4dc3a4eab55623df4", - "from": "github:macbre/jshint#master", + "version": "2.9.7", + "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.9.7.tgz", + "integrity": "sha512-Q8XN38hGsVQhdlM+4gd1Xl7OB1VieSuCJf+fEJjpo59JH99bVJhXRXAh26qQ15wfdd1VPMuDWNeSWoNl53T4YA==", "dev": true, "requires": { "cli": "~1.0.0", "console-browserify": "1.1.x", "exit": "0.1.x", "htmlparser2": "3.8.x", - "lodash": "^4.17.10", + "lodash": "~4.17.10", "minimatch": "~3.0.2", - "phantom": "~4.0.1", - "phantomjs-prebuilt": "~2.1.7", "shelljs": "0.3.x", "strip-json-comments": "1.0.x" - }, - "dependencies": { - "phantomjs-prebuilt": { - "version": "2.1.16", - "resolved": "https://registry.npmjs.org/phantomjs-prebuilt/-/phantomjs-prebuilt-2.1.16.tgz", - "integrity": "sha1-79ISpKOWbTZHaE6ouniFSb4q7+8=", - "dev": true, - "optional": true, - "requires": { - "es6-promise": "^4.0.3", - "extract-zip": "^1.6.5", - "fs-extra": "^1.0.0", - "hasha": "^2.2.0", - "kew": "^0.7.0", - "progress": "^1.1.8", - "request": "^2.81.0", - "request-progress": "^2.0.1", - "which": "^1.2.10" - } - }, - "progress": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", - "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=", - "dev": true, - "optional": true - } - } - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "jsonfile": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", - "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", - "requires": { - "graceful-fs": "^4.1.6" - } - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "kew": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", - "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=" - }, - "klaw": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", - "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", - "requires": { - "graceful-fs": "^4.1.9" } }, "lodash": { - "version": "4.17.10", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz", - "integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg==" + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "dev": true }, "lru-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-3.2.0.tgz", - "integrity": "sha1-cXibO39Tmb7IVl3aOKow0qCX7+4=", + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", "dev": true, "requires": { - "pseudomap": "^1.0.1" + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" } }, "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true - }, - "mime-db": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", - "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==" - }, - "mime-types": { - "version": "2.1.18", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", - "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", - "requires": { - "mime-db": "~1.33.0" - } + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz", + "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==" }, "minimatch": { "version": "3.0.4", @@ -869,23 +527,16 @@ } }, "minimist": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", - "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" + "version": "0.0.8", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - } } }, "mockery": { @@ -895,34 +546,20 @@ "dev": true }, "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "node-fetch": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", - "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" - }, - "node-statsd": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.1.1.tgz", - "integrity": "sha1-J6WTSHY9CvegN6wqAx/vPwUQE9M=" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, "nopt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "dev": true, "requires": { - "abbrev": "1" + "abbrev": "1", + "osenv": "^0.1.4" } }, - "oauth-sign": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", - "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" - }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -931,11 +568,6 @@ "wrappy": "1" } }, - "onecolor": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/onecolor/-/onecolor-3.0.5.tgz", - "integrity": "sha1-Nu/zIgE3nv3xGA+0ReUajiQl+fY=" - }, "opener": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/opener/-/opener-1.4.3.tgz", @@ -946,35 +578,37 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dev": true, "requires": { "minimist": "~0.0.1", "wordwrap": "~0.0.2" } }, - "optimist-config-file": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/optimist-config-file/-/optimist-config-file-1.0.1.tgz", - "integrity": "sha1-tUBUiKztcKjnhJ/V+Jx4aCckvpM=", + "os-homedir": { + "version": "1.0.2", + "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "dev": true + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dev": true, "requires": { - "ansistyles": "^0.1.3", - "debug": "^2.6.8", - "js-yaml": "^3.8.4", - "optimist": "^0.6.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - } + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" } }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "pend": { @@ -982,95 +616,10 @@ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "phantom": { - "version": "4.0.12", - "resolved": "https://registry.npmjs.org/phantom/-/phantom-4.0.12.tgz", - "integrity": "sha512-Tz82XhtPmwCk1FFPmecy7yRGZG2btpzY2KI9fcoPT7zT9det0CcMyfBFPp1S8DqzsnQnm8ZYEfdy528mwVtksA==", - "dev": true, - "optional": true, - "requires": { - "phantomjs-prebuilt": "^2.1.16", - "split": "^1.0.1", - "winston": "^2.4.0" - }, - "dependencies": { - "phantomjs-prebuilt": { - "version": "2.1.16", - "resolved": "https://registry.npmjs.org/phantomjs-prebuilt/-/phantomjs-prebuilt-2.1.16.tgz", - "integrity": "sha1-79ISpKOWbTZHaE6ouniFSb4q7+8=", - "dev": true, - "optional": true, - "requires": { - "es6-promise": "^4.0.3", - "extract-zip": "^1.6.5", - "fs-extra": "^1.0.0", - "hasha": "^2.2.0", - "kew": "^0.7.0", - "progress": "^1.1.8", - "request": "^2.81.0", - "request-progress": "^2.0.1", - "which": "^1.2.10" - } - }, - "progress": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", - "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=", - "dev": true, - "optional": true - } - } - }, - "phantomjs-prebuilt-macbre": { - "version": "2.5.0-dev2", - "resolved": "https://registry.npmjs.org/phantomjs-prebuilt-macbre/-/phantomjs-prebuilt-macbre-2.5.0-dev2.tgz", - "integrity": "sha512-QNNogRkEBdchBqgbvnY+0ZPFaeu0mfaKDaRGvXZWwj4OM8Mq/fLsZUZmlzTbiLrFkYfgzLwTyMbsooKqMrBTig==", - "requires": { - "es6-promise": "~4.0.3", - "extract-zip": "^1.6.7", - "fs-extra": "~1.0.0", - "hasha": "~2.2.0", - "kew": "~0.7.0", - "progress": "~1.1.8", - "request": "^2.87.0", - "request-progress": "~2.0.1", - "which": "~1.2.10" - }, - "dependencies": { - "es6-promise": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.0.5.tgz", - "integrity": "sha1-eILzCt3lskDM+n99eMVIMwlRrkI=" - }, - "progress": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", - "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" - } - } - }, - "pinkie": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", - "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" - }, - "pinkie-promise": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", - "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", - "requires": { - "pinkie": "^2.0.0" - } - }, "portfinder": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.13.tgz", - "integrity": "sha1-uzLs2HwnEErm7kS1o8y/Drsa7ek=", + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.20.tgz", + "integrity": "sha512-Yxe4mTyDzTd59PZJY4ojZR8F+E5e97iq2ZOHPz3HDgSvYC5siNad2tLooQ5y5QHyQhc3xVqvyk/eNA3wuoa7Sw==", "dev": true, "requires": { "async": "^1.5.2", @@ -1078,12 +627,6 @@ "mkdirp": "0.5.x" }, "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", - "dev": true - }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -1092,6 +635,12 @@ "requires": { "ms": "2.0.0" } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true } } }, @@ -1101,9 +650,9 @@ "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, "progress": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", - "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" }, "proto-list": { "version": "1.2.4", @@ -1111,16 +660,31 @@ "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", "dev": true }, + "proxy-from-env": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", + "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", "dev": true }, - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "puppeteer": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.11.0.tgz", + "integrity": "sha512-iG4iMOHixc2EpzqRV+pv7o3GgmU2dNYEMkvKwSaQO/vMZURakwSOn/EYJ6OIRFYOque1qorzIBvrytPIQB3YzQ==", + "requires": { + "debug": "^4.1.0", + "extract-zip": "^1.6.6", + "https-proxy-agent": "^2.2.1", + "mime": "^2.0.3", + "progress": "^2.0.1", + "proxy-from-env": "^1.0.0", + "rimraf": "^2.6.1", + "ws": "^6.1.0" + } }, "q": { "version": "1.5.1", @@ -1128,13 +692,14 @@ "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "2.3.3", + "resolved": "http://registry.npmjs.org/qs/-/qs-2.3.3.tgz", + "integrity": "sha1-6eha2+ddoLvkyOBHaghikPhjtAQ=", + "dev": true }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -1146,51 +711,19 @@ "util-deprecate": "~1.0.1" } }, - "request": { - "version": "2.87.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz", - "integrity": "sha512-fcogkm7Az5bsS6Sl0sibkbhcKsnyon/jV1kF3ajGmF0c8HrttdKTPRT9hieOaQHA5HEq6r8OyWOo/o781C1tNw==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.6.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.5", - "extend": "~3.0.1", - "forever-agent": "~0.6.1", - "form-data": "~2.3.1", - "har-validator": "~5.0.3", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.17", - "oauth-sign": "~0.8.2", - "performance-now": "^2.1.0", - "qs": "~6.5.1", - "safe-buffer": "^5.1.1", - "tough-cookie": "~2.3.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.1.0" - } - }, - "request-progress": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-2.0.1.tgz", - "integrity": "sha1-XTa7V5YcZzqlt4jbyBQf3yO0Tgg=", - "requires": { - "throttleit": "^1.0.0" - } - }, "requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", "dev": true }, - "resolve-url": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" + "rimraf": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", + "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", + "requires": { + "glob": "^7.0.5" + } }, "safe-buffer": { "version": "5.1.2", @@ -1198,14 +731,14 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "semver": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", - "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", "dev": true }, "shelljs": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -1215,87 +748,9 @@ "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=", "dev": true }, - "slick": { - "version": "1.12.2", - "resolved": "https://registry.npmjs.org/slick/-/slick-1.12.2.tgz", - "integrity": "sha1-vQSN23TefRymkV+qSldXCzVQwtc=" - }, - "slimerjs": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/slimerjs/-/slimerjs-0.10.3.tgz", - "integrity": "sha1-v9lesGNOgaI7I4M6Qi9ovjLP/sg=", - "optional": true - }, - "source-map": { - "version": "0.1.43", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", - "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=", - "requires": { - "amdefine": ">=0.0.4" - } - }, - "source-map-resolve": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", - "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", - "requires": { - "atob": "^2.1.1", - "decode-uri-component": "^0.2.0", - "resolve-url": "^0.2.1", - "source-map-url": "^0.4.0", - "urix": "^0.1.0" - } - }, - "source-map-url": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", - "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=" - }, - "specificity": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/specificity/-/specificity-0.3.2.tgz", - "integrity": "sha512-Nc/QN/A425Qog7j9aHmwOrlwX2e7pNI47ciwxwy4jOlvbbMHkNNJchit+FX+UjF3IAdiaaV5BKeWuDUnws6G1A==" - }, - "split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", - "dev": true, - "optional": true, - "requires": { - "through": "2" - } - }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, - "sshpk": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz", - "integrity": "sha1-Ew9Zde3a2WPx1W+SuaxsUfqfg+s=", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "tweetnacl": "~0.14.0" - } - }, - "stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "dev": true, - "optional": true - }, "string_decoder": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" @@ -1307,54 +762,6 @@ "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=", "dev": true }, - "tap-producer-macbre": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tap-producer-macbre/-/tap-producer-macbre-0.0.3.tgz", - "integrity": "sha1-M6FLw1xKvEtbtGw8eMEIE861XX4=", - "requires": { - "inherits": "^2.0.1", - "yamlish": "^0.0.7" - } - }, - "throttleit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz", - "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=" - }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true, - "optional": true - }, - "tough-cookie": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", - "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", - "requires": { - "punycode": "^1.4.1" - } - }, - "travis-fold": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/travis-fold/-/travis-fold-0.1.2.tgz", - "integrity": "sha1-/sAF+dyqJZo/lFnOWmkGq6TFRdo=" - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true - }, "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -1362,26 +769,13 @@ }, "union": { "version": "0.4.6", - "resolved": "https://registry.npmjs.org/union/-/union-0.4.6.tgz", + "resolved": "http://registry.npmjs.org/union/-/union-0.4.6.tgz", "integrity": "sha1-GY+9rrolTniLDvy2MLwR8kopWeA=", "dev": true, "requires": { "qs": "~2.3.3" - }, - "dependencies": { - "qs": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz", - "integrity": "sha1-6eha2+ddoLvkyOBHaghikPhjtAQ=", - "dev": true - } } }, - "urix": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" - }, "url-join": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/url-join/-/url-join-2.0.5.tgz", @@ -1393,21 +787,6 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, - "uuid": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", - "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, "vows": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/vows/-/vows-0.7.0.tgz", @@ -1418,52 +797,30 @@ "eyes": ">=0.1.6" } }, - "which": { - "version": "1.2.14", - "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz", - "integrity": "sha1-mofEN48D6CfOyvGs31bHNsAcFOU=", - "requires": { - "isexe": "^2.0.0" - } - }, - "winston": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.2.tgz", - "integrity": "sha512-4S/Ad4ZfSNl8OccCLxnJmNISWcm2joa6Q0YGDxlxMzH0fgSwWsjMt+SmlNwCqdpaPg3ev1HKkMBsIiXeSUwpbA==", - "dev": true, - "optional": true, - "requires": { - "async": "~1.0.0", - "colors": "1.0.x", - "cycle": "1.0.x", - "eyes": "0.1.x", - "isstream": "0.1.x", - "stack-trace": "0.0.x" - }, - "dependencies": { - "async": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", - "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", - "dev": true, - "optional": true - } - } - }, "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, - "yamlish": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/yamlish/-/yamlish-0.0.7.tgz", - "integrity": "sha1-tK+aHcxjYYhzw9bkUewyE8OaV/s=" + "ws": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.2.tgz", + "integrity": "sha512-rfUqzvz0WxmSXtJpPMX2EeASXabOrSMk1ruMOV3JBTBjo4ac2lDjGGsbQSyxj8Odhw5fBib8ZKEjDNvgouNKYw==", + "requires": { + "async-limiter": "~1.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true }, "yauzl": { "version": "2.4.1", diff --git a/package.json b/package.json index 67430e869..2979eeb77 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "phantomas", - "version": "1.20.1", + "version": "2.0.0-beta", "author": "macbre (http://macbre.net)", "description": "PhantomJS-based web performance metrics collector", "main": "./lib/index.js", @@ -19,42 +19,23 @@ ], "license": "BSD-2-Clause", "engines": { - "node": ">=4.0" + "node": ">=8.0" }, "dependencies": { - "analyze-css": "^0.12.6", - "ansicolors": "~0.3.2", - "ansistyles": "~0.1.0", - "ascii-table": "0.0.9", - "async": "^2.6.1", - "csv-string": "^3.1.3", - "debug": "^3.1.0", - "js-yaml": "^3.11.0", - "node-statsd": "0.1.1", - "optimist": "^0.6.1", - "optimist-config-file": "^1.0.1", - "progress": "~2.0.0", - "q": "^1.5.0", - "tap-producer-macbre": "0.0.3", - "travis-fold": ">=0.1.2", - "uuid": "^3.0.1" + "debug": "^4.1.1", + "puppeteer": "^1.11.0", + "q": "^1.5.0" }, "devDependencies": { "glob": "^7.1.2", "http-server": "^0.11.1", "js-beautify": "^1.6.14", - "jshint": "macbre/jshint#master", + "jshint": "^2.9.7", "mockery": "^2.0.0", "vows": "^0.7.0" }, - "optionalDependencies": { - "phantomjs-prebuilt-macbre": "^2.5.0-dev2", - "slimerjs": "^0.10.3" - }, - "bin": "./bin/phantomas.js", - "preferGlobal": true, "scripts": { - "test": "PHANTOMAS_ENGINE=webkit vows --spec", + "test": "vows --spec", "unit-test": "vows test/modules/*-test.js --spec", "lint": "jshint --verbose core/ modules/ scripts/ test/*.js test/*/*-test.js lib/*.js lib/metadata/*.js reporters/ examples/", "beautify": "js-beautify -r bin/phantomas.js core/*.js examples/*.js extensions/*/*.js lib/*.js lib/engines/*.js lib/metadata/*.js modules/*/*.js reporters/*.js scripts/*.js test/*.js test/*/*.js", From ddf2101a21b9f72fbaf8c5bb1fbf139d5c23b709 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 16:29:45 +0100 Subject: [PATCH 003/248] Remove lib/engines --- lib/engines/gecko.js | 47 ------------------------------------------- lib/engines/webkit.js | 22 -------------------- 2 files changed, 69 deletions(-) delete mode 100644 lib/engines/gecko.js delete mode 100644 lib/engines/webkit.js diff --git a/lib/engines/gecko.js b/lib/engines/gecko.js deleted file mode 100644 index cd2dc7de6..000000000 --- a/lib/engines/gecko.js +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Defines SlimerJS engine - */ -var pkg = require('slimerjs'), - debug = require('debug')('phantomas:slimerjs'), - spawn = require('child_process').spawn; - -module.exports = { - name: 'SlimerJS', - engine: 'gecko', - path: pkg.path, - version: pkg.version, - getUserAgent: function() { - return 'SlimerJS/' + pkg.version - }, - spawn: function(path, engineArgs) { - // xvfb arguments - var args = [ - '--auto-servernum', - '--server-num=1', - //'--server-args=\':1 -noreset -screen 1 1600x1200x24\'' - ]; - - // SlimerJS binary - args.push(path); - - // filter out --ssl-protocol option - // supoorted by PhantomJS, but not by SlimerJS - engineArgs = engineArgs.filter(function(opt) { - return (opt.indexOf('--ssl-protocol=') === 0) ? false /* remove */ : true; - }); - - // pass runner and phantomas options - args = args.concat(engineArgs); - - debug('xvfb-run %s', args.join(' ')); - - // spawn SlimerJS using xvfb - // - // sudo aptitude install xvfb libasound2 libgtk2.0-0 - // - // @see https://gist.github.com/macbre/e7e2e35caf9d91af5ecf - return spawn('xvfb-run', args, { - env: process.env - }); - } -}; diff --git a/lib/engines/webkit.js b/lib/engines/webkit.js deleted file mode 100644 index cd048f508..000000000 --- a/lib/engines/webkit.js +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Defines PhantomJS 2.x engine - * - * @see https://github.com/macbre/phantomas/issues/488 - */ -var pkg = require('phantomjs-prebuilt-macbre'); - -module.exports = { - name: 'PhantomJS', - engine: 'webkit', - path: pkg.path, - version: pkg.version, - getUserAgent: function() { - return 'PhantomJS/' + pkg.version - }, - getEngineArgs: function() { - return [ - // enable local storage in PhantomJS 2.5 (#710) - '--local-storage-quota=5000', - ]; - } -}; From 3527430b79b0f607c2f7cfc31025609037e61f7e Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 16:30:00 +0100 Subject: [PATCH 004/248] Require Node.js 8+ --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3cbbfb233..e8a515dda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ dist: trusty node_js: - 10 - 8 - - 6 sudo: false before_script: - sh test/server-start.sh & From 7db6109284115d1f3b185f07c7625b0549807433 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 17:29:16 +0100 Subject: [PATCH 005/248] Code cleanup and Promise-based async handling --- examples/promise.js | 41 ++- lib/ansicolors.js | 37 --- lib/browser.js | 23 ++ lib/engines.js | 134 ---------- lib/fast-stats.js | 611 -------------------------------------------- lib/index.js | 216 ++-------------- lib/ipc.js | 70 ----- lib/simple-queue.js | 59 ----- lib/stats.js | 80 ------ 9 files changed, 59 insertions(+), 1212 deletions(-) delete mode 100644 lib/ansicolors.js create mode 100644 lib/browser.js delete mode 100644 lib/engines.js delete mode 100644 lib/fast-stats.js delete mode 100644 lib/ipc.js delete mode 100644 lib/simple-queue.js delete mode 100644 lib/stats.js diff --git a/examples/promise.js b/examples/promise.js index 468852c7b..1106b66cf 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -3,47 +3,46 @@ /** * Example script that uses phantomas npm module with promise pattern */ -var phantomas = require('..'), - run; +const phantomas = require('..'); console.log('phantomas v%s loaded from %s', phantomas.version, phantomas.path); -run = phantomas('http://google.is', { +const promise = phantomas('http://google.is', { 'analyze-css': true, 'assert-requests': 1 }); -console.log('Running phantomas: pid %d', run.pid); +console.log('Results: %s', promise); // metrics metadata console.log('Number of available metrics: %d', phantomas.metadata.metricsCount); // handle the promise -run. -then(function(res) { - console.log('Exit code: %d', res.code); - console.log('Number of requests: %d', res.results.getMetric('requests')); - console.log('Failed asserts: %j', res.results.getFailedAsserts()); -}). -fail(function(res) { - console.log('Exit code #%d', res.code); - process.exit(res.code); -}). -progress(function(progress) { - console.log('Loading progress: %d%', progress * 100); -}). -done(); +promise. + then(function(res) { + console.log('Resolved: %s', res); + /** + console.log('Exit code: %d', res.code); + console.log('Number of requests: %d', res.results.getMetric('requests')); + console.log('Failed asserts: %j', res.results.getFailedAsserts()); + **/ + }). + catch(function(res) { + console.error(res); + console.log('Error code #%d', res.code); + process.exit(res.code); + }); // events handling -run.on('milestone', function(milestone) { +promise.on('milestone', function(milestone) { console.log('Milestone reached: %s', milestone); }); -run.on('recv', function(response) { +promise.on('recv', function(response) { console.log('Response #%d: %s %s [HTTP %d]', response.id, response.method, response.url, response.status); }); // including the custom once emitted by phantomas modules -run.on('domQuery', function(type, query) { +promise.on('domQuery', function(type, query) { console.log('DOM query by %s - "%s"', type, query); }); diff --git a/lib/ansicolors.js b/lib/ansicolors.js deleted file mode 100644 index 7290171b1..000000000 --- a/lib/ansicolors.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * A simple wrapper for ansicolors npm module - * - * Makes it noop when the stdout is not a TTY - */ -'use strict'; - -var colors = require('ansicolors'), - keys = Object.keys(colors), - disableColors; - -if (typeof process !== 'undefined' && typeof process.stdout !== 'undefined') { - // nodejs - disableColors = !!process.env.BW; -} else { - // PhantomJS - disableColors = !!(require('system').env.BW); -} - -function nop(str) { - return str; -} - -function disable() { - keys.forEach(function(key) { - colors[key] = nop; - }); -} - -if (disableColors) { - disable(); -} - -// expose disable() function -colors.disable = disable; - -module.exports = colors; diff --git a/lib/browser.js b/lib/browser.js new file mode 100644 index 000000000..7e47762b6 --- /dev/null +++ b/lib/browser.js @@ -0,0 +1,23 @@ +/** + * Expose puppeteer API and events emitter object for lib/index.js + */ +const debug = require('debug')('phantomas:browser'), + puppeteer = require("puppeteer"); + +function browser() { + this.events = require('events').EventEmitter; +} + +browser.prototype.init = async () => { + debug('Launching Puppeter'); + + const args = []; // ['--no-sandbox']; + const browser = await puppeteer.launch({args: args}); + + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion + debug('Browser: %s', await browser.version()); + + await browser.close(); +}; + +module.exports = browser; diff --git a/lib/engines.js b/lib/engines.js deleted file mode 100644 index 9d549546b..000000000 --- a/lib/engines.js +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Provides abstractor layer for using different engines to run phantomas - */ -'use strict'; - -var debug = require('debug')('phantomas:engines'), - spawn = require('child_process').spawn, - VERSION = require('../package').version; - -function engines(options) { - /* jshint validthis: true */ - this.options = options; - - //--debug can be either 'true' or 'false' - this.options.debug = (this.options.debug === true) ? 'true' : 'false'; - - // engines storage - this.engines = {}; - [ - // the first one is a default one - 'webkit', - 'gecko' - ].forEach(this.registerEngine, this); -} - -engines.prototype.registerEngine = function(engineName) { - var def; - - try { - def = require('./engines/' + engineName); - this.engines[engineName] = def; - - debug('Engine "%s": %s v%s installed in %s', engineName, def.name, def.version, def.path); - } catch (ex) { - debug('Engine "%s": failed to import - %s', engineName, ex); - } -}; - -engines.prototype.getEngines = function() { - return Object.keys(this.engines); -}; - -engines.prototype.getEngine = function(engine) { - return this.engines[engine]; -}; - -engines.prototype.run = function(scriptFile) { - // select the engine - var engines = this.getEngines(), - engine = this.getEngine(engines[0]); - - // --engine=[gecko|webkit] - if (typeof this.options.engine === 'string') { - engine = this.getEngine(this.options.engine); - - if (typeof engine === 'undefined') { - throw 'Engine "' + this.options.engine + '" not found!'; - } - } - // PHANTOMAS_ENGINE=gecko phantomas "http://example.com" - else if (typeof process.env.PHANTOMAS_ENGINE === 'string') { - engine = this.getEngine(process.env.PHANTOMAS_ENGINE); - - if (typeof engine === 'undefined') { - throw 'Engine "' + process.env.PHANTOMAS_ENGINE + '" not found!'; - } - } - // --gecko / --webkit - else { - engines.forEach(function(key) { - if (this.options[key] === true) { - engine = this.getEngine(key); - } - }, this); - } - - // customize user agent - if (typeof this.options['user-agent'] === 'undefined') { - this.options['user-agent'] = "phantomas/" + VERSION + " (" + engine.getUserAgent() + "; " + process.platform + " " + process.arch + ")"; - } - - // build engine specific command-line arguments - var engineArgs = engine.getEngineArgs ? engine.getEngineArgs() : []; - - Object.keys(this.options).forEach(function(key) { - var val = this.options[key], - nativeOptions = [ - 'cookies-file', - 'debug', - 'ignore-ssl-errors', - 'ssl-protocol', - 'proxy', - 'proxy-auth', - 'proxy-type' - ]; - - if (val === false) { - return; - } - - // handle native PhantomJS options (#163) - // @see http://phantomjs.org/api/command-line.html - if (nativeOptions.indexOf(key) > -1) { - engineArgs.push('--' + key + '=' + val); - } - }, this); - - debug('Running %s (using %s)', scriptFile, engine.name); - debug('Passing phantomas options: %j', this.options); - debug('Passing engine options: %j', engineArgs); - - // add a script to run - engineArgs.push(scriptFile); - - // pass JSON encoded options - engineArgs.push(JSON.stringify(this.options)); - - // spawn the process - var proc; - - if (typeof engine.spawn === 'function') { - debug('Using custom spawn function'); - proc = engine.spawn(engine.path, engineArgs); - } else { - proc = spawn(engine.path, engineArgs, { - env: process.env - }); - } - - return proc; -}; - -// public API -module.exports = engines; diff --git a/lib/fast-stats.js b/lib/fast-stats.js deleted file mode 100644 index 53ba0c945..000000000 --- a/lib/fast-stats.js +++ /dev/null @@ -1,611 +0,0 @@ -/* jshint -W004: false, -W014: false, -W041: false */ -/** - * Taken from https://github.com/bluesmoon/node-faststats - */ - -/* -Note that if your data is too large, there _will_ be overflow. -*/ - - -function asc(a, b) { - return a - b; -} - -var config_params = { - bucket_precision: function(o, s) { - if (typeof s != "number" || s <= 0) { - throw new Error("bucket_precision must be a positive number"); - } - o._config.bucket_precision = s; - o.buckets = []; - }, - - buckets: function(o, b) { - if (!Array.isArray(b) || b.length == 0) { - throw new Error("buckets must be an array of bucket limits"); - } - - o._config.buckets = b; - o.buckets = []; - }, - - bucket_extension_interval: function(o, s) { - if (s === undefined) - return; - if (typeof s != "number" || s <= 0) { - throw new Error("bucket_extension_interval must be a positive number"); - } - o._config.bucket_extension_interval = s; - }, - - store_data: function(o, s) { - if (typeof s != "boolean") { - throw new Error("store_data must be a true or false"); - } - o._config.store_data = s; - }, - - sampling: function(o, s) { - if (typeof s != "boolean") { - throw new Error("sampling must be a true or false"); - } - o._config.sampling = s; - } -}; - -function Stats(c) { - this._config = { - store_data: true - }; - - if (c) { - for (var k in config_params) { - if (c.hasOwnProperty(k)) { - config_params[k](this, c[k]); - } - } - } - - this.reset(); - - return this; -} - -Stats.prototype = { - - reset: function() { - if (this._config.store_data) - this.data = []; - - this.length = 0; - - this.sum = 0; - this.sum_of_squares = 0; - this.sum_of_logs = 0; - this.sum_of_square_of_logs = 0; - this.zeroes = 0; - this.max = this.min = null; - - this._reset_cache(); - - return this; - }, - - _reset_cache: function() { - this._stddev = null; - - if (this._config.store_data) - this._data_sorted = null; - }, - - _find_bucket: function(a) { - var b = 0, - e, l; - if (this._config.buckets) { - l = this._config.buckets.length; - if (this._config.bucket_extension_interval && a >= this._config.buckets[l - 1]) { - e = a - this._config.buckets[l - 1]; - b = parseInt(e / this._config.bucket_extension_interval) + l; - if (this._config.buckets[b] === undefined) - this._config.buckets[b] = this._config.buckets[l - 1] + (parseInt(e / this._config.bucket_extension_interval) + 1) * this._config.bucket_extension_interval; - if (this._config.buckets[b - 1] === undefined) - this._config.buckets[b - 1] = this._config.buckets[l - 1] + parseInt(e / this._config.bucket_extension_interval) * this._config.bucket_extension_interval; - } - for (; b < l; b++) { - if (a < this._config.buckets[b]) { - break; - } - } - } else if (this._config.bucket_precision) { - b = Math.floor(a / this._config.bucket_precision); - } - - return b; - }, - - _add_cache: function(a) { - var tuple = [1], - i; - if (a instanceof Array) { - tuple = a; - a = tuple.shift(); - } - - this.sum += a * tuple[0]; - this.sum_of_squares += a * a * tuple[0]; - if (a === 0) { - this.zeroes++; - } else { - this.sum_of_logs += Math.log(a) * tuple[0]; - this.sum_of_square_of_logs += Math.pow(Math.log(a), 2) * tuple[0]; - } - this.length += tuple[0]; - - if (tuple[0] > 0) { - if (this.max === null || this.max < a) - this.max = a; - if (this.min === null || this.min > a) - this.min = a; - } - - if (this.buckets) { - var b = this._find_bucket(a); - if (!this.buckets[b]) - this.buckets[b] = [0]; - this.buckets[b][0] += tuple.shift(); - - for (i = 0; i < tuple.length; i++) - this.buckets[b][i + 1] = (this.buckets[b][i + 1] | 0) + (tuple[i] | 0); - } - - this._reset_cache(); - }, - - _del_cache: function(a) { - var tuple = [1], - i; - if (a instanceof Array) { - tuple = a; - a = tuple.shift(); - } - - this.sum -= a * tuple[0]; - this.sum_of_squares -= a * a * tuple[0]; - if (a === 0) { - this.zeroes--; - } else { - this.sum_of_logs -= Math.log(a) * tuple[0]; - this.sum_of_square_of_logs -= Math.pow(Math.log(a), 2) * tuple[0]; - } - this.length -= tuple[0]; - - if (this._config.store_data) { - if (this.length === 0) { - this.max = this.min = null; - } - if (this.length === 1) { - this.max = this.min = this.data[0]; - } else if (tuple[0] > 0 && (this.max === a || this.min === a)) { - var i = this.length - 1; - if (i >= 0) { - this.max = this.min = this.data[i--]; - while (i-- >= 0) { - if (this.max < this.data[i]) - this.max = this.data[i]; - if (this.min > this.data[i]) - this.min = this.data[i]; - } - } - } - } - - if (this.buckets) { - var b = this._find_bucket(a); - if (this.buckets[b]) { - this.buckets[b][0] -= tuple.shift(); - - if (this.buckets[b][0] === 0) - delete this.buckets[b]; - else - for (i = 0; i < tuple.length; i++) - this.buckets[b][i + 1] = (this.buckets[b][i + 1] | 0) - (tuple[i] | 0); - } - } - - this._reset_cache(); - }, - - push: function() { - var i, a, args = Array.prototype.slice.call(arguments, 0); - if (args.length && args[0] instanceof Array) - args = args[0]; - for (i = 0; i < args.length; i++) { - a = args[i]; - if (this._config.store_data) - this.data.push(a); - this._add_cache(a); - } - - return this; - }, - - push_tuple: function(tuple) { - if (!this.buckets) { - throw new Error("push_tuple is only valid when using buckets"); - } - this._add_cache(tuple); - }, - - pop: function() { - if (this.length === 0 || this._config.store_data === false) - return undefined; - - var a = this.data.pop(); - this._del_cache(a); - - return a; - }, - - remove_tuple: function(tuple) { - if (!this.buckets) { - throw new Error("remove_tuple is only valid when using buckets"); - } - this._del_cache(tuple); - }, - - reset_tuples: function(tuple) { - var b, l, t, ts = tuple.length; - if (!this.buckets) { - throw new Error("reset_tuple is only valid when using buckets"); - } - - for (b = 0, l = this.buckets.length; b < l; b++) { - if (!this.buckets[b] || this.buckets[b].length <= 1) { - continue; - } - for (t = 0; t < ts; t++) { - if (typeof tuple[t] !== 'undefined') { - this.buckets[b][t] = tuple[t]; - } - } - } - }, - - unshift: function() { - var i, a, args = Array.prototype.slice.call(arguments, 0); - if (args.length && args[0] instanceof Array) - args = args[0]; - i = args.length; - while (i--) { - a = args[i]; - if (this._config.store_data) - this.data.unshift(a); - this._add_cache(a); - } - - return this; - }, - - shift: function() { - if (this.length === 0 || this._config.store_data === false) - return undefined; - - var a = this.data.shift(); - this._del_cache(a); - - return a; - }, - - amean: function() { - if (this.length === 0) - return NaN; - return this.sum / this.length; - }, - - gmean: function() { - if (this.length === 0) - return NaN; - if (this.zeroes > 0) - return NaN; - return Math.exp(this.sum_of_logs / this.length); - }, - - stddev: function() { - if (this.length === 0) - return NaN; - var n = this.length; - if (this._config.sampling) - n--; - if (this._stddev === null) - this._stddev = Math.sqrt((this.length * this.sum_of_squares - this.sum * this.sum) / (this.length * n)); - - return this._stddev; - }, - - gstddev: function() { - if (this.length === 0) - return NaN; - if (this.zeroes > 0) - return NaN; - var n = this.length; - if (this._config.sampling) - n--; - return Math.exp(Math.sqrt((this.length * this.sum_of_square_of_logs - this.sum_of_logs * this.sum_of_logs) / (this.length * n))); - }, - - moe: function() { - if (this.length === 0) - return NaN; - // see http://en.wikipedia.org/wiki/Standard_error_%28statistics%29 - return 1.96 * this.stddev() / Math.sqrt(this.length); - }, - - range: function() { - if (this.length === 0) - return [NaN, NaN]; - return [this.min, this.max]; - }, - - distribution: function() { - if (this.length === 0) - return []; - if (!this.buckets) - throw new Error("bucket_precision or buckets not configured."); - - var d = [], - i, j, k, l; - - if (this._config.buckets) { - j = this.min; - l = Math.min(this.buckets.length, this._config.buckets.length); - - for (i = 0; i < l; j = this._config.buckets[i++]) { // this has to be i++ and not ++i - if (this._config.buckets[i] === undefined && this._config.bucket_extension_interval) - this._config.buckets[i] = this._config.buckets[i - 1] + this._config.bucket_extension_interval; - if (this.min > this._config.buckets[i]) - continue; - - d[i] = { - bucket: (j + this._config.buckets[i]) / 2, - range: [j, this._config.buckets[i]], - count: (this.buckets[i] ? this.buckets[i][0] : 0), - tuple: this.buckets[i] ? this.buckets[i].slice(1) : [] - }; - - if (this.max < this._config.buckets[i]) - break; - } - if (i == l && this.buckets[i]) { - d[i] = { - bucket: (j + this.max) / 2, - range: [j, this.max], - count: this.buckets[i][0], - tuple: this.buckets[i] ? this.buckets[i].slice(1) : [] - }; - } - } else if (this._config.bucket_precision) { - i = Math.floor(this.min / this._config.bucket_precision); - l = Math.floor(this.max / this._config.bucket_precision) + 1; - for (j = 0; i < l && i < this.buckets.length; i++, j++) { - if (!this.buckets[i]) { - continue; - } - d[j] = { - bucket: (i + 0.5) * this._config.bucket_precision, - range: [i * this._config.bucket_precision, (i + 1) * this._config.bucket_precision], - count: this.buckets[i][0], - tuple: this.buckets[i] ? this.buckets[i].slice(1) : [] - }; - } - } - - return d; - - }, - - percentile: function(p) { - if (this.length === 0 || (!this._config.store_data && !this.buckets)) - return NaN; - - // If we come here, we either have sorted data or sorted buckets - - var v; - - if (p <= 0) - v = 0; - else if (p == 25) - v = [Math.floor((this.length - 1) * 0.25), Math.ceil((this.length - 1) * 0.25)]; - else if (p == 50) - v = [Math.floor((this.length - 1) * 0.5), Math.ceil((this.length - 1) * 0.5)]; - else if (p == 75) - v = [Math.floor((this.length - 1) * 0.75), Math.ceil((this.length - 1) * 0.75)]; - else if (p >= 100) - v = this.length - 1; - else - v = Math.floor(this.length * p / 100); - - if (v === 0) - return this.min; - if (v === this.length - 1) - return this.max; - - if (this._config.store_data) { - if (this._data_sorted === null) - this._data_sorted = this.data.slice(0).sort(asc); - - if (typeof v == 'number') - return this._data_sorted[v]; - else - return (this._data_sorted[v[0]] + this._data_sorted[v[1]]) / 2; - } else { - var j; - if (typeof v != 'number') - v = (v[0] + v[1]) / 2; - - if (this._config.buckets) - j = 0; - else if (this._config.bucket_precision) - j = Math.floor(this.min / this._config.bucket_precision); - - for (; j < this.buckets.length; j++) { - if (!this.buckets[j]) - continue; - if (v <= this.buckets[j][0]) { - break; - } - v -= this.buckets[j][0]; - } - - return this._get_nth_in_bucket(v, j); - } - }, - - _get_nth_in_bucket: function(n, b) { - var range = []; - if (this._config.buckets) { - range[0] = (b > 0 ? this._config.buckets[b - 1] : this.min); - range[1] = (b < this._config.buckets.length ? this._config.buckets[b] : this.max); - } else if (this._config.bucket_precision) { - range[0] = Math.max(b * this._config.bucket_precision, this.min); - range[1] = Math.min((b + 1) * this._config.bucket_precision, this.max); - } - return range[0] + (range[1] - range[0]) * n / this.buckets[b][0]; - }, - - median: function() { - return this.percentile(50); - }, - - iqr: function() { - var q1, q3, fw; - - q1 = this.percentile(25); - q3 = this.percentile(75); - - fw = (q3 - q1) * 1.5; - - return this.band_pass(q1 - fw, q3 + fw, true); - }, - - band_pass: function(low, high, open, config) { - var i, j, b, b_val, i_val; - - if (!config) - config = this._config; - - b = new Stats(config); - - if (this.length === 0) - return b; - - if (this._config.store_data) { - if (this._data_sorted === null) - this._data_sorted = this.data.slice(0).sort(asc); - - for (i = 0; i < this.length && (this._data_sorted[i] < high || (!open && this._data_sorted[i] === high)); i++) { - if (this._data_sorted[i] > low || (!open && this._data_sorted[i] === low)) { - b.push(this._data_sorted[i]); - } - } - } else if (this._config.buckets) { - for (i = 0; i <= this._config.buckets.length; i++) { - if (this._config.buckets[i] < this.min) - continue; - - b_val = (i == 0 ? this.min : this._config.buckets[i - 1]); - if (b_val < this.min) - b_val = this.min; - if (b_val > this.max) - b_val = this.max; - - if (high < b_val || (open && high === b_val)) { - break; - } - if (low < b_val || (!open && low === b_val)) { - for (j = 0; j < (this.buckets[i] ? this.buckets[i][0] : 0); j++) { - i_val = Stats.prototype._get_nth_in_bucket.call(this, j, i); - if ((i_val > low || (!open && i_val === low)) && (i_val < high || (!open && i_val === high))) { - b.push(i_val); - } - } - } - } - - b.min = Math.max(low, b.min); - b.max = Math.min(high, b.max); - } else if (this._config.bucket_precision) { - var low_i = Math.floor(low / this._config.bucket_precision), - high_i = Math.floor(high / this._config.bucket_precision) + 1; - - for (i = low_i; i < Math.min(this.buckets.length, high_i); i++) { - for (j = 0; j < (this.buckets[i] ? this.buckets[i][0] : 0); j++) { - i_val = Stats.prototype._get_nth_in_bucket.call(this, j, i); - if ((i_val > low || (!open && i_val === low)) && (i_val < high || (!open && i_val === high))) { - b.push(i_val); - } - } - } - - b.min = Math.max(low, b.min); - b.max = Math.min(high, b.max); - } - - return b; - }, - - copy: function(config) { - var b = Stats.prototype.band_pass.call(this, this.min, this.max, false, config); - - b.sum = this.sum; - b.sum_of_squares = this.sum_of_squares; - b.sum_of_logs = this.sum_of_logs; - b.sum_of_square_of_logs = this.sum_of_square_of_logs; - b.zeroes = this.zeroes; - - return b; - }, - - Σ: function() { - return this.sum; - }, - - Π: function() { - return this.zeroes > 0 ? 0 : Math.exp(this.sum_of_logs); - } -}; - -Stats.prototype.σ = Stats.prototype.stddev; -Stats.prototype.μ = Stats.prototype.amean; - - -exports.Stats = Stats; - -/** -if(process.argv[1] && process.argv[1].match(__filename)) { - var s = new Stats({store_data:false, buckets: [ 1, 5, 10, 15, 20, 25, 30, 35 ]}).push(1, 2, 3); - var l = process.argv.slice(2); - if(!l.length) l = [10, 11, 15, 8, 13, 12, 19, 32, 17, 16]; - l.forEach(function(e, i, a) { a[i] = parseFloat(e, 10); }); - Stats.prototype.push.apply(s, l); - console.log(s.data); - console.log(s.amean().toFixed(2), s.μ().toFixed(2), s.stddev().toFixed(2), s.σ().toFixed(2), s.gmean().toFixed(2), s.median().toFixed(2), s.moe().toFixed(2), s.distribution()); - var t=s.copy({buckets: [0, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 25, 30, 35] }); - console.log(t.amean().toFixed(2), t.μ().toFixed(2), t.stddev().toFixed(2), t.σ().toFixed(2), t.gmean().toFixed(2), t.median().toFixed(2), t.moe().toFixed(2), t.distribution()); - - s = new Stats({store_data: false, buckets: [1, 5, 10, 15, 20, 25, 30, 35]}); - s.push_tuple([1, 1, 3, 4]); - s.push_tuple([2, 1, 5, 8]); - s.push_tuple([3, 1, 4, 9]); - s.push_tuple([1, 1, 13, 14]); - - console.log(s.amean(), s.median()); - console.log(s.distribution()); - - s.remove_tuple([1, 1, 3, 4]); - s.push_tuple([4, 1, 3, 3]); - console.log(s.amean(), s.median()); - console.log(s.distribution()); - -} -**/ diff --git a/lib/index.js b/lib/index.js index 4d81ce766..e6954f094 100644 --- a/lib/index.js +++ b/lib/index.js @@ -3,50 +3,23 @@ */ 'use strict'; -var debug = require('debug')('phantomas'), +var debug = require('debug')('phantomas:core'), emitter = require('events').EventEmitter, - Engines = require('./engines'), - net = require('net'), + puppeteer = require("puppeteer"), path = require('path'), - Q = require('q'), - Stream = require('stream'), util = require('util'), VERSION = require('./../package').version; -/** - * Return temporary directory for the current phantomas run - */ -function getTmpDir() { - var tmpdir = require('os').tmpdir(), - uuidV4 = require('uuid/v4'); - - // example: /tmp/phantomas/58aea8b5-2c97-48ee-9885-fcd81d38561f/ - return tmpdir + '/phantomas/' + uuidV4() + '/'; -} - /** * Main CommonJS module entry point - * - * FIXME: split into seperate functions */ -function phantomas(url, opts, callback) { - var engineArgs = [], - deferred = Q.defer(), - events = new emitter(), - proc, - engine, - data = '', - tmpDir, - options; - - // options can be omitted - if (typeof opts === 'function') { - callback = opts; - opts = {}; - } +function phantomas(url, opts) { + var events = new emitter(), + browser, options; - debug('nodejs %s', process.version); - debug('phantomas v%s installed in %s', VERSION, phantomas.path); + debug('OS: %s %s', process.platform, process.arch); + debug('Node.js: %s', process.version); + debug('Puppeteer: preferred revision, r%s installed in %s', puppeteer._launcher._preferredRevision, puppeteer._projectRoot); debug('URL: <%s>', url); debug('Options: %s', JSON.stringify(opts)); @@ -54,179 +27,22 @@ function phantomas(url, opts, callback) { options = util._extend({}, opts || {}); // use util._extend to avoid #563 options.url = options.url || url || false; - // generate unique temporary directory name - // and pass it to PhantomJS script as an environment variable - tmpDir = getTmpDir(); - process.env.PHANTOMAS_TMP_DIR = tmpDir; - - // path to analyze-css main script (#664) - process.env.ANALYZE_CSS_BIN = require('analyze-css').pathBin; - debug('Environment: %j', process.env); - // select the engine and run phantomas - engine = new Engines(options); - proc = engine.run(phantomas.path + '/scripts/phantomas.js'); - - debug('Spawned with pid #%d', proc.pid); - - proc.on('error', function(err) { - debug('Error: %s', err); - }); - - /** - * Set up IPC channel - * - * It will receive JSON-formatted messages from phantomjs / slimerjs process - * (send over stdout stream of spawned process) and emit events - */ - var ipc = new(require('./ipc'))(proc.stdout); - ipc.setEventEmitter(events); - - // pipe log messages to error stream - var errStream = new Stream.Readable(); - errStream._read = function() { - return true; - }; - - ipc.on('log', function(msg) { - errStream.push(msg + "\n"); - }); - - // push raw messages from stderr (PhantomJS run in --debug=true mode) to errStream - proc.stderr.on('data', function(data) { - var msg = data.toString(). - split('[DEBUG]').pop(). - trim(); - - errStream.push('Debug: ' + msg + "\n"); - }); - - // handle loading progress - var progressDebug = require('debug')('phantomas:progress'); - - ipc.on('progress', function(progress, inc) { - progressDebug('%d% (+%d%)', progress, inc); - deferred.notify(progress / 100); - }); - - // handle --socket option - // @see http://nodejs.org/api/net.html#net_new_net_socket_options - if (typeof options.socket === 'string') { - var socket, - socketName = options.socket; - - // @see http://nodejs.org/api/net.html#net_net_connect_options_connectionlistener - debug('Using UNIX socket for IPC: %s', socketName); - - socket = new net.Socket(); - socket.connect(socketName, function() { - debug('Socket connected'); - }); - - socket.on('error', function(err) { - debug('Socket error: %s', err); + // set up Puppeteer + browser = new (require('./browser')); - throw 'Socket "' + socketName + '" error - ' + err; - }); - - ipc.on('_msg', function(data) { - // example: ["metric","bodyHTMLSize",75193] - socket.write(JSON.stringify(data) + "\n"); - }); - } - - // get JSON results - ipc.on('json', function(raw) { - data = raw; - }); - - // failures handling - var exitCode = 0; - ipc.on('exit', function(code, msg) { - if ((code > 0) && (typeof msg === 'string')) { - exitCode = code; - errStream.push('phantomas: (' + code + ') ' + msg + "\n"); - } - }); - - // process results - proc.on('close', function(code) { - var debug = require('debug')('phantomas:results'), - json, - results; - - // PhantomJS made a core dump (#382) - if (code === null) { - code = 255; // EXIT_ERROR - errStream.push('phantomas: (' + code + ') Engine crashed unexpectedly\n'); - } - - // apply the code from 'exit' event - code = code || exitCode; - - debug('Process returned code #%d', code); - - // (try to) parse to JSON + // promise handling + var promise = new Promise(async (resolve, reject) => { try { - json = JSON.parse(data); - } catch (ex) { - debug('Error when parsing JSON (%s): "%s"', ex, data); - errStream.push(data); - - // Return EXIT_ERROR when JSON can not be parsed (and there was no error earlier) - if (code < 250) { - code = 255; - } - - errStream.push('phantomas: (' + code + ') Failed to parse JSON with the results\n'); - } - - // finish the readable stream - errStream.emit('end'); - - if (typeof json !== 'undefined') { - events.emit('data', json); - - // wrap JSON data into results object - results = new(require('../core/results'))(json); - events.emit('results', results); + await browser.init(); + resolve('foo'); } - - if (code > 0) { - if (events.listeners('error').length > 0) { - events.emit('error', code); - } - } - - if (typeof callback === 'function') { - var err = code === 0 ? null : new Error(code); - callback(err, json, results); - } - - // either reject or resolve the promise - if (code > 250) { - debug('Rejecting a promise with %d exit code', code); - deferred.reject({ - code: code, - json: json, - results: results - }); - } else { - debug('Resolving a promise with %d exit code', code); - deferred.resolve({ - code: code, - json: json, - results: results - }); + catch(ex) { + reject(ex); } }); - var promise = deferred.promise; - - promise.pid = proc.pid; - promise.stdout = proc.stdout; - promise.stderr = errStream; promise.on = events.on.bind(events); return promise; diff --git a/lib/ipc.js b/lib/ipc.js deleted file mode 100644 index f9319f451..000000000 --- a/lib/ipc.js +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Simple IPC implementation using JSON-encoded messages sent over stdout stream - * - * Implements consumer of the data (for nodejs environment) - */ -'use strict'; - -var debug = require('debug')('phantomas:ipc'), - emitter = require('events').EventEmitter, - SEPARATOR = "\xFF\xFF"; - -function ipc(stream) { - /* jshint validthis: true */ - this.stream = stream; - this.events = new emitter(); - - this.init(); -} - -ipc.prototype.setEventEmitter = function(emitter) { - this.events = emitter; -}; - -ipc.prototype.init = function() { - var self = this, - buffer = ''; - - this.stream.on('data', function(data) { - buffer += data.toString().trim(); - - // check for the separator at the of the buffer - parse the data - if (buffer.substr(-2) === SEPARATOR) { - self.parse(buffer); - buffer = ''; - } - }); -}; - -ipc.prototype.parse = function(data) { - // split by separator - data.split(SEPARATOR).forEach(function(msg) { - msg = msg.trim(); - - if (msg === '') return; - - try { - msg = JSON.parse(msg); - debug('%s: %j', msg.event, msg.data); - - // send event name and the rest of the data - var args = msg.data; - args.unshift(msg.event); - - this.events.emit.apply(this.events, args); - - // send generic "_msg" event for each message received (issue #354) - this.events.emit('_msg', args); - } catch (e) { - // send PhantomJS errors to stderr to ease debugging of issues like #302 - debug('message parsing failed: "%s"', msg); - process.stderr.write(msg + "\n"); - } - }, this); -}; - -ipc.prototype.on = function(event, fn) { - this.events.on(event, fn); -}; - -module.exports = ipc; diff --git a/lib/simple-queue.js b/lib/simple-queue.js deleted file mode 100644 index 9bf43704d..000000000 --- a/lib/simple-queue.js +++ /dev/null @@ -1,59 +0,0 @@ -'use strict'; - -function queue() { - var remaining = 0, - jobs = 0, - doneFn = []; - - function preCheck() { - if (remaining === false) { - throw new Error('Can\'t push jobs to completed queue'); - } - } - - function postCheck() { - if (remaining === 0 && doneFn.length > 0) { - doneFn.forEach(function(fn) { - fn(null, { - jobs: jobs - }); - }); - - remaining = false; - } - } - - // public API - return { - // add a job to the queue - push: function(fn) { - preCheck(); - - remaining++; - jobs++; - - // pass an anonymous function to the callback - // call it to mark a job as done - var called = false; - fn(function() { - if (!called) { - called = true; - remaining--; - postCheck(); - } - }); - - return this; - }, - - // call given function when all jobs added to the queue are done - done: function(fn, scope) { - doneFn.push(fn.bind(scope)); - postCheck(); - - return this; - } - }; -} - -module.exports = queue; diff --git a/lib/stats.js b/lib/stats.js deleted file mode 100644 index 44f569c0c..000000000 --- a/lib/stats.js +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Metrics stats implementation to be used by reporters - * when phantomas is executed in multiple runs mode - */ -'use strict'; - -var Stats = require('./fast-stats').Stats; - -function stats() { - /* jshint validthis: true */ - // use pushMetrics() to add results for each run - this.metrics = []; - this.runs = 0; - - // stats to be calculated - // @see https://github.com/bluesmoon/node-faststats - this.stats = { - min: function(values) { - return values.range()[0]; - }, - max: function(values) { - return values.range()[1]; - }, - average: function(values) { - return values.amean().toFixed(2); - }, - median: function(values) { - return values.median().toFixed(2); - }, - stddev: function(values) { - return values.stddev().toFixed(2); - } - }; - - this.statsNames = Object.keys(this.stats); -} - -// add key/value collection of metrics values -stats.prototype.pushMetrics = function(metrics) { - this.metrics.push(metrics); - this.runs++; -}; - -// get list of metrics name -stats.prototype.getMetrics = function() { - if (this.runs > 0) { - return Object.keys(this.metrics[0]); - } else { - return []; - } -}; - -// get stats for given metric -stats.prototype.getMetricStats = function(metricName) { - var i, - stats = {}, - value, - values = new Stats(); - - for (i = 0; i < this.runs; i++) { - value = this.metrics[i][metricName]; - - if (typeof value === 'number') { - values.push(this.metrics[i][metricName]); - } - } - - // apply stats functions - this.getAvailableStats().forEach(function(fnName) { - stats[fnName] = parseFloat(this.stats[fnName](values)); - }, this); - - return stats; -}; - -stats.prototype.getAvailableStats = function() { - return this.statsNames; -}; - -module.exports = stats; From eda86029637115507450f726be6bbe699d4c8c39 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 17:29:38 +0100 Subject: [PATCH 006/248] Remove q module --- package-lock.json | 5 ----- package.json | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8b53128b9..2a0a049e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -686,11 +686,6 @@ "ws": "^6.1.0" } }, - "q": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" - }, "qs": { "version": "2.3.3", "resolved": "http://registry.npmjs.org/qs/-/qs-2.3.3.tgz", diff --git a/package.json b/package.json index 2979eeb77..86d147639 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,7 @@ }, "dependencies": { "debug": "^4.1.1", - "puppeteer": "^1.11.0", - "q": "^1.5.0" + "puppeteer": "^1.11.0" }, "devDependencies": { "glob": "^7.1.2", From cb94c16983f2057d61250e9266e33d3d6a9c8492 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 18:17:16 +0100 Subject: [PATCH 007/248] browser: emit more events --- examples/promise.js | 16 +++++++------ lib/browser.js | 57 +++++++++++++++++++++++++++++++++++++++------ lib/index.js | 7 ++++-- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/examples/promise.js b/examples/promise.js index 1106b66cf..b6b3ff0bc 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -12,14 +12,14 @@ const promise = phantomas('http://google.is', { 'assert-requests': 1 }); -console.log('Results: %s', promise); +//console.log('Results: %s', promise); // metrics metadata -console.log('Number of available metrics: %d', phantomas.metadata.metricsCount); +//console.log('Number of available metrics: %d', phantomas.metadata.metricsCount); // handle the promise promise. - then(function(res) { + then(res => { console.log('Resolved: %s', res); /** console.log('Exit code: %d', res.code); @@ -27,22 +27,24 @@ promise. console.log('Failed asserts: %j', res.results.getFailedAsserts()); **/ }). - catch(function(res) { + catch(res => { console.error(res); console.log('Error code #%d', res.code); process.exit(res.code); }); // events handling -promise.on('milestone', function(milestone) { +//promise.on('init', (browser, page) => console.log('Init', browser, page)); + +promise.on('milestone', milestone => { console.log('Milestone reached: %s', milestone); }); -promise.on('recv', function(response) { +promise.on('recv', response => { console.log('Response #%d: %s %s [HTTP %d]', response.id, response.method, response.url, response.status); }); // including the custom once emitted by phantomas modules -promise.on('domQuery', function(type, query) { +promise.on('domQuery', (type, query) => { console.log('DOM query by %s - "%s"', type, query); }); diff --git a/lib/browser.js b/lib/browser.js index 7e47762b6..f83a47164 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -4,20 +4,63 @@ const debug = require('debug')('phantomas:browser'), puppeteer = require("puppeteer"); -function browser() { - this.events = require('events').EventEmitter; -} +function browser() {} + +// use the provided events emitter +browser.prototype.bind = events => this.events = events; +// initialize puppeter instance browser.prototype.init = async () => { debug('Launching Puppeter'); - + const args = []; // ['--no-sandbox']; - const browser = await puppeteer.launch({args: args}); + this.browser = await puppeteer.launch({args: args}); + this.page = await this.browser.newPage(); + + this.events.emit('init', this.browser, this.page); // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion - debug('Browser: %s', await browser.version()); + debug('Browser: %s', await this.browser.version()); + debug('Viewport:', await this.page.viewport()); + + // bind events + this.page.on('console', msg => { + debug('console.log:', msg.text()); + this.events.emit('consoleLog', msg); + }); + + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-request + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request + this.page.on('request', req => { + debug('> [%s] %s %s', req.resourceType(), req.method(), req.url()); + + this.events.emit('request', req); + }); + + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-response + this.page.on('response', async resp => { + debug('< HTTP %s %s', resp.status(), resp.url()); + + this.events.emit('response', resp); + }); +}; + +// visit the page and emit all necessary events +browser.prototype.visit = async url => { + debug('Go to URL: <%s>', url); + await this.page.goto(url); + debug('URL opened: <%s>', url); + + const metrics = await this.page.metrics(); + debug('Metrics: %s', JSON.stringify(metrics)); + + this.events.emit('metrics', metrics); +} - await browser.close(); +// we're done +browser.prototype.close = async () => { + await this.browser.close(); + this.events.emit('close'); }; module.exports = browser; diff --git a/lib/index.js b/lib/index.js index e6954f094..31d547545 100644 --- a/lib/index.js +++ b/lib/index.js @@ -4,7 +4,6 @@ 'use strict'; var debug = require('debug')('phantomas:core'), - emitter = require('events').EventEmitter, puppeteer = require("puppeteer"), path = require('path'), util = require('util'), @@ -14,7 +13,7 @@ var debug = require('debug')('phantomas:core'), * Main CommonJS module entry point */ function phantomas(url, opts) { - var events = new emitter(), + var events = new (require('events').EventEmitter)(), browser, options; debug('OS: %s %s', process.platform, process.arch); @@ -31,11 +30,15 @@ function phantomas(url, opts) { // set up Puppeteer browser = new (require('./browser')); + browser.bind(events); // promise handling var promise = new Promise(async (resolve, reject) => { try { await browser.init(); + await browser.visit(url); + await browser.close(); + resolve('foo'); } catch(ex) { From 34bcbc4e9fdeb4639095109a2b6734387e65aaaf Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 21:05:06 +0100 Subject: [PATCH 008/248] Resolve a promise with a results wrapper --- examples/promise.js | 6 ++---- lib/browser.js | 41 ++++++++++++++++++++++++++++++++++------- lib/index.js | 15 +++++++++++++-- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/examples/promise.js b/examples/promise.js index b6b3ff0bc..b28b6476f 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -20,12 +20,10 @@ const promise = phantomas('http://google.is', { // handle the promise promise. then(res => { - console.log('Resolved: %s', res); - /** - console.log('Exit code: %d', res.code); + console.log('Resolved', res); + console.log('Number of requests: %d', res.results.getMetric('requests')); console.log('Failed asserts: %j', res.results.getFailedAsserts()); - **/ }). catch(res => { console.error(res); diff --git a/lib/browser.js b/lib/browser.js index f83a47164..6f7eec0f4 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -6,7 +6,10 @@ const debug = require('debug')('phantomas:browser'), function browser() {} -// use the provided events emitter +/** + * Use the provided events emitter + * @param {EventEmitter} events + */ browser.prototype.bind = events => this.events = events; // initialize puppeter instance @@ -20,7 +23,7 @@ browser.prototype.init = async () => { this.events.emit('init', this.browser, this.page); // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion - debug('Browser: %s', await this.browser.version()); + debug('Browser: %s', await this.browser.userAgent()); debug('Viewport:', await this.page.viewport()); // bind events @@ -29,23 +32,47 @@ browser.prototype.init = async () => { this.events.emit('consoleLog', msg); }); - // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-request - // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request + /** + * Emit an event when browser makes a request + * + * @param {Puppeteer.Request} req + * + * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-request + * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request + */ this.page.on('request', req => { debug('> [%s] %s %s', req.resourceType(), req.method(), req.url()); this.events.emit('request', req); }); - // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-response + /** + * Emit an event when browser makes a request + * + * @param {Puppeteer.Response} resp + * + * https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-response + */ this.page.on('response', async resp => { - debug('< HTTP %s %s', resp.status(), resp.url()); + var responseLength = 0; + + try { + const buffer = await resp.buffer(); + responseLength = buffer.length; + } + catch {}; + + debug('< HTTP %s %s (%f kB)', resp.status(), resp.url(), 1. * responseLength / 1024); this.events.emit('response', resp); }); }; -// visit the page and emit all necessary events +/** + * Opens the provided URL and emits all necessary events + * + * @param {string} url + */ browser.prototype.visit = async url => { debug('Go to URL: <%s>', url); await this.page.goto(url); diff --git a/lib/index.js b/lib/index.js index 31d547545..1554c8561 100644 --- a/lib/index.js +++ b/lib/index.js @@ -6,11 +6,16 @@ var debug = require('debug')('phantomas:core'), puppeteer = require("puppeteer"), path = require('path'), + Results = require('../core/results'), util = require('util'), VERSION = require('./../package').version; /** * Main CommonJS module entry point + * + * @param {string} url + * @param {Object} opts + * @returns {browser} */ function phantomas(url, opts) { var events = new (require('events').EventEmitter)(), @@ -37,9 +42,15 @@ function phantomas(url, opts) { try { await browser.init(); await browser.visit(url); - await browser.close(); - resolve('foo'); + // build results + const json = false; // TODO + var results = new Results(json); + + results.setUrl(url); + + await browser.close(); + resolve({json, results}); } catch(ex) { reject(ex); From d26844acc132f82b2d1171ab6680ff552879d157 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 21:19:05 +0100 Subject: [PATCH 009/248] Linting fixes --- lib/browser.js | 2 +- lib/index.js | 4 ++-- package.json | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 6f7eec0f4..ad6cf7f48 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -24,7 +24,7 @@ browser.prototype.init = async () => { // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion debug('Browser: %s', await this.browser.userAgent()); - debug('Viewport:', await this.page.viewport()); + debug('Viewport: %j', await this.page.viewport()); // bind events this.page.on('console', msg => { diff --git a/lib/index.js b/lib/index.js index 1554c8561..628a2fbf9 100644 --- a/lib/index.js +++ b/lib/index.js @@ -31,10 +31,10 @@ function phantomas(url, opts) { options = util._extend({}, opts || {}); // use util._extend to avoid #563 options.url = options.url || url || false; - debug('Environment: %j', process.env); + //debug('Environment: %j', process.env); // set up Puppeteer - browser = new (require('./browser')); + browser = new (require('./browser'))(); browser.bind(events); // promise handling diff --git a/package.json b/package.json index 86d147639..3dd5758b3 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ }, "jshintConfig": { "node": true, + "esversion": 6, "-W020": false, "-W030": false } From ba0c1f5e35cc66871898ef7af10605bf0d1282c3 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 22:42:42 +0100 Subject: [PATCH 010/248] Create a small scope object and pass it to a module --- examples/promise.js | 8 +- lib/fast-stats.js | 611 ++++++++++++++++++++++++++++++++++++++++++++ lib/index.js | 35 ++- 3 files changed, 637 insertions(+), 17 deletions(-) create mode 100644 lib/fast-stats.js diff --git a/examples/promise.js b/examples/promise.js index b28b6476f..cf4536822 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -19,11 +19,9 @@ const promise = phantomas('http://google.is', { // handle the promise promise. - then(res => { - console.log('Resolved', res); - - console.log('Number of requests: %d', res.results.getMetric('requests')); - console.log('Failed asserts: %j', res.results.getFailedAsserts()); + then(results => { + console.log('Number of requests: %d', results.getMetric('requests')); + console.log('Failed asserts: %j', results.getFailedAsserts()); }). catch(res => { console.error(res); diff --git a/lib/fast-stats.js b/lib/fast-stats.js new file mode 100644 index 000000000..53ba0c945 --- /dev/null +++ b/lib/fast-stats.js @@ -0,0 +1,611 @@ +/* jshint -W004: false, -W014: false, -W041: false */ +/** + * Taken from https://github.com/bluesmoon/node-faststats + */ + +/* +Note that if your data is too large, there _will_ be overflow. +*/ + + +function asc(a, b) { + return a - b; +} + +var config_params = { + bucket_precision: function(o, s) { + if (typeof s != "number" || s <= 0) { + throw new Error("bucket_precision must be a positive number"); + } + o._config.bucket_precision = s; + o.buckets = []; + }, + + buckets: function(o, b) { + if (!Array.isArray(b) || b.length == 0) { + throw new Error("buckets must be an array of bucket limits"); + } + + o._config.buckets = b; + o.buckets = []; + }, + + bucket_extension_interval: function(o, s) { + if (s === undefined) + return; + if (typeof s != "number" || s <= 0) { + throw new Error("bucket_extension_interval must be a positive number"); + } + o._config.bucket_extension_interval = s; + }, + + store_data: function(o, s) { + if (typeof s != "boolean") { + throw new Error("store_data must be a true or false"); + } + o._config.store_data = s; + }, + + sampling: function(o, s) { + if (typeof s != "boolean") { + throw new Error("sampling must be a true or false"); + } + o._config.sampling = s; + } +}; + +function Stats(c) { + this._config = { + store_data: true + }; + + if (c) { + for (var k in config_params) { + if (c.hasOwnProperty(k)) { + config_params[k](this, c[k]); + } + } + } + + this.reset(); + + return this; +} + +Stats.prototype = { + + reset: function() { + if (this._config.store_data) + this.data = []; + + this.length = 0; + + this.sum = 0; + this.sum_of_squares = 0; + this.sum_of_logs = 0; + this.sum_of_square_of_logs = 0; + this.zeroes = 0; + this.max = this.min = null; + + this._reset_cache(); + + return this; + }, + + _reset_cache: function() { + this._stddev = null; + + if (this._config.store_data) + this._data_sorted = null; + }, + + _find_bucket: function(a) { + var b = 0, + e, l; + if (this._config.buckets) { + l = this._config.buckets.length; + if (this._config.bucket_extension_interval && a >= this._config.buckets[l - 1]) { + e = a - this._config.buckets[l - 1]; + b = parseInt(e / this._config.bucket_extension_interval) + l; + if (this._config.buckets[b] === undefined) + this._config.buckets[b] = this._config.buckets[l - 1] + (parseInt(e / this._config.bucket_extension_interval) + 1) * this._config.bucket_extension_interval; + if (this._config.buckets[b - 1] === undefined) + this._config.buckets[b - 1] = this._config.buckets[l - 1] + parseInt(e / this._config.bucket_extension_interval) * this._config.bucket_extension_interval; + } + for (; b < l; b++) { + if (a < this._config.buckets[b]) { + break; + } + } + } else if (this._config.bucket_precision) { + b = Math.floor(a / this._config.bucket_precision); + } + + return b; + }, + + _add_cache: function(a) { + var tuple = [1], + i; + if (a instanceof Array) { + tuple = a; + a = tuple.shift(); + } + + this.sum += a * tuple[0]; + this.sum_of_squares += a * a * tuple[0]; + if (a === 0) { + this.zeroes++; + } else { + this.sum_of_logs += Math.log(a) * tuple[0]; + this.sum_of_square_of_logs += Math.pow(Math.log(a), 2) * tuple[0]; + } + this.length += tuple[0]; + + if (tuple[0] > 0) { + if (this.max === null || this.max < a) + this.max = a; + if (this.min === null || this.min > a) + this.min = a; + } + + if (this.buckets) { + var b = this._find_bucket(a); + if (!this.buckets[b]) + this.buckets[b] = [0]; + this.buckets[b][0] += tuple.shift(); + + for (i = 0; i < tuple.length; i++) + this.buckets[b][i + 1] = (this.buckets[b][i + 1] | 0) + (tuple[i] | 0); + } + + this._reset_cache(); + }, + + _del_cache: function(a) { + var tuple = [1], + i; + if (a instanceof Array) { + tuple = a; + a = tuple.shift(); + } + + this.sum -= a * tuple[0]; + this.sum_of_squares -= a * a * tuple[0]; + if (a === 0) { + this.zeroes--; + } else { + this.sum_of_logs -= Math.log(a) * tuple[0]; + this.sum_of_square_of_logs -= Math.pow(Math.log(a), 2) * tuple[0]; + } + this.length -= tuple[0]; + + if (this._config.store_data) { + if (this.length === 0) { + this.max = this.min = null; + } + if (this.length === 1) { + this.max = this.min = this.data[0]; + } else if (tuple[0] > 0 && (this.max === a || this.min === a)) { + var i = this.length - 1; + if (i >= 0) { + this.max = this.min = this.data[i--]; + while (i-- >= 0) { + if (this.max < this.data[i]) + this.max = this.data[i]; + if (this.min > this.data[i]) + this.min = this.data[i]; + } + } + } + } + + if (this.buckets) { + var b = this._find_bucket(a); + if (this.buckets[b]) { + this.buckets[b][0] -= tuple.shift(); + + if (this.buckets[b][0] === 0) + delete this.buckets[b]; + else + for (i = 0; i < tuple.length; i++) + this.buckets[b][i + 1] = (this.buckets[b][i + 1] | 0) - (tuple[i] | 0); + } + } + + this._reset_cache(); + }, + + push: function() { + var i, a, args = Array.prototype.slice.call(arguments, 0); + if (args.length && args[0] instanceof Array) + args = args[0]; + for (i = 0; i < args.length; i++) { + a = args[i]; + if (this._config.store_data) + this.data.push(a); + this._add_cache(a); + } + + return this; + }, + + push_tuple: function(tuple) { + if (!this.buckets) { + throw new Error("push_tuple is only valid when using buckets"); + } + this._add_cache(tuple); + }, + + pop: function() { + if (this.length === 0 || this._config.store_data === false) + return undefined; + + var a = this.data.pop(); + this._del_cache(a); + + return a; + }, + + remove_tuple: function(tuple) { + if (!this.buckets) { + throw new Error("remove_tuple is only valid when using buckets"); + } + this._del_cache(tuple); + }, + + reset_tuples: function(tuple) { + var b, l, t, ts = tuple.length; + if (!this.buckets) { + throw new Error("reset_tuple is only valid when using buckets"); + } + + for (b = 0, l = this.buckets.length; b < l; b++) { + if (!this.buckets[b] || this.buckets[b].length <= 1) { + continue; + } + for (t = 0; t < ts; t++) { + if (typeof tuple[t] !== 'undefined') { + this.buckets[b][t] = tuple[t]; + } + } + } + }, + + unshift: function() { + var i, a, args = Array.prototype.slice.call(arguments, 0); + if (args.length && args[0] instanceof Array) + args = args[0]; + i = args.length; + while (i--) { + a = args[i]; + if (this._config.store_data) + this.data.unshift(a); + this._add_cache(a); + } + + return this; + }, + + shift: function() { + if (this.length === 0 || this._config.store_data === false) + return undefined; + + var a = this.data.shift(); + this._del_cache(a); + + return a; + }, + + amean: function() { + if (this.length === 0) + return NaN; + return this.sum / this.length; + }, + + gmean: function() { + if (this.length === 0) + return NaN; + if (this.zeroes > 0) + return NaN; + return Math.exp(this.sum_of_logs / this.length); + }, + + stddev: function() { + if (this.length === 0) + return NaN; + var n = this.length; + if (this._config.sampling) + n--; + if (this._stddev === null) + this._stddev = Math.sqrt((this.length * this.sum_of_squares - this.sum * this.sum) / (this.length * n)); + + return this._stddev; + }, + + gstddev: function() { + if (this.length === 0) + return NaN; + if (this.zeroes > 0) + return NaN; + var n = this.length; + if (this._config.sampling) + n--; + return Math.exp(Math.sqrt((this.length * this.sum_of_square_of_logs - this.sum_of_logs * this.sum_of_logs) / (this.length * n))); + }, + + moe: function() { + if (this.length === 0) + return NaN; + // see http://en.wikipedia.org/wiki/Standard_error_%28statistics%29 + return 1.96 * this.stddev() / Math.sqrt(this.length); + }, + + range: function() { + if (this.length === 0) + return [NaN, NaN]; + return [this.min, this.max]; + }, + + distribution: function() { + if (this.length === 0) + return []; + if (!this.buckets) + throw new Error("bucket_precision or buckets not configured."); + + var d = [], + i, j, k, l; + + if (this._config.buckets) { + j = this.min; + l = Math.min(this.buckets.length, this._config.buckets.length); + + for (i = 0; i < l; j = this._config.buckets[i++]) { // this has to be i++ and not ++i + if (this._config.buckets[i] === undefined && this._config.bucket_extension_interval) + this._config.buckets[i] = this._config.buckets[i - 1] + this._config.bucket_extension_interval; + if (this.min > this._config.buckets[i]) + continue; + + d[i] = { + bucket: (j + this._config.buckets[i]) / 2, + range: [j, this._config.buckets[i]], + count: (this.buckets[i] ? this.buckets[i][0] : 0), + tuple: this.buckets[i] ? this.buckets[i].slice(1) : [] + }; + + if (this.max < this._config.buckets[i]) + break; + } + if (i == l && this.buckets[i]) { + d[i] = { + bucket: (j + this.max) / 2, + range: [j, this.max], + count: this.buckets[i][0], + tuple: this.buckets[i] ? this.buckets[i].slice(1) : [] + }; + } + } else if (this._config.bucket_precision) { + i = Math.floor(this.min / this._config.bucket_precision); + l = Math.floor(this.max / this._config.bucket_precision) + 1; + for (j = 0; i < l && i < this.buckets.length; i++, j++) { + if (!this.buckets[i]) { + continue; + } + d[j] = { + bucket: (i + 0.5) * this._config.bucket_precision, + range: [i * this._config.bucket_precision, (i + 1) * this._config.bucket_precision], + count: this.buckets[i][0], + tuple: this.buckets[i] ? this.buckets[i].slice(1) : [] + }; + } + } + + return d; + + }, + + percentile: function(p) { + if (this.length === 0 || (!this._config.store_data && !this.buckets)) + return NaN; + + // If we come here, we either have sorted data or sorted buckets + + var v; + + if (p <= 0) + v = 0; + else if (p == 25) + v = [Math.floor((this.length - 1) * 0.25), Math.ceil((this.length - 1) * 0.25)]; + else if (p == 50) + v = [Math.floor((this.length - 1) * 0.5), Math.ceil((this.length - 1) * 0.5)]; + else if (p == 75) + v = [Math.floor((this.length - 1) * 0.75), Math.ceil((this.length - 1) * 0.75)]; + else if (p >= 100) + v = this.length - 1; + else + v = Math.floor(this.length * p / 100); + + if (v === 0) + return this.min; + if (v === this.length - 1) + return this.max; + + if (this._config.store_data) { + if (this._data_sorted === null) + this._data_sorted = this.data.slice(0).sort(asc); + + if (typeof v == 'number') + return this._data_sorted[v]; + else + return (this._data_sorted[v[0]] + this._data_sorted[v[1]]) / 2; + } else { + var j; + if (typeof v != 'number') + v = (v[0] + v[1]) / 2; + + if (this._config.buckets) + j = 0; + else if (this._config.bucket_precision) + j = Math.floor(this.min / this._config.bucket_precision); + + for (; j < this.buckets.length; j++) { + if (!this.buckets[j]) + continue; + if (v <= this.buckets[j][0]) { + break; + } + v -= this.buckets[j][0]; + } + + return this._get_nth_in_bucket(v, j); + } + }, + + _get_nth_in_bucket: function(n, b) { + var range = []; + if (this._config.buckets) { + range[0] = (b > 0 ? this._config.buckets[b - 1] : this.min); + range[1] = (b < this._config.buckets.length ? this._config.buckets[b] : this.max); + } else if (this._config.bucket_precision) { + range[0] = Math.max(b * this._config.bucket_precision, this.min); + range[1] = Math.min((b + 1) * this._config.bucket_precision, this.max); + } + return range[0] + (range[1] - range[0]) * n / this.buckets[b][0]; + }, + + median: function() { + return this.percentile(50); + }, + + iqr: function() { + var q1, q3, fw; + + q1 = this.percentile(25); + q3 = this.percentile(75); + + fw = (q3 - q1) * 1.5; + + return this.band_pass(q1 - fw, q3 + fw, true); + }, + + band_pass: function(low, high, open, config) { + var i, j, b, b_val, i_val; + + if (!config) + config = this._config; + + b = new Stats(config); + + if (this.length === 0) + return b; + + if (this._config.store_data) { + if (this._data_sorted === null) + this._data_sorted = this.data.slice(0).sort(asc); + + for (i = 0; i < this.length && (this._data_sorted[i] < high || (!open && this._data_sorted[i] === high)); i++) { + if (this._data_sorted[i] > low || (!open && this._data_sorted[i] === low)) { + b.push(this._data_sorted[i]); + } + } + } else if (this._config.buckets) { + for (i = 0; i <= this._config.buckets.length; i++) { + if (this._config.buckets[i] < this.min) + continue; + + b_val = (i == 0 ? this.min : this._config.buckets[i - 1]); + if (b_val < this.min) + b_val = this.min; + if (b_val > this.max) + b_val = this.max; + + if (high < b_val || (open && high === b_val)) { + break; + } + if (low < b_val || (!open && low === b_val)) { + for (j = 0; j < (this.buckets[i] ? this.buckets[i][0] : 0); j++) { + i_val = Stats.prototype._get_nth_in_bucket.call(this, j, i); + if ((i_val > low || (!open && i_val === low)) && (i_val < high || (!open && i_val === high))) { + b.push(i_val); + } + } + } + } + + b.min = Math.max(low, b.min); + b.max = Math.min(high, b.max); + } else if (this._config.bucket_precision) { + var low_i = Math.floor(low / this._config.bucket_precision), + high_i = Math.floor(high / this._config.bucket_precision) + 1; + + for (i = low_i; i < Math.min(this.buckets.length, high_i); i++) { + for (j = 0; j < (this.buckets[i] ? this.buckets[i][0] : 0); j++) { + i_val = Stats.prototype._get_nth_in_bucket.call(this, j, i); + if ((i_val > low || (!open && i_val === low)) && (i_val < high || (!open && i_val === high))) { + b.push(i_val); + } + } + } + + b.min = Math.max(low, b.min); + b.max = Math.min(high, b.max); + } + + return b; + }, + + copy: function(config) { + var b = Stats.prototype.band_pass.call(this, this.min, this.max, false, config); + + b.sum = this.sum; + b.sum_of_squares = this.sum_of_squares; + b.sum_of_logs = this.sum_of_logs; + b.sum_of_square_of_logs = this.sum_of_square_of_logs; + b.zeroes = this.zeroes; + + return b; + }, + + Σ: function() { + return this.sum; + }, + + Π: function() { + return this.zeroes > 0 ? 0 : Math.exp(this.sum_of_logs); + } +}; + +Stats.prototype.σ = Stats.prototype.stddev; +Stats.prototype.μ = Stats.prototype.amean; + + +exports.Stats = Stats; + +/** +if(process.argv[1] && process.argv[1].match(__filename)) { + var s = new Stats({store_data:false, buckets: [ 1, 5, 10, 15, 20, 25, 30, 35 ]}).push(1, 2, 3); + var l = process.argv.slice(2); + if(!l.length) l = [10, 11, 15, 8, 13, 12, 19, 32, 17, 16]; + l.forEach(function(e, i, a) { a[i] = parseFloat(e, 10); }); + Stats.prototype.push.apply(s, l); + console.log(s.data); + console.log(s.amean().toFixed(2), s.μ().toFixed(2), s.stddev().toFixed(2), s.σ().toFixed(2), s.gmean().toFixed(2), s.median().toFixed(2), s.moe().toFixed(2), s.distribution()); + var t=s.copy({buckets: [0, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 25, 30, 35] }); + console.log(t.amean().toFixed(2), t.μ().toFixed(2), t.stddev().toFixed(2), t.σ().toFixed(2), t.gmean().toFixed(2), t.median().toFixed(2), t.moe().toFixed(2), t.distribution()); + + s = new Stats({store_data: false, buckets: [1, 5, 10, 15, 20, 25, 30, 35]}); + s.push_tuple([1, 1, 3, 4]); + s.push_tuple([2, 1, 5, 8]); + s.push_tuple([3, 1, 4, 9]); + s.push_tuple([1, 1, 13, 14]); + + console.log(s.amean(), s.median()); + console.log(s.distribution()); + + s.remove_tuple([1, 1, 3, 4]); + s.push_tuple([4, 1, 3, 3]); + console.log(s.amean(), s.median()); + console.log(s.distribution()); + +} +**/ diff --git a/lib/index.js b/lib/index.js index 628a2fbf9..62f879f1e 100644 --- a/lib/index.js +++ b/lib/index.js @@ -3,7 +3,8 @@ */ 'use strict'; -var debug = require('debug')('phantomas:core'), +var EventEmitter = require('events').EventEmitter, + debug = require('debug')('phantomas:core'), puppeteer = require("puppeteer"), path = require('path'), Results = require('../core/results'), @@ -18,12 +19,13 @@ var debug = require('debug')('phantomas:core'), * @returns {browser} */ function phantomas(url, opts) { - var events = new (require('events').EventEmitter)(), + var events = new EventEmitter(), browser, options; debug('OS: %s %s', process.platform, process.arch); debug('Node.js: %s', process.version); - debug('Puppeteer: preferred revision, r%s installed in %s', puppeteer._launcher._preferredRevision, puppeteer._projectRoot); + debug('phantomas: %s', VERSION); + debug('Puppeteer: preferred revision r%s', puppeteer._launcher._preferredRevision); debug('URL: <%s>', url); debug('Options: %s', JSON.stringify(opts)); @@ -31,26 +33,35 @@ function phantomas(url, opts) { options = util._extend({}, opts || {}); // use util._extend to avoid #563 options.url = options.url || url || false; - //debug('Environment: %j', process.env); + // TODO: prepare a small instance object that will be passed to modules and extensions on init + var results = new Results(); + results.setUrl(url); - // set up Puppeteer + var scope = { + on: events.on.bind(events), + setMetric: results.setMetric, + log: require('debug')('phantomas:module') + }; + + // TODO: set up modules and extensions + var module = require('../modules/requestsStats/requestsStats'); + module.module(scope); + + // set up and run Puppeteer browser = new (require('./browser'))(); browser.bind(events); - // promise handling var promise = new Promise(async (resolve, reject) => { try { await browser.init(); await browser.visit(url); - // build results - const json = false; // TODO - var results = new Results(json); - - results.setUrl(url); + // your last chance to add metrics + events.emit('report'); + // resolve our run await browser.close(); - resolve({json, results}); + resolve(results); } catch(ex) { reject(ex); From ee48d67b0e8b83a050ad519d57d848560d0c97c9 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 22:47:38 +0100 Subject: [PATCH 011/248] Simplify modules export structure --- lib/index.js | 5 ++--- modules/requestsStats/requestsStats.js | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/index.js b/lib/index.js index 62f879f1e..029c81e16 100644 --- a/lib/index.js +++ b/lib/index.js @@ -40,12 +40,11 @@ function phantomas(url, opts) { var scope = { on: events.on.bind(events), setMetric: results.setMetric, - log: require('debug')('phantomas:module') + log: require('debug')('phantomas:modules:requestsStats') }; // TODO: set up modules and extensions - var module = require('../modules/requestsStats/requestsStats'); - module.module(scope); + require('../modules/requestsStats/requestsStats')(scope); // set up and run Puppeteer browser = new (require('./browser'))(); diff --git a/modules/requestsStats/requestsStats.js b/modules/requestsStats/requestsStats.js index c7e1a7a57..9b3840c82 100644 --- a/modules/requestsStats/requestsStats.js +++ b/modules/requestsStats/requestsStats.js @@ -12,11 +12,9 @@ */ 'use strict'; -exports.version = '0.3'; - var Stats = require('../../lib/fast-stats').Stats; -exports.module = function(phantomas) { +module.exports = function(phantomas) { var stack = {}; // adds given entry under the "type" if given check function returns true From b8920f472661a54afecc8074b14a7f83b756fff3 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 27 Dec 2018 22:54:09 +0100 Subject: [PATCH 012/248] phantomas.emit - log events --- lib/index.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/index.js b/lib/index.js index 029c81e16..c259cc57b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -33,6 +33,14 @@ function phantomas(url, opts) { options = util._extend({}, opts || {}); // use util._extend to avoid #563 options.url = options.url || url || false; + // log events emitted + events._emit = events.emit; + events.log = require('debug')('phantomas:events'); + events.emit = function() { + this.log('emit: %s', arguments[0]); + this._emit.apply(this, arguments); + } + // TODO: prepare a small instance object that will be passed to modules and extensions on init var results = new Results(); results.setUrl(url); From 827e15f28745cdfadd8178fdca2b20a07149185f Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 12:34:05 +0100 Subject: [PATCH 013/248] Browser: log a path to Chromium binary --- lib/browser.js | 14 ++++++++------ lib/index.js | 5 +++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index ad6cf7f48..2e5c15fac 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -4,16 +4,16 @@ const debug = require('debug')('phantomas:browser'), puppeteer = require("puppeteer"); -function browser() {} +function Browser() {} /** * Use the provided events emitter * @param {EventEmitter} events */ -browser.prototype.bind = events => this.events = events; +Browser.prototype.bind = events => this.events = events; // initialize puppeter instance -browser.prototype.init = async () => { +Browser.prototype.init = async () => { debug('Launching Puppeter'); const args = []; // ['--no-sandbox']; @@ -22,6 +22,8 @@ browser.prototype.init = async () => { this.events.emit('init', this.browser, this.page); + debug('Using binary from: %s', this.browser.process().spawnfile); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion debug('Browser: %s', await this.browser.userAgent()); debug('Viewport: %j', await this.page.viewport()); @@ -73,7 +75,7 @@ browser.prototype.init = async () => { * * @param {string} url */ -browser.prototype.visit = async url => { +Browser.prototype.visit = async url => { debug('Go to URL: <%s>', url); await this.page.goto(url); debug('URL opened: <%s>', url); @@ -85,9 +87,9 @@ browser.prototype.visit = async url => { } // we're done -browser.prototype.close = async () => { +Browser.prototype.close = async () => { await this.browser.close(); this.events.emit('close'); }; -module.exports = browser; +module.exports = Browser; diff --git a/lib/index.js b/lib/index.js index c259cc57b..1c378797c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -3,7 +3,8 @@ */ 'use strict'; -var EventEmitter = require('events').EventEmitter, +var Browser = require('./browser'), + EventEmitter = require('events').EventEmitter, debug = require('debug')('phantomas:core'), puppeteer = require("puppeteer"), path = require('path'), @@ -55,7 +56,7 @@ function phantomas(url, opts) { require('../modules/requestsStats/requestsStats')(scope); // set up and run Puppeteer - browser = new (require('./browser'))(); + browser = new Browser(); browser.bind(events); var promise = new Promise(async (resolve, reject) => { From f9e54314ef584adbf822c9a6b9d2c0cc9b5d153d Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 12:35:40 +0100 Subject: [PATCH 014/248] Remove old code required by PhantomJS layer --- lib/modules/_coffee-script.js | 2 - lib/modules/assert.js | 326 ------------------ lib/modules/events.js | 216 ------------ lib/modules/http.js | 55 --- lib/modules/path.js | 441 ------------------------ lib/modules/punycode.js | 510 --------------------------- lib/modules/querystring.js | 214 ------------ lib/modules/tty.js | 7 - lib/modules/url.js | 625 ---------------------------------- lib/modules/util.js | 520 ---------------------------- 10 files changed, 2916 deletions(-) delete mode 100644 lib/modules/_coffee-script.js delete mode 100644 lib/modules/assert.js delete mode 100644 lib/modules/events.js delete mode 100644 lib/modules/http.js delete mode 100644 lib/modules/path.js delete mode 100644 lib/modules/punycode.js delete mode 100644 lib/modules/querystring.js delete mode 100644 lib/modules/tty.js delete mode 100644 lib/modules/url.js delete mode 100644 lib/modules/util.js diff --git a/lib/modules/_coffee-script.js b/lib/modules/_coffee-script.js deleted file mode 100644 index a385606b4..000000000 --- a/lib/modules/_coffee-script.js +++ /dev/null @@ -1,2 +0,0 @@ -require.stub('vm'); -module.exports = require('coffee-script'); diff --git a/lib/modules/assert.js b/lib/modules/assert.js deleted file mode 100644 index 8340d08e7..000000000 --- a/lib/modules/assert.js +++ /dev/null @@ -1,326 +0,0 @@ -// http://wiki.commonjs.org/wiki/Unit_Testing/1.0 -// -// THIS IS NOT TESTED NOR LIKELY TO WORK OUTSIDE V8! -// -// Originally from narwhal.js (http://narwhaljs.org) -// Copyright (c) 2009 Thomas Robinson <280north.com> -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the 'Software'), to -// deal in the Software without restriction, including without limitation the -// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -// sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// UTILITY -var util = require('util'); -var pSlice = Array.prototype.slice; - -// 1. The assert module provides functions that throw -// AssertionError's when particular conditions are not met. The -// assert module must conform to the following interface. - -var assert = module.exports = ok; - -// 2. The AssertionError is defined in assert. -// new assert.AssertionError({ message: message, -// actual: actual, -// expected: expected }) - -assert.AssertionError = function AssertionError(options) { - this.name = 'AssertionError'; - this.message = options.message; - this.actual = options.actual; - this.expected = options.expected; - this.operator = options.operator; - var stackStartFunction = options.stackStartFunction || fail; - - if (Error.captureStackTrace) { - Error.captureStackTrace(this, stackStartFunction); - } -}; -util.inherits(assert.AssertionError, Error); - -function replacer(key, value) { - if (value === undefined) { - return '' + value; - } - if (typeof value === 'number' && (isNaN(value) || !isFinite(value))) { - return value.toString(); - } - if (typeof value === 'function' || value instanceof RegExp) { - return value.toString(); - } - return value; -} - -function truncate(s, n) { - if (typeof s == 'string') { - return s.length < n ? s : s.slice(0, n); - } else { - return s; - } -} - -assert.AssertionError.prototype.toString = function() { - if (this.message) { - return [this.name + ':', this.message].join(' '); - } else { - return [ - this.name + ':', - truncate(JSON.stringify(this.actual, replacer), 128), - this.operator, - truncate(JSON.stringify(this.expected, replacer), 128) - ].join(' '); - } -}; - -// assert.AssertionError instanceof Error - -assert.AssertionError.__proto__ = Error.prototype; - -// At present only the three keys mentioned above are used and -// understood by the spec. Implementations or sub modules can pass -// other keys to the AssertionError's constructor - they will be -// ignored. - -// 3. All of the following functions must throw an AssertionError -// when a corresponding condition is not met, with a message that -// may be undefined if not provided. All assertion methods provide -// both the actual and expected values to the assertion error for -// display purposes. - -function fail(actual, expected, message, operator, stackStartFunction) { - throw new assert.AssertionError({ - message: message, - actual: actual, - expected: expected, - operator: operator, - stackStartFunction: stackStartFunction - }); -} - -// EXTENSION! allows for well behaved errors defined elsewhere. -assert.fail = fail; - -// 4. Pure assertion tests whether a value is truthy, as determined -// by !!guard. -// assert.ok(guard, message_opt); -// This statement is equivalent to assert.equal(true, guard, -// message_opt);. To test strictly for the value true, use -// assert.strictEqual(true, guard, message_opt);. - -function ok(value, message) { - if (!!!value) fail(value, true, message, '==', assert.ok); -} -assert.ok = ok; - -// 5. The equality assertion tests shallow, coercive equality with -// ==. -// assert.equal(actual, expected, message_opt); - -assert.equal = function equal(actual, expected, message) { - if (actual != expected) fail(actual, expected, message, '==', assert.equal); -}; - -// 6. The non-equality assertion tests for whether two objects are not equal -// with != assert.notEqual(actual, expected, message_opt); - -assert.notEqual = function notEqual(actual, expected, message) { - if (actual == expected) { - fail(actual, expected, message, '!=', assert.notEqual); - } -}; - -// 7. The equivalence assertion tests a deep equality relation. -// assert.deepEqual(actual, expected, message_opt); - -assert.deepEqual = function deepEqual(actual, expected, message) { - if (!_deepEqual(actual, expected)) { - fail(actual, expected, message, 'deepEqual', assert.deepEqual); - } -}; - -function _deepEqual(actual, expected) { - // 7.1. All identical values are equivalent, as determined by ===. - if (actual === expected) { - return true; - - } else if (Buffer.isBuffer(actual) && Buffer.isBuffer(expected)) { - if (actual.length != expected.length) return false; - - for (var i = 0; i < actual.length; i++) { - if (actual[i] !== expected[i]) return false; - } - - return true; - - // 7.2. If the expected value is a Date object, the actual value is - // equivalent if it is also a Date object that refers to the same time. - } else if (actual instanceof Date && expected instanceof Date) { - return actual.getTime() === expected.getTime(); - - // 7.3. Other pairs that do not both pass typeof value == 'object', - // equivalence is determined by ==. - } else if (typeof actual != 'object' && typeof expected != 'object') { - return actual == expected; - - // 7.4. For all other Object pairs, including Array objects, equivalence is - // determined by having the same number of owned properties (as verified - // with Object.prototype.hasOwnProperty.call), the same set of keys - // (although not necessarily the same order), equivalent values for every - // corresponding key, and an identical 'prototype' property. Note: this - // accounts for both named and indexed properties on Arrays. - } else { - return objEquiv(actual, expected); - } -} - -function isUndefinedOrNull(value) { - return value === null || value === undefined; -} - -function isArguments(object) { - return Object.prototype.toString.call(object) == '[object Arguments]'; -} - -function objEquiv(a, b) { - if (isUndefinedOrNull(a) || isUndefinedOrNull(b)) - return false; - // an identical 'prototype' property. - if (a.prototype !== b.prototype) return false; - //~~~I've managed to break Object.keys through screwy arguments passing. - // Converting to array solves the problem. - if (isArguments(a)) { - if (!isArguments(b)) { - return false; - } - a = pSlice.call(a); - b = pSlice.call(b); - return _deepEqual(a, b); - } - try { - var ka = Object.keys(a), - kb = Object.keys(b), - key, i; - } catch (e) {//happens when one is a string literal and the other isn't - return false; - } - // having the same number of owned properties (keys incorporates - // hasOwnProperty) - if (ka.length != kb.length) - return false; - //the same set of keys (although not necessarily the same order), - ka.sort(); - kb.sort(); - //~~~cheap key test - for (i = ka.length - 1; i >= 0; i--) { - if (ka[i] != kb[i]) - return false; - } - //equivalent values for every corresponding key, and - //~~~possibly expensive deep test - for (i = ka.length - 1; i >= 0; i--) { - key = ka[i]; - if (!_deepEqual(a[key], b[key])) return false; - } - return true; -} - -// 8. The non-equivalence assertion tests for any deep inequality. -// assert.notDeepEqual(actual, expected, message_opt); - -assert.notDeepEqual = function notDeepEqual(actual, expected, message) { - if (_deepEqual(actual, expected)) { - fail(actual, expected, message, 'notDeepEqual', assert.notDeepEqual); - } -}; - -// 9. The strict equality assertion tests strict equality, as determined by ===. -// assert.strictEqual(actual, expected, message_opt); - -assert.strictEqual = function strictEqual(actual, expected, message) { - if (actual !== expected) { - fail(actual, expected, message, '===', assert.strictEqual); - } -}; - -// 10. The strict non-equality assertion tests for strict inequality, as -// determined by !==. assert.notStrictEqual(actual, expected, message_opt); - -assert.notStrictEqual = function notStrictEqual(actual, expected, message) { - if (actual === expected) { - fail(actual, expected, message, '!==', assert.notStrictEqual); - } -}; - -function expectedException(actual, expected) { - if (!actual || !expected) { - return false; - } - - if (expected instanceof RegExp) { - return expected.test(actual); - } else if (actual instanceof expected) { - return true; - } else if (expected.call({}, actual) === true) { - return true; - } - - return false; -} - -function _throws(shouldThrow, block, expected, message) { - var actual; - - if (typeof expected === 'string') { - message = expected; - expected = null; - } - - try { - block(); - } catch (e) { - actual = e; - } - - message = (expected && expected.name ? ' (' + expected.name + ').' : '.') + - (message ? ' ' + message : '.'); - - if (shouldThrow && !actual) { - fail('Missing expected exception' + message); - } - - if (!shouldThrow && expectedException(actual, expected)) { - fail('Got unwanted exception' + message); - } - - if ((shouldThrow && actual && expected && - !expectedException(actual, expected)) || (!shouldThrow && actual)) { - throw actual; - } -} - -// 11. Expected to throw an error: -// assert.throws(block, Error_opt, message_opt); - -assert.throws = function(block, /*optional*/error, /*optional*/message) { - _throws.apply(this, [true].concat(pSlice.call(arguments))); -}; - -// EXTENSION! This is annoying to write outside this module. -assert.doesNotThrow = function(block, /*optional*/error, /*optional*/message) { - _throws.apply(this, [false].concat(pSlice.call(arguments))); -}; - -assert.ifError = function(err) { if (err) {throw err;}}; diff --git a/lib/modules/events.js b/lib/modules/events.js deleted file mode 100644 index 21701dcff..000000000 --- a/lib/modules/events.js +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -var isArray = Array.isArray; - -function EventEmitter() { } -exports.EventEmitter = EventEmitter; - -// By default EventEmitters will print a warning if more than -// 10 listeners are added to it. This is a useful default which -// helps finding memory leaks. -// -// Obviously not all Emitters should be limited to 10. This function allows -// that to be increased. Set to zero for unlimited. -var defaultMaxListeners = 10; -EventEmitter.prototype.setMaxListeners = function(n) { - if (!this._events) this._events = {}; - this._maxListeners = n; -}; - - -EventEmitter.prototype.emit = function() { - var type = arguments[0]; - // If there is no 'error' event listener then throw. - if (type === 'error') { - if (!this._events || !this._events.error || - (isArray(this._events.error) && !this._events.error.length)) - { - if (arguments[1] instanceof Error) { - throw arguments[1]; // Unhandled 'error' event - } else { - throw new Error("Uncaught, unspecified 'error' event."); - } - return false; - } - } - - if (!this._events) return false; - var handler = this._events[type]; - if (!handler) return false; - - if (typeof handler == 'function') { - switch (arguments.length) { - // fast cases - case 1: - handler.call(this); - break; - case 2: - handler.call(this, arguments[1]); - break; - case 3: - handler.call(this, arguments[1], arguments[2]); - break; - // slower - default: - var l = arguments.length; - var args = new Array(l - 1); - for (var i = 1; i < l; i++) args[i - 1] = arguments[i]; - handler.apply(this, args); - } - return true; - - } else if (isArray(handler)) { - var l = arguments.length; - var args = new Array(l - 1); - for (var i = 1; i < l; i++) args[i - 1] = arguments[i]; - - var listeners = handler.slice(); - for (var i = 0, l = listeners.length; i < l; i++) { - listeners[i].apply(this, args); - } - return true; - - } else { - return false; - } -}; - -// EventEmitter is defined in src/node_events.cc -// EventEmitter.prototype.emit() is also defined there. -EventEmitter.prototype.addListener = function(type, listener) { - if ('function' !== typeof listener) { - throw new Error('addListener only takes instances of Function'); - } - - if (!this._events) this._events = {}; - - // To avoid recursion in the case that type == "newListeners"! Before - // adding it to the listeners, first emit "newListeners". - this.emit('newListener', type, listener); - - if (!this._events[type]) { - // Optimize the case of one listener. Don't need the extra array object. - this._events[type] = listener; - } else if (isArray(this._events[type])) { - - // If we've already got an array, just append. - this._events[type].push(listener); - - // Check for listener leak - if (!this._events[type].warned) { - var m; - if (this._maxListeners !== undefined) { - m = this._maxListeners; - } else { - m = defaultMaxListeners; - } - - if (m && m > 0 && this._events[type].length > m) { - this._events[type].warned = true; - console.error('(node) warning: possible EventEmitter memory ' + - 'leak detected. %d listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit.', - this._events[type].length); - console.trace(); - } - } - } else { - // Adding the second element, need to change to array. - this._events[type] = [this._events[type], listener]; - } - - return this; -}; - -EventEmitter.prototype.on = EventEmitter.prototype.addListener; - -EventEmitter.prototype.once = function(type, listener) { - if ('function' !== typeof listener) { - throw new Error('.once only takes instances of Function'); - } - - var self = this; - function g() { - self.removeListener(type, g); - listener.apply(this, arguments); - }; - - g.listener = listener; - self.on(type, g); - - return this; -}; - -EventEmitter.prototype.removeListener = function(type, listener) { - if ('function' !== typeof listener) { - throw new Error('removeListener only takes instances of Function'); - } - - // does not use listeners(), so no side effect of creating _events[type] - if (!this._events || !this._events[type]) return this; - - var list = this._events[type]; - - if (isArray(list)) { - var position = -1; - for (var i = 0, length = list.length; i < length; i++) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) - { - position = i; - break; - } - } - - if (position < 0) return this; - list.splice(position, 1); - if (list.length == 0) - delete this._events[type]; - } else if (list === listener || - (list.listener && list.listener === listener)) - { - delete this._events[type]; - } - - return this; -}; - -EventEmitter.prototype.removeAllListeners = function(type) { - if (arguments.length === 0) { - this._events = {}; - return this; - } - - // does not use listeners(), so no side effect of creating _events[type] - if (type && this._events && this._events[type]) this._events[type] = null; - return this; -}; - -EventEmitter.prototype.listeners = function(type) { - if (!this._events) this._events = {}; - if (!this._events[type]) this._events[type] = []; - if (!isArray(this._events[type])) { - this._events[type] = [this._events[type]]; - } - return this._events[type]; -}; - diff --git a/lib/modules/http.js b/lib/modules/http.js deleted file mode 100644 index 0123fdee6..000000000 --- a/lib/modules/http.js +++ /dev/null @@ -1,55 +0,0 @@ -exports.STATUS_CODES = { - '100': 'Continue', - '101': 'Switching Protocols', - '102': 'Processing', - '200': 'OK', - '201': 'Created', - '202': 'Accepted', - '203': 'Non-Authoritative Information', - '204': 'No Content', - '205': 'Reset Content', - '206': 'Partial Content', - '207': 'Multi-Status', - '300': 'Multiple Choices', - '301': 'Moved Permanently', - '302': 'Moved Temporarily', - '303': 'See Other', - '304': 'Not Modified', - '305': 'Use Proxy', - '307': 'Temporary Redirect', - '400': 'Bad Request', - '401': 'Unauthorized', - '402': 'Payment Required', - '403': 'Forbidden', - '404': 'Not Found', - '405': 'Method Not Allowed', - '406': 'Not Acceptable', - '407': 'Proxy Authentication Required', - '408': 'Request Time-out', - '409': 'Conflict', - '410': 'Gone', - '411': 'Length Required', - '412': 'Precondition Failed', - '413': 'Request Entity Too Large', - '414': 'Request-URI Too Large', - '415': 'Unsupported Media Type', - '416': 'Requested Range Not Satisfiable', - '417': 'Expectation Failed', - '418': 'I\'m a teapot', - '422': 'Unprocessable Entity', - '423': 'Locked', - '424': 'Failed Dependency', - '425': 'Unordered Collection', - '426': 'Upgrade Required', - '500': 'Internal Server Error', - '501': 'Not Implemented', - '502': 'Bad Gateway', - '503': 'Service Unavailable', - '504': 'Gateway Time-out', - '505': 'HTTP Version not supported', - '506': 'Variant Also Negotiates', - '507': 'Insufficient Storage', - '509': 'Bandwidth Limit Exceeded', - '510': 'Not Extended' -}; - diff --git a/lib/modules/path.js b/lib/modules/path.js deleted file mode 100644 index 8fdd9d0c6..000000000 --- a/lib/modules/path.js +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - - -var isWindows = process.platform === 'win32'; - - -// resolves . and .. elements in a path array with directory names there -// must be no slashes, empty elements, or device names (c:\) in the array -// (so also no leading and trailing slashes - it does not distinguish -// relative and absolute paths) -function normalizeArray(parts, allowAboveRoot) { - // if the path tries to go above the root, `up` ends up > 0 - var up = 0; - for (var i = parts.length - 1; i >= 0; i--) { - var last = parts[i]; - if (last == '.') { - parts.splice(i, 1); - } else if (last === '..') { - parts.splice(i, 1); - up++; - } else if (up) { - parts.splice(i, 1); - up--; - } - } - - // if the path is allowed to go above the root, restore leading ..s - if (allowAboveRoot) { - for (; up--; up) { - parts.unshift('..'); - } - } - - return parts; -} - - -if (isWindows) { - // Regex to split a windows path into three parts: [*, device, slash, - // tail] windows-only - var splitDeviceRe = - /^([a-zA-Z]:|[\\\/]{2}[^\\\/]+[\\\/][^\\\/]+)?([\\\/])?([\s\S]*?)$/; - - // Regex to split the tail part of the above into [*, dir, basename, ext] - var splitTailRe = /^([\s\S]+[\\\/](?!$)|[\\\/])?((?:[\s\S]+?)?(\.[^.]*)?)$/; - - // Function to split a filename into [root, dir, basename, ext] - // windows version - var splitPath = function(filename) { - // Separate device+slash from tail - var result = splitDeviceRe.exec(filename), - device = (result[1] || '') + (result[2] || ''), - tail = result[3] || ''; - // Split the tail into dir, basename and extension - var result2 = splitTailRe.exec(tail), - dir = result2[1] || '', - basename = result2[2] || '', - ext = result2[3] || ''; - return [device, dir, basename, ext]; - }; - - // path.resolve([from ...], to) - // windows version - exports.resolve = function() { - var resolvedDevice = '', - resolvedTail = '', - resolvedAbsolute = false; - - for (var i = arguments.length - 1; i >= -1; i--) { - var path; - if (i >= 0) { - path = arguments[i]; - } else if (!resolvedDevice) { - path = process.cwd(); - } else { - // Windows has the concept of drive-specific current working - // directories. If we've resolved a drive letter but not yet an - // absolute path, get cwd for that drive. We're sure the device is not - // an unc path at this points, because unc paths are always absolute. - path = process._cwdForDrive(resolvedDevice[0]); - } - - // Skip empty and invalid entries - if (typeof path !== 'string' || !path) { - continue; - } - - var result = splitDeviceRe.exec(path), - device = result[1] || '', - isUnc = device && device.charAt(1) !== ':', - isAbsolute = !!result[2] || isUnc, // UNC paths are always absolute - tail = result[3]; - - if (device && - resolvedDevice && - device.toLowerCase() !== resolvedDevice.toLowerCase()) { - // This path points to another device so it is not applicable - continue; - } - - if (!resolvedDevice) { - resolvedDevice = device; - } - if (!resolvedAbsolute) { - resolvedTail = tail + '\\' + resolvedTail; - resolvedAbsolute = isAbsolute; - } - - if (resolvedDevice && resolvedAbsolute) { - break; - } - } - - // Replace slashes (in UNC share name) by backslashes - resolvedDevice = resolvedDevice.replace(/\//g, '\\'); - - // At this point the path should be resolved to a full absolute path, - // but handle relative paths to be safe (might happen when process.cwd() - // fails) - - // Normalize the tail path - - function f(p) { - return !!p; - } - - resolvedTail = normalizeArray(resolvedTail.split(/[\\\/]+/).filter(f), - !resolvedAbsolute).join('\\'); - - return (resolvedDevice + (resolvedAbsolute ? '\\' : '') + resolvedTail) || - '.'; - }; - - // windows version - exports.normalize = function(path) { - var result = splitDeviceRe.exec(path), - device = result[1] || '', - isUnc = device && device.charAt(1) !== ':', - isAbsolute = !!result[2] || isUnc, // UNC paths are always absolute - tail = result[3], - trailingSlash = /[\\\/]$/.test(tail); - - // Normalize the tail path - tail = normalizeArray(tail.split(/[\\\/]+/).filter(function(p) { - return !!p; - }), !isAbsolute).join('\\'); - - if (!tail && !isAbsolute) { - tail = '.'; - } - if (tail && trailingSlash) { - tail += '\\'; - } - - return device + (isAbsolute ? '\\' : '') + tail; - }; - - // windows version - exports.join = function() { - function f(p) { - return p && typeof p === 'string'; - } - - var paths = Array.prototype.slice.call(arguments, 0).filter(f); - var joined = paths.join('\\'); - - // Make sure that the joined path doesn't start with two slashes - // - it will be mistaken for an unc path by normalize() - - // unless the paths[0] also starts with two slashes - if (/^[\\\/]{2}/.test(joined) && !/^[\\\/]{2}/.test(paths[0])) { - joined = joined.slice(1); - } - - return exports.normalize(joined); - }; - - // path.relative(from, to) - // it will solve the relative path from 'from' to 'to', for instance: - // from = 'C:\\orandea\\test\\aaa' - // to = 'C:\\orandea\\impl\\bbb' - // The output of the function should be: '..\\..\\impl\\bbb' - // windows version - exports.relative = function(from, to) { - from = exports.resolve(from); - to = exports.resolve(to); - - // windows is not case sensitive - var lowerFrom = from.toLowerCase(); - var lowerTo = to.toLowerCase(); - - function trim(arr) { - var start = 0; - for (; start < arr.length; start++) { - if (arr[start] !== '') break; - } - - var end = arr.length - 1; - for (; end >= 0; end--) { - if (arr[end] !== '') break; - } - - if (start > end) return []; - return arr.slice(start, end - start + 1); - } - - var fromParts = trim(from.split('\\')); - var toParts = trim(to.split('\\')); - - var lowerFromParts = trim(lowerFrom.split('\\')); - var lowerToParts = trim(lowerTo.split('\\')); - - var length = Math.min(lowerFromParts.length, lowerToParts.length); - var samePartsLength = length; - for (var i = 0; i < length; i++) { - if (lowerFromParts[i] !== lowerToParts[i]) { - samePartsLength = i; - break; - } - } - - if (samePartsLength == 0) { - return to; - } - - var outputParts = []; - for (var i = samePartsLength; i < lowerFromParts.length; i++) { - outputParts.push('..'); - } - - outputParts = outputParts.concat(toParts.slice(samePartsLength)); - - return outputParts.join('\\'); - }; - - -} else /* posix */ { - - // Split a filename into [root, dir, basename, ext], unix version - // 'root' is just a slash, or nothing. - var splitPathRe = /^(\/?)([\s\S]+\/(?!$)|\/)?((?:[\s\S]+?)?(\.[^.]*)?)$/; - var splitPath = function(filename) { - var result = splitPathRe.exec(filename); - return [result[1] || '', result[2] || '', result[3] || '', result[4] || '']; - }; - - // path.resolve([from ...], to) - // posix version - exports.resolve = function() { - var resolvedPath = '', - resolvedAbsolute = false; - - for (var i = arguments.length - 1; i >= -1 && !resolvedAbsolute; i--) { - var path = (i >= 0) ? arguments[i] : process.cwd(); - - // Skip empty and invalid entries - if (typeof path !== 'string' || !path) { - continue; - } - - resolvedPath = path + '/' + resolvedPath; - resolvedAbsolute = path.charAt(0) === '/'; - } - - // At this point the path should be resolved to a full absolute path, but - // handle relative paths to be safe (might happen when process.cwd() fails) - - // Normalize the path - resolvedPath = normalizeArray(resolvedPath.split('/').filter(function(p) { - return !!p; - }), !resolvedAbsolute).join('/'); - - return ((resolvedAbsolute ? '/' : '') + resolvedPath) || '.'; - }; - - // path.normalize(path) - // posix version - exports.normalize = function(path) { - var isAbsolute = path.charAt(0) === '/', - trailingSlash = path.slice(-1) === '/'; - - // Normalize the path - path = normalizeArray(path.split('/').filter(function(p) { - return !!p; - }), !isAbsolute).join('/'); - - if (!path && !isAbsolute) { - path = '.'; - } - if (path && trailingSlash) { - path += '/'; - } - - return (isAbsolute ? '/' : '') + path; - }; - - - // posix version - exports.join = function() { - var paths = Array.prototype.slice.call(arguments, 0); - return exports.normalize(paths.filter(function(p, index) { - return p && typeof p === 'string'; - }).join('/')); - }; - - - // path.relative(from, to) - // posix version - exports.relative = function(from, to) { - from = exports.resolve(from).substr(1); - to = exports.resolve(to).substr(1); - - function trim(arr) { - var start = 0; - for (; start < arr.length; start++) { - if (arr[start] !== '') break; - } - - var end = arr.length - 1; - for (; end >= 0; end--) { - if (arr[end] !== '') break; - } - - if (start > end) return []; - return arr.slice(start, end - start + 1); - } - - var fromParts = trim(from.split('/')); - var toParts = trim(to.split('/')); - - var length = Math.min(fromParts.length, toParts.length); - var samePartsLength = length; - for (var i = 0; i < length; i++) { - if (fromParts[i] !== toParts[i]) { - samePartsLength = i; - break; - } - } - - var outputParts = []; - for (var i = samePartsLength; i < fromParts.length; i++) { - outputParts.push('..'); - } - - outputParts = outputParts.concat(toParts.slice(samePartsLength)); - - return outputParts.join('/'); - }; - -} - - -exports.dirname = function(path) { - var result = splitPath(path), - root = result[0], - dir = result[1]; - - if (!root && !dir) { - // No dirname whatsoever - return '.'; - } - - if (dir) { - // It has a dirname, strip trailing slash - dir = dir.substring(0, dir.length - 1); - } - - return root + dir; -}; - - -exports.basename = function(path, ext) { - var f = splitPath(path)[2]; - // TODO: make this comparison case-insensitive on windows? - if (ext && f.substr(-1 * ext.length) === ext) { - f = f.substr(0, f.length - ext.length); - } - return f; -}; - - -exports.extname = function(path) { - return splitPath(path)[3]; -}; - - -exports.exists = function(path, callback) { - process.binding('fs').stat(path, function(err, stats) { - if (callback) callback(err ? false : true); - }); -}; - - -exports.existsSync = function(path) { - try { - process.binding('fs').stat(path); - return true; - } catch (e) { - return false; - } -}; - - -exports._makeLong = isWindows ? - function(path) { - var resolvedPath = exports.resolve(path); - - if (resolvedPath.match(/^[a-zA-Z]\:\\/)) { - // path is local filesystem path, which needs to be converted - // to long UNC path. - return '\\\\?\\' + resolvedPath; - } else if (resolvedPath.match(/^\\\\[^?.]/)) { - // path is network UNC path, which needs to be converted - // to long UNC path. - return '\\\\?\\UNC\\' + resolvedPath.substring(2); - } - - return path; - } : - function(path) { - return path; - }; - diff --git a/lib/modules/punycode.js b/lib/modules/punycode.js deleted file mode 100644 index 5ca1254f6..000000000 --- a/lib/modules/punycode.js +++ /dev/null @@ -1,510 +0,0 @@ -/*! http://mths.be/punycode by @mathias */ -;(function(root) { - - /** - * The `punycode` object. - * @name punycode - * @type Object - */ - var punycode, - - /** Detect free variables `define`, `exports`, `module` and `require` */ - freeDefine = typeof define == 'function' && typeof define.amd == 'object' && - define.amd && define, - freeExports = typeof exports == 'object' && exports, - freeModule = typeof module == 'object' && module, - freeRequire = typeof require == 'function' && require, - - /** Highest positive signed 32-bit float value */ - maxInt = 2147483647, // aka. 0x7FFFFFFF or 2^31-1 - - /** Bootstring parameters */ - base = 36, - tMin = 1, - tMax = 26, - skew = 38, - damp = 700, - initialBias = 72, - initialN = 128, // 0x80 - delimiter = '-', // '\x2D' - - /** Regular expressions */ - regexNonASCII = /[^ -~]/, // unprintable ASCII chars + non-ASCII chars - regexPunycode = /^xn--/, - - /** Error messages */ - errors = { - 'overflow': 'Overflow: input needs wider integers to process.', - 'not-basic': 'Illegal input >= 0x80 (not a basic code point)', - 'invalid-input': 'Invalid input' - }, - - /** Convenience shortcuts */ - baseMinusTMin = base - tMin, - floor = Math.floor, - stringFromCharCode = String.fromCharCode, - - /** Temporary variable */ - key; - - /*--------------------------------------------------------------------------*/ - - /** - * A generic error utility function. - * @private - * @param {String} type The error type. - * @returns {Error} Throws a `RangeError` with the applicable error message. - */ - function error(type) { - throw RangeError(errors[type]); - } - - /** - * A generic `Array#map` utility function. - * @private - * @param {Array} array The array to iterate over. - * @param {Function} callback The function that gets called for every array - * item. - * @returns {Array} A new array of values returned by the callback function. - */ - function map(array, fn) { - var length = array.length; - while (length--) { - array[length] = fn(array[length]); - } - return array; - } - - /** - * A simple `Array#map`-like wrapper to work with domain name strings. - * @private - * @param {String} domain The domain name. - * @param {Function} callback The function that gets called for every - * character. - * @returns {Array} A new string of characters returned by the callback - * function. - */ - function mapDomain(string, fn) { - var glue = '.'; - return map(string.split(glue), fn).join(glue); - } - - /** - * Creates an array containing the decimal code points of each Unicode - * character in the string. While JavaScript uses UCS-2 internally, - * this function will convert a pair of surrogate halves (each of which - * UCS-2 exposes as separate characters) into a single code point, - * matching UTF-16. - * @see `punycode.ucs2.encode` - * @see - * @memberOf punycode.ucs2 - * @name decode - * @param {String} string The Unicode input string (UCS-2). - * @returns {Array} The new array of code points. - */ - function ucs2decode(string) { - var output = [], - counter = 0, - length = string.length, - value, - extra; - while (counter < length) { - value = string.charCodeAt(counter++); - if ((value & 0xF800) == 0xD800 && counter < length) { - // high surrogate, and there is a next character - extra = string.charCodeAt(counter++); - if ((extra & 0xFC00) == 0xDC00) { // low surrogate - output.push(((value & 0x3FF) << 10) + (extra & 0x3FF) + 0x10000); - } else { - output.push(value, extra); - } - } else { - output.push(value); - } - } - return output; - } - - /** - * Creates a string based on an array of decimal code points. - * @see `punycode.ucs2.decode` - * @memberOf punycode.ucs2 - * @name encode - * @param {Array} codePoints The array of decimal code points. - * @returns {String} The new Unicode string (UCS-2). - */ - function ucs2encode(array) { - return map(array, function(value) { - var output = ''; - if (value > 0xFFFF) { - value -= 0x10000; - output += stringFromCharCode(value >>> 10 & 0x3FF | 0xD800); - value = 0xDC00 | value & 0x3FF; - } - output += stringFromCharCode(value); - return output; - }).join(''); - } - - /** - * Converts a basic code point into a digit/integer. - * @see `digitToBasic()` - * @private - * @param {Number} codePoint The basic (decimal) code point. - * @returns {Number} The numeric value of a basic code point (for use in - * representing integers) in the range `0` to `base - 1`, or `base` if - * the code point does not represent a value. - */ - function basicToDigit(codePoint) { - return codePoint - 48 < 10 - ? codePoint - 22 - : codePoint - 65 < 26 - ? codePoint - 65 - : codePoint - 97 < 26 - ? codePoint - 97 - : base; - } - - /** - * Converts a digit/integer into a basic code point. - * @see `basicToDigit()` - * @private - * @param {Number} digit The numeric value of a basic code point. - * @returns {Number} The basic code point whose value (when used for - * representing integers) is `digit`, which needs to be in the range - * `0` to `base - 1`. If `flag` is non-zero, the uppercase form is - * used; else, the lowercase form is used. The behavior is undefined - * if flag is non-zero and `digit` has no uppercase form. - */ - function digitToBasic(digit, flag) { - // 0..25 map to ASCII a..z or A..Z - // 26..35 map to ASCII 0..9 - return digit + 22 + 75 * (digit < 26) - ((flag != 0) << 5); - } - - /** - * Bias adaptation function as per section 3.4 of RFC 3492. - * http://tools.ietf.org/html/rfc3492#section-3.4 - * @private - */ - function adapt(delta, numPoints, firstTime) { - var k = 0; - delta = firstTime ? floor(delta / damp) : delta >> 1; - delta += floor(delta / numPoints); - for (/* no initialization */; delta > baseMinusTMin * tMax >> 1; k += base) { - delta = floor(delta / baseMinusTMin); - } - return floor(k + (baseMinusTMin + 1) * delta / (delta + skew)); - } - - /** - * Converts a basic code point to lowercase is `flag` is falsy, or to - * uppercase if `flag` is truthy. The code point is unchanged if it's - * caseless. The behavior is undefined if `codePoint` is not a basic code - * point. - * @private - * @param {Number} codePoint The numeric value of a basic code point. - * @returns {Number} The resulting basic code point. - */ - function encodeBasic(codePoint, flag) { - codePoint -= (codePoint - 97 < 26) << 5; - return codePoint + (!flag && codePoint - 65 < 26) << 5; - } - - /** - * Converts a Punycode string of ASCII code points to a string of Unicode - * code points. - * @memberOf punycode - * @param {String} input The Punycode string of ASCII code points. - * @returns {String} The resulting string of Unicode code points. - */ - function decode(input) { - // Don't use UCS-2 - var output = [], - inputLength = input.length, - out, - i = 0, - n = initialN, - bias = initialBias, - basic, - j, - index, - oldi, - w, - k, - digit, - t, - length, - /** Cached calculation results */ - baseMinusT; - - // Handle the basic code points: let `basic` be the number of input code - // points before the last delimiter, or `0` if there is none, then copy - // the first basic code points to the output. - - basic = input.lastIndexOf(delimiter); - if (basic < 0) { - basic = 0; - } - - for (j = 0; j < basic; ++j) { - // if it's not a basic code point - if (input.charCodeAt(j) >= 0x80) { - error('not-basic'); - } - output.push(input.charCodeAt(j)); - } - - // Main decoding loop: start just after the last delimiter if any basic code - // points were copied; start at the beginning otherwise. - - for (index = basic > 0 ? basic + 1 : 0; index < inputLength; /* no final expression */) { - - // `index` is the index of the next character to be consumed. - // Decode a generalized variable-length integer into `delta`, - // which gets added to `i`. The overflow checking is easier - // if we increase `i` as we go, then subtract off its starting - // value at the end to obtain `delta`. - for (oldi = i, w = 1, k = base; /* no condition */; k += base) { - - if (index >= inputLength) { - error('invalid-input'); - } - - digit = basicToDigit(input.charCodeAt(index++)); - - if (digit >= base || digit > floor((maxInt - i) / w)) { - error('overflow'); - } - - i += digit * w; - t = k <= bias ? tMin : (k >= bias + tMax ? tMax : k - bias); - - if (digit < t) { - break; - } - - baseMinusT = base - t; - if (w > floor(maxInt / baseMinusT)) { - error('overflow'); - } - - w *= baseMinusT; - - } - - out = output.length + 1; - bias = adapt(i - oldi, out, oldi == 0); - - // `i` was supposed to wrap around from `out` to `0`, - // incrementing `n` each time, so we'll fix that now: - if (floor(i / out) > maxInt - n) { - error('overflow'); - } - - n += floor(i / out); - i %= out; - - // Insert `n` at position `i` of the output - output.splice(i++, 0, n); - - } - - return ucs2encode(output); - } - - /** - * Converts a string of Unicode code points to a Punycode string of ASCII - * code points. - * @memberOf punycode - * @param {String} input The string of Unicode code points. - * @returns {String} The resulting Punycode string of ASCII code points. - */ - function encode(input) { - var n, - delta, - handledCPCount, - basicLength, - bias, - j, - m, - q, - k, - t, - currentValue, - output = [], - /** `inputLength` will hold the number of code points in `input`. */ - inputLength, - /** Cached calculation results */ - handledCPCountPlusOne, - baseMinusT, - qMinusT; - - // Convert the input in UCS-2 to Unicode - input = ucs2decode(input); - - // Cache the length - inputLength = input.length; - - // Initialize the state - n = initialN; - delta = 0; - bias = initialBias; - - // Handle the basic code points - for (j = 0; j < inputLength; ++j) { - currentValue = input[j]; - if (currentValue < 0x80) { - output.push(stringFromCharCode(currentValue)); - } - } - - handledCPCount = basicLength = output.length; - - // `handledCPCount` is the number of code points that have been handled; - // `basicLength` is the number of basic code points. - - // Finish the basic string - if it is not empty - with a delimiter - if (basicLength) { - output.push(delimiter); - } - - // Main encoding loop: - while (handledCPCount < inputLength) { - - // All non-basic code points < n have been handled already. Find the next - // larger one: - for (m = maxInt, j = 0; j < inputLength; ++j) { - currentValue = input[j]; - if (currentValue >= n && currentValue < m) { - m = currentValue; - } - } - - // Increase `delta` enough to advance the decoder's state to , - // but guard against overflow - handledCPCountPlusOne = handledCPCount + 1; - if (m - n > floor((maxInt - delta) / handledCPCountPlusOne)) { - error('overflow'); - } - - delta += (m - n) * handledCPCountPlusOne; - n = m; - - for (j = 0; j < inputLength; ++j) { - currentValue = input[j]; - - if (currentValue < n && ++delta > maxInt) { - error('overflow'); - } - - if (currentValue == n) { - // Represent delta as a generalized variable-length integer - for (q = delta, k = base; /* no condition */; k += base) { - t = k <= bias ? tMin : (k >= bias + tMax ? tMax : k - bias); - if (q < t) { - break; - } - qMinusT = q - t; - baseMinusT = base - t; - output.push( - stringFromCharCode(digitToBasic(t + qMinusT % baseMinusT, 0)) - ); - q = floor(qMinusT / baseMinusT); - } - - output.push(stringFromCharCode(digitToBasic(q, 0))); - bias = adapt(delta, handledCPCountPlusOne, handledCPCount == basicLength); - delta = 0; - ++handledCPCount; - } - } - - ++delta; - ++n; - - } - return output.join(''); - } - - /** - * Converts a Punycode string representing a domain name to Unicode. Only the - * Punycoded parts of the domain name will be converted, i.e. it doesn't - * matter if you call it on a string that has already been converted to - * Unicode. - * @memberOf punycode - * @param {String} domain The Punycode domain name to convert to Unicode. - * @returns {String} The Unicode representation of the given Punycode - * string. - */ - function toUnicode(domain) { - return mapDomain(domain, function(string) { - return regexPunycode.test(string) - ? decode(string.slice(4).toLowerCase()) - : string; - }); - } - - /** - * Converts a Unicode string representing a domain name to Punycode. Only the - * non-ASCII parts of the domain name will be converted, i.e. it doesn't - * matter if you call it with a domain that's already in ASCII. - * @memberOf punycode - * @param {String} domain The domain name to convert, as a Unicode string. - * @returns {String} The Punycode representation of the given domain name. - */ - function toASCII(domain) { - return mapDomain(domain, function(string) { - return regexNonASCII.test(string) - ? 'xn--' + encode(string) - : string; - }); - } - - /*--------------------------------------------------------------------------*/ - - /** Define the public API */ - punycode = { - /** - * A string representing the current Punycode.js version number. - * @memberOf punycode - * @type String - */ - 'version': '1.1.1', - /** - * An object of methods to convert from JavaScript's internal character - * representation (UCS-2) to decimal Unicode code points, and back. - * @see - * @memberOf punycode - * @type Object - */ - 'ucs2': { - 'decode': ucs2decode, - 'encode': ucs2encode - }, - 'decode': decode, - 'encode': encode, - 'toASCII': toASCII, - 'toUnicode': toUnicode - }; - - /** Expose `punycode` */ - if (freeExports) { - if (freeModule && freeModule.exports == freeExports) { - // in Node.js or Ringo 0.8+ - freeModule.exports = punycode; - } else { - // in Narwhal or Ringo 0.7- - for (key in punycode) { - punycode.hasOwnProperty(key) && (freeExports[key] = punycode[key]); - } - } - } else if (freeDefine) { - // via curl.js or RequireJS - define('punycode', punycode); - } else { - // in a browser or Rhino - root.punycode = punycode; - } - -}(this)); diff --git a/lib/modules/querystring.js b/lib/modules/querystring.js deleted file mode 100644 index 0ab739a52..000000000 --- a/lib/modules/querystring.js +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -// Query String Utilities - -var QueryString = exports; - - -// If obj.hasOwnProperty has been overridden, then calling -// obj.hasOwnProperty(prop) will break. -// See: https://github.com/joyent/node/issues/1707 -function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); -} - - -function charCode(c) { - return c.charCodeAt(0); -} - - -// a safe fast alternative to decodeURIComponent -QueryString.unescapeBuffer = function(s, decodeSpaces) { - var out = new Buffer(s.length); - var state = 'CHAR'; // states: CHAR, HEX0, HEX1 - var n, m, hexchar; - - for (var inIndex = 0, outIndex = 0; inIndex <= s.length; inIndex++) { - var c = s.charCodeAt(inIndex); - switch (state) { - case 'CHAR': - switch (c) { - case charCode('%'): - n = 0; - m = 0; - state = 'HEX0'; - break; - case charCode('+'): - if (decodeSpaces) c = charCode(' '); - // pass thru - default: - out[outIndex++] = c; - break; - } - break; - - case 'HEX0': - state = 'HEX1'; - hexchar = c; - if (charCode('0') <= c && c <= charCode('9')) { - n = c - charCode('0'); - } else if (charCode('a') <= c && c <= charCode('f')) { - n = c - charCode('a') + 10; - } else if (charCode('A') <= c && c <= charCode('F')) { - n = c - charCode('A') + 10; - } else { - out[outIndex++] = charCode('%'); - out[outIndex++] = c; - state = 'CHAR'; - break; - } - break; - - case 'HEX1': - state = 'CHAR'; - if (charCode('0') <= c && c <= charCode('9')) { - m = c - charCode('0'); - } else if (charCode('a') <= c && c <= charCode('f')) { - m = c - charCode('a') + 10; - } else if (charCode('A') <= c && c <= charCode('F')) { - m = c - charCode('A') + 10; - } else { - out[outIndex++] = charCode('%'); - out[outIndex++] = hexchar; - out[outIndex++] = c; - break; - } - out[outIndex++] = 16 * n + m; - break; - } - } - - // TODO support returning arbitrary buffers. - - return out.slice(0, outIndex - 1); -}; - - -QueryString.unescape = function(s, decodeSpaces) { - return QueryString.unescapeBuffer(s, decodeSpaces).toString(); -}; - - -QueryString.escape = function(str) { - return encodeURIComponent(str); -}; - -var stringifyPrimitive = function(v) { - switch (typeof v) { - case 'string': - return v; - - case 'boolean': - return v ? 'true' : 'false'; - - case 'number': - return isFinite(v) ? v : ''; - - default: - return ''; - } -}; - - -QueryString.stringify = QueryString.encode = function(obj, sep, eq, name) { - sep = sep || '&'; - eq = eq || '='; - if (obj === null) { - obj = undefined; - } - - if (typeof obj === 'object') { - return Object.keys(obj).map(function(k) { - var ks = QueryString.escape(stringifyPrimitive(k)) + eq; - if (Array.isArray(obj[k])) { - return obj[k].map(function(v) { - return ks + QueryString.escape(stringifyPrimitive(v)); - }).join(sep); - } else { - return ks + QueryString.escape(stringifyPrimitive(obj[k])); - } - }).join(sep); - - } - - if (!name) return ''; - return QueryString.escape(stringifyPrimitive(name)) + eq + - QueryString.escape(stringifyPrimitive(obj)); -}; - -// Parse a key=val string. -QueryString.parse = QueryString.decode = function(qs, sep, eq, options) { - sep = sep || '&'; - eq = eq || '='; - var obj = {}; - - if (typeof qs !== 'string' || qs.length === 0) { - return obj; - } - - var regexp = /\+/g; - qs = qs.split(sep); - - var maxKeys = 1000; - if (options && typeof options.maxKeys === 'number') { - maxKeys = options.maxKeys; - } - - var len = qs.length; - // maxKeys <= 0 means that we should not limit keys count - if (maxKeys > 0 && len > maxKeys) { - len = maxKeys; - } - - for (var i = 0; i < len; ++i) { - var x = qs[i].replace(regexp, '%20'), - idx = x.indexOf(eq), - kstr, vstr, k, v; - - if (idx >= 0) { - kstr = x.substr(0, idx); - vstr = x.substr(idx + 1); - } else { - kstr = x; - vstr = ''; - } - - try { - k = decodeURIComponent(kstr); - v = decodeURIComponent(vstr); - } catch (e) { - k = QueryString.unescape(kstr, true); - v = QueryString.unescape(vstr, true); - } - - if (!hasOwnProperty(obj, k)) { - obj[k] = v; - } else if (Array.isArray(obj[k])) { - obj[k].push(v); - } else { - obj[k] = [obj[k], v]; - } - } - - return obj; -}; diff --git a/lib/modules/tty.js b/lib/modules/tty.js deleted file mode 100644 index 54fd3cae1..000000000 --- a/lib/modules/tty.js +++ /dev/null @@ -1,7 +0,0 @@ -exports.isatty = function(){ - return true; -}; - -exports.getWindowSize = function(){ - return [window.innerHeight, window.innerWidth]; -}; diff --git a/lib/modules/url.js b/lib/modules/url.js deleted file mode 100644 index 162afa2fc..000000000 --- a/lib/modules/url.js +++ /dev/null @@ -1,625 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -var punycode = require('./punycode'); - -exports.parse = urlParse; -exports.resolve = urlResolve; -exports.resolveObject = urlResolveObject; -exports.format = urlFormat; - -// Reference: RFC 3986, RFC 1808, RFC 2396 - -// define these here so at least they only have to be -// compiled once on the first module load. -var protocolPattern = /^([a-z0-9.+-]+:)/i, - portPattern = /:[0-9]*$/, - - // RFC 2396: characters reserved for delimiting URLs. - // We actually just auto-escape these. - delims = ['<', '>', '"', '`', ' ', '\r', '\n', '\t'], - - // RFC 2396: characters not allowed for various reasons. - unwise = ['{', '}', '|', '\\', '^', '~', '`'].concat(delims), - - // Allowed by RFCs, but cause of XSS attacks. Always escape these. - autoEscape = ['\''].concat(delims), - // Characters that are never ever allowed in a hostname. - // Note that any invalid chars are also handled, but these - // are the ones that are *expected* to be seen, so we fast-path - // them. - nonHostChars = ['%', '/', '?', ';', '#'] - .concat(unwise).concat(autoEscape), - nonAuthChars = ['/', '@', '?', '#'].concat(delims), - hostnameMaxLen = 255, - hostnamePartPattern = /^[a-zA-Z0-9][a-z0-9A-Z_-]{0,62}$/, - hostnamePartStart = /^([a-zA-Z0-9][a-z0-9A-Z_-]{0,62})(.*)$/, - // protocols that can allow "unsafe" and "unwise" chars. - unsafeProtocol = { - 'javascript': true, - 'javascript:': true - }, - // protocols that never have a hostname. - hostlessProtocol = { - 'javascript': true, - 'javascript:': true - }, - // protocols that always have a path component. - pathedProtocol = { - 'http': true, - 'https': true, - 'ftp': true, - 'gopher': true, - 'file': true, - 'http:': true, - 'ftp:': true, - 'gopher:': true, - 'file:': true - }, - // protocols that always contain a // bit. - slashedProtocol = { - 'http': true, - 'https': true, - 'ftp': true, - 'gopher': true, - 'file': true, - 'http:': true, - 'https:': true, - 'ftp:': true, - 'gopher:': true, - 'file:': true - }, - querystring = require('./querystring'); - -function urlParse(url, parseQueryString, slashesDenoteHost) { - if (url && typeof(url) === 'object' && url.href) return url; - - if (typeof url !== 'string') { - throw new TypeError("Parameter 'url' must be a string, not " + typeof url); - } - - var out = {}, - rest = url; - - // trim before proceeding. - // This is to support parse stuff like " http://foo.com \n" - rest = rest.trim(); - - var proto = protocolPattern.exec(rest); - if (proto) { - proto = proto[0]; - var lowerProto = proto.toLowerCase(); - out.protocol = lowerProto; - rest = rest.substr(proto.length); - } - - // figure out if it's got a host - // user@server is *always* interpreted as a hostname, and url - // resolution will treat //foo/bar as host=foo,path=bar because that's - // how the browser resolves relative URLs. - if (slashesDenoteHost || proto || rest.match(/^\/\/[^@\/]+@[^@\/]+/)) { - var slashes = rest.substr(0, 2) === '//'; - if (slashes && !(proto && hostlessProtocol[proto])) { - rest = rest.substr(2); - out.slashes = true; - } - } - - if (!hostlessProtocol[proto] && - (slashes || (proto && !slashedProtocol[proto]))) { - // there's a hostname. - // the first instance of /, ?, ;, or # ends the host. - // don't enforce full RFC correctness, just be unstupid about it. - - // If there is an @ in the hostname, then non-host chars *are* allowed - // to the left of the first @ sign, unless some non-auth character - // comes *before* the @-sign. - // URLs are obnoxious. - var atSign = rest.indexOf('@'); - if (atSign !== -1) { - var auth = rest.slice(0, atSign); - - // there *may be* an auth - var hasAuth = true; - for (var i = 0, l = nonAuthChars.length; i < l; i++) { - if (auth.indexOf(nonAuthChars[i]) !== -1) { - // not a valid auth. Something like http://foo.com/bar@baz/ - hasAuth = false; - break; - } - } - - if (hasAuth) { - // pluck off the auth portion. - out.auth = decodeURIComponent(auth); - rest = rest.substr(atSign + 1); - } - } - - var firstNonHost = -1; - for (var i = 0, l = nonHostChars.length; i < l; i++) { - var index = rest.indexOf(nonHostChars[i]); - if (index !== -1 && - (firstNonHost < 0 || index < firstNonHost)) firstNonHost = index; - } - - if (firstNonHost !== -1) { - out.host = rest.substr(0, firstNonHost); - rest = rest.substr(firstNonHost); - } else { - out.host = rest; - rest = ''; - } - - // pull out port. - var p = parseHost(out.host); - var keys = Object.keys(p); - for (var i = 0, l = keys.length; i < l; i++) { - var key = keys[i]; - out[key] = p[key]; - } - - // we've indicated that there is a hostname, - // so even if it's empty, it has to be present. - out.hostname = out.hostname || ''; - - // if hostname begins with [ and ends with ] - // assume that it's an IPv6 address. - var ipv6Hostname = out.hostname[0] === '[' && - out.hostname[out.hostname.length - 1] === ']'; - - // validate a little. - if (out.hostname.length > hostnameMaxLen) { - out.hostname = ''; - } else if (!ipv6Hostname) { - var hostparts = out.hostname.split(/\./); - for (var i = 0, l = hostparts.length; i < l; i++) { - var part = hostparts[i]; - if (!part) continue; - if (!part.match(hostnamePartPattern)) { - var newpart = ''; - for (var j = 0, k = part.length; j < k; j++) { - if (part.charCodeAt(j) > 127) { - // we replace non-ASCII char with a temporary placeholder - // we need this to make sure size of hostname is not - // broken by replacing non-ASCII by nothing - newpart += 'x'; - } else { - newpart += part[j]; - } - } - // we test again with ASCII char only - if (!newpart.match(hostnamePartPattern)) { - var validParts = hostparts.slice(0, i); - var notHost = hostparts.slice(i + 1); - var bit = part.match(hostnamePartStart); - if (bit) { - validParts.push(bit[1]); - notHost.unshift(bit[2]); - } - if (notHost.length) { - rest = '/' + notHost.join('.') + rest; - } - out.hostname = validParts.join('.'); - break; - } - } - } - } - - // hostnames are always lower case. - out.hostname = out.hostname.toLowerCase(); - - if (!ipv6Hostname) { - // IDNA Support: Returns a puny coded representation of "domain". - // It only converts the part of the domain name that - // has non ASCII characters. I.e. it dosent matter if - // you call it with a domain that already is in ASCII. - var domainArray = out.hostname.split('.'); - var newOut = []; - for (var i = 0; i < domainArray.length; ++i) { - var s = domainArray[i]; - newOut.push(s.match(/[^A-Za-z0-9_-]/) ? - 'xn--' + punycode.encode(s) : s); - } - out.hostname = newOut.join('.'); - } - - out.host = (out.hostname || '') + - ((out.port) ? ':' + out.port : ''); - out.href += out.host; - - // strip [ and ] from the hostname - if (ipv6Hostname) { - out.hostname = out.hostname.substr(1, out.hostname.length - 2); - if (rest[0] !== '/') { - rest = '/' + rest; - } - } - } - - // now rest is set to the post-host stuff. - // chop off any delim chars. - if (!unsafeProtocol[lowerProto]) { - - // First, make 100% sure that any "autoEscape" chars get - // escaped, even if encodeURIComponent doesn't think they - // need to be. - for (var i = 0, l = autoEscape.length; i < l; i++) { - var ae = autoEscape[i]; - var esc = encodeURIComponent(ae); - if (esc === ae) { - esc = escape(ae); - } - rest = rest.split(ae).join(esc); - } - } - - - // chop off from the tail first. - var hash = rest.indexOf('#'); - if (hash !== -1) { - // got a fragment string. - out.hash = rest.substr(hash); - rest = rest.slice(0, hash); - } - var qm = rest.indexOf('?'); - if (qm !== -1) { - out.search = rest.substr(qm); - out.query = rest.substr(qm + 1); - if (parseQueryString) { - out.query = querystring.parse(out.query); - } - rest = rest.slice(0, qm); - } else if (parseQueryString) { - // no query string, but parseQueryString still requested - out.search = ''; - out.query = {}; - } - if (rest) out.pathname = rest; - if (slashedProtocol[proto] && - out.hostname && !out.pathname) { - out.pathname = '/'; - } - - //to support http.request - if (out.pathname || out.search) { - out.path = (out.pathname ? out.pathname : '') + - (out.search ? out.search : ''); - } - - // finally, reconstruct the href based on what has been validated. - out.href = urlFormat(out); - return out; -} - -// format a parsed object into a url string -function urlFormat(obj) { - // ensure it's an object, and not a string url. - // If it's an obj, this is a no-op. - // this way, you can call url_format() on strings - // to clean up potentially wonky urls. - if (typeof(obj) === 'string') obj = urlParse(obj); - - var auth = obj.auth || ''; - if (auth) { - auth = encodeURIComponent(auth); - auth = auth.replace(/%3A/i, ':'); - auth += '@'; - } - - var protocol = obj.protocol || '', - pathname = obj.pathname || '', - hash = obj.hash || '', - host = false, - query = ''; - - if (obj.host !== undefined) { - host = auth + obj.host; - } else if (obj.hostname !== undefined) { - host = auth + (obj.hostname.indexOf(':') === -1 ? - obj.hostname : - '[' + obj.hostname + ']'); - if (obj.port) { - host += ':' + obj.port; - } - } - - if (obj.query && typeof obj.query === 'object' && - Object.keys(obj.query).length) { - query = querystring.stringify(obj.query); - } - - var search = obj.search || (query && ('?' + query)) || ''; - - if (protocol && protocol.substr(-1) !== ':') protocol += ':'; - - // only the slashedProtocols get the //. Not mailto:, xmpp:, etc. - // unless they had them to begin with. - if (obj.slashes || - (!protocol || slashedProtocol[protocol]) && host !== false) { - host = '//' + (host || ''); - if (pathname && pathname.charAt(0) !== '/') pathname = '/' + pathname; - } else if (!host) { - host = ''; - } - - if (hash && hash.charAt(0) !== '#') hash = '#' + hash; - if (search && search.charAt(0) !== '?') search = '?' + search; - - return protocol + host + pathname + search + hash; -} - -function urlResolve(source, relative) { - return urlFormat(urlResolveObject(source, relative)); -} - -function urlResolveObject(source, relative) { - if (!source) return relative; - - source = urlParse(urlFormat(source), false, true); - relative = urlParse(urlFormat(relative), false, true); - - // hash is always overridden, no matter what. - source.hash = relative.hash; - - if (relative.href === '') { - source.href = urlFormat(source); - return source; - } - - // hrefs like //foo/bar always cut to the protocol. - if (relative.slashes && !relative.protocol) { - relative.protocol = source.protocol; - //urlParse appends trailing / to urls like http://www.example.com - if (slashedProtocol[relative.protocol] && - relative.hostname && !relative.pathname) { - relative.path = relative.pathname = '/'; - } - relative.href = urlFormat(relative); - return relative; - } - - if (relative.protocol && relative.protocol !== source.protocol) { - // if it's a known url protocol, then changing - // the protocol does weird things - // first, if it's not file:, then we MUST have a host, - // and if there was a path - // to begin with, then we MUST have a path. - // if it is file:, then the host is dropped, - // because that's known to be hostless. - // anything else is assumed to be absolute. - if (!slashedProtocol[relative.protocol]) { - relative.href = urlFormat(relative); - return relative; - } - source.protocol = relative.protocol; - if (!relative.host && !hostlessProtocol[relative.protocol]) { - var relPath = (relative.pathname || '').split('/'); - while (relPath.length && !(relative.host = relPath.shift())); - if (!relative.host) relative.host = ''; - if (!relative.hostname) relative.hostname = ''; - if (relPath[0] !== '') relPath.unshift(''); - if (relPath.length < 2) relPath.unshift(''); - relative.pathname = relPath.join('/'); - } - source.pathname = relative.pathname; - source.search = relative.search; - source.query = relative.query; - source.host = relative.host || ''; - source.auth = relative.auth; - source.hostname = relative.hostname || relative.host; - source.port = relative.port; - //to support http.request - if (source.pathname !== undefined || source.search !== undefined) { - source.path = (source.pathname ? source.pathname : '') + - (source.search ? source.search : ''); - } - source.slashes = source.slashes || relative.slashes; - source.href = urlFormat(source); - return source; - } - - var isSourceAbs = (source.pathname && source.pathname.charAt(0) === '/'), - isRelAbs = ( - relative.host !== undefined || - relative.pathname && relative.pathname.charAt(0) === '/' - ), - mustEndAbs = (isRelAbs || isSourceAbs || - (source.host && relative.pathname)), - removeAllDots = mustEndAbs, - srcPath = source.pathname && source.pathname.split('/') || [], - relPath = relative.pathname && relative.pathname.split('/') || [], - psychotic = source.protocol && - !slashedProtocol[source.protocol]; - - // if the url is a non-slashed url, then relative - // links like ../.. should be able - // to crawl up to the hostname, as well. This is strange. - // source.protocol has already been set by now. - // Later on, put the first path part into the host field. - if (psychotic) { - - delete source.hostname; - delete source.port; - if (source.host) { - if (srcPath[0] === '') srcPath[0] = source.host; - else srcPath.unshift(source.host); - } - delete source.host; - if (relative.protocol) { - delete relative.hostname; - delete relative.port; - if (relative.host) { - if (relPath[0] === '') relPath[0] = relative.host; - else relPath.unshift(relative.host); - } - delete relative.host; - } - mustEndAbs = mustEndAbs && (relPath[0] === '' || srcPath[0] === ''); - } - - if (isRelAbs) { - // it's absolute. - source.host = (relative.host || relative.host === '') ? - relative.host : source.host; - source.hostname = (relative.hostname || relative.hostname === '') ? - relative.hostname : source.hostname; - source.search = relative.search; - source.query = relative.query; - srcPath = relPath; - // fall through to the dot-handling below. - } else if (relPath.length) { - // it's relative - // throw away the existing file, and take the new path instead. - if (!srcPath) srcPath = []; - srcPath.pop(); - srcPath = srcPath.concat(relPath); - source.search = relative.search; - source.query = relative.query; - } else if ('search' in relative) { - // just pull out the search. - // like href='?foo'. - // Put this after the other two cases because it simplifies the booleans - if (psychotic) { - source.hostname = source.host = srcPath.shift(); - //occationaly the auth can get stuck only in host - //this especialy happens in cases like - //url.resolveObject('mailto:local1@domain1', 'local2@domain2') - var authInHost = source.host && source.host.indexOf('@') > 0 ? - source.host.split('@') : false; - if (authInHost) { - source.auth = authInHost.shift(); - source.host = source.hostname = authInHost.shift(); - } - } - source.search = relative.search; - source.query = relative.query; - //to support http.request - if (source.pathname !== undefined || source.search !== undefined) { - source.path = (source.pathname ? source.pathname : '') + - (source.search ? source.search : ''); - } - source.href = urlFormat(source); - return source; - } - if (!srcPath.length) { - // no path at all. easy. - // we've already handled the other stuff above. - delete source.pathname; - //to support http.request - if (!source.search) { - source.path = '/' + source.search; - } else { - delete source.path; - } - source.href = urlFormat(source); - return source; - } - // if a url ENDs in . or .., then it must get a trailing slash. - // however, if it ends in anything else non-slashy, - // then it must NOT get a trailing slash. - var last = srcPath.slice(-1)[0]; - var hasTrailingSlash = ( - (source.host || relative.host) && (last === '.' || last === '..') || - last === ''); - - // strip single dots, resolve double dots to parent dir - // if the path tries to go above the root, `up` ends up > 0 - var up = 0; - for (var i = srcPath.length; i >= 0; i--) { - last = srcPath[i]; - if (last == '.') { - srcPath.splice(i, 1); - } else if (last === '..') { - srcPath.splice(i, 1); - up++; - } else if (up) { - srcPath.splice(i, 1); - up--; - } - } - - // if the path is allowed to go above the root, restore leading ..s - if (!mustEndAbs && !removeAllDots) { - for (; up--; up) { - srcPath.unshift('..'); - } - } - - if (mustEndAbs && srcPath[0] !== '' && - (!srcPath[0] || srcPath[0].charAt(0) !== '/')) { - srcPath.unshift(''); - } - - if (hasTrailingSlash && (srcPath.join('/').substr(-1) !== '/')) { - srcPath.push(''); - } - - var isAbsolute = srcPath[0] === '' || - (srcPath[0] && srcPath[0].charAt(0) === '/'); - - // put the host back - if (psychotic) { - source.hostname = source.host = isAbsolute ? '' : - srcPath.length ? srcPath.shift() : ''; - //occationaly the auth can get stuck only in host - //this especialy happens in cases like - //url.resolveObject('mailto:local1@domain1', 'local2@domain2') - var authInHost = source.host && source.host.indexOf('@') > 0 ? - source.host.split('@') : false; - if (authInHost) { - source.auth = authInHost.shift(); - source.host = source.hostname = authInHost.shift(); - } - } - - mustEndAbs = mustEndAbs || (source.host && srcPath.length); - - if (mustEndAbs && !isAbsolute) { - srcPath.unshift(''); - } - - source.pathname = srcPath.join('/'); - //to support request.http - if (source.pathname !== undefined || source.search !== undefined) { - source.path = (source.pathname ? source.pathname : '') + - (source.search ? source.search : ''); - } - source.auth = relative.auth || source.auth; - source.slashes = source.slashes || relative.slashes; - source.href = urlFormat(source); - return source; -} - -function parseHost(host) { - var out = {}; - var port = portPattern.exec(host); - if (port) { - port = port[0]; - if (port !== ':') { - out.port = port.substr(1); - } - host = host.substr(0, host.length - port.length); - } - if (host) out.hostname = host; - return out; -} diff --git a/lib/modules/util.js b/lib/modules/util.js deleted file mode 100644 index 311125ef7..000000000 --- a/lib/modules/util.js +++ /dev/null @@ -1,520 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -var formatRegExp = /%[sdj%]/g; -exports.format = function(f) { - if (typeof f !== 'string') { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': return JSON.stringify(args[i++]); - case '%%': return '%'; - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (x === null || typeof x !== 'object') { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; -}; - - -exports.print = function() { - for (var i = 0, len = arguments.length; i < len; ++i) { - process.stdout.write(String(arguments[i])); - } -}; - - -exports.puts = function() { - for (var i = 0, len = arguments.length; i < len; ++i) { - process.stdout.write(arguments[i] + '\n'); - } -}; - - -exports.debug = function(x) { - process.stderr.write('DEBUG: ' + x + '\n'); -}; - - -var error = exports.error = function(x) { - for (var i = 0, len = arguments.length; i < len; ++i) { - process.stderr.write(arguments[i] + '\n'); - } -}; - - -/** -* Echos the value of a value. Trys to print the value out -* in the best way possible given the different types. -* -* @param {Object} obj The object to print out. -* @param {Boolean} showHidden Flag that shows hidden (not enumerable) -* properties of objects. -* @param {Number} depth Depth in which to descend in object. Default is 2. -* @param {Boolean} colors Flag to turn on ANSI escape codes to color the -* output. Default is false (no coloring). -*/ -function inspect(obj, showHidden, depth, colors) { - var ctx = { - showHidden: showHidden, - seen: [], - stylize: colors ? stylizeWithColor : stylizeNoColor - }; - return formatValue(ctx, obj, (typeof depth === 'undefined' ? 2 : depth)); -} -exports.inspect = inspect; - - -// http://en.wikipedia.org/wiki/ANSI_escape_code#graphics -var colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] -}; - -// Don't use 'blue' not visible on cmd.exe -var styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' -}; - - -function stylizeWithColor(str, styleType) { - var style = styles[styleType]; - - if (style) { - return '\033[' + colors[style][0] + 'm' + str + - '\033[' + colors[style][1] + 'm'; - } else { - return str; - } -} - - -function stylizeNoColor(str, styleType) { - return str; -} - - -function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (value && typeof value.inspect === 'function' && - // Filter out the util module, it's inspect function is special - value !== exports && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - return value.inspect(recurseTimes); - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var visibleKeys = Object.keys(value); - var keys = ctx.showHidden ? Object.getOwnPropertyNames(value) : visibleKeys; - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (typeof value === 'function') { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toUTCString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (typeof value === 'function') { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); -} - - -function formatPrimitive(ctx, value) { - switch (typeof value) { - case 'undefined': - return ctx.stylize('undefined', 'undefined'); - - case 'string': - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - - case 'number': - return ctx.stylize('' + value, 'number'); - - case 'boolean': - return ctx.stylize('' + value, 'boolean'); - } - // For some reason typeof null is "object", so special case here. - if (value === null) { - return ctx.stylize('null', 'null'); - } -} - - -function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; -} - - -function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (Object.prototype.hasOwnProperty.call(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; -} - - -function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str; - if (value.__lookupGetter__) { - if (value.__lookupGetter__(key)) { - if (value.__lookupSetter__(key)) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (value.__lookupSetter__(key)) { - str = ctx.stylize('[Setter]', 'special'); - } - } - } - if (visibleKeys.indexOf(key) < 0) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(value[key]) < 0) { - if (recurseTimes === null) { - str = formatValue(ctx, value[key], null); - } else { - str = formatValue(ctx, value[key], recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (typeof name === 'undefined') { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; -} - - -function reduceToSingleString(output, base, braces) { - var numLinesEst = 0; - var length = output.reduce(function(prev, cur) { - numLinesEst++; - if (cur.indexOf('\n') >= 0) numLinesEst++; - return prev + cur.length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; -} - - -// NOTE: These type checking functions intentionally don't use `instanceof` -// because it is fragile and can be easily faked with `Object.create()`. -function isArray(ar) { - return Array.isArray(ar) || - (typeof ar === 'object' && objectToString(ar) === '[object Array]'); -} -exports.isArray = isArray; - - -function isRegExp(re) { - return typeof re === 'object' && objectToString(re) === '[object RegExp]'; -} -exports.isRegExp = isRegExp; - - -function isDate(d) { - return typeof d === 'object' && objectToString(d) === '[object Date]'; -} -exports.isDate = isDate; - - -function isError(e) { - return typeof e === 'object' && objectToString(e) === '[object Error]'; -} -exports.isError = isError; - - -function objectToString(o) { - return Object.prototype.toString.call(o); -} - - -var pWarning; - -exports.p = function() { - if (!pWarning) { - pWarning = 'util.p will be removed in future versions of Node. ' + - 'Use util.puts(util.inspect()) instead.\n'; - exports.error(pWarning); - } - for (var i = 0, len = arguments.length; i < len; ++i) { - error(exports.inspect(arguments[i])); - } -}; - - -function pad(n) { - return n < 10 ? '0' + n.toString(10) : n.toString(10); -} - - -var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', - 'Oct', 'Nov', 'Dec']; - -// 26 Feb 16:19:34 -function timestamp() { - var d = new Date(); - var time = [pad(d.getHours()), - pad(d.getMinutes()), - pad(d.getSeconds())].join(':'); - return [d.getDate(), months[d.getMonth()], time].join(' '); -} - - -exports.log = function(msg) { - exports.puts(timestamp() + ' - ' + msg.toString()); -}; - - -var execWarning; -exports.exec = function() { - if (!execWarning) { - execWarning = 'util.exec has moved to the "child_process" module.' + - ' Please update your source code.'; - error(execWarning); - } - return require('child_process').exec.apply(this, arguments); -}; - - -exports.pump = function(readStream, writeStream, callback) { - var callbackCalled = false; - - function call(a, b, c) { - if (callback && !callbackCalled) { - callback(a, b, c); - callbackCalled = true; - } - } - - readStream.addListener('data', function(chunk) { - if (writeStream.write(chunk) === false) readStream.pause(); - }); - - writeStream.addListener('drain', function() { - readStream.resume(); - }); - - readStream.addListener('end', function() { - writeStream.end(); - }); - - readStream.addListener('close', function() { - call(); - }); - - readStream.addListener('error', function(err) { - writeStream.end(); - call(err); - }); - - writeStream.addListener('error', function(err) { - readStream.destroy(); - call(err); - }); -}; - - -/** -* Inherit the prototype methods from one constructor into another. -* -* The Function.prototype.inherits from lang.js rewritten as a standalone -* function (not on Function.prototype). NOTE: If this file is to be loaded -* during bootstrapping this function needs to be revritten using some native -* functions as prototype setup using normal JavaScript does not work as -* expected during bootstrapping (see mirror.js in r114903). -* -* @param {function} ctor Constructor function which needs to inherit the -* prototype. -* @param {function} superCtor Constructor function to inherit prototype from. -*/ -exports.inherits = function(ctor, superCtor) { - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); -}; From 923b5e1721eb631dc39c7127b546354eb2e95baa Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 14:13:38 +0100 Subject: [PATCH 015/248] Loader: extensions and core modules --- .../navigationTiming/navigationTiming.js | 5 +- .../requestsMonitor/requestsMonitor.js | 8 +-- .../timeToFirstByte/timeToFirstByte.js | 4 +- extensions/cookies/cookies.js | 6 +- extensions/devices/devices.js | 4 +- extensions/filmStrip/filmStrip.js | 4 +- extensions/har/har.js | 8 ++- extensions/httpAuth/httpAuth.js | 4 +- extensions/pageSource/pageSource.js | 4 +- extensions/postLoadDelay/postLoadDelay.js | 4 +- extensions/screenshot/screenshot.js | 6 +- extensions/scroll/scroll.js | 4 +- extensions/waitForEvent/waitForEvent.js | 4 +- extensions/waitForSelector/waitForSelector.js | 4 +- lib/index.js | 26 +++++--- lib/loader.js | 60 +++++++++++++++++++ 16 files changed, 103 insertions(+), 52 deletions(-) create mode 100644 lib/loader.js diff --git a/core/modules/navigationTiming/navigationTiming.js b/core/modules/navigationTiming/navigationTiming.js index 10f419f65..38ff3200a 100644 --- a/core/modules/navigationTiming/navigationTiming.js +++ b/core/modules/navigationTiming/navigationTiming.js @@ -10,10 +10,9 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.once('init', function() { + return; // TODO phantomas.evaluate(function () { (function(phantomas) { function emit(eventName) { diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 81daecbc2..8faa60077 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -3,12 +3,10 @@ */ 'use strict'; -exports.version = '1.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // imports - var HTTP_STATUS_CODES = phantomas.require('http').STATUS_CODES, - parseUrl = phantomas.require('url').parse; + var HTTP_STATUS_CODES = require('http').STATUS_CODES, + parseUrl = require('url').parse; var requests = []; diff --git a/core/modules/timeToFirstByte/timeToFirstByte.js b/core/modules/timeToFirstByte/timeToFirstByte.js index e5768ec46..da2613549 100644 --- a/core/modules/timeToFirstByte/timeToFirstByte.js +++ b/core/modules/timeToFirstByte/timeToFirstByte.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '1.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var measured = false, reqId = 1; // request ID to consider when calculating TTFB / TTLB diff --git a/extensions/cookies/cookies.js b/extensions/cookies/cookies.js index 49a3f797c..bdbbfd17b 100644 --- a/extensions/cookies/cookies.js +++ b/extensions/cookies/cookies.js @@ -4,9 +4,7 @@ /* global phantom: true */ 'use strict'; -exports.version = '1.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var cookiesJar = phantomas.getParam('cookies', [], 'object'), COOKIE_SEPARATOR = '|'; @@ -14,7 +12,7 @@ exports.module = function(phantomas) { // setup cookies handling function initCookies() { // cookie handling via command line and config.json - phantom.cookiesEnabled = true; + //phantom.cookiesEnabled = true; // handles multiple cookies from config.json, and used for storing // constructed cookies from command line. diff --git a/extensions/devices/devices.js b/extensions/devices/devices.js index 76d96cb02..34b9b479a 100644 --- a/extensions/devices/devices.js +++ b/extensions/devices/devices.js @@ -5,9 +5,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var device, // @see https://developers.google.com/chrome/mobile/docs/user-agent?hl=pl // @see http://viewportsizes.com/ diff --git a/extensions/filmStrip/filmStrip.js b/extensions/filmStrip/filmStrip.js index e5567fe39..17e618d58 100644 --- a/extensions/filmStrip/filmStrip.js +++ b/extensions/filmStrip/filmStrip.js @@ -14,9 +14,7 @@ */ 'use strict'; -exports.version = '0.3'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { if (!phantomas.getParam('film-strip')) { phantomas.log('filmStrip: to enable screenshots of page being loaded run phantomas with --film-strip option'); return; diff --git a/extensions/har/har.js b/extensions/har/har.js index fe411f1a7..bc81b4d82 100644 --- a/extensions/har/har.js +++ b/extensions/har/har.js @@ -7,8 +7,6 @@ */ 'use strict'; -exports.version = '0.1'; - var fs = require('fs'); /** @@ -106,12 +104,16 @@ function createHAR(page, creator) { } /** End **/ -exports.module = function(phantomas) { +module.exports = function(phantomas) { var param = phantomas.getParam('har'), path = '', timeToLastByte = undefined; + if (param === false) { + return; + } + var page = { origin: undefined, resources: [], diff --git a/extensions/httpAuth/httpAuth.js b/extensions/httpAuth/httpAuth.js index dcd197ff6..dec8c8005 100644 --- a/extensions/httpAuth/httpAuth.js +++ b/extensions/httpAuth/httpAuth.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var userName = phantomas.getParam('auth-user', '', 'string'), password = phantomas.getParam('auth-pass', '', 'string'); diff --git a/extensions/pageSource/pageSource.js b/extensions/pageSource/pageSource.js index 76138417f..63b5145f6 100644 --- a/extensions/pageSource/pageSource.js +++ b/extensions/pageSource/pageSource.js @@ -8,9 +8,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { if (!phantomas.getParam('page-source')) { phantomas.log('To enable page-source of page being loaded run phantomas with --page-source option'); return; diff --git a/extensions/postLoadDelay/postLoadDelay.js b/extensions/postLoadDelay/postLoadDelay.js index 13c2fa0bf..ca17b0bb5 100644 --- a/extensions/postLoadDelay/postLoadDelay.js +++ b/extensions/postLoadDelay/postLoadDelay.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // e.g. --post-load-delay 5 var delay = parseInt(phantomas.getParam('post-load-delay'), 10); diff --git a/extensions/screenshot/screenshot.js b/extensions/screenshot/screenshot.js index 67e28722f..75979fad9 100644 --- a/extensions/screenshot/screenshot.js +++ b/extensions/screenshot/screenshot.js @@ -5,12 +5,12 @@ */ 'use strict'; -exports.version = '0.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var param = phantomas.getParam('screenshot'), path = ''; + return; // TODO + if (typeof param === 'undefined') { phantomas.log('Screenshot: to enable screenshot of the fully loaded page run phantomas with --screenshot option'); return; diff --git a/extensions/scroll/scroll.js b/extensions/scroll/scroll.js index deca11115..40fdfa31a 100644 --- a/extensions/scroll/scroll.js +++ b/extensions/scroll/scroll.js @@ -6,9 +6,7 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var scroll = phantomas.getParam('scroll') === true; if (!scroll) { diff --git a/extensions/waitForEvent/waitForEvent.js b/extensions/waitForEvent/waitForEvent.js index 69104e244..dea96926f 100644 --- a/extensions/waitForEvent/waitForEvent.js +++ b/extensions/waitForEvent/waitForEvent.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // e.g. --wait-for-event "done" var eventName = phantomas.getParam('wait-for-event'); diff --git a/extensions/waitForSelector/waitForSelector.js b/extensions/waitForSelector/waitForSelector.js index 4a9328f0c..281599f29 100644 --- a/extensions/waitForSelector/waitForSelector.js +++ b/extensions/waitForSelector/waitForSelector.js @@ -4,8 +4,6 @@ /* global document: true */ 'use strict'; -exports.version = '0.2'; - function checkSelector(phantomas, selector) { var res = phantomas.evaluate(function(selector) { return (function(phantomas) { @@ -27,7 +25,7 @@ function checkSelector(phantomas, selector) { return res; } -exports.module = function(phantomas) { +module.exports = function(phantomas) { // e.g. --wait-for-selector "body.loaded" var selector = phantomas.getParam('wait-for-selector'); diff --git a/lib/index.js b/lib/index.js index 1c378797c..bcdb82d5c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -6,6 +6,7 @@ var Browser = require('./browser'), EventEmitter = require('events').EventEmitter, debug = require('debug')('phantomas:core'), + loader = require('./loader'), puppeteer = require("puppeteer"), path = require('path'), Results = require('../core/results'), @@ -34,7 +35,7 @@ function phantomas(url, opts) { options = util._extend({}, opts || {}); // use util._extend to avoid #563 options.url = options.url || url || false; - // log events emitted + // log emitted events events._emit = events.emit; events.log = require('debug')('phantomas:events'); events.emit = function() { @@ -42,18 +43,29 @@ function phantomas(url, opts) { this._emit.apply(this, arguments); } - // TODO: prepare a small instance object that will be passed to modules and extensions on init var results = new Results(); results.setUrl(url); + results.setGenerator('phantomas v' + VERSION); - var scope = { + // TODO: prepare a small instance object that will be passed to modules and extensions on init + const scope = { + getParam: () => false, // TODO + getVersion: () => VERSION, on: events.on.bind(events), - setMetric: results.setMetric, - log: require('debug')('phantomas:modules:requestsStats') + once: events.once.bind(events), + setMetric: results.setMetric }; - // TODO: set up modules and extensions - require('../modules/requestsStats/requestsStats')(scope); + // load modules and extensions + debug('Loading core modules...'); + loader.loadCoreModules(scope); + + debug('Loading extensions...'); + loader.loadExtensions(scope); + + debug('Loading modules...'); + //loader.loadCoreModules(scope); + // set up and run Puppeteer browser = new Browser(); diff --git a/lib/loader.js b/lib/loader.js new file mode 100644 index 000000000..25983164d --- /dev/null +++ b/lib/loader.js @@ -0,0 +1,60 @@ +/** + * Handles loading of modules and extensions + */ +const debug = require('debug'), + extend = require('util')._extend; + +function listModulesInDirectory(modulesDir) { + const log = debug('phantomas:modules'); + log('Getting the list of all modules in ' + modulesDir); + + // https://nodejs.org/api/fs.html#fs_fs_readdirsync_path_options + var fs = require('fs'), + ls = fs.readdirSync(modulesDir, {withFileTypes: true}), + modules = []; + + ls.forEach(function(entry) { + // First check whether an entry is a directory, than check if it contains a module file + // https://nodejs.org/api/fs.html#fs_fs_existssync_path + if (entry.isDirectory() && fs.existsSync(modulesDir + '/' + entry.name + '/' + entry.name + '.js')) { + modules.push(entry.name); + } + }); + + return modules.sort(); +}; + + +function loadCoreModules(scope) { + const modules = [ + 'navigationTiming', + 'requestsMonitor', + 'timeToFirstByte', + ]; + + modules.forEach(name => { + var log = debug('phantomas:modules:' + name), + _scope = extend(scope,{log}); + + _scope.log('Loading...'); + + var module = require(__dirname + '/../core/modules/' + name + '/' + name); + module(_scope); + }); +} + +function loadExtensions(scope) { + const extensions = listModulesInDirectory(__dirname + '/../extensions/'); + + extensions.forEach(name => { + var log = debug('phantomas:extensions:' + name), + _scope = extend(scope,{log}); + + _scope.log('Loading...'); + + var module = require(__dirname + '/../extensions/' + name + '/' + name); + module(_scope); + }); +} + +module.exports = {loadCoreModules, loadExtensions}; From 1b8c39cde2ab8ea722e1d56d1430e5910dd47640 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 14:25:47 +0100 Subject: [PATCH 016/248] Extensions loaded --- lib/index.js | 3 +-- lib/loader.js | 16 +++++++++++++++- modules/ajaxRequests/ajaxRequests.js | 6 +++--- modules/alerts/alerts.js | 4 +--- modules/analyzeCss/analyzeCss.js | 4 +--- modules/assetsTypes/assetsTypes.js | 4 +--- modules/blockDomains/blockDomains.js | 4 +--- modules/cacheHits/cacheHits.js | 4 +--- modules/caching/caching.js | 4 +--- modules/console/console.js | 4 +--- modules/cookies/cookies.js | 6 +++--- modules/documentHeight/documentHeight.js | 6 +++--- modules/domComplexity/domComplexity.js | 6 +++--- modules/domHiddenContent/domHiddenContent.js | 6 +++--- modules/domMutations/domMutations.js | 6 +++--- modules/domQueries/domQueries.js | 6 +++--- modules/domains/domains.js | 4 +--- modules/events/events.js | 6 +++--- modules/globalVariables/globalVariables.js | 6 +++--- modules/headers/headers.js | 4 +--- modules/jQuery/jQuery.js | 6 +++--- .../javaScriptBottlenecks.js | 6 +++--- modules/jserrors/jserrors.js | 4 +--- modules/keepAlive/keepAlive.js | 4 +--- modules/lazyLoadableImages/lazyLoadableImages.js | 6 +++--- modules/localStorage/localStorage.js | 6 +++--- modules/mainRequest/mainRequest.js | 4 +--- modules/redirects/redirects.js | 4 +--- modules/repaints/repaints.js | 6 +++--- modules/requestsTo/requestsTo.js | 4 +--- modules/staticAssets/staticAssets.js | 4 +--- modules/timeToFirst/timeToFirst.js | 4 +--- modules/windowPerformance/windowPerformance.js | 6 +++--- 33 files changed, 77 insertions(+), 96 deletions(-) diff --git a/lib/index.js b/lib/index.js index bcdb82d5c..8826fc31a 100644 --- a/lib/index.js +++ b/lib/index.js @@ -64,8 +64,7 @@ function phantomas(url, opts) { loader.loadExtensions(scope); debug('Loading modules...'); - //loader.loadCoreModules(scope); - + loader.loadModules(scope); // set up and run Puppeteer browser = new Browser(); diff --git a/lib/loader.js b/lib/loader.js index 25983164d..249d1395a 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -57,4 +57,18 @@ function loadExtensions(scope) { }); } -module.exports = {loadCoreModules, loadExtensions}; +function loadModules(scope) { + const extensions = listModulesInDirectory(__dirname + '/../modules/'); + + extensions.forEach(name => { + var log = debug('phantomas:modules:' + name), + _scope = extend(scope,{log}); + + _scope.log('Loading...'); + + var module = require(__dirname + '/../modules/' + name + '/' + name); + module(_scope); + }); +} + +module.exports = {loadCoreModules, loadExtensions, loadModules}; diff --git a/modules/ajaxRequests/ajaxRequests.js b/modules/ajaxRequests/ajaxRequests.js index f17e391e5..ef530e28f 100644 --- a/modules/ajaxRequests/ajaxRequests.js +++ b/modules/ajaxRequests/ajaxRequests.js @@ -4,11 +4,11 @@ /* global window: true */ 'use strict'; -exports.version = '0.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests + return; // TODO + phantomas.on('init', function() { phantomas.evaluate(function() { (function(phantomas) { diff --git a/modules/alerts/alerts.js b/modules/alerts/alerts.js index 5238d2c6c..0f6045cc5 100644 --- a/modules/alerts/alerts.js +++ b/modules/alerts/alerts.js @@ -4,9 +4,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var alerts = [], confirms = [], prompts = []; diff --git a/modules/analyzeCss/analyzeCss.js b/modules/analyzeCss/analyzeCss.js index c8fb38e68..ef0c50761 100644 --- a/modules/analyzeCss/analyzeCss.js +++ b/modules/analyzeCss/analyzeCss.js @@ -44,9 +44,7 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '0.6'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { if (!phantomas.getParam('analyze-css')) { phantomas.log('To enable CSS in-depth metrics please run phantomas with --analyze-css option'); return; diff --git a/modules/assetsTypes/assetsTypes.js b/modules/assetsTypes/assetsTypes.js index 7b1fa8dad..7769bc4a7 100644 --- a/modules/assetsTypes/assetsTypes.js +++ b/modules/assetsTypes/assetsTypes.js @@ -22,9 +22,7 @@ */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { ['html', 'css', 'js', 'json', 'image', 'video', 'webfont', 'base64', 'other'].forEach(function(key) { phantomas.setMetric(key + 'Count'); phantomas.setMetric(key + 'Size'); diff --git a/modules/blockDomains/blockDomains.js b/modules/blockDomains/blockDomains.js index 228d9806c..e0d35009a 100644 --- a/modules/blockDomains/blockDomains.js +++ b/modules/blockDomains/blockDomains.js @@ -5,9 +5,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var ourDomain = false, // --no-externals diff --git a/modules/cacheHits/cacheHits.js b/modules/cacheHits/cacheHits.js index c4ddf4354..0794e65d5 100644 --- a/modules/cacheHits/cacheHits.js +++ b/modules/cacheHits/cacheHits.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.3'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('cacheHits'); // @desc number of cache hits @offenders phantomas.setMetric('cacheMisses'); // @desc number of cache misses @offenders phantomas.setMetric('cachePasses'); // @desc number of cache passes @offenders diff --git a/modules/caching/caching.js b/modules/caching/caching.js index 8aefae9fe..a84f4a563 100644 --- a/modules/caching/caching.js +++ b/modules/caching/caching.js @@ -5,9 +5,7 @@ */ 'use strict'; -exports.version = '0.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var cacheControlRegExp = /max-age=(\d+)/; function getCachingTime(url, headers) { diff --git a/modules/console/console.js b/modules/console/console.js index 80d1a24af..b75fd205a 100644 --- a/modules/console/console.js +++ b/modules/console/console.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('consoleMessages'); // @desc number of calls to console.* functions phantomas.on('consoleLog', function(msg) { diff --git a/modules/cookies/cookies.js b/modules/cookies/cookies.js index a8766559d..4a55bd490 100644 --- a/modules/cookies/cookies.js +++ b/modules/cookies/cookies.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.3'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // monitor cookies in HTTP headers var Collection = require('../../lib/collection'), cookiesDomains = new Collection(); @@ -16,6 +14,8 @@ exports.module = function(phantomas) { phantomas.setMetric('documentCookiesLength'); // @desc length of document.cookie phantomas.setMetric('documentCookiesCount'); //@desc number of cookies in document.cookie + return; // TODO + phantomas.on('send', function(entry, res) { res.headers.forEach(function(header) { switch (header.name) { diff --git a/modules/documentHeight/documentHeight.js b/modules/documentHeight/documentHeight.js index 83ccaa341..b6461b87c 100644 --- a/modules/documentHeight/documentHeight.js +++ b/modules/documentHeight/documentHeight.js @@ -4,11 +4,11 @@ /* global document: true */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('documentHeight'); // @desc the page height [px] + return; // TODO + phantomas.on('report', function() { phantomas.setMetricEvaluate('documentHeight', function() { // @see https://github.com/HTTPArchive/httparchive/blob/master/custom_metrics/document_height.js diff --git a/modules/domComplexity/domComplexity.js b/modules/domComplexity/domComplexity.js index 388d94768..d166a3747 100644 --- a/modules/domComplexity/domComplexity.js +++ b/modules/domComplexity/domComplexity.js @@ -4,9 +4,7 @@ /* global document: true, Node: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // total length of HTML comments (including brackets) phantomas.setMetric('commentsSize'); // @desc the size of HTML comments on the page @offenders @@ -26,6 +24,8 @@ exports.module = function(phantomas) { phantomas.setMetric('imagesScaledDown'); // @desc number of nodes that have images scaled down in HTML @offenders phantomas.setMetric('imagesWithoutDimensions'); // @desc number of nodes without both width and height attribute @offenders + return; // TODO + // keep the track of SVG graphics (#479) var svgResources = []; phantomas.on('recv', function(entry) { diff --git a/modules/domHiddenContent/domHiddenContent.js b/modules/domHiddenContent/domHiddenContent.js index 08ac511ea..ddb0b07ad 100644 --- a/modules/domHiddenContent/domHiddenContent.js +++ b/modules/domHiddenContent/domHiddenContent.js @@ -4,14 +4,14 @@ /* global document: true, Node: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // total length of HTML of hidden elements (i.e. display: none) phantomas.setMetric('hiddenContentSize'); // @desc the size of content of hidden elements on the page (with CSS display: none) @offenders phantomas.setMetric('hiddenImages'); // @desc number of hidden images that can be lazy-loaded @offenders + return; // TODO + // HTML size phantomas.on('report', function() { phantomas.evaluate(function() { diff --git a/modules/domMutations/domMutations.js b/modules/domMutations/domMutations.js index 7f4a5509f..fcc64bc47 100644 --- a/modules/domMutations/domMutations.js +++ b/modules/domMutations/domMutations.js @@ -6,14 +6,14 @@ /* global window: true, document: true, MutationObserver: true */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // SlimerJS only metrics phantomas.setMetric('DOMmutationsInserts'); // @desc number of node inserts phantomas.setMetric('DOMmutationsRemoves'); // @desc number of node removes phantomas.setMetric('DOMmutationsAttributes'); // @desc number of DOM nodes attributes changes + return; // TODO + phantomas.on('init', function() { phantomas.evaluate(function() { (function(phantomas) { diff --git a/modules/domQueries/domQueries.js b/modules/domQueries/domQueries.js index b71fe8329..2c859d000 100644 --- a/modules/domQueries/domQueries.js +++ b/modules/domQueries/domQueries.js @@ -4,9 +4,7 @@ /* global Element: true, Document: true, Node: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('DOMqueries'); // @desc number of all DOM queries @offenders phantomas.setMetric('DOMqueriesWithoutResults'); // @desc number of DOM queries that returned nothing @offenders phantomas.setMetric('DOMqueriesById'); // @desc number of document.getElementById calls @@ -17,6 +15,8 @@ exports.module = function(phantomas) { phantomas.setMetric('DOMqueriesDuplicated'); // @desc number of DOM queries called more than once phantomas.setMetric('DOMqueriesAvoidable'); // @desc number of repeated uses of a duplicated query + return; // TODO + // fake native DOM functions phantomas.on('init', function() { phantomas.evaluate(function() { diff --git a/modules/domains/domains.js b/modules/domains/domains.js index ba5f16ece..2cef05395 100644 --- a/modules/domains/domains.js +++ b/modules/domains/domains.js @@ -3,11 +3,9 @@ */ 'use strict'; -exports.version = '0.3'; - var Stats = require('../../lib/fast-stats').Stats; -exports.module = function(phantomas) { +module.exports = function(phantomas) { var Collection = require('../../lib/collection'), domains = new Collection(); diff --git a/modules/events/events.js b/modules/events/events.js index ef72eea6e..f15f93b0d 100644 --- a/modules/events/events.js +++ b/modules/events/events.js @@ -4,13 +4,13 @@ /* global Document: true, Element: true, window: true */ 'use strict'; -exports.version = '0.4'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('eventsBound'); // @desc number of EventTarget.addEventListener calls phantomas.setMetric('eventsDispatched'); // @desc number of EventTarget.dispatchEvent calls phantomas.setMetric('eventsScrollBound'); // @desc number of scroll event bounds + return; // TODO + phantomas.on('init', function() { phantomas.evaluate(function() { (function(phantomas) { diff --git a/modules/globalVariables/globalVariables.js b/modules/globalVariables/globalVariables.js index 0644f73a9..1056c5aca 100644 --- a/modules/globalVariables/globalVariables.js +++ b/modules/globalVariables/globalVariables.js @@ -4,12 +4,12 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '0.3'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('globalVariables'); // @desc number of JS globals variables @offenders phantomas.setMetric('globalVariablesFalsy'); // @desc number of JS globals variables with falsy value @offenders + return; // TODO + phantomas.on('report', function() { phantomas.evaluate(function() { (function(phantomas) { diff --git a/modules/headers/headers.js b/modules/headers/headers.js index b4271f05c..830eb38af 100644 --- a/modules/headers/headers.js +++ b/modules/headers/headers.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('headersCount'); // @desc number of requests and responses headers phantomas.setMetric('headersSentCount'); // @desc number of headers sent in requests phantomas.setMetric('headersRecvCount'); // @desc number of headers received in responses diff --git a/modules/jQuery/jQuery.js b/modules/jQuery/jQuery.js index 370f83d02..cfd730986 100644 --- a/modules/jQuery/jQuery.js +++ b/modules/jQuery/jQuery.js @@ -7,9 +7,7 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var lastUrl; phantomas.setMetric('jQueryVersion', ''); // @desc version of jQuery framework (if loaded) [string] @@ -22,6 +20,8 @@ exports.module = function(phantomas) { phantomas.setMetric('jQueryDOMWrites'); // @desc number of DOM write operations phantomas.setMetric('jQueryDOMWriteReadSwitches'); // @desc number of read operations that follow a series of write operations (will cause repaint and can cause reflow) + return; // TODO + // spy calls to jQuery functions phantomas.on('init', function() { phantomas.evaluate(function() { diff --git a/modules/javaScriptBottlenecks/javaScriptBottlenecks.js b/modules/javaScriptBottlenecks/javaScriptBottlenecks.js index 1e3ec8371..704e1183d 100644 --- a/modules/javaScriptBottlenecks/javaScriptBottlenecks.js +++ b/modules/javaScriptBottlenecks/javaScriptBottlenecks.js @@ -10,12 +10,12 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '0.2'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('documentWriteCalls'); //@desc number of calls to either document.write or document.writeln @offenders phantomas.setMetric('evalCalls'); // @desc number of calls to eval (either direct or via setTimeout / setInterval) @offenders + return; // TODO + // spy calls to eval only when requested (issue #467) var spyEval = phantomas.getParam('spy-eval') === true; if (!spyEval) { diff --git a/modules/jserrors/jserrors.js b/modules/jserrors/jserrors.js index 5d5df0926..04d1fee5b 100644 --- a/modules/jserrors/jserrors.js +++ b/modules/jserrors/jserrors.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.3'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('jsErrors'); // @desc number of JavaScript errors function formatTrace(trace) { diff --git a/modules/keepAlive/keepAlive.js b/modules/keepAlive/keepAlive.js index fb0fa3b95..60a5405d9 100644 --- a/modules/keepAlive/keepAlive.js +++ b/modules/keepAlive/keepAlive.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var closedConnectionHosts = {}; phantomas.setMetric('closedConnections'); // @desc number of requests not keeping the connection alive and slowing down the next request diff --git a/modules/lazyLoadableImages/lazyLoadableImages.js b/modules/lazyLoadableImages/lazyLoadableImages.js index 40d7077ca..4eb8ee712 100644 --- a/modules/lazyLoadableImages/lazyLoadableImages.js +++ b/modules/lazyLoadableImages/lazyLoadableImages.js @@ -6,11 +6,11 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('lazyLoadableImagesBelowTheFold'); // @desc number of images displayed below the fold that can be lazy-loaded + return; // TODO + phantomas.on('report', function() { phantomas.log('lazyLoadableImages: analyzing which images can be lazy-loaded...'); diff --git a/modules/localStorage/localStorage.js b/modules/localStorage/localStorage.js index 14f2db93c..57d1d8cbf 100644 --- a/modules/localStorage/localStorage.js +++ b/modules/localStorage/localStorage.js @@ -4,11 +4,11 @@ /* global window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('localStorageEntries'); // @desc number of entries in local storage + return; // TODO + phantomas.on('report', function() { phantomas.evaluate(function() { (function(phantomas) { diff --git a/modules/mainRequest/mainRequest.js b/modules/mainRequest/mainRequest.js index 5b99b205d..3372d3279 100644 --- a/modules/mainRequest/mainRequest.js +++ b/modules/mainRequest/mainRequest.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var isMainRequest = true; var statusCodes = []; diff --git a/modules/redirects/redirects.js b/modules/redirects/redirects.js index e8d8c8ab2..0f8350e83 100644 --- a/modules/redirects/redirects.js +++ b/modules/redirects/redirects.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('redirects'); // @desc number of HTTP redirects (either 301, 302 or 303) phantomas.setMetric('redirectsTime'); // @desc time it took to send and receive redirects diff --git a/modules/repaints/repaints.js b/modules/repaints/repaints.js index c1ac507fc..6bd892e93 100644 --- a/modules/repaints/repaints.js +++ b/modules/repaints/repaints.js @@ -7,12 +7,12 @@ /* global window: true */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('repaints'); // @desc number of repaints of the current document @gecko phantomas.setMetric('firstPaint'); // @desc time it took to perform the first paint @gecko @unreliable + return; // TODO + phantomas.on('init', function() { phantomas.evaluate(function() { (function(phantomas) { diff --git a/modules/requestsTo/requestsTo.js b/modules/requestsTo/requestsTo.js index 160dbb022..3b05d5502 100644 --- a/modules/requestsTo/requestsTo.js +++ b/modules/requestsTo/requestsTo.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { phantomas.setMetric('requestsToFirstPaint'); // @desc number of HTTP requests it took to make the first paint @gecko phantomas.setMetric('domainsToFirstPaint'); // @desc number of domains used to make the first paint @offenders @gecko @offenders phantomas.setMetric('requestsToDomContentLoaded'); // @desc number of HTTP requests it took to make the page reach DomContentLoaded state diff --git a/modules/staticAssets/staticAssets.js b/modules/staticAssets/staticAssets.js index f15b909d9..8d331f164 100644 --- a/modules/staticAssets/staticAssets.js +++ b/modules/staticAssets/staticAssets.js @@ -3,9 +3,7 @@ */ 'use strict'; -exports.version = '0.5'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { var SIZE_THRESHOLD = 2 * 1024; // count requests for each asset diff --git a/modules/timeToFirst/timeToFirst.js b/modules/timeToFirst/timeToFirst.js index 8528e8ff5..caa94c6b7 100644 --- a/modules/timeToFirst/timeToFirst.js +++ b/modules/timeToFirst/timeToFirst.js @@ -7,9 +7,7 @@ */ 'use strict'; -exports.version = '0.1'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { function capitalize(txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); } diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index abcde7569..ad32c53b1 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -7,9 +7,7 @@ /* global document: true, window: true */ 'use strict'; -exports.version = '1.0'; - -exports.module = function(phantomas) { +module.exports = function(phantomas) { // times below are calculated relative to performance.timing.responseEnd (#117) phantomas.setMetric('domInteractive'); // @desc time it took to parse the HTML and construct the DOM phantomas.setMetric('domContentLoaded'); // @desc time it took to construct both DOM and CSSOM, no stylesheets that are blocking JavaScript execution (i.e. onDOMReady) @@ -20,6 +18,8 @@ exports.module = function(phantomas) { phantomas.setMetric('timeBackend'); // @desc time to the first byte compared to the total loading time [%] phantomas.setMetric('timeFrontend'); // @desc time to window.load compared to the total loading time [%] + return; // TODO + // measure dom... metrics from the moment HTML response was fully received var responseEndTime = Date.now(); From 3e7136be52ddecaa7c84ee8cb35350c1411097cf Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 14:27:13 +0100 Subject: [PATCH 017/248] MaxListenersExceededWarning: Possible EventEmitter memory leak detected --- lib/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/index.js b/lib/index.js index 8826fc31a..eb47f6767 100644 --- a/lib/index.js +++ b/lib/index.js @@ -43,6 +43,8 @@ function phantomas(url, opts) { this._emit.apply(this, arguments); } + events.setMaxListeners(100); // MaxListenersExceededWarning: Possible EventEmitter memory leak detected. + var results = new Results(); results.setUrl(url); results.setGenerator('phantomas v' + VERSION); From ffb5db3cae2ef6cbcff17645dcb5f9f04956c964 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 22:52:00 +0100 Subject: [PATCH 018/248] Use low-level Chrome Devtools Protocol when handling network traffic events --- lib/browser.js | 48 +++++++++++++++++++++++++++++++++++++++++++++--- lib/index.js | 2 ++ lib/loader.js | 19 ++++++++++++------- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 2e5c15fac..6495a536d 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -14,12 +14,17 @@ Browser.prototype.bind = events => this.events = events; // initialize puppeter instance Browser.prototype.init = async () => { + const networkDebug = require('debug')('phantomas:network'); + debug('Launching Puppeter'); const args = []; // ['--no-sandbox']; this.browser = await puppeteer.launch({args: args}); this.page = await this.browser.newPage(); + // A Chrome Devtools Protocol session attached to the target + this.cdp = this.page._client; + this.events.emit('init', this.browser, this.page); debug('Using binary from: %s', this.browser.process().spawnfile); @@ -41,12 +46,14 @@ Browser.prototype.init = async () => { * * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-request * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request - */ + * this.page.on('request', req => { - debug('> [%s] %s %s', req.resourceType(), req.method(), req.url()); + networkDebug('> [%s] %s %s', req.resourceType(), req.method(), req.url()); + console.log(req); this.events.emit('request', req); }); + **/ /** * Emit an event when browser makes a request @@ -54,7 +61,7 @@ Browser.prototype.init = async () => { * @param {Puppeteer.Response} resp * * https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-response - */ + * this.page.on('response', async resp => { var responseLength = 0; @@ -64,10 +71,45 @@ Browser.prototype.init = async () => { } catch {}; + networkDebug('response', resp); debug('< HTTP %s %s (%f kB)', resp.status(), resp.url(), 1. * responseLength / 1024); this.events.emit('response', resp); }); + **/ + + /** + * Bind to low-level network events + * + * https://chromedevtools.github.io/devtools-protocol/tot/Network + */ + + // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-requestWillBeSent + this.cdp.on('Network.requestWillBeSent', data => { + /** @type {Request} request */ + var request = data.request; + request._requestId = data.requestId; + request._timestamp = data.timestamp; + request._type = data.type; + request._initiator = data.initiator; + + networkDebug('> %s %s [%s]', request.method, request.url, request._initiator.type); + + this.events.emit('request', request); + }); + + // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-responseReceived + this.cdp.on('Network.responseReceived', data => { + /** @type {Response} response */ + var response = data.response; + response._requestId = data.requestId; + response._timestamp = data.timestamp; + + //networkDebug('responseReceived', response); + networkDebug('< %s %s %s %s', response.protocol, response.status, response.statusText, response.url); + + this.events.emit('response', response); + }); }; /** diff --git a/lib/index.js b/lib/index.js index eb47f6767..f47188744 100644 --- a/lib/index.js +++ b/lib/index.js @@ -51,10 +51,12 @@ function phantomas(url, opts) { // TODO: prepare a small instance object that will be passed to modules and extensions on init const scope = { + emit: events.emit.bind(events), getParam: () => false, // TODO getVersion: () => VERSION, on: events.on.bind(events), once: events.once.bind(events), + incrMetric: results.incrMetric.bind(results), setMetric: results.setMetric }; diff --git a/lib/loader.js b/lib/loader.js index 249d1395a..592cbdce1 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -34,9 +34,10 @@ function loadCoreModules(scope) { modules.forEach(name => { var log = debug('phantomas:modules:' + name), - _scope = extend(scope,{log}); - - _scope.log('Loading...'); + _scope = extend({}, scope); + + _scope.log = log; + //_scope.log('Loading...'); var module = require(__dirname + '/../core/modules/' + name + '/' + name); module(_scope); @@ -48,9 +49,11 @@ function loadExtensions(scope) { extensions.forEach(name => { var log = debug('phantomas:extensions:' + name), - _scope = extend(scope,{log}); + _scope = extend({}, scope); + + _scope.log = log; - _scope.log('Loading...'); + //_scope.log('Loading...'); var module = require(__dirname + '/../extensions/' + name + '/' + name); module(_scope); @@ -62,9 +65,11 @@ function loadModules(scope) { extensions.forEach(name => { var log = debug('phantomas:modules:' + name), - _scope = extend(scope,{log}); + _scope = extend({}, scope); + + _scope.log = log; - _scope.log('Loading...'); + //_scope.log('Loading...'); var module = require(__dirname + '/../modules/' + name + '/' + name); module(_scope); From 41a458988606046e7c4da944f06667d2a93a32b5 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 28 Dec 2018 22:52:19 +0100 Subject: [PATCH 019/248] More work on network traffic handling --- .../requestsMonitor/requestsMonitor.js | 73 ++++++------------- core/results.js | 5 ++ examples/promise.js | 3 +- modules/headers/headers.js | 10 ++- 4 files changed, 35 insertions(+), 56 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 8faa60077..d692a0925 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -44,62 +44,33 @@ module.exports = function(phantomas) { } } - phantomas.on('onResourceRequested', function(res, request) { + phantomas.on('request', /** @param {Request} request **/ (request) => { + const resId = request._requestId; + // store current request data - var entry = requests[res.id] = { - id: res.id, - url: res.url, - method: res.method, - requestHeaders: {}, - sendTime: res.time, - bodySize: 0, - isBlocked: false - }; - - // allow modules to block requests - entry.block = function() { - this.isBlocked = true; - }; - - res.headers.forEach(function(header) { - entry.requestHeaders[header.name] = header.value; - }); - - parseEntryUrl(entry); - - if (entry.isBase64) { - return; - } + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request + var entry = requests[resId] = request; - // give modules a chance to block requests using entry.block() - // @see https://github.com/ariya/phantomjs/issues/10230 - phantomas.emitInternal('beforeSend', entry, res); // @desc allows the request to be blocked + phantomas.log('Request: %j', entry); + phantomas.emit('send', entry); // @desc request has been sent + }); - if ( (entry.isBlocked === true) && (typeof request !== 'undefined') ) { - phantomas.log('Blocked request: <' + entry.url + '>'); - request.abort(); - return; - } + phantomas.on('response', /** @param {Response} resp **/ resp => { + const resId = resp._requestId; + var entry = requests[resId]; - phantomas.log('req: <%s>', entry.url); + phantomas.log('Response: %j', resp); - phantomas.emit('send', entry, res); // @desc request has been sent - }); + // get navigation timing details + // https://www.w3.org/TR/navigation-timing/#performancetiming + // https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceTiming + phantomas.log('Timing', entry._timestamp, resp.timing); - phantomas.on('onResourceReceived', function(res) { - // current request data - var entry = requests[res.id] || {}; - - // fix for blocked requests still "emitting" onResourceReceived with "stage" = 'end' and empty "status" (issue #122) - // or empty URL (broken in PhantomJS 2.0.1 (PR #573) - if (res.status === null || res.url === '' ) { - if (entry.isBlocked) { - return; - } else if (!entry.isBase64) { - phantomas.log('Got a response with no status or URL set: <%s> (%j)', res.url, res); - phantomas.emitInternal('abort', entry, res); // @desc request has been blocked - } - } + return; // TODO + + entry.timeToFirstByte = resp.timing.sendStart; + entry.timeToLastByte = resp.timing.sendEnd; + entry.receiveTime = entry.recvEndTime - entry.recvStartTime; switch(res.stage) { // the beginning of response @@ -327,7 +298,7 @@ module.exports = function(phantomas) { loadStartedTime = Date.now(); }); - phantomas.on('recv', function(entry, res) { + phantomas.on('recv', function(entry, res) { phantomas.setMetric('httpTrafficCompleted', entry.recvEndTime - loadStartedTime); }); }; diff --git a/core/results.js b/core/results.js index d4b3cd3d9..8eaa3b3f4 100644 --- a/core/results.js +++ b/core/results.js @@ -31,6 +31,11 @@ module.exports = function(data) { offenders[metricName] = undefined; } }, + // increements given metric by given number (default is one) + incrMetric: function(name, incr /* =1 */ ) { + var currVal = this.getMetric(name) || 0; + this.setMetric(name, currVal + (typeof incr === 'number' ? incr : 1)); + }, getMetric: function(name) { return metrics[name]; }, diff --git a/examples/promise.js b/examples/promise.js index cf4536822..63f4aa18a 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -7,7 +7,7 @@ const phantomas = require('..'); console.log('phantomas v%s loaded from %s', phantomas.version, phantomas.path); -const promise = phantomas('http://google.is', { +const promise = phantomas('http://0.0.0.0', { 'analyze-css': true, 'assert-requests': 1 }); @@ -20,6 +20,7 @@ const promise = phantomas('http://google.is', { // handle the promise promise. then(results => { + console.log('Metrics: %j', results.getMetrics()); console.log('Number of requests: %d', results.getMetric('requests')); console.log('Failed asserts: %j', results.getFailedAsserts()); }). diff --git a/modules/headers/headers.js b/modules/headers/headers.js index 830eb38af..24eaabf0a 100644 --- a/modules/headers/headers.js +++ b/modules/headers/headers.js @@ -14,6 +14,8 @@ module.exports = function(phantomas) { phantomas.setMetric('headersBiggerThanContent'); // @desc number of responses with headers part bigger than the response body + return; // TODO + function processHeaders(headers) { var res = { count: 0, @@ -30,8 +32,8 @@ module.exports = function(phantomas) { return res; } - phantomas.on('send', function(entry, res) { - var headers = processHeaders(res.headers); + phantomas.on('send', function(entry) { + var headers = processHeaders(entry.headers); phantomas.incrMetric('headersCount', headers.count); phantomas.incrMetric('headersSize', headers.size); @@ -40,8 +42,8 @@ module.exports = function(phantomas) { phantomas.incrMetric('headersSentSize', headers.size); }); - phantomas.on('recv', function(entry, res) { - var headers = processHeaders(res.headers); + phantomas.on('recv', function(entry) { + var headers = processHeaders(entry.headers); phantomas.incrMetric('headersCount', headers.count); phantomas.incrMetric('headersSize', headers.size); From c65d5db3aa75003c138f2d07ad78569c5593e10b Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 29 Dec 2018 13:34:13 +0100 Subject: [PATCH 020/248] Improved handling of HTTP responses + types detection --- .../requestsMonitor/requestsMonitor.js | 337 +++++++++--------- lib/browser.js | 79 ++-- lib/index.js | 5 +- 3 files changed, 213 insertions(+), 208 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index d692a0925..bf8f5767e 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -3,21 +3,120 @@ */ 'use strict'; +const debug = require('debug')('phantomas:modules:requestsMonitor'); + +/** + * Detect response content type using "Content-Type header value" + * + * @param {string} headerValue + * @param {object} entry + */ +function addContentType(headerValue, entry) { + var value = headerValue.split(';').shift().toLowerCase(); + entry.contentType = value; + + switch(value) { + case 'text/html': + entry.type = 'html'; + entry.isHTML = true; + break; + + case 'text/xml': + entry.type = 'xml'; + entry.isXML = true; + break; + + case 'text/css': + entry.type = 'css'; + entry.isCSS = true; + break; + + case 'application/x-javascript': + case 'application/javascript': + case 'text/javascript': + entry.type = 'js'; + entry.isJS = true; + break; + + case 'application/json': + entry.type = 'json'; + entry.isJSON = true; + break; + + case 'image/png': + case 'image/jpeg': + case 'image/gif': + case 'image/svg+xml': + case 'image/webp': + entry.type = 'image'; + entry.isImage = true; + + if (value === 'image/svg+xml') { + entry.isSVG = true; + } + break; + + case 'video/webm': + entry.type = 'video'; + entry.isVideo = true; + break; + + // @see http://stackoverflow.com/questions/2871655/proper-mime-type-for-fonts + case 'application/font-wof': + case 'application/font-woff': + case 'application/vnd.ms-fontobject': + case 'application/x-font-opentype': + case 'application/x-font-truetype': + case 'application/x-font-ttf': + case 'application/x-font-woff': + case 'font/opentype': + case 'font/ttf': + case 'font/woff': + entry.type = 'webfont'; + entry.isWebFont = true; + + if (/ttf|truetype$/.test(value)) { + entry.isTTF = true; + } + break; + + case 'application/octet-stream': + var ext = (entry.url || '').split('.').pop(); + + switch(ext) { + // @see http://stackoverflow.com/questions/2871655/proper-mime-type-for-fonts#comment-8077637 + case 'otf': + entry.type = 'webfont'; + entry.isWebFont = true; + break; + } + break; + + case 'image/x-icon': + case 'image/vnd.microsoft.icon': + entry.type = 'favicon'; + entry.isFavicon = true; + break; + + default: + debug('Unknown content type found: ' + value + ' for <' + entry.url + '>'); + } + + return entry; +} + module.exports = function(phantomas) { // imports var HTTP_STATUS_CODES = require('http').STATUS_CODES, parseUrl = require('url').parse; - var requests = []; - // register metric phantomas.setMetric('requests'); // @desc total number of HTTP requests made phantomas.setMetric('gzipRequests'); // @desc number of gzipped HTTP responses @unreliable phantomas.setMetric('postRequests'); // @desc number of POST requests phantomas.setMetric('httpsRequests'); // @desc number of HTTPS requests phantomas.setMetric('notFound'); // @desc number of HTTP 404 responses - phantomas.setMetric('bodySize'); // @desc size of the uncompressed content of all responses @unreliable - phantomas.setMetric('contentLength'); // @desc size of the content of all responses (based on Content-Length header) @unreliable + phantomas.setMetric('bodySize'); // @desc size of the compressed content of all responses, equals Content-Length header value phantomas.setMetric('httpTrafficCompleted'); // @desc time it took to receive the last byte of the last HTTP response // parse given URL to get protocol and domain @@ -44,68 +143,70 @@ module.exports = function(phantomas) { } } + var requests = {}; + phantomas.on('request', /** @param {Request} request **/ (request) => { const resId = request._requestId; + requests[resId] = request; - // store current request data + // request data // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request - var entry = requests[resId] = request; - - phantomas.log('Request: %j', entry); - phantomas.emit('send', entry); // @desc request has been sent + phantomas.emit('send', request); // @desc request has been sent }); phantomas.on('response', /** @param {Response} resp **/ resp => { - const resId = resp._requestId; - var entry = requests[resId]; - - phantomas.log('Response: %j', resp); - - // get navigation timing details - // https://www.w3.org/TR/navigation-timing/#performancetiming - // https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceTiming - phantomas.log('Timing', entry._timestamp, resp.timing); - - return; // TODO - - entry.timeToFirstByte = resp.timing.sendStart; - entry.timeToLastByte = resp.timing.sendEnd; - entry.receiveTime = entry.recvEndTime - entry.recvStartTime; - - switch(res.stage) { - // the beginning of response - case 'start': - entry.recvStartTime = res.time; - entry.timeToFirstByte = res.time - entry.sendTime; - - // FIXME: buggy - // @see https://github.com/ariya/phantomjs/issues/10169 - entry.bodySize = res.bodySize || 0; - break; - - // the end of response - case 'end': - // SlimerJS sets res.bodySize at stage = end - entry.bodySize = entry.bodySize || res.bodySize; - - // timing - entry.recvEndTime = res.time; - entry.timeToLastByte = res.time - entry.sendTime; - entry.receiveTime = entry.recvEndTime - entry.recvStartTime; - - // issue #295 - if (typeof entry.recvStartTime === 'undefined') { - phantomas.log('recv: "start" stage not registered for <%s>!', res.url); - entry.receiveTime = entry.recvEndTime - entry.sendTime; - } + const resId = resp._requestId, + request = requests[resId]; + + var entry = { + id: resId, + url: resp.url, + method: request.method, + headers: resp.headers, + bodySize: resp.encodedDataLength, + }; + + /** + * Time to First Byte is the amount of time it takes for the browser + * to receive the first byte of data from the server + * after the browser makes the request. + * + * https://developers.google.com/web/tools/chrome-devtools/network-performance/reference#timing-explanation + * https://www.w3.org/TR/navigation-timing/#performancetiming + * https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceTiming + * + * All times are in seconds, while times in resp.timing are in miliseconds. + */ + + // how long a given request stalled waiting for DNS, proxy, connection, SSL negotation, etc. + entry.stalled = 1.0 * resp.timing.sendStart / 1000; + + // how it took to receive a first byte of the response after making a request + entry.timeToFirstByte = 1.0 * (resp.timing.receiveHeadersEnd - resp.timing.sendEnd) / 1000; + + // difference between when the request was sent and when it was received + entry.timeToLastByte = resp._timestamp - request._timestamp; + + // POST requests + if (entry.method === 'POST') { + phantomas.incrMetric('postRequests'); + phantomas.addOffender('postRequests', entry.url); + } - // request method - switch(entry.method) { - case 'POST': - phantomas.incrMetric('postRequests'); - phantomas.addOffender('postRequests', entry.url); - break; - } + // response content type + // https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceType + Object.keys(entry.headers).forEach(headerName => { + const headerValue = entry.headers[headerName]; + + switch(headerName.toLowerCase()) { + // detect content type + case 'content-type': + entry = addContentType(headerValue, entry); + break; + } + }); + + /** // asset type entry.type = 'other'; @@ -116,112 +217,6 @@ module.exports = function(phantomas) { entry.headers[header.name] = header.value; switch (header.name.toLowerCase()) { - // TODO: why it's not gzipped? - // because: https://github.com/ariya/phantomjs/issues/10156 - case 'content-length': - entry.contentLength = parseInt(header.value, 10); - - /** - if (entry.bodySize !== entry.contentLength) { - phantomas.log('%s: %j', 'bodySize vs contentLength', {url: entry.url, bodySize: entry.bodySize, contentLength: entry.contentLength}); - } - **/ - break; - - // detect content type - case 'content-type': - // parse header value - var value = header.value.split(';').shift().toLowerCase(); - entry.contentType = value; - - switch(value) { - case 'text/html': - entry.type = 'html'; - entry.isHTML = true; - break; - - case 'text/xml': - entry.type = 'xml'; - entry.isXML = true; - break; - - case 'text/css': - entry.type = 'css'; - entry.isCSS = true; - break; - - case 'application/x-javascript': - case 'application/javascript': - case 'text/javascript': - entry.type = 'js'; - entry.isJS = true; - break; - - case 'application/json': - entry.type = 'json'; - entry.isJSON = true; - break; - - case 'image/png': - case 'image/jpeg': - case 'image/gif': - case 'image/svg+xml': - case 'image/webp': - entry.type = 'image'; - entry.isImage = true; - - if (value === 'image/svg+xml') { - entry.isSVG = true; - } - break; - - case 'video/webm': - entry.type = 'video'; - entry.isVideo = true; - break; - - // @see http://stackoverflow.com/questions/2871655/proper-mime-type-for-fonts - case 'application/font-wof': - case 'application/font-woff': - case 'application/vnd.ms-fontobject': - case 'application/x-font-opentype': - case 'application/x-font-truetype': - case 'application/x-font-ttf': - case 'application/x-font-woff': - case 'font/opentype': - case 'font/ttf': - case 'font/woff': - entry.type = 'webfont'; - entry.isWebFont = true; - - if (/ttf|truetype$/.test(value)) { - entry.isTTF = true; - } - break; - - case 'application/octet-stream': - var ext = (entry.url || '').split('.').pop(); - - switch(ext) { - // @see http://stackoverflow.com/questions/2871655/proper-mime-type-for-fonts#comment-8077637 - case 'otf': - entry.type = 'webfont'; - entry.isWebFont = true; - break; - } - break; - - case 'image/x-icon': - case 'image/vnd.microsoft.icon': - entry.type = 'favicon'; - entry.isFavicon = true; - break; - - default: - phantomas.log('Unknown content type found: ' + value + ' for <' + entry.url + '>'); - } - break; - // detect content encoding case 'content-encoding': if (header.value === 'gzip') { @@ -239,7 +234,7 @@ module.exports = function(phantomas) { parseEntryUrl(entry); // HTTP code - entry.status = res.status || 200 /* for base64 data */; + entry.status = res.status || 200 // for base64 data; entry.statusText = HTTP_STATUS_CODES[entry.status]; switch(entry.status) { @@ -287,9 +282,17 @@ module.exports = function(phantomas) { phantomas.log('recv: HTTP %d <%s> [%s]', entry.status, entry.url, entry.contentType); phantomas.emit('recv' , entry, res); // @desc response has been received } - break; - } + //break; + **/ + + phantomas.log('Response metadata: %j', entry); + + phantomas.incrMetric('requests'); + phantomas.incrMetric('bodySize', entry.bodySize); + + phantomas.emit('recv' , entry, resp); // @desc response has been received }); + // completion of the last HTTP request var loadStartedTime; diff --git a/lib/browser.js b/lib/browser.js index 6495a536d..d0f09a508 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -39,44 +39,8 @@ Browser.prototype.init = async () => { this.events.emit('consoleLog', msg); }); - /** - * Emit an event when browser makes a request - * - * @param {Puppeteer.Request} req - * - * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-request - * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-request - * - this.page.on('request', req => { - networkDebug('> [%s] %s %s', req.resourceType(), req.method(), req.url()); - - console.log(req); - this.events.emit('request', req); - }); - **/ - - /** - * Emit an event when browser makes a request - * - * @param {Puppeteer.Response} resp - * - * https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#class-response - * - this.page.on('response', async resp => { - var responseLength = 0; - - try { - const buffer = await resp.buffer(); - responseLength = buffer.length; - } - catch {}; - - networkDebug('response', resp); - debug('< HTTP %s %s (%f kB)', resp.status(), resp.url(), 1. * responseLength / 1024); - - this.events.emit('response', resp); - }); - **/ + // storage for requests metadata + var responses = {}; /** * Bind to low-level network events @@ -85,6 +49,7 @@ Browser.prototype.init = async () => { */ // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-requestWillBeSent + // Fired when page is about to send HTTP request this.cdp.on('Network.requestWillBeSent', data => { /** @type {Request} request */ var request = data.request; @@ -93,12 +58,18 @@ Browser.prototype.init = async () => { request._type = data.type; request._initiator = data.initiator; - networkDebug('> %s %s [%s]', request.method, request.url, request._initiator.type); + networkDebug('Network.requestWillBeSent > %s %s [%s]', request.method, request.url, request._initiator.type); this.events.emit('request', request); + + responses[data.requestId] = { + _chunks: 0, + _encodedDataLength: 0, + }; }); // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-responseReceived + // Fired when HTTP response is available (HTTP headers are present) this.cdp.on('Network.responseReceived', data => { /** @type {Response} response */ var response = data.response; @@ -106,10 +77,38 @@ Browser.prototype.init = async () => { response._timestamp = data.timestamp; //networkDebug('responseReceived', response); - networkDebug('< %s %s %s %s', response.protocol, response.status, response.statusText, response.url); + //networkDebug('Network.responseReceived < %s %s %s %s', response.protocol, response.status, response.statusText, response.url); + + // next event tells us that the response was fully fetched + responses[data.requestId].response = response; + }); + + // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-loadingFinished + // Fired when HTTP request has finished loading + this.cdp.on('Network.loadingFinished', data => { + var meta = responses[data.requestId], + response = meta.response; + + response.encodedDataLength = meta._encodedDataLength; + response.chunks = meta._chunks; + response._timestamp = data.timestamp; + + //networkDebug('Network.loadingFinished', data); + networkDebug('Network.loadingFinished < %s %s %s %s (%f kB fetched)', + response.protocol, response.status, response.statusText, response.url, 1.0 * response.encodedDataLength / 1024); this.events.emit('response', response); }); + + // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-dataReceived + // Fired when data chunk was received over the network + this.cdp.on('Network.dataReceived', data => { + //networkDebug('Network.dataReceived', data); + + responses[data.requestId]._chunks++; + // Actual bytes received (might be less than dataLength for compressed encodings) + responses[data.requestId]._encodedDataLength += data.encodedDataLength; + }); }; /** diff --git a/lib/index.js b/lib/index.js index f47188744..da04c62d2 100644 --- a/lib/index.js +++ b/lib/index.js @@ -51,11 +51,14 @@ function phantomas(url, opts) { // TODO: prepare a small instance object that will be passed to modules and extensions on init const scope = { - emit: events.emit.bind(events), getParam: () => false, // TODO getVersion: () => VERSION, + + emit: events.emit.bind(events), on: events.on.bind(events), once: events.once.bind(events), + + addOffender: results.addOffender.bind(results), incrMetric: results.incrMetric.bind(results), setMetric: results.setMetric }; From 6054f6839eb1c58fcd02d18bf24c018c38f9a2e7 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 29 Dec 2018 13:34:33 +0100 Subject: [PATCH 021/248] modules/assetsTypes: bodySize is now reliable, yay! --- modules/assetsTypes/assetsTypes.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/assetsTypes/assetsTypes.js b/modules/assetsTypes/assetsTypes.js index 7769bc4a7..d6638ec00 100644 --- a/modules/assetsTypes/assetsTypes.js +++ b/modules/assetsTypes/assetsTypes.js @@ -2,23 +2,23 @@ * Analyzes number of requests and sizes of different types of assets * * setMetric('htmlCount') @desc number of HTML responses @offenders - * setMetric('htmlSize') @desc size of HTML responses @unreliable + * setMetric('htmlSize') @desc size of HTML responses * setMetric('cssCount') @desc number of CSS responses @offenders - * setMetric('cssSize') @desc size of CSS responses @unreliable + * setMetric('cssSize') @desc size of CSS responses * setMetric('jsCount') @desc number of JS responses @offenders - * setMetric('jsSize') @desc size of JS responses @unreliable + * setMetric('jsSize') @desc size of JS responses * setMetric('jsonCount') @desc number of JSON responses @offenders - * setMetric('jsonSize') @desc size of JSON responses @unreliable + * setMetric('jsonSize') @desc size of JSON responses * setMetric('imageCount') @desc number of image responses @offenders - * setMetric('imageSize') @desc size of image responses @unreliable + * setMetric('imageSize') @desc size of image responses * setMetric('webfontCount') @desc number of web font responses @offenders - * setMetric('webfontSize') @desc size of web font responses @unreliable - * setMetric('videoCount') @desc number of video responses @offenders @gecko - * setMetric('videoSize') @desc size of video responses @gecko + * setMetric('webfontSize') @desc size of web font responses + * setMetric('videoCount') @desc number of video responses @offenders + * setMetric('videoSize') @desc size of video responses * setMetric('base64Count') @desc number of base64 encoded "responses" (no HTTP request was actually made) @offenders - * setMetric('base64Size') @desc size of base64 encoded responses @unreliable + * setMetric('base64Size') @desc size of base64 encoded responses * setMetric('otherCount') @desc number of other responses @offenders - * setMetric('otherSize') @desc size of other responses @unreliable + * setMetric('otherSize') @desc size of other responses */ 'use strict'; @@ -29,7 +29,7 @@ module.exports = function(phantomas) { }); phantomas.on('recv', function(entry, res) { - var size = entry.contentLength; + var size = entry.bodySize; phantomas.incrMetric(entry.type + 'Count'); phantomas.incrMetric(entry.type + 'Size', size); From 92e08bedd6158e4cfd8c1223804b505076541b05 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 29 Dec 2018 14:17:41 +0100 Subject: [PATCH 022/248] Add js-yaml (required by tests) and get rid of simple-queue --- core/phantomas.js | 143 -------------------------------------- package-lock.json | 31 +++++++++ package.json | 1 + test/simple-queue-test.js | 141 ------------------------------------- 4 files changed, 32 insertions(+), 284 deletions(-) delete mode 100644 test/simple-queue-test.js diff --git a/core/phantomas.js b/core/phantomas.js index 6b234ed97..168d0d199 100644 --- a/core/phantomas.js +++ b/core/phantomas.js @@ -104,10 +104,6 @@ var phantomas = function(params) { this.log('phantomas v%s: %s', this.getVersion(), this.dir); this.log('Options: %j', this.params); - // queue of jobs that needs to be done before report can be generated - var Queue = require('../lib/simple-queue'); - this.reportQueue = new Queue(); - // set up results wrapper var Results = require('./results'); this.results = new Results(); @@ -239,9 +235,6 @@ phantomas.prototype = { emit: this.emit.bind(this), emitInternal: this.emitInternal.bind(this), - // reports - reportQueuePush: this.reportQueue.push.bind(this.reportQueue), - // metrics setMetric: this.setMetric.bind(this), setMetricEvaluate: this.setMetricEvaluate.bind(this), @@ -460,64 +453,6 @@ phantomas.prototype = { this.initLoadingProgress(); - // do not wait for any requests, stop immediately after onload event (issue #513) - if (this.getParam('stop-at-onload', false) === true) { - this.log('stop-at-onload: --stop-at-onload passed, will stop immediately after onload event'); - } else { - // observe HTTP requests - // finish when the last request is completed + one second timeout - this.reportQueue.push(function(done) { - var currentRequests = 0, - requestsUrls = {}, - onFinished = function(entry) { - currentRequests--; - delete requestsUrls[entry.url]; - - if (currentRequests < 1) { - timeoutId = setTimeout(function() { - done(); - }, 1000); - } - }, - timeoutId; - - // update HTTP requests counter - self.on('send', function(entry) { - clearTimeout(timeoutId); - - currentRequests++; - requestsUrls[entry.url] = true; - }); - - self.on('recv', onFinished); - self.on('abort', onFinished); - - // add debug info about pending responses (issue #216) - self.on('timeout', function() { - var timedOutRequests = Object.keys(requestsUrls); - - self.log('Timeout: gave up waiting for %d HTTP response(s): <%s>', currentRequests, timedOutRequests.join('>, <')); - - // emit timed out requests as a fake metric (#539) - self.results.setMetric('requestsWithTimeout', timedOutRequests.length); - - timedOutRequests.forEach(function(url) { - self.results.addOffender('requestsWithTimeout', url); - }); - }); - - // always register the metric (issue #581) - self.results.setMetric('requestsWithTimeout', 0); - }); - } - - this.reportQueue.push(function(done) { - self.on('loadFinished', done); - }); - - // generate a report when all jobs are done - this.reportQueue.done(this.report, this); - // last time changes? this.emitInternal('pageBeforeOpen', this.page); // @desc page.open is about to be called @@ -847,84 +782,6 @@ phantomas.prototype = { return require('../lib/modules/' + module); }, - // runs a given helper script from phantomas main directory - // tries to parse it's output (assumes JSON formatted output) - runScript: function(script, args, callback) { - var execFile = require("child_process").execFile, - fs = require('fs'), - osName = require('system').os.name, // linux / windows - start = Date.now(), - self = this; - - if (typeof args === 'function') { - callback = args; - } - - // execFile(file, args, options, callback) - // @see https://github.com/ariya/phantomjs/wiki/API-Reference-ChildProcess - args = args || []; - - // handle relative paths to binaries (issue #672) - if (!fs.isAbsolute(script)) { - script = this.dir + script; - } - - // Windows fix: escape '&' (#587) and ')' (#687) - // @see http://superuser.com/questions/550048/is-there-an-escape-for-character-in-the-command-prompt - if (osName === 'windows') { - args = args.map(function(arg) { - return arg.replace(/[&)]/g, '^^^$&'); // $& - Inserts the matched substring - }); - } - - // always wait for runScript to finish (issue #417) - this.reportQueue.push(function(done) { - var ctx, pid; - - ctx = execFile(script, args, null, function(err, stdout, stderr) { - var time = Date.now() - start; - var result = stderr !== "" ? stderr : stdout; - - if (err || stderr) { - var errMessage = (err || stderr || 'unknown error').trim(); - self.log('runScript: pid #%d failed - %s (took %d ms)!', pid, errMessage, time); - - callback(errMessage, result); - - done(); - return; - } else if (!pid) { - self.log('runScript: failed running %s %s!', script, args.join(' ')); - - done(); - return; - } else { - self.log('runScript: pid #%d done (took %d ms)', pid, time); - } - - // (try to) parse JSON-encoded output - try { - result = JSON.parse(stdout); - } catch (ex) { - self.log('runScript: JSON parsing failed! - %s', ex); - err = ex; - } - - callback(err, result); - - done(); - }); - - pid = ctx.pid; - - if (pid) { - self.log('runScript: %s %s (pid #%d)', script, args.join(' '), pid); - } else { - done(); - } - }); - }, - // return temporary directory for the current phantomas run // passed as PHANTOMAS_TMP_DIR environment variable by phantomas' node.js runner tmpdir: function() { diff --git a/package-lock.json b/package-lock.json index 2a0a049e4..40918e72e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,15 @@ "es6-promisify": "^5.0.0" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "async": { "version": "1.5.2", "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", @@ -256,6 +265,12 @@ "es6-promise": "^4.0.3" } }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, "eventemitter3": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", @@ -481,6 +496,16 @@ "nopt": "~4.0.1" } }, + "js-yaml": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", + "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "jshint": { "version": "2.9.7", "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.9.7.tgz", @@ -743,6 +768,12 @@ "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=", "dev": true }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, "string_decoder": { "version": "1.1.1", "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", diff --git a/package.json b/package.json index 3dd5758b3..99cb279a1 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "glob": "^7.1.2", "http-server": "^0.11.1", "js-beautify": "^1.6.14", + "js-yaml": "^3.12.0", "jshint": "^2.9.7", "mockery": "^2.0.0", "vows": "^0.7.0" diff --git a/test/simple-queue-test.js b/test/simple-queue-test.js deleted file mode 100644 index 9ddd61852..000000000 --- a/test/simple-queue-test.js +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Tests simple-queue lib - */ -var vows = require('vows'), - assert = require('assert'), - Queue = require('../lib/simple-queue'); - -// run the test -vows.describe('simple-queue').addBatch({ - 'done()': { - topic: function() { - var q = new Queue(); - q.done(this.callback); - }, - 'should be completed immediately if there are no jobs pushed': function(err, stats) { - assert.strictEqual(stats && stats.jobs, 0); - } - }, - 'done() + push()': { - topic: function() { - var q = new Queue(); - q.done(function() {}); - this.callback(null, q); - }, - 'should throw an error': function(err, q) { - assert.throws(q.push); - } - }, - 'push() + done() + done()': { - topic: function() { - var q = new Queue(), - called = 0; - - q. - push(function(done) { - called++; - done(); - }). - push(function(done) { - setTimeout(function() { - called++; - done(); - }); - }). - done(function() { - called++; - }). - done(function(err, stats) { - called++; - this.callback(null, called, stats); - }, this); - }, - 'should call all done() callbacks': function(err, called) { - assert.equal(called, 4); - }, - 'should pass stats to done() callback': function(err, called, stats) { - assert.equal(stats && stats.jobs, 2); - } - }, - 'push() + async + done()': { - topic: function() { - var q = new Queue(), - called = 0; - - q. - push(function(done) { - setTimeout(function() { - called++; - done(); - }, 0); - }). - push(function(done) { - called++; - done(); - }). - done(function() { - this.callback(null, called); - }, this); - }, - 'should be completed when all jobs are done': function(err, called) { - assert.equal(called, 2); - } - }, - 'push() + async + done() + delayed push()': { - topic: function() { - var q = new Queue(), - called = 0; - - q. - push(function(done) { - setTimeout(function() { - called++; - done(); - }, 0); - }). - push(function(done) { - called++; - done(); - }). - done(function() { - this.callback(null, called); - }, this). - push(function(done) { - setTimeout(function() { - called++; - done(); - }, 0); - }); - }, - 'should be completed when all jobs are done': function(err, called) { - assert.equal(called, 3); - } - }, - 'push() + multiple calls of the same done()': { - topic: function() { - var q = new Queue(), - callbackOneIsCalled = false, - callbackTwoIsCalled = false; - - - q.push(function(done) { - callbackOneIsCalled = true; - done(); - done(); - done(); - }). - push(function(done) { - callbackTwoIsCalled = true; - done(); - }). - done(function() { - this.callback(null, callbackOneIsCalled, callbackTwoIsCalled); - }, this); - }, - 'should call both callbacks, even though the first callback calls done() multiple times': function(err, callbackOneIsCalled, callbackTwoIsCalled) { - assert.equal(callbackOneIsCalled, true); - assert.equal(callbackTwoIsCalled, true); - } - } - -}).export(module); From da3b1fb8952e2786c8ac86aeeb2c3b1a06aa9046 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 29 Dec 2018 14:20:46 +0100 Subject: [PATCH 023/248] test/modules/mock.js: use new, simpler module structure --- test/modules/mock.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/modules/mock.js b/test/modules/mock.js index 29e31efdb..5bfef535c 100644 --- a/test/modules/mock.js +++ b/test/modules/mock.js @@ -138,7 +138,7 @@ function initModule(name, isCore) { instance = new phantomas(name); def = require('../../' + (isCore ? 'core/modules' : 'modules') + '/' + name + '/' + name + '.js'); - new(def.module)(instance); + new(def)(instance); } catch (ex) { console.log(ex); } From d66b726ba013771bb6a434bf8803aad7ec84b798 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 29 Dec 2018 14:48:26 +0100 Subject: [PATCH 024/248] Some work on tests --- test/integration-test.js | 55 ++++++++++++++++++---------------------- test/public-api-test.js | 28 -------------------- 2 files changed, 25 insertions(+), 58 deletions(-) diff --git a/test/integration-test.js b/test/integration-test.js index 8643fd780..5b89c1d37 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -9,11 +9,10 @@ var vows = require('vows'), yaml = require('js-yaml'), phantomas = require('..'); -var WEBROOT = 'http://127.0.0.1:8888', // see start-server.sh - ENGINE = process.env.PHANTOMAS_ENGINE; // currently used engine (either PhantomJS or SlimerJS) +var WEBROOT = 'http://127.0.0.1:8888'; // see start-server.sh // run the test -var suite = vows.describe('Integration tests - ' + ENGINE).addBatch({ +var suite = vows.describe('Integration tests').addBatch({ 'test server': { topic: function() { var http = require('http'), @@ -36,35 +35,31 @@ var raw = fs.readFileSync(__dirname + '/integration-spec.yaml').toString(), spec.forEach(function(test) { var batch = {}, - batchName = test.label || test.url, - shouldSkip = test.skip && (test.skip === ENGINE); + batchName = test.label || test.url; - if (shouldSkip) { - batch[batchName] = { - topic: 'foo', - 'should be skipped': function() {} - }; - } else { - batch[batchName] = { - topic: function() { - phantomas(WEBROOT + test.url, test.options || {}, this.callback); - }, - 'should be generated': function(err, data, results) { - if (test.exitCode) { - assert.ok(err instanceof Error); - assert.strictEqual(err.message, test.exitCode.toString(), 'Exit code matches the expected value'); - } else { - assert.equal(err, null, 'Exit code matches the expected value'); - } - }, - }; + batch[batchName] = { + topic: function() { + phantomas(WEBROOT + test.url, test.options || {}). + then(res => this.callback(null, res)). + catch(err => this.callback(err)); + }, + 'should be generated': function(err) { + if (test.exitCode) { + assert.ok(err instanceof Error); + assert.strictEqual(err.message, test.exitCode.toString(), 'Exit code matches the expected value'); + } else { + assert.equal(err, null, 'Exit code matches the expected value'); + } + }, + }; - Object.keys(test.metrics || {}).forEach(function(name) { - batch[batchName]['should have "' + name + '" metric properly set'] = function(err, data, results) { - assert.strictEqual(results.getMetric(name), test.metrics[name]); - }; - }); - } + Object.keys(test.metrics || {}).forEach(function(name) { + batch[batchName]['should have "' + name + '" metric properly set'] = function(err, results) { + assert.ok(!(err instanceof Error), 'Error should not be thrown: ' + err); + assert.ok(false, results); + assert.strictEqual(results.getMetric(name), test.metrics[name]); + }; + }); suite.addBatch(batch); }); diff --git a/test/public-api-test.js b/test/public-api-test.js index 5edb1cec7..c5c2a4bce 100644 --- a/test/public-api-test.js +++ b/test/public-api-test.js @@ -3,36 +3,8 @@ */ var vows = require('vows'), assert = require('assert'), - mockery = require('mockery'), phantomas = require('../core/phantomas'); -// mock PhantomJS-specific modules and globals -GLOBAL.phantom = { - version: {} -}; -mockery.registerMock('fs', { - list: function() {} -}); -mockery.registerMock('system', { - os: {}, - stdout: { - writeLine: function() {} - } -}); -mockery.registerMock('webpage', { - create: function() { - return { - evaluate: function() {}, - injectJs: function() {}, - render: function() {}, - content: '' - }; - }, -}); -mockery.enable({ - warnOnUnregistered: false -}); - // helper function getPhantomasAPI(params) { var instance = new phantomas(params || {}); From 772699a260dd3cdec75d6dcd2abc536bfbd6ae40 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 10:16:12 +0100 Subject: [PATCH 025/248] Travis: install Chrome --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index e8a515dda..bec032817 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ # https://docs.travis-ci.com/user/languages/javascript-with-nodejs/ language: node_js dist: trusty +# https://docs.travis-ci.com/user/gui-and-headless-browsers/#using-the-chrome-addon-in-the-headless-mode +addons: + chrome: stable node_js: - 10 - 8 @@ -9,6 +12,7 @@ before_script: - sh test/server-start.sh & - SERVER_PID=$! - sleep 1 + - which google-chrome-stable && google-chrome-stable --version after_script: - kill $SERVER_PID script: "npm test && npm run-script lint" From f7dd1f523917da3b2dcf666f186cc682de28d3a4 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 13:55:57 +0100 Subject: [PATCH 026/248] Set up Travis for Chrome stable --- .travis.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index bec032817..1f9b419a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,30 @@ # https://docs.travis-ci.com/user/languages/javascript-with-nodejs/ language: node_js dist: trusty -# https://docs.travis-ci.com/user/gui-and-headless-browsers/#using-the-chrome-addon-in-the-headless-mode -addons: - chrome: stable node_js: - 10 - 8 -sudo: false +addons: + chrome: stable + +# allow headful tests +# https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/troubleshooting.md#running-puppeteer-on-travis-ci +before_install: + # Enable user namespace cloning + - "sysctl kernel.unprivileged_userns_clone=1" + # Launch XVFB + - "export DISPLAY=:99.0" + - "sh -e /etc/init.d/xvfb start" + # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. + - "export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true" + # Point Phantomas to Chrome stable binary provided by Travis + - "export PHANTOMAS_CHROMIUM_EXECUTABLE=`which google-chrome-stable`" + - env && google-chrome-stable --version + before_script: - sh test/server-start.sh & - SERVER_PID=$! - sleep 1 - - which google-chrome-stable && google-chrome-stable --version after_script: - kill $SERVER_PID script: "npm test && npm run-script lint" From ab1de079bfc1c7feb8a15125f4727a3c5a92afe1 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 14:42:40 +0100 Subject: [PATCH 027/248] Improve tests code --- test/README.md | 7 ++++++ test/cli-test.js | 47 ---------------------------------------- test/integration-test.js | 7 +++--- test/module-test.js | 2 ++ 4 files changed, 12 insertions(+), 51 deletions(-) delete mode 100644 test/cli-test.js diff --git a/test/README.md b/test/README.md index a831c09dd..71f0d0db7 100644 --- a/test/README.md +++ b/test/README.md @@ -13,3 +13,10 @@ And then run the test suite: phantomas$ npm test ``` +### Travis CI + +``` +$ which google-chrome-stable && google-chrome-stable --version +/usr/bin/google-chrome-stable +Google Chrome 71.0.3578.98 +``` diff --git a/test/cli-test.js b/test/cli-test.js deleted file mode 100644 index 0c8c7dbba..000000000 --- a/test/cli-test.js +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Tests CommonJS module - */ -'use strict'; - -var vows = require('vows'), - assert = require('assert'), - debug = require('debug')('phantomas:test'), - exec = require('child_process').exec, - phantomas = require('..'); - -function runPhantomasCLI(url, opts, callback) { - var cmd = __dirname + '/../bin/phantomas.js ' + url + ' ' + opts; - debug(cmd); - - exec(cmd, callback); -} - -// run the test -vows.describe('CLI command').addBatch({ - 'should return zero exit code when everything is fine': { - topic: function() { - runPhantomasCLI('http://example.com', '', this.callback); - }, - 'should return exit code 0': function(err, stdout) { - assert.equal(err, null); - } - }, - 'should pass error code with loading error': { - topic: function() { - runPhantomasCLI('http://foo.bar.test', '', this.callback); - }, - 'should fail with err #254': function(err, stdout) { - assert.ok(err instanceof Error); - assert.strictEqual(err.code, 254); - } - }, - 'should pass error code when assert fails': { - topic: function() { - runPhantomasCLI('http://example.com', '--assert-requests=0', this.callback); - }, - 'should fail with err #1': function(err, stdout) { - assert.ok(err instanceof Error); - assert.strictEqual(err.code, 1); - } - } -}).export(module); diff --git a/test/integration-test.js b/test/integration-test.js index 5b89c1d37..afb77203b 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -43,12 +43,12 @@ spec.forEach(function(test) { then(res => this.callback(null, res)). catch(err => this.callback(err)); }, - 'should be generated': function(err) { + 'should be generated': (err, res) => { if (test.exitCode) { assert.ok(err instanceof Error); - assert.strictEqual(err.message, test.exitCode.toString(), 'Exit code matches the expected value'); } else { - assert.equal(err, null, 'Exit code matches the expected value'); + assert.equal(err, null, 'No error should be thrown'); + assert.ok(res.getMetric instanceof Function, 'Results wrapper should be passed'); } }, }; @@ -56,7 +56,6 @@ spec.forEach(function(test) { Object.keys(test.metrics || {}).forEach(function(name) { batch[batchName]['should have "' + name + '" metric properly set'] = function(err, results) { assert.ok(!(err instanceof Error), 'Error should not be thrown: ' + err); - assert.ok(false, results); assert.strictEqual(results.getMetric(name), test.metrics[name]); }; }); diff --git a/test/module-test.js b/test/module-test.js index 8fd278575..258e08db7 100644 --- a/test/module-test.js +++ b/test/module-test.js @@ -7,6 +7,8 @@ var vows = require('vows'), assert = require('assert'), phantomas = require('..'); +return; // TODO + // run the test vows.describe('CommonJS module').addBatch({ 'when not provided with URL': { From 3e5b9e6c10ca8a6ca4216e7785e1d654b8e62534 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 14:43:03 +0100 Subject: [PATCH 028/248] Browser: handle PHANTOMAS_CHROMIUM_EXECUTABLE env variable --- .travis.yml | 1 - examples/promise.js | 2 +- lib/browser.js | 15 +++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f9b419a0..0707ba6f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,6 @@ before_install: - "export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true" # Point Phantomas to Chrome stable binary provided by Travis - "export PHANTOMAS_CHROMIUM_EXECUTABLE=`which google-chrome-stable`" - - env && google-chrome-stable --version before_script: - sh test/server-start.sh & diff --git a/examples/promise.js b/examples/promise.js index 63f4aa18a..e4420d888 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -38,7 +38,7 @@ promise.on('milestone', milestone => { }); promise.on('recv', response => { - console.log('Response #%d: %s %s [HTTP %d]', response.id, response.method, response.url, response.status); + console.log('Response: %s %s [%s]', response.method, response.url, response.contentType); }); // including the custom once emitted by phantomas modules diff --git a/lib/browser.js b/lib/browser.js index d0f09a508..1947c2234 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -14,12 +14,19 @@ Browser.prototype.bind = events => this.events = events; // initialize puppeter instance Browser.prototype.init = async () => { - const networkDebug = require('debug')('phantomas:network'); + const networkDebug = require('debug')('phantomas:network'), + env = require('process').env; - debug('Launching Puppeter'); + var options = {}; - const args = []; // ['--no-sandbox']; - this.browser = await puppeteer.launch({args: args}); + // customize path to Chromium binary + if (env['PHANTOMAS_CHROMIUM_EXECUTABLE']) { + options.executablePath = env['PHANTOMAS_CHROMIUM_EXECUTABLE']; + } + + debug('Launching Puppeter: %j', options); + + this.browser = await puppeteer.launch(options); this.page = await this.browser.newPage(); // A Chrome Devtools Protocol session attached to the target From 84b4473a0a27336eb7508131ac4e747e345dd7d7 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 15:11:45 +0100 Subject: [PATCH 029/248] loader: be compatible with Node.js v8 --- lib/loader.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/loader.js b/lib/loader.js index 592cbdce1..c913d5cfe 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -10,14 +10,14 @@ function listModulesInDirectory(modulesDir) { // https://nodejs.org/api/fs.html#fs_fs_readdirsync_path_options var fs = require('fs'), - ls = fs.readdirSync(modulesDir, {withFileTypes: true}), + ls = fs.readdirSync(modulesDir), modules = []; ls.forEach(function(entry) { // First check whether an entry is a directory, than check if it contains a module file // https://nodejs.org/api/fs.html#fs_fs_existssync_path - if (entry.isDirectory() && fs.existsSync(modulesDir + '/' + entry.name + '/' + entry.name + '.js')) { - modules.push(entry.name); + if (fs.existsSync(modulesDir + '/' + entry + '/' + entry + '.js')) { + modules.push(entry); } }); From 81bc6c9d27b968cd0740a94b5881b790fd0d5a77 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 17:30:58 +0100 Subject: [PATCH 030/248] requestsMonitor: all tests now pass --- .../requestsMonitor/requestsMonitor.js | 198 ++++++++---------- lib/browser.js | 13 +- test/modules/mock.js | 8 +- test/modules/requestsMonitor-test.js | 95 ++++----- 4 files changed, 144 insertions(+), 170 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index bf8f5767e..20752da0a 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -5,6 +5,33 @@ const debug = require('debug')('phantomas:modules:requestsMonitor'); +// parse given URL to get protocol and domain +function parseEntryUrl(entry) { + const parseUrl = require('url').parse; + var parsed; + + if (entry.url.indexOf('data:') !== 0) { + // @see http://nodejs.org/api/url.html#url_url + parsed = parseUrl(entry.url) || {}; + + entry.protocol = parsed.protocol.replace(':', ''); + entry.domain = parsed.hostname; + entry.query = parsed.query; + + if (entry.protocol === 'https') { + entry.isSSL = true; + } + } + else { + // base64 encoded data + entry.domain = false; + entry.protocol = false; + entry.isBase64 = true; + } + + return entry; +} + /** * Detect response content type using "Content-Type header value" * @@ -116,33 +143,10 @@ module.exports = function(phantomas) { phantomas.setMetric('postRequests'); // @desc number of POST requests phantomas.setMetric('httpsRequests'); // @desc number of HTTPS requests phantomas.setMetric('notFound'); // @desc number of HTTP 404 responses - phantomas.setMetric('bodySize'); // @desc size of the compressed content of all responses, equals Content-Length header value + phantomas.setMetric('bodySize'); // @desc size of the uncompressed content of all responses + phantomas.setMetric('transferedSize'); // @desc size of the compressed content of all responses, i.e. what was transfered in packets phantomas.setMetric('httpTrafficCompleted'); // @desc time it took to receive the last byte of the last HTTP response - // parse given URL to get protocol and domain - function parseEntryUrl(entry) { - var parsed; - - if (entry.url.indexOf('data:') !== 0) { - // @see http://nodejs.org/api/url.html#url_url - parsed = parseUrl(entry.url) || {}; - - entry.protocol = parsed.protocol.replace(':', ''); - entry.domain = parsed.hostname; - entry.query = parsed.query; - - if (entry.protocol === 'https') { - entry.isSSL = true; - } - } - else { - // base64 encoded data - entry.domain = false; - entry.protocol = false; - entry.isBase64 = true; - } - } - var requests = {}; phantomas.on('request', /** @param {Request} request **/ (request) => { @@ -163,7 +167,8 @@ module.exports = function(phantomas) { url: resp.url, method: request.method, headers: resp.headers, - bodySize: resp.encodedDataLength, + bodySize: resp.dataLength, + transferedSize: resp.encodedDataLength, }; /** @@ -203,94 +208,73 @@ module.exports = function(phantomas) { case 'content-type': entry = addContentType(headerValue, entry); break; + + // content compression + case 'content-encoding': + if (headerValue === 'gzip') { + entry.gzip = true; + + phantomas.log('Response compressed with %s, %f kB -> %f kB (x%f)', + headerValue, entry.bodySize / 1024, entry.transferedSize / 1024, entry.bodySize / entry.transferedSize); + } + break; + + // detect cookies (issue #92) + case 'set-cookie': + entry.hasCookies = true; + break; } }); - /** + // asset type + entry.type = 'other'; - // asset type - entry.type = 'other'; - - // analyze HTTP headers - entry.headers = {}; - res.headers.forEach(function(header) { - entry.headers[header.name] = header.value; - - switch (header.name.toLowerCase()) { - // detect content encoding - case 'content-encoding': - if (header.value === 'gzip') { - entry.gzip = true; - } - break; - - // detect cookies (issue #92) - case 'set-cookie': - entry.hasCookies = true; - break; - } - }); - - parseEntryUrl(entry); - - // HTTP code - entry.status = res.status || 200 // for base64 data; - entry.statusText = HTTP_STATUS_CODES[entry.status]; - - switch(entry.status) { - case 301: // Moved Permanently - case 302: // Found - case 303: // See Other - entry.isRedirect = true; - break; - - case 404: // Not Found - phantomas.incrMetric('notFound'); - phantomas.addOffender('notFound', entry.url); - break; - } - - // default value (if Content-Length header is not present in the response or it's base64-encoded) - // see issue #137 - if (typeof entry.contentLength === 'undefined') { - entry.contentLength = entry.bodySize; - phantomas.log('%s: %j', 'contentLength missing', {url: entry.url, bodySize: entry.bodySize}); - } - - // requests stats - if (!entry.isBase64) { - phantomas.incrMetric('requests'); - - phantomas.incrMetric('bodySize', entry.bodySize); - phantomas.incrMetric('contentLength', entry.contentLength); - } - - if (entry.gzip) { - phantomas.incrMetric('gzipRequests'); - phantomas.addOffender('gzipRequests', '%s (gzip: %s kB / uncompressed: %s kB)', entry.url, (entry.contentLength/1024).toFixed(2), (entry.bodySize/1024).toFixed(2)); - } - - if (entry.isSSL) { - phantomas.incrMetric('httpsRequests'); - phantomas.addOffender('httpsRequests', entry.url); - } - - if (entry.isBase64) { - phantomas.emit('base64recv', entry, res); // @desc base64-encoded "response" has been received - } - else { - phantomas.log('recv: HTTP %d <%s> [%s]', entry.status, entry.url, entry.contentType); - phantomas.emit('recv' , entry, res); // @desc response has been received - } - //break; - **/ + entry = parseEntryUrl(entry); - phantomas.log('Response metadata: %j', entry); + // HTTP code + entry.status = resp.status || 200 // for base64 data; + entry.statusText = HTTP_STATUS_CODES[entry.status]; + + switch(entry.status) { + case 301: // Moved Permanently + case 302: // Found + case 303: // See Other + entry.isRedirect = true; + break; - phantomas.incrMetric('requests'); - phantomas.incrMetric('bodySize', entry.bodySize); + case 404: // Not Found + phantomas.incrMetric('notFound'); + phantomas.addOffender('notFound', entry.url); + break; + } - phantomas.emit('recv' , entry, resp); // @desc response has been received + // requests stats + if (!entry.isBase64) { + phantomas.incrMetric('requests'); + + phantomas.incrMetric('bodySize', entry.bodySize); + phantomas.incrMetric('contentLength', entry.contentLength); + } + + if (entry.gzip) { + phantomas.incrMetric('gzipRequests'); + phantomas.addOffender('gzipRequests', '%s (gzip: %s kB / uncompressed: %s kB)', entry.url, (entry.contentLength/1024).toFixed(2), (entry.bodySize/1024).toFixed(2)); + } + + if (entry.isSSL) { + phantomas.incrMetric('httpsRequests'); + phantomas.addOffender('httpsRequests', entry.url); + } + + if (entry.isBase64) { + phantomas.emit('base64recv', entry, resp); // @desc base64-encoded "response" has been received + } + else { + phantomas.log('recv: HTTP %d <%s> [%s]', entry.status, entry.url, entry.contentType); + phantomas.emit('recv' , entry, resp); // @desc response has been received + } + + phantomas.log('Response metadata: %j', entry); }); diff --git a/lib/browser.js b/lib/browser.js index 1947c2234..dd43f11c4 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -71,6 +71,8 @@ Browser.prototype.init = async () => { responses[data.requestId] = { _chunks: 0, + _dataLength: 0, + // Actual bytes received (might be less than dataLength for compressed encodings). _encodedDataLength: 0, }; }); @@ -81,7 +83,6 @@ Browser.prototype.init = async () => { /** @type {Response} response */ var response = data.response; response._requestId = data.requestId; - response._timestamp = data.timestamp; //networkDebug('responseReceived', response); //networkDebug('Network.responseReceived < %s %s %s %s', response.protocol, response.status, response.statusText, response.url); @@ -96,13 +97,16 @@ Browser.prototype.init = async () => { var meta = responses[data.requestId], response = meta.response; + response.dataLength = meta._dataLength; response.encodedDataLength = meta._encodedDataLength; response.chunks = meta._chunks; response._timestamp = data.timestamp; //networkDebug('Network.loadingFinished', data); - networkDebug('Network.loadingFinished < %s %s %s %s (%f kB fetched)', - response.protocol, response.status, response.statusText, response.url, 1.0 * response.encodedDataLength / 1024); + networkDebug('Network.loadingFinished < %s %s %s %s (%f kB fetched, %f kB uncompressed)', + response.protocol, response.status, response.statusText, response.url, + 1.0 * response.encodedDataLength / 1024, + 1.0 * response.dataLength / 1024); this.events.emit('response', response); }); @@ -110,9 +114,10 @@ Browser.prototype.init = async () => { // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-dataReceived // Fired when data chunk was received over the network this.cdp.on('Network.dataReceived', data => { - //networkDebug('Network.dataReceived', data); + // networkDebug('Network.dataReceived', data); responses[data.requestId]._chunks++; + responses[data.requestId]._dataLength += data.dataLength; // Actual bytes received (might be less than dataLength for compressed encodings) responses[data.requestId]._encodedDataLength += data.encodedDataLength; }); diff --git a/test/modules/mock.js b/test/modules/mock.js index 5bfef535c..77309b33a 100644 --- a/test/modules/mock.js +++ b/test/modules/mock.js @@ -64,17 +64,17 @@ phantomas.prototype = { // mock core PhantomJS events sendRequest: function(req) { req = req || {}; - req.id = req.id || 1; + req._requestId = req._requestId || 1; req.method = req.method || 'GET'; req.url = req.url || 'http://example.com'; req.headers = req.headers || []; + req.timing = {}; this.log('sendRequest: %j', req); try { - this.emitter.emit('onResourceRequested', req, { - abort: noop - }); + this.emitter.emit('request', req); + this.emitter.emit('response', req); } catch (ex) { console.log(ex); } diff --git a/test/modules/requestsMonitor-test.js b/test/modules/requestsMonitor-test.js index a8b2c3e92..14fe2ae51 100644 --- a/test/modules/requestsMonitor-test.js +++ b/test/modules/requestsMonitor-test.js @@ -3,45 +3,27 @@ */ var vows = require('vows'), assert = require('assert'), - mock = require('./mock'); + mock = require('./mock'), + extend = require('util')._extend; -function sendReq(url) { +function sendReq(url, extra) { return function() { var phantomas = mock.initCoreModule('requestsMonitor'), ret = false; - phantomas.on('send', function(entry, res) { + phantomas.on('recv', function(entry, _) { ret = entry; }); - phantomas.sendRequest({ + phantomas.sendRequest(extend({ url: url - }); - return ret; - }; -} - -function recvReq(url, req, ev) { - req = req || {}; - req.url = req.url || url; - - return function() { - var phantomas = mock.initCoreModule('requestsMonitor'), - ret = false; - - phantomas.on(ev || 'recv', function(entry, res) { - ret = entry; - }); - phantomas.recvRequest(req); + }, extra || {})); return ret; }; } -function recvContentType(contentType, url) { - return recvReq(url, { - headers: [{ - name: 'Content-Type', - value: contentType - }] +function sendContentType(contentType, url) { + return sendReq(url, { + headers: {'Content-Type': contentType} }); } @@ -58,13 +40,16 @@ vows.describe('requestMonitor').addBatch({ phantomas.sendRequest(); return phantomas; }, + /** 'beforeSend is fired': function(phantomas) { assert.isTrue(phantomas.emitted('beforeSend')); }, + **/ 'send is fired': function(phantomas) { assert.isTrue(phantomas.emitted('send')); } }, + /** 'request can be aborted': { topic: function() { var phantomas = mock.initCoreModule('requestsMonitor'); @@ -81,6 +66,7 @@ vows.describe('requestMonitor').addBatch({ assert.isFalse(phantomas.emitted('send')); } }, + **/ 'URLs are properly parsed when sent': { topic: sendReq('http://example.com/foo?bar=test&a=b'), 'protocol is set': assertField('protocol', 'http'), @@ -94,6 +80,7 @@ vows.describe('requestMonitor').addBatch({ 'protocol is set': assertField('protocol', 'https'), 'isSSL is set': assertField('isSSL', true) }, + /** 'base64-encoded data is property detected': { topic: recvReq('data:image/png;base64,iVBORw0KGgoAAAA', {}, 'base64recv'), 'protocol is not set': assertField('protocol', false), @@ -101,123 +88,121 @@ vows.describe('requestMonitor').addBatch({ 'isSSL is not set': assertField('isSSL', undefined), 'isBase64 is set': assertField('isBase64', true) }, + **/ }).addBatch({ 'content type is properly passed': { - topic: recvContentType('text/html'), + topic: sendContentType('text/html'), 'entry.contentType is set': assertField('contentType', 'text/html') }, 'HTML is properly detected': { - topic: recvContentType('text/html'), + topic: sendContentType('text/html'), 'isHTML is set': assertField('isHTML', true) }, 'XML is properly detected': { - topic: recvContentType('text/xml'), + topic: sendContentType('text/xml'), 'isXML is set': assertField('isXML', true) }, 'CSS is properly detected': { - topic: recvContentType('text/css'), + topic: sendContentType('text/css'), 'isCSS is set': assertField('isCSS', true) }, 'JS is properly detected': { - topic: recvContentType('text/javascript'), + topic: sendContentType('text/javascript'), 'isJS is set': assertField('isJS', true) }, 'JSON is properly detected': { - topic: recvContentType('application/json'), + topic: sendContentType('application/json'), 'isJSON is set': assertField('isJSON', true) }, 'PNG image is properly detected': { - topic: recvContentType('image/png'), + topic: sendContentType('image/png'), 'isImage is set': assertField('isImage', true) }, 'JPEG image is properly detected': { - topic: recvContentType('image/jpeg'), + topic: sendContentType('image/jpeg'), 'isImage is set': assertField('isImage', true) }, 'GIF image is properly detected': { - topic: recvContentType('image/gif'), + topic: sendContentType('image/gif'), 'isImage is set': assertField('isImage', true) }, 'SVG image is properly detected': { - topic: recvContentType('image/svg+xml'), + topic: sendContentType('image/svg+xml'), 'isImage is set': assertField('isImage', true), 'isSVG is set': assertField('isSVG', true) }, 'WEBP image is properly detected': { - topic: recvContentType('image/webp'), + topic: sendContentType('image/webp'), 'isImage is set': assertField('isImage', true) }, 'WebM video is properly detected': { - topic: recvContentType('video/webm'), + topic: sendContentType('video/webm'), 'isVideo is set': assertField('isVideo', true) }, 'Web font is properly detected (via MIME)': { - topic: recvContentType('application/font-woff'), + topic: sendContentType('application/font-woff'), 'isWebFont is set': assertField('isWebFont', true), 'isTTF is not set': assertField('isTTF', undefined) }, 'Web font is properly detected (via URL)': { - topic: recvContentType('application/octet-stream', 'http://foo.bar/font.otf'), + topic: sendContentType('application/octet-stream', 'http://foo.bar/font.otf'), 'isWebFont is set': assertField('isWebFont', true), 'isTTF is not set': assertField('isTTF', undefined) }, 'TTF font is properly detected': { - topic: recvContentType('application/x-font-ttf'), + topic: sendContentType('application/x-font-ttf'), 'isWebFont is set': assertField('isWebFont', true), 'isTTF is set': assertField('isTTF', true) }, 'favicon is properly detected': { - topic: recvContentType('image/x-icon'), + topic: sendContentType('image/x-icon'), 'isFavicon is set': assertField('isFavicon', true) }, 'favicon is properly detected (Microsoft\'s MIME type)': { - topic: recvContentType('image/vnd.microsoft.icon'), + topic: sendContentType('image/vnd.microsoft.icon'), 'isFavicon is set': assertField('isFavicon', true) }, }).addBatch({ 'redirects are detected (HTTP 301)': { - topic: recvReq('', { + topic: sendReq('', { status: 301 }), 'isRedirect field is set': assertField('isRedirect', true) }, 'redirects are detected (HTTP 302)': { - topic: recvReq('', { + topic: sendReq('', { status: 302 }), 'isRedirect field is set': assertField('isRedirect', true) }, 'redirects are detected (HTTP 303)': { - topic: recvReq('', { + topic: sendReq('', { status: 303 }), 'isRedirect field is set': assertField('isRedirect', true) }, 'redirects are detected (HTTP 200)': { - topic: recvReq('', { + topic: sendReq('', { status: 200 }), 'isRedirect field is not set': assertField('isRedirect', undefined) } }).addBatch({ 'POST requests are detected': { - topic: mock.initCoreModule('requestsMonitor').recvRequest({ + topic: mock.initCoreModule('requestsMonitor').sendRequest({ method: 'POST' }), 'postRequests metric is set': mock.assertMetric('postRequests', 1) }, 'not found responses are detected (HTTP 404)': { - topic: mock.initCoreModule('requestsMonitor').recvRequest({ + topic: mock.initCoreModule('requestsMonitor').sendRequest({ status: 404 }), 'notFound metric is set': mock.assertMetric('notFound', 1) }, 'GZIP responses are detected': { - topic: recvReq(undefined, { - headers: [{ - name: 'Content-Encoding', - value: 'gzip' - }] + topic: sendReq(undefined, { + headers: {'Content-Encoding': 'gzip'} }), 'gzip is set': assertField('gzip', true) } From 4a2fdc01ced52697947e128eba421488970f0927 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 20:41:01 +0100 Subject: [PATCH 031/248] phantomas object will probably go away --- test/public-api-test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/public-api-test.js b/test/public-api-test.js index c5c2a4bce..4a9994887 100644 --- a/test/public-api-test.js +++ b/test/public-api-test.js @@ -12,6 +12,7 @@ function getPhantomasAPI(params) { } // run the test +/** vows.describe('phantomas public API').addBatch({ 'exposes values and methods': { topic: function() { @@ -192,3 +193,4 @@ vows.describe('phantomas public API').addBatch({ } } }).export(module); +**/ \ No newline at end of file From afbab88d9614ca4c11eda1459d7845051c2b5e91 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 20:42:34 +0100 Subject: [PATCH 032/248] vows: v0.8.2 --- package-lock.json | 11 ++++++----- package.json | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 40918e72e..8af34572a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -814,13 +814,14 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "vows": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/vows/-/vows-0.7.0.tgz", - "integrity": "sha1-3QBl8RC6DAptY+hEhRwyCBdtWGc=", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/vows/-/vows-0.8.2.tgz", + "integrity": "sha1-aR95qybM3oC6cm3en+yOc9a88us=", "dev": true, "requires": { - "diff": "~1.0.3", - "eyes": ">=0.1.6" + "diff": "~1.0.8", + "eyes": "~0.1.6", + "glob": "^7.1.2" } }, "wordwrap": { diff --git a/package.json b/package.json index 99cb279a1..f1f5b09fc 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "js-yaml": "^3.12.0", "jshint": "^2.9.7", "mockery": "^2.0.0", - "vows": "^0.7.0" + "vows": "^0.8.2" }, "scripts": { "test": "vows --spec", From 4da2f274981ab130e8331d9935eb28b177f68b4c Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 20:45:03 +0100 Subject: [PATCH 033/248] loader: linting + JSdoc --- lib/loader.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/loader.js b/lib/loader.js index c913d5cfe..e3f87f36e 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -4,6 +4,12 @@ const debug = require('debug'), extend = require('util')._extend; +/** + * Lists all modules / extensions in a given directory + * + * @param {string} modulesDir + * @returns {Array Date: Sun, 30 Dec 2018 20:56:56 +0100 Subject: [PATCH 034/248] examples: show offenders --- core/modules/requestsMonitor/requestsMonitor.js | 11 +++++------ core/results.js | 2 +- examples/promise.js | 3 ++- modules/requestsStats/requestsStats.js | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 20752da0a..b0763698e 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -134,8 +134,7 @@ function addContentType(headerValue, entry) { module.exports = function(phantomas) { // imports - var HTTP_STATUS_CODES = require('http').STATUS_CODES, - parseUrl = require('url').parse; + var HTTP_STATUS_CODES = require('http').STATUS_CODES; // register metric phantomas.setMetric('requests'); // @desc total number of HTTP requests made @@ -180,14 +179,14 @@ module.exports = function(phantomas) { * https://www.w3.org/TR/navigation-timing/#performancetiming * https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceTiming * - * All times are in seconds, while times in resp.timing are in miliseconds. + * All times are in seconds! */ // how long a given request stalled waiting for DNS, proxy, connection, SSL negotation, etc. - entry.stalled = 1.0 * resp.timing.sendStart / 1000; + entry.stalled = resp.timing.sendStart; // how it took to receive a first byte of the response after making a request - entry.timeToFirstByte = 1.0 * (resp.timing.receiveHeadersEnd - resp.timing.sendEnd) / 1000; + entry.timeToFirstByte = resp.timing.receiveHeadersEnd - resp.timing.sendEnd; // difference between when the request was sent and when it was received entry.timeToLastByte = resp._timestamp - request._timestamp; @@ -232,7 +231,7 @@ module.exports = function(phantomas) { entry = parseEntryUrl(entry); // HTTP code - entry.status = resp.status || 200 // for base64 data; + entry.status = resp.status || 200; // for base64 data entry.statusText = HTTP_STATUS_CODES[entry.status]; switch(entry.status) { diff --git a/core/results.js b/core/results.js index 8eaa3b3f4..96487f1bc 100644 --- a/core/results.js +++ b/core/results.js @@ -5,7 +5,7 @@ */ 'use strict'; -module.exports = function(data) { +module.exports = function (data) { var asserts = {}, generator = '', metrics = {}, diff --git a/examples/promise.js b/examples/promise.js index e4420d888..a1a183b8f 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -20,7 +20,8 @@ const promise = phantomas('http://0.0.0.0', { // handle the promise promise. then(results => { - console.log('Metrics: %j', results.getMetrics()); + console.log('Metrics', results.getMetrics()); + console.log('Offenders', results.getAllOffenders()); console.log('Number of requests: %d', results.getMetric('requests')); console.log('Failed asserts: %j', results.getFailedAsserts()); }). diff --git a/modules/requestsStats/requestsStats.js b/modules/requestsStats/requestsStats.js index 9b3840c82..c0f5cdeeb 100644 --- a/modules/requestsStats/requestsStats.js +++ b/modules/requestsStats/requestsStats.js @@ -44,11 +44,11 @@ module.exports = function(phantomas) { // size pushToStack('smallestResponse', entry, function(stack, entry) { - return stack.contentLength > entry.contentLength; + return stack.transferedSize > entry.transferedSize; }); pushToStack('biggestResponse', entry, function(stack, entry) { - return stack.contentLength < entry.contentLength; + return stack.transferedSize < entry.transferedSize; }); // time (from sent to last byte) @@ -97,8 +97,8 @@ module.exports = function(phantomas) { switch (metric) { case 'smallestResponse': case 'biggestResponse': - phantomas.setMetric(metric, entry.contentLength); - details = (entry.contentLength / 1024).toFixed(2) + ' kB'; + phantomas.setMetric(metric, entry.transferedSize); + details = (entry.transferedSize / 1024).toFixed(2) + ' kB'; break; case 'fastestResponse': From a5eba26022ac25579cdbdf8ea21bc92cd196cc38 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 21:00:36 +0100 Subject: [PATCH 035/248] Fix test/modules/requestsStats-test.js --- test/modules/requestsStats-test.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/modules/requestsStats-test.js b/test/modules/requestsStats-test.js index 3aaf23719..ec9d4cdb3 100644 --- a/test/modules/requestsStats-test.js +++ b/test/modules/requestsStats-test.js @@ -2,27 +2,26 @@ * Test requestsStats core module */ var vows = require('vows'), - assert = require('assert'), mock = require('./mock'); vows.describe('requestsStats').addBatch({ - 'module': mock.getContext('requestsStats', function(phantomas) { + 'module': mock.getContext('requestsStats', phantomas => { var requests = [ { status: 200, - contentLength: 25, + transferedSize: 25, timeToFirstByte: 3, timeToLastByte: 5, }, { status: 200, - contentLength: 2245, + transferedSize: 2245, timeToFirstByte: 1, timeToLastByte: 11, }, { status: 200, - contentLength: 205, + transferedSize: 205, timeToFirstByte: 2, timeToLastByte: 2, } From 82168ef6d789e61d18bf9f4ad9aa1e52234fda1a Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 22:24:42 +0100 Subject: [PATCH 036/248] test/results-test.js: offenders can be objects too --- test/results-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/results-test.js b/test/results-test.js index 815fb63a2..3ee86732b 100644 --- a/test/results-test.js +++ b/test/results-test.js @@ -35,11 +35,11 @@ vows.describe('Results wrapper').addBatch({ topic: topic, 'should be registered': function(results) { results.addOffender('metric', 'foo'); - results.addOffender('metric', 'bar'); + results.addOffender('metric', {'url': 'bar', 'size': 42}); results.addOffender('metric2', 'test'); }, 'should be kept in order': function(results) { - assert.deepEqual(results.getOffenders('metric'), ['foo', 'bar']); + assert.deepEqual(results.getOffenders('metric'), ['foo', {'url': 'bar', 'size': 42}]); assert.deepEqual(results.getOffenders('metric2'), ['test']); assert.equal('undefined', typeof results.getOffenders('metric3')); From 0931e8870449ec57e4249da70b2ebe992e8dfbe2 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 30 Dec 2018 22:25:11 +0100 Subject: [PATCH 037/248] requestsMonitor: handle base64-encoded assets --- .../requestsMonitor/requestsMonitor.js | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index b0763698e..7cd64223c 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -3,7 +3,8 @@ */ 'use strict'; -const debug = require('debug')('phantomas:modules:requestsMonitor'); +const assert = require('assert'), + debug = require('debug')('phantomas:modules:requestsMonitor'); // parse given URL to get protocol and domain function parseEntryUrl(entry) { @@ -170,6 +171,8 @@ module.exports = function(phantomas) { transferedSize: resp.encodedDataLength, }; + entry = parseEntryUrl(entry); + /** * Time to First Byte is the amount of time it takes for the browser * to receive the first byte of data from the server @@ -181,15 +184,19 @@ module.exports = function(phantomas) { * * All times are in seconds! */ + if (!entry.isBase64) { + // resp.timing is empty when handling data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D + assert(typeof resp.timing !== 'undefined', 'resp.timing is empty when handling ' + resp.url); - // how long a given request stalled waiting for DNS, proxy, connection, SSL negotation, etc. - entry.stalled = resp.timing.sendStart; + // how long a given request stalled waiting for DNS, proxy, connection, SSL negotation, etc. + entry.stalled = resp.timing.sendStart; - // how it took to receive a first byte of the response after making a request - entry.timeToFirstByte = resp.timing.receiveHeadersEnd - resp.timing.sendEnd; + // how it took to receive a first byte of the response after making a request + entry.timeToFirstByte = resp.timing.receiveHeadersEnd - resp.timing.sendEnd; - // difference between when the request was sent and when it was received - entry.timeToLastByte = resp._timestamp - request._timestamp; + // difference between when the request was sent and when it was received + entry.timeToLastByte = resp._timestamp - request._timestamp; + } // POST requests if (entry.method === 'POST') { @@ -228,8 +235,6 @@ module.exports = function(phantomas) { // asset type entry.type = 'other'; - entry = parseEntryUrl(entry); - // HTTP code entry.status = resp.status || 200; // for base64 data entry.statusText = HTTP_STATUS_CODES[entry.status]; From c3fb16840dafeafa5e77409672d69e4851208d3c Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 31 Dec 2018 19:57:19 +0100 Subject: [PATCH 038/248] Use http-serve (with gzip compression) --- package-lock.json | 54 +++++++++++++++----------------------------- package.json | 2 +- test/server-start.sh | 2 +- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8af34572a..b30d560e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,9 +40,9 @@ } }, "async": { - "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", + "version": "0.9.0", + "resolved": "http://registry.npmjs.org/async/-/async-0.9.0.tgz", + "integrity": "sha1-rDYTsdqb7RtHUQu0ZRuJMeRxRsc=", "dev": true }, "async-limiter": { @@ -207,15 +207,15 @@ } }, "ecstatic": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-3.3.0.tgz", - "integrity": "sha512-EblWYTd+wPIAMQ0U4oYJZ7QBypT9ZUIwpqli0bKDjeIIQnXDBK2dXtZ9yzRCOlkW1HkO8gn7/FxLK1yPIW17pw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-2.2.1.tgz", + "integrity": "sha512-ztE4WqheoWLh3wv+HQwy7dACnvNY620coWpa+XqY6R2cVWgaAT2lUISU1Uf7JpdLLJCURktJOaA9av2AOzsyYQ==", "dev": true, "requires": { "he": "^1.1.1", - "mime": "^1.6.0", + "mime": "^1.2.11", "minimist": "^1.1.0", - "url-join": "^2.0.5" + "url-join": "^2.0.2" }, "dependencies": { "mime": { @@ -423,19 +423,19 @@ "requires-port": "^1.0.0" } }, - "http-server": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.11.1.tgz", - "integrity": "sha512-6JeGDGoujJLmhjiRGlt8yK8Z9Kl0vnl/dQoQZlc4oeqaUoAKQg94NILLfrY3oWzSyFaQCVNTcKE5PZ3cH8VP9w==", + "http-serve": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/http-serve/-/http-serve-1.0.1.tgz", + "integrity": "sha1-u4JL+P7SCS6RlYN7C/eztvo8ovg=", "dev": true, "requires": { "colors": "1.0.3", "corser": "~2.0.0", - "ecstatic": "^3.0.0", + "ecstatic": "^2.0.0", "http-proxy": "^1.8.1", "opener": "~1.4.0", "optimist": "0.6.x", - "portfinder": "^1.0.13", + "portfinder": "0.4.x", "union": "~0.4.3" } }, @@ -642,31 +642,13 @@ "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, "portfinder": { - "version": "1.0.20", - "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.20.tgz", - "integrity": "sha512-Yxe4mTyDzTd59PZJY4ojZR8F+E5e97iq2ZOHPz3HDgSvYC5siNad2tLooQ5y5QHyQhc3xVqvyk/eNA3wuoa7Sw==", + "version": "0.4.0", + "resolved": "http://registry.npmjs.org/portfinder/-/portfinder-0.4.0.tgz", + "integrity": "sha1-o/+t/6/k+5jgYBqF7aJ8J86Eyh4=", "dev": true, "requires": { - "async": "^1.5.2", - "debug": "^2.2.0", + "async": "0.9.0", "mkdirp": "0.5.x" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - } } }, "process-nextick-args": { diff --git a/package.json b/package.json index f1f5b09fc..626cf1ffe 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ }, "devDependencies": { "glob": "^7.1.2", - "http-server": "^0.11.1", + "http-serve": "^1.0.1", "js-beautify": "^1.6.14", "js-yaml": "^3.12.0", "jshint": "^2.9.7", diff --git a/test/server-start.sh b/test/server-start.sh index d11ca7be2..56cb82c1f 100755 --- a/test/server-start.sh +++ b/test/server-start.sh @@ -1,5 +1,5 @@ PORT=8888 DIR=`dirname $0` -$DIR/../node_modules/.bin/http-server $DIR/webroot -p $PORT +$DIR/../node_modules/.bin/http-serve $DIR/webroot -p $PORT -c 84600 --gzip From 048ea8d1ae5f2d9851cca5854ed1744a61737499 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 31 Dec 2018 19:58:36 +0100 Subject: [PATCH 039/248] assetsTypes: more tests do pass now --- .../requestsMonitor/requestsMonitor.js | 8 +++--- examples/promise.js | 2 +- modules/assetsTypes/assetsTypes.js | 24 ++++++++++-------- test/integration-spec.yaml | 2 ++ test/webroot/static/jquery-2.1.1.min.js.gz | Bin 0 -> 29415 bytes 5 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 test/webroot/static/jquery-2.1.1.min.js.gz diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 7cd64223c..78fe44e12 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -11,6 +11,9 @@ function parseEntryUrl(entry) { const parseUrl = require('url').parse; var parsed; + // asset type + entry.type = 'other'; + if (entry.url.indexOf('data:') !== 0) { // @see http://nodejs.org/api/url.html#url_url parsed = parseUrl(entry.url) || {}; @@ -182,7 +185,7 @@ module.exports = function(phantomas) { * https://www.w3.org/TR/navigation-timing/#performancetiming * https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceTiming * - * All times are in seconds! + * "Throughout this work, time is measured in milliseconds" */ if (!entry.isBase64) { // resp.timing is empty when handling data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D @@ -232,9 +235,6 @@ module.exports = function(phantomas) { } }); - // asset type - entry.type = 'other'; - // HTTP code entry.status = resp.status || 200; // for base64 data entry.statusText = HTTP_STATUS_CODES[entry.status]; diff --git a/examples/promise.js b/examples/promise.js index a1a183b8f..9e6b5079c 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -7,7 +7,7 @@ const phantomas = require('..'); console.log('phantomas v%s loaded from %s', phantomas.version, phantomas.path); -const promise = phantomas('http://0.0.0.0', { +const promise = phantomas('http://127.0.0.1:8888/dom-operations.html', { 'analyze-css': true, 'assert-requests': 1 }); diff --git a/modules/assetsTypes/assetsTypes.js b/modules/assetsTypes/assetsTypes.js index d6638ec00..c281fab0f 100644 --- a/modules/assetsTypes/assetsTypes.js +++ b/modules/assetsTypes/assetsTypes.js @@ -2,23 +2,23 @@ * Analyzes number of requests and sizes of different types of assets * * setMetric('htmlCount') @desc number of HTML responses @offenders - * setMetric('htmlSize') @desc size of HTML responses + * setMetric('htmlSize') @desc size of HTML responses (with compression) * setMetric('cssCount') @desc number of CSS responses @offenders - * setMetric('cssSize') @desc size of CSS responses + * setMetric('cssSize') @desc size of CSS responses (with compression) * setMetric('jsCount') @desc number of JS responses @offenders - * setMetric('jsSize') @desc size of JS responses + * setMetric('jsSize') @desc size of JS responses (with compression) * setMetric('jsonCount') @desc number of JSON responses @offenders - * setMetric('jsonSize') @desc size of JSON responses + * setMetric('jsonSize') @desc size of JSON responses (with compression) * setMetric('imageCount') @desc number of image responses @offenders - * setMetric('imageSize') @desc size of image responses + * setMetric('imageSize') @desc size of image responses (with compression) * setMetric('webfontCount') @desc number of web font responses @offenders - * setMetric('webfontSize') @desc size of web font responses + * setMetric('webfontSize') @desc size of web font responses (with compression) * setMetric('videoCount') @desc number of video responses @offenders - * setMetric('videoSize') @desc size of video responses + * setMetric('videoSize') @desc size of video responses (with compression) * setMetric('base64Count') @desc number of base64 encoded "responses" (no HTTP request was actually made) @offenders * setMetric('base64Size') @desc size of base64 encoded responses * setMetric('otherCount') @desc number of other responses @offenders - * setMetric('otherSize') @desc size of other responses + * setMetric('otherSize') @desc size of other responses (with compression) */ 'use strict'; @@ -29,12 +29,16 @@ module.exports = function(phantomas) { }); phantomas.on('recv', function(entry, res) { - var size = entry.bodySize; + var size = entry.transferedSize; phantomas.incrMetric(entry.type + 'Count'); phantomas.incrMetric(entry.type + 'Size', size); - phantomas.addOffender(entry.type + 'Count', entry.url + ' (size: ' + (size / 1024).toFixed(2) + ' kB, latency: ' + entry.timeToFirstByte + ' ms)'); + phantomas.addOffender(entry.type + 'Count', { + url: entry.url, + size: size, + latency: entry.timeToFirstByte + }); }); phantomas.on('base64recv', function(entry, res) { diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 87932a169..541b49c39 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -14,7 +14,9 @@ metrics: requests: 3 cssCount: 1 + cssSize: 22 jsCount: 1 + jsSize: 29415 domains: 1 DOMqueries: 20 DOMqueriesById: 7 diff --git a/test/webroot/static/jquery-2.1.1.min.js.gz b/test/webroot/static/jquery-2.1.1.min.js.gz new file mode 100644 index 0000000000000000000000000000000000000000..4cef7b8ff6603be9302147f824afccbd42425d69 GIT binary patch literal 29415 zcmV(zK<2+6iwFqJ_GnrF18Q+~Wpa5fGA=PLF)nRsZZ2wb0IXYiliN0y|9?M)#JFUl zg=R~>szL>=@|k$XS7IkKyOKP`1CrP+k!q6a!%g|K@9zP;L{Z61?Z&Z4;&>Mx-UayV zq}#dpbt~h$&h;Z}Xbn4A$Mjg|(O~eD<9GO^=zbGzL*Ge*DCC{Dp=V*>#TE5fQM@=? z1)dBO**Q7;ukL&sdNj^-xXbR1$i0wWY6y|uZDcg>tRsKBlD*!M4olu{qBu>)`=7$G z{K(s`Wtfg#K<*BhT_Ixk!QAXtLs+_uqnl1BZ#u8zIEqap=OUI@+aQ)n$LZVzp^q^) zLAu0OG1Fk9Sf<-J>;Mbf+3MFcz`L9Wq4bSzL9U)Z)(<;f1_|Hi{Mv~-o|w$IFFY$D z32})xFA6;;<#W7lw#kw&@B$d}_T!u{#r=*4Vriw(=QIw&1;4<@(n&tvgrDMQBjfar zU(%peO(1+N47GR$55+zWIWHuL(3*!p6r`$W$0Kp}buvw++c&S@yqVrUAI$n$U% z_L@T&Ng8jxG>YwzCvqj#t6}h!3>WFr9`H1J9>>mIwb~^j-}0Q*$|TW)pM^~KG@@XA zyIOUHGaj5fV;VVe`t-0g{F$xWX7)ZXTA9zO<6Z87q!o3+$y&w>sfJoL2294CN_)Vr zy#A2NQ^=I+H-5r;Xx3t^W&7wYqiR)EcJDWBx708lk6C^qtb6+Q2?v=wG z%`BP-C<9!jpDi+O0QN0svsvBc6vYdre+dDY2XT@f10b)=0miJHKOJ&9#29(iBJ$db zaZmJ}zDdjJ+JjQw4LZGZG3fQ&b8kFR3+&Bi_GCujL;on3<$`6|!P=4<<$7Vyc>n6Q#K0Dmm>lMJF9n;>Fi-*jOcPZ0hEh*Vli>a?m6^WbcOJVyk3tUg!R+P2|_Jv z(PuTSVS;K>!`rwqOsL)AtOOW-g}Jl!~PBHyec2mXX%x6$<4Qnb;Zt6;ZAeU>>^82R!G zKD9z>s?jy>4sEBt2YbElg|=IJ&6Qu_21qDAKF$pVqN^x-z22mEEtDzqGODR=&h0Q) z3z|*Pj!6;X8#{f&un+c^lV$YZHf=CSF2>=*7$rUThi=~sw*DFc;)ie?>4mRk7Q&Jwu;VWwi z+{ zW3>Z(`2i5xy`NSM>W9LW?}#qmMK?us0mj|^mKJ|Xo{!`Qdn*R8C{AHe7a{?$8+p#eS= zLY4|K^3MH{kHvrt`~8`4Cw|FQF-Abb*rgE90aXP8(lf;UbHH9y%C;0;Z?rg{kLEz& zi!M@&$sD7wRxpXnUQen3Mzxl1Nj5{ix;7smAZHVJr}mO6oO&BMdZB zpN&e@nrqei%LxUExweDjfa36N;M+qEt9;w4D-`qOQxEjOh0b_7YH(qc96^ZdQHn#6 zfyiU^GXCI?7-Vr6);T$4t`6w9%#$y(34o~x)*{j&&f0@~5ubcY~xev^yWx0{aMCOS4SR3r&2 z-GPDL7vOSYOti=77=5>I%#7KA`D3GEPA91%Xr~IFOXoXki7UJoFRZ;w;ieH*kGlig zaEOU~%7Dx;yjiID-CIxhoC5LFq0*@x5tE7*zgEP;e8~fT!7urWuX)HLzTt2cJmD$d z3L^==eOpPRe@cNA$&X*x^WYoohg}PW%6kx7;~rS)KX9V7MUK zt7kbdMDEDdUO^==2v)HOtjI3&r8^z&_&Z^EOX*!oKhya$UYz7E^fG6gMsvs}QE#y3 zcNt}!akNS@Feu~94-&^+VZt)-eHms!0%Mb{5SM1_?J5m6E17}WFoPHIqi}VXB_G@eew>2y9b_y=K_V|$AKtPBHgPG{NVD+Ue*r}%XSGuAiw zNAaQbGsca?B($;NG&imhq z6nwC9=23iluN8NV|Ak0vrm_W{QJQ{2COW$*R5#QgT(x z$f#P8h3S(1p3(<9H9d-dvuJ)w1};~UTw3xP`y}#xtlp##ZD;0m>YuPMtMoMQ$iLCu zhZ2;cRSIDQ6qUn$H25a{e+NDvL!{p6*tAvJj;aC({TiYj@PEx?oJsA6!R2Yg_6 z`fRL7uMuXB#pEk!e~^>7!CAon2jiAgpI#qO z#qsO?tUtY`ZUY_`fu?AIu3^3W&v=-ryIykT(DpcU*Bc0b6I>#T)kE z6q&c@?rEOdZ`f|Wb2^{h5Bam5;$oa>7P0pf2c*KjD_X-UVGM5JT~D7qefs!W5#&&a zhG@W}qSLuQ)*juO$I<%b(urTfH=2H5&0uzG=lS_?kUf3+=*MS#IC%89=Vwo!J$^J` zyHOh7-M`Hvl{eyVIyAYql&Vqcl`y{l^=4B4E-IO&gWe&+K@cM0wPQy@V2$`Cx;4gd~J%uk8>o_(_rTEb;s+ z9E!imqgl~ksGQI1{3I{#zBmiYvq=VAQLp6icm^Ea{>YcGAc@Y;-6K7JiLD$-+<;Wd z#jUG^vd8WfWO`+#GEoVnl71p$u|=TgBG6N7((5^Vs9$OYCih6WfZa!8K}nJs>t%eN zLV;ke;7tM_@xT}eN(bY*90uh<&tSG|CC$McTQNcO1F=6e`bpn-IO`aEC88pR)7xZq zddi~7Y9^BYm21))8{G)UE%@s7A{UEa(;O&1xQncdCYV1tNf)=+zxpeDQ zrO9@NU>#X_*&n#|S2nQ?Dj`yLRCjkB@0KkWs!-hXhP+bBzJXI2u^GAUH_lpn>7UR9Xd);U+f6Ju#OK}SVvi? zsp!K1*Se6)YI-D<8mTPe70weXvk>!rHCR9mu<|gmi`k6oAJ78PHGO)}J3KP|DM^a< z_5$pG?)X}bI85P7sLjq3bEt%_RM7G9oW6tPpYPutCgZAd=QMq>!^%86$5I}3j*VZw zynnZEc5Un@zE)UeT0m(@{3R984{+Jnkdw5Kq&i4L{|MwwS!~kr0GWPS(|Fk=Zfu4^ z@<<$X@QvusO^Jo*_2^`30ZrM^d64D!{%b_b>AN6FCFmI+Hqk|ZRph|-*NRPRsyTOu z%wEuDhMiMW0KNWc8c}#;E>${}8Bi8E5kXCt>a}#Q@RR}(IHI41`zpI(&k16iIvj;v zulvi~fWC~@+JM3>=I9QC1WF%seLlom`gZX^dpmh7O6c{DQ5J-dJL#7&eu5?MC^p4I z=i&KJXZSwfYwQF?v%wupWjTtchU^|G`*|b`q4cJ~5d~XEL>KLe4x0qgz?OF#j1pkP zGA*stvK@w2;D( z)mG-D@|?sk&wm!}`>Bym6}Jfv7iixikB>&&MTxn!aPCvRQJ-KP1wdXLdr}n(?+a%> zSEm__!UV5oH1iC>D<$9;q1Sv@@UAK}P3~)P|9>EWa#pUXO(|4cu#y9hiRUZ&(P=?t zNt$slKA^SAk|D=+$w9p$2gav zs%y7#UUYH8F3R6Wpqy~~7G1?mZk^bMa@SDBHDwPnDaIFSnILV;yRs~@Zlrq|r0@5~ zQpY;_*3ufajZ7Mi?9f?y7C7-Ds5)tm8ZVOkh>Pp|hgaW|q)V!*c#Cu1y29YUk}h0{ zt5Wc_y4qEqjAj%I#7L1_#>Dk8ZXuD#@}f04n&Ui}ooZ7H3b%HQq;}+Ye za{Vgh89U9e&ZgAJ-Otv_Bt@v&UDxd5mr6o@?aoQp^h(s@>J^ERx_#10Ix?u3WK~?Nd+fY^OUIl?f?-A>8o+9H*9vb?DfN4@SCh**!|rY1cj0RT}@X z58o`>B7Y!kmZ!4JW#i)_aG5>i!yZXpUz2+!DHN7ItN$3}s?p?x|2yH@*4%sMUKLEXT9((?(g&mu zV&cr`5Z+bA#qJz3Sg@KZ%94CgI9GQU{+!g2s!SqPDWK$afsrSiQI)XQbZTjBLtR<% zXq4MNDyWarBbF1N4jKbInBVVLx&f5b$5oG=54$GfgW`Q?!(U5ghe$8o!V0z@x~iHm zl}_{zU2LtUls5HXS=w~7s1G};K3v9%zjH%8d@v-B%(3wg3^o>JlVx$Q7i;_DGPnrx zHTf4)h@J;$1dAo^alF;?cj?s6tNVj2t(BU%`Ru0P>wUAwHW7rbt z4$>bleYK_g!icaEekpP*`8lO{gi;lCaljlGg%W_F5d9Jl3e6D9n2o2mhd@ z*W)U=sk15tqsk;bWIMW2rENnZ?rY_0`{{t|Q20|Kw|-uvvE#cv7zxfm=WPmW}V@xE|>oW`+GW*eq zY0y!)2KHFpy!HL!i~VnNZ{FRumF0{6-=9K4rPYLaL{U!qzHTruU&cx*b|qDIDoN2y z7A6i7Cy|t8N96tNZ~yi_BLLc|e(SDvyE+yK92}f+pFRFI-u+K`_x~AF^Z%<+Ai*CZ zF10Tnje9}c&N;{DX*1L4n5Fh}kHe zPv>bptJ__M|CrA&wN9Kzn(cZw%Gi=*_0xw#Cv+0%{AwLwsHk`bz_kH`3S)RnyR^Bk zg{&1$8RGpo99p$eZJH>px{YGqVn^C1wJjUCyxQBw$dKw|y;FMON8%V;4I$WJxRQL> zsk)%$k=z(U$yFrg`4BCmpTFv#wCeh-VmHgHh?D4vd+0xSMoeJzk^f9qkz0cDssF8u zz_GDgF@31la_}2>I8=5$nyR|jTv<1|j1s8aX%#ieiixe!{st*i5$4C-@+Pi5g8z$u z|MC;WzC_x)hg)LC69-$V$7I-C9_Q!2MBpuKv}7*r2%H+z2}}>e$UE7h7HWSNiWXaV z37@iYsa<+LzqmM`;dH(ecLu{TY~sg-l>!#&Vze1U?wk!5r?=Ra{XQ3U*7f{)uueBL zN~;BXe`fT_^~FWdlcU|@^pSNsnhx*0Lp{0cVdVLG3^U9we!NZZM)ySUI{3puNH^&% zal!29u}W`1z|!>L;^Kx*0<{coJRPeA*ws7FDQ2`9+unA&^l(A{PraruEYnrmkD*%G ztT!F@^9Vb7anbEAsW{7kk7z|YdZ1}rjtAm{Z+VX$2VHe?X?m`k9)o*F!=#BZT6;Yn zc#Oy&Y16+rIl9r@q|QPbwD1~)E-dN9l3s;j0n5v@qA_J9$@1UfMA9eb|Efe(&#~o% z^IW;Xbuw49=VX=Zq&pC@4Svk@DyVtan(|RaMae!M@=6xr(YR4=z2)<>nKy1 zA}WUHGVfb8&)iN{mpgp>Z00-Avp{mfDqE)C8OaaESsa-qOtS_4@SS7ac7~zOKB5q) z_29*1fRe}6dY{bNbgeKEOQ7(q>3`w0^B9w`BOouK9$Zu1|LE;CLuP=6-(jeG+=PWU-^XJq1s?^wbN$E1-4>uSfk*GTNt zjx<#JY#YQ=`+k7yA`>O3yT<;T1$Hz!QLiOFYhIF2v|hKyXG0^lCRP)E>ZhKsdRYO1 zO6z*z8pfaenw{GZ?z#gG?))74pOw2=Z?>&SI{e>sBg@(Gg=2tO>)+0P8k+!zPmy4# ze}1?yiDcTS(#k9ULC2RI?I%7`d-RhOvN53<`jIgZ9hJFiZ=3JK3=Fi1; zrN0{qoruP*jcFS8lUu9z91!!uk&Gk=ls&7?GK@_8*f}p4H{hi>Y|moWh*a!0J=4`x zP^l+HPO2;lvs9tzo0bhWscx9{L@WaYDrK*V(e-tHhA(7x*smG(8u?^jQ)JnZF z7gGejnSqSY>2p65=V-rpINZ=rqU`hlSDgy?KHQ3_(gqD;F;9FQ#-~Du4cjrE>F3F8 z=yX|e@;gn|Jhr2Mbj#u{Zw9J)!2nTYL)FSIM{bJB-MN(5nznfuW|7nJxdkK+Q94ik z2|0uq^S93Wl*`HEPFl&%rua?iADtu2{;qb8V7r6s&7As@pyM5*`+aZNeL` z37$k$(vF2ZMs<}bm7!5l6~aj5UWI!o9|xYnng#~?pp+FH^v^n-n!apc(V#&@NbQqI z&79u@^(UFJaCes?{^fjvaNJEoBehXWl8sTl+NQ;jmRe^r%#t-p+}2)B?O7eu3Oc|^CP6pXR#(q9gi#S>0K8bzwXfnD% zGpG22Hp({La<5@jA8w6}Y^}^k;$%znIn>RV(rC@&!~|VT0bSuaWIu!yQt@f$V@#Ai zV-Xs$5*`E`>vsx;iG73vkrp(30s0uOc7Ir5EoZO*sKtpw-&rUGmV9+-T4}xt1XHi~ zFR+7P)ILY_5G@hf|NdZqQ)id7PK_3=ip(w36ptuAd1Kk$w!}&8>2GR_^#EE7#!Sl=O z*q{RF6P6}|%|7$Xr&8sRHenJ)M-TPK#KzEEKbv1%Jh&?7ATf#>DR-@M+1}*?H9js4 z*J{pYZlPOMZ^PIRt1a>3NS^h5)C0p{-O4*qHAB(dCrT0ZBY?GCT+nvfVNdPp;J62o zVJ}eQLskQ&=q&E+hz#$lHnm*APfw?@sPfWj|D3@Q|O!* zwDUKYmudnp4qcpS(59Q-DJqDDY{GBl@~w3Tp@;aCvt@6d^|goA9TAr;HTrPa@xLAX zN&hU{YEI|G4%A~89}P*EBNmLB2y(o$vqP=3KqFXqfwKVBV~z3BH#)-hM7xjnAY^4< zoJ>r)SvnEtDQaUYm#w`G7<%M0kHhL6`^_q_gI$I{tG+KXx zq%v0ns+PX@7H1nyyooiLr%PBEI);B;j_gHr_^ zs^I!X-2`)G!tHntZ_+imr?qG6q%+>k!BgTizx@)syK~DXgjfsK*7054R(kn zGdFCbW-`MreW`QQ@7x-i;?K#aW+KJxIp)$7}S){iCxdfNHOrzMdEb|nJh(fflbsg|Si z9c+kv>A<`wHtsdrd6nZ7?p4!CCp|x>^$JyM;Lms>Eadte;hMMSl_%kr(X}>V{ArGAfChxoC=ao!0LL;<3PGIzBe3E z@B9b{zZLKKbkFdIn{B#xeo*?>h`1ktQ`~H=9%!Y{GAi#5HM%PZr^cg<(VKpL@tigR z-Q-KUWt|!K_LBV!@6AV;=ma#5n3AhsKV?)1U`JdHpLK@Gr?0PGUq3ql8{&Z;_eQ@a ze|kN7-BaTyA6+>W##=M40I&4Xq;hLfS@i(u6}WzQ4Dz`FSX1z^fTzTfkD>%M}23b^fr(1SW8Iri4kUSa|f#p(qkhk?9jCf|mS`S^d^T@7u zZ9Z5&a;#>LaLb>7CU!&}O&4)}ddNN9Icr*pwe+Bx+|uq;05M}UVW*VQDnb7x83ou| z88Ouvd(#G@?e);?IRqz^F{@%V`^AOLuVL`3%N}`%kFXA1EdyNp+FOnAD?l69Jl^g0 zyZ64>j~Sn|&E#{gOEx{KE-vmZM}s#K+X@pYc@($gcj|Y6jUkHA`E>O9;q_1B?z4FG z>$CBbgNBlM!pJe7r563VopEL^Ihg;}Eh;*Vg3c-t5&L7}D4)r-k;*Qw$7~uEmNWf$ zI1-_jP3Pm}cLOk@pp-fLsILwzBlhV+Wv*9ygV2HqST?Pgfiqge|enW&P37(GL0$J_7eHSa&j1GHIu(Cu`g zf-RC}%Kw}fXVeK^Gu27)@H25z2P$j>jepkzlG?kW{>!Vov!WB+I>2YPRi-ztmG(01 zO=hkDKxlC~JDDaw$H8ZwO$Qr6y-QaRt+Dt&mn|fNxI4fFjeh>P+RqmUj;g#;U!;%h zSw$t-JS9FCJ6yv8%;qV24Hl+>Bgt@0Za-R>3fm4B#z7%t^MOX_adTjl=-wTzMqs); z-*pP(&!k0GcPPS0F81k@P@to6i!r+nu#ug!$+s}|J6$z{K6(ccJYkx4<&~KXU-b3~ zX#43RAS*a5sELgeMo{J9Z04OXBP>(nQxQ}%bxQ2RmI-H8biuM7m2PvQ6Bp4xkN7V>1hz;_roO_0N<7`MY)#hVDU-)tNdge)d zArT)t2vMizkOQoW%F41fk0&@3)2kpi0g+6SSLH=UIHl@7uje%IE2QTg0|Z=y#7}C! z&5DI8(}x`WQkq%c(%0dZcE)jynEKe_8)95CngN};4zTi1EFI&c=~yeoG3<>9fc1=x zv3-Ln-btL{mRA{OcvbiD&@W&Z0+-nMC^Yg^*}ChmuA5fGd%M!|i4VPU;!A(5|Ni%Q z08rZ>9J;!5GhWx!`NVwbr-=J=1CN{ru_yII22%EXdT%|xvH&lcgRE6Ep`SV)$m}g5 zmYI*ohd_y7kK#ap%dNfZnOD8c9Jw=kMm&UpFZzGRY+Edt$!UO|@m;AXOtLRcu)7uhOg6FS}P$^>zB_IR*UL?a!~#(?_ec z*84dVg5CLw^`8Ft(8>Ux2UoLL<+T65AmW+RS9d}58Ht-s^7}P+2p><}iF;H!N-;nY z#*sEhK=hi}U^aZ*xcV1Hc%x=k%k}o%#l-^NAVAZm_0`uAdow3Ep7#eN;WJ>>#g-31t8_8H{pz5ROg{p#y%u_Io$E1P6ELpG`1nr%z@3kbT;kqGUTbqYbjua2h$+Xv;qV8=W>|o<#KUx z3%5!taX8G(DVwlwr)jw35)d0Pg)J*SdZ5~>9J)bFa$d59x?zw>pUYQ*s@9U11potj zL0PQt7WvI^kifRqi17eLq~3z1h-XIiTtM}VtuuKBuan8;rJC{;yuFXI4UVeL!dZsE zRi3FCGb`sEjr(X+^L6h@*AoK>FW=C!+gfm*=9)VW3%4Y$IiDTXV(p*kww}4EgP1mN zHR&qxQfq~C_7kjrn(y0F5~05|VCQo-Co$~5%Hfuu3*j2-za3i2V_E>(j+e&?&%6FS zKGD+V+n)J$J;9LoUR$fri4ivx=+;l<@+Dxdx2-O@qfxZ2cI(9(sFTa}ivD=S2zXsZ zbtdEi%4S`*jK&HtGSH-oCK-|K54?B7-nIPRYQ3LN-bFYJ*HeTN*WNX+jS&zK0Y!yR z$})WlesI_@++J7wUswj@)7OcV~TK8Ur;Inmgmz3jx&>7u76Om`Z;F%e+2 zW$0NF!E_K+qsH{d-3Ti$8uNFV78b5nN$@t6JQcP{K;~1KfI!BsFxVo#d|yH`yGWsF zPRb)fvE4aJqU@YKFflo*ImVeAV}QESfd`1zK`FzUGISkI8ouyHna+zNNsZUPW@CEK z5C*#Qlz)RT#H!)}@i-W_rdo%p8kR}>jt$FGyUY$}cr7zx6{oS$S;;D5W?->o$kPuYB*5TCVPl=2{8EQh*clr=6Bf>Bwh(Q2%9N5n-1P>-fg5 zqMR*9wUx!|@h==^RM5h%5db4pXm8;*w^L{_VK1;rz#M-aKUR<)uySzW_34g&xmIkTnP(hdmr<7!UaO=Ba9a_g(C=Dtx4dP9jt0u_3aVGY3Sp~zIY3;pr zpv^f#5RSz}-VHs&otZL^V&^nl5h1-9ns&s=9C=zH8WnJ18c44<`!BP7*0$Y4m8(FC z_`lD)p8d})_U~x1YmJmB+ieV}Xe!~P0l?e7dJF>E%y%D;N6q;6bkaYuLE|b+`FH%T zhrB@7>O0+&UF!FVZhHw%Z>x}uz`SJNgVId*T41Gev=DiZ{C14B13#VE{N-+!F|6~j zP-?;gj;d`K=4@HDXxC-jc)w5{BSHlO6)F!ON$a%Qg5T`1ULXQ(-_ZM*1X%n6jRV%# zB=uge#wX0*6F4{&1G5LP;=R0mK(?_>4XOh!lTBV#itT9v?|46b*QA~y);U*tm3fV} zZ^~X2bb|yZC9&MRf!n*@eC=?^TU0o7&lV|$9c}Q1&XyyR9R<+5m43A9*_3+>QJNuRM7W;B!85R{?9}UWL+0_2)6t1TZO+Oruf=FC z)K-2M6lY!}5|fT{RgK}yQX$8K+=a7187Pz3f3a-(FKSrX>8M1u*CWgE*qEo1S^7)M z>sD+p!BdXM-gjf2Y!?m#I(6Vr?f9wr;Upq&|FoYy@o7=k*rNoeEUSA)H!}8>6*Ta?Z(DxI1M$@H$%Jz_}C~ot+Gx2=|$b3x8xd zZTL~6H>#%iD^Ijc)v}^%xxX88px3&c23hcCwcp)W_tpDP-wb(gIe=>N774=m6=>3) zyCW8EZRe#VJha)RQNr7lzC#6ZyNSijjumGia!A!>)od1&|GEu=A4+S6OGyJtr?i0n z!UlX&so%KRH|zE7&LWh!0b2F+*Mu4j;JRuwicrUhs@+yt((XfeS_chDwJU?c{`B9a zqH=^y*o1+EnIct8IhO65q!A_{tmS4^4wPM9TicVv>fz@S(ZJF^nTUc3u^sY4J&F$f zv{JRM((*{-D7RBxj3seKom95XlC<-wvBovLAJ|PF`A9K<3dF3j=0BywP0y{Nwid>m zc}Lt$-m^RXw|RpN!d0O6g3@zHZSrRhTa^6_+|qEHr)6!ve*?BwQY@-$`?vSjqt~RY zc!VjoX+r&Ym1|XnY63Z{3>Rt>+j?!gyexqX<||Vs3Mq4G8=RgvLpQTjx6aSLqKN_^ zeN+}KRwfdKtalts-LBHVcoBLrqcduJ<*OOAn(J@KS$8=eg1sxByzU+1*JYP}jH<83 z{D6KQ;wzn-NBun=MX%PINJW2#cRv07e7)bVmvr&Gn(U*odU-03DG?f2=wZoV1w_I7 z3I66t{Rvc=bI*IIhWg4@^jE@$8vKC+BN8BspEE65J-x{(ZYubCn@u^K0s4m!1HO@d zTml}5#cVt=AoE2#$UHYF;uwq&Ku@FDD|^)}qk-oo>$%@N5}9rVaOH~y4Iac7{AC|{ zI>BncEOeQ{Xi<3(#9(qDK70G=mA#JW3#&C(|AoAjvQznqnuF*9t)BYPPfI(v?J;1^ zeD(Q#o-e8$@YwX72#I$2m4DE5l2mu4ev)}zuJ3o%VY8mE_SN>Fow#V+@8d&3{MGH1 z`dJFM&6!|BGABeQrzc295qu(Qfq*we4rl2v;-j38 zZpPxju}<$&fDx!o+5!H6RHkF7dM1b}R%=eAY}&?=C)r}OW+9p?=~Tl-4;;pn_B@1$ zWYNos;bigja9FF8@7|{F0B8J*wOZ4la29_SFJZ=zeo?p)Xt@ z$wo~PN^*V;n5w!08^*APQhQBH8v1O-%5e-e{+b$ZLB{kDn57!tNM%F-RWMu{pNLwM z>457~Rf!HyzGb$XXGFFgItsSFRjc&J_MelvFD`=J2)`J&lfXOwE6bKsbN+O2V3NduIK!KZeXK-EKUsXa@lR#;HevTaE$2#T9)^TI}1im7+V$~RCU{L_^% zA`9=vqtyeip+B6*W=gGuu@RQaCX&8(n$A+riMvoA2gyw?F8-_b z`}}8F5_jFk(Sm;8Iq9FOe+C%P&NjXCK1r?LTCsAgHxKO2-m6Izf7(Yj)>YKKLsEv3 zwWy5Yr%U@{)su<5Qw_BY0#U%B&3CV)X=yclntZIhAxO857W?;q+k$zsKZmV5aQnd> zn21&b5(Y70){Uwsn27Lu(w7J;DUAJZILZ;z=4lHQ++EbVK~?UAs=%hTHscTISAA(y zyB4zeC43tDo{F#5!tl`#tlXby(AN(@pLrDLfOSBn4u?CXXMEnh6YgW``{NI$gW8>q z(ETa=Af|&ELgmYPFc9M|XwV!#M#7g+35Fm)6G;w8{Xt6o5gw zy||c;ZpYpea01SfN1cb*Xy!k7+ujvV&zwEt;Uz4i&~fsh?#G?tBsKG$F7bnuM^`cD zzv~%X4GqSpo#ZsTM3mIcjekd&@q3JPJk@ zM{8wnSIEZYis`JQ+51wr9A1*%Pd}zZ@59HL{ha2ic77>!14i2}_6e~8xnAtqsm1=7 z^@{r~c8qS=q(=>1s>0tiRv_7N!gW*Ox65T!&XJa)t>zAu@d+cCO#in~g{j9HFe0%2 z6di`?6vGB%{gnTZG}P(ThkI(d5V4c#*Wm)B+Z^lFtUEUMG}je>Q17S8I8U>eR=B5> zI8=N6?8MH6n5J#L+nafV+p_;Vm`9bl6>G-ywAIAGjO8J#!`>fq0B~yei;KCRdYQ&u zP(X>(WT{%b`WYWXyI0T+i7-v)L0khzh+)-QTv>J|YN?2+eJQxn{lScpFuU<2g=c*r zwj%gfStTTr!0-mH#<1jg0?%;OsU$*82V>1~{OcHP{Ya-Kn%mSw#Cmas7Aa;>S`JSm ztUY4Zleru1@@OY+_G#W;oIhsYFb>5t_*za`p}5&4QMTBBSG_ySMgMVDAl!9)DI+kK##o7P=LWus!|tVt_HVdZqN4fW%Y8OEjMRx=xhV7UondeD}>x8 zMASs*4f(Wsr>}MMiWwFX{qqG)?HN7Bf4HtyQH#g+6nX|dfVv!wkx((wfb5A z{pD=G)>@hmcgeR!=6Zh&YW?I-8Pq7~#<*pS$Z7UlgArj1N%{`81r)cRI>E-rJ&K`q46)q5!bYDBaDvhNI5Z zc>Z`d(`ovXQILoY)mvx(2%orp0e|0=-9R@x6I@7_##KZ=i4*q-j@6#5Fv0O!^ZUqI zdtU-NA%k1eg#NJ>K*z9~%apjDj|ERz){{rKZ5jK}2W6V-NUP!020p4uoMcC(kHX{; z9*4mA7Y3bEGzqBrV?#^~j3z|Ye*@aH3^U%)QRfpuH0RXf&dBKLAR?pjJt`z2ER+4s|D4@p{gns2N_FmWnXy3rt%vae$Cv)%A?)2pAtfIH? z#YmU2n=kJdL9m9_jJXg%t;zIOH_V=_1IW~T_fpruV9&ut`g z7uMfzKdzIhsj5(P0rxk?~xYhdAIS@(_e{s=wY2(Z;(#9c zz{PlPlZdZr(`_T+_ZC(5rZTsQt}1Qt)5Y~Kc|rfsub4+n>FXiQH932nJ_d(Gu8gLJ zE`@h*#O}c0&wE5DrpJ6UUal2A^$FfN2VA!J!26bK1LB-%O_7yBedAqw2$|CF!1;$A zzw35X>`obCIlJuP6U3YEU2IoQ9Z9M)ftRJ%ww$ zupU_JNmH^{G!TXxb|g62AauYls{q;DkGr7LMjih@|M2DYpc)3%S3s$2Jsw06{D~m(5%!!4YZ4dM1 z^uRL2VKHCb9-!Viknol*;|@UJl5tls;o{d<&#tC(m2+kK$(1Vb^KkK};p>O);7XP9 zsf6VA`DTCE?cXgbt~kD$tBM|KzaN0k4U^HY>G+V+wQ&dPq1c3q{rb`2^%a%4$=+m# zs#s>R)JTt~cnptuD$sjEO_<>ZRr2XMTxv#NeEHd{&t8uXB;6ixWBhuI>t|HrqpPW! z=jo%Qg-S-(D*9B`Lcr46pfFA zdETSWb4yaWrh%X_>G9jRwV$lldt8_n^L54r`8h^`-5YH#A=ABn}lhZFk0b!ed8>abS zKBptvv{UJ#y3tkC)l!pJ(;|1#*3(?$6^i27TDOLVlNw!*6C?LEzTnro%w*3SVa~(q|iC)oRQ&- zA?*RA0T5)IbBYN!GCl%R=kqnNg#Sk4qM@fZ$WT!0tbc$49vx)>f_QupHfol>@ESGu zoS}|t7ONM!jdSo#b|}_C27&f9cxq`%<`G(7J%%$guxpZ75QG7b;*Qod|zX%9~1V{_@#hzWaTQoQjJWqr8#SlT>#NymFZCrQIm!oS7Z>-$`$ zA;5F&#TK#4<|#Hsn>Dl0v6JWPJ$|+{h#n3hW}1dP^Fv}CwlX40d8tiPn)Mmn zs4@Go2K%$QO$}NfjWPlx)f8bgjAYEvd|br@%4T_ciWDVM;fab?K}F1|@(!H2Tsqt$ z70KZT|0XX#0?~x2Le^qltF}z6fKhQzgwS{uw{Vh%j)bYj^3!g~3--{XeyknUSwCA2 zmV)S1ArHkE=6cTZ9Jj`ESBOECwYyN2mzSp+Csk}&^&&TFaB3&U^EROF`3Z;UY1+v( zy{*L84M$$gIR}E^Je*r3nH&yue3>(nxO?Y^441cVWn=uZoX*&fuIXs~=;^a4?!rTj z$Qd$Cc^-5V%&&1m9=Lu^e6%xKTDSfQH}UhZrcopwSPKpNqdN*HaNZ6yAcNc58_MP9 zsM3Ya1;U{@m)AV@26u_mPOzJxkF;FdQI%f);h0ykw+83D0nRcp&GaU4u0Vbw_xZ`_ z2A`jp`7?4scSR(6bKh&5dW>zjWloqOVRvqOQE=aO3eDtSWVhJ?Z=JeP2E^19{Tf7frt25#41 zoSBPq%Nh|s0Mz6KcEaA$E7!{x*=oLN1ux?X`yV$}b7#b^YS)HA7}Jm;{)EvP|1WAF zZPseL|D267pyL4=BbqDrn*U*I#c=3hLDz27KX2vMY_dOTGy6aB(Ew@qcq%^M2kgy; zD`$3en*cC{1D$25VCRmQGh^k#I}`z{U@29HNF%}*ix4=Qxim{tiyBP@5h99-SN&t! zuS3%j6H^l^IGVV47T=hirgSev-44G=#x1JDFn4O*$iy8!s~sq{Pd8aMFKEiZ-JV}3 znFdZluwIskVuJw;noDCP99|dY-UUrIuU?>vyW-_9T?oJ`y30vEoxm;$>51j&uNc&#U z|1sCw$>lnV1 zz46RcdXb^}VCINTd;j$5FRwq1p0PJ)y_3^Kf4imJgs1mJEqsgEfnzi zHtyCFjpiFtMRhCD+GZawp4|+Hsf=g9J|(ieW_jLF5aecL^T!nQS2*s83PmQ~D0UPO zy)aWZvE-^1x9`cweD3nP-brcWL@jb?F#3b3Z4WYID#Pl|d{+0|>Of|prX@He<|O8! zGripwn%v(iD)QrYfimV=9)inswct~ZURw$?%aPTo1`R7=|7?(x-T!KxL~<1w?gqE? zWu*M`Hi?9xFn2%WgFfQ(J~D&?aM_VZG+IobH8kfeJe)b3!AivC%FY7P)6!6B>ISVQUp4{ar{YP9*C;D_g*=Eb?jQ`Eo+p^lG z|Iz=CGtP3xSCaM&msfOs76K|2{ibk2b>MqN=SfQ%r|MISNtVh`Jey_BGubp zL6AFRMDW4SOD?R3@q^mlRh7O?L<6d#G?Ja0^ango^jCH*UF_Ez`gc)H_Q~ac)BkPW z4rDuB;>k7r+~`D$NtW-{3tIk>ssiD7;(=vOJ8lf@*DoKc{C3WbzTC}!NBS6P5bg{v zm+Rk8JZisdh++G}YUr3}m`0J~k`!+~021<7WQ0T9`zaEjwZgib-<*`t3^| zrN38;`3W3!>tca-3h?z(^aT$egJabB` z@N_snZ1<3D8aS!i!<9p4v>9w7aEw#H1?p1;Va)C_&4DrqpM8U1Ah=5Y+g@o@WcdgI z)B7DqY6mRAfi1Ljd|no#DjUZ=nD;+BameOx--k7ajZ^fH|R#CX2kWCbRe=Z+S}PeRzDPx^f-_f?*b zqMvxt(PzJ>|6Z(rr+=5bXzYmA*ZTB<%2)hLj5X~jbB(ijdILzRxzPG6Oy>+xl$;24K) zr<7=XTCZ`EyE?zgeeW*AXiw6DLs5tNWQY=TvPALllO&QPwv4lM&CuTc!6(lqgGo1i z8b`9IwXv`>?OJS{qWgD_sqb_8lycaym721b4o64t%qV!r)P3G8uN%t6fq%|fcICA= z@mtYWv%7iTX!Uz*wq`dsqTkkL2_;j1#Il}*`*2N(`SlGiOvP!zS-6SZkrn_9S!y0Z zpq?;)(SHU2cA?=qUt6k~O<-sApLtK&ykWLD7|1uydS-U|phRMT&TX%8bRdqlVm+a& zph`Vz2TNMtq5*cz`U3OWwwjqZUp~QcU{m{9Zs9pXMS04rJ2qUQ?f+|K4V&X z0=AUKo+(wdUX)fUVl-2s;rgD+hg_a^hZS_FxwTTM&4FBdaB=Zc!hZEPtrw>MSw-Sj zX|3yVEXb@c8RgSJNYT2h;B>KTG+I@|Ld$riC*Z|8!ReHP5>Jc-uy9Sc8ZD1n?zVJ2 zAL{X;nd7?RFr}C=*6(&Y`e+TROB6cf)GZM;{sz26BVE`EtQxE|x{eZf4IdTN``7zr zP6zg5L8riS#et!S)|ku}i|_Tf@N-eU{quJHV6I=zw)533|8k_Cel6y!>KprO?J5wq zfG;+)Y{fYjY3HsV`0Dp>IeQ(h*6SsQ98uGL;7Gs6XoBUBrM=tnPKe<4{%+Ccl_~C( zAJ-fX{;T#oGeU!BcCUl^&g-&cE`)*`HJXhXFW>u}(pd&Q5(o@GkSMd96Ia=vv#|?e zErvu1BAQ14T}?fRN?N~DjVVZUsEm>2VGdM=&cx!?dJwjNLZw$e*JoA{i-%>Wg6aG^ zep1o-)b;3yp?45dCuy|e1pqvLHeEmyp$`r7=$ym$_%&HA(*2^F@}bw1mav{>ulL!h z%pz(yX(?iWDfos{?`PF^&K?!e!`8+lWwY{Bry`YFXZT;3SG^Mc z$n1tw>d$0fx%OIUOMR38%82&=c9|_S>?QLoUrMT}FLOQ&BJK235K#@!1@WNmiz5w9 z5e;WS!r_o+t)EnJQhDmfQ6v&5@nUjvjvH!1uX|~o&iKC1#qf81z%?=YXbnm%Y<1t7u{g$wy|$5ayOwrH7+QI8!6&4HwPGr`_REkqW;@c_=BOqQ;k4F%M1~&Gv@ZzEqM5cbCy{0Ov`R}80jOyW- zaX8Swmrwa`-xF_+>fYM!%m#BU!`@E+49iF8X;I(Nrwx%c)gL;7qg!L|8$@@5HXwXp z<*>IZeZ)^ja7G&21%h8ff*tsN;M2j_^q%5Rv4y~p*CQ;dEttXx+nUi?+S~$v(Ow2@ z3?mt=O`o94z)yWWoi5riOle`SVbt?6qBTl1KCF{K_3#06U3WiuDIXb1zU$b$XMW17 z$1q>ZcY(xX)E~fw&t1G9=Pvik5tsAop{iEtce(P*rH@{U@Pr=Xy_Uckq>qvwEUKl6 zqD0trT9{CR9W7^Y@zL&q_qxO|uov|%%}tP6w%r@oj|ZJPfWwKUl2w_Oa(U0W*Uc){ z2VyZEf`!9zNGH+#wrR1IzHMaH+eW$|Z*V0B zB${NlXm~H=-!R!oqx?8cF+HGBF)hW1)KM(SN4=`qD@1-CsK` z^@P*`BGBqlCQ||fI({3eLMK)8?wOLn0oSxe`YrciYeevFl-=*w;JWf*PKQ_$ex`nY zvNiB652vlaafg$-2d(S0G|%If^m`#wq;JNKP6o{fU;fbLWS#qZtC?wkINFKP^Jjr5 z(w5tRwcvQe5Wz`W?27<=Bk&w*`?v{KWg7B`lRMkCIWqu2;UP@VZl3l2^OAK(#hRw> z319y3ZG82VYo5GK8;efanSk<<(`@AGMXvsu3#zTq{R2qy0fd;-YScACO&1CPQ=u&XLa!fqSEtt`s6FMA9{CpOLwHU5D&r?Lr(%e$ zY1)M=`k-8&&C(ftqW7PXnqH+dW!9ysS53QI1hV0@8wh4e7yrUAF!s!uFLQU^ds)T2L*=;%pChK}IS2!X z;yX>xt1x^-*gI`|%0#1`i;ht#5d zJWcLD+6}=5zg66VmMMdw8nJE@C*=9NO-=mo6`rOV_|nweDN9 zFa-jqPiuH98j%q;4B9y&H?lzknuZ=i@Fxp8;74nFt=#3$-t{l=)^AFS>Nd5?x|^XL zdSAPEIKZ#MDf*Hdr9~;cbZc~RzIZ2kBeZetbd7=d#h#z0ZCuXw#hO6chSn?R-iMEf z*K7aDwg2Q7|H;$wQR^CYzAx4=Bi|R1dlNyydM#~x=SNK5A~^?GVW6Nn?rer=bwz|# zmMp&~*OEx|{LVuPxkIQ6)W!4FeKpu1k@*tI#_ldI?pT&{f=#LMXm3oiqL!>?%Q*HX zq~pzJed59yxshm-$Gf2!avBGC&27@Vp4^F(J3pC;*^&xaOcRECm2&C%CRUpv&R!$1 zEF{^cANq@@Mt)Jv=fbl2NV%2*FxA8~En{EkTS7f^^k#7xPgA0Nn<2(A>64hUpc4{L z?%?EuL7r-ccY~vYPX}{H|Md(TLX-@ytJrtTv1{fe zmevC1%0OT#gh-~9UVUhA>K!gWH)%Z|9)HVSz70PoN6SVwkjs4ZAEEX29%3uT2I+rb zV`0+gLiAhjP-GknbkPkRK_<6{fD~pLh|mwk zq(+m(<=s2)t4AI3^A+>L0B{11tO1m9X&RH<-uzyqwxm#YXc9YkW;~L@4vmr?CxC-= z3}%2zZESy9Ye0RwfpEBEb%0#mybl_t7e4Lzur^f* z)4?hLv*3tXihOiMzx}AyP8ByBm7f zBUcJI=JvjeFV(0wrQzrG1=~>AMN|J4roQYE8wzk|JuJ#yhMCNK{A#=t10Zv~D4o6v zm2uUL>{j|dLLiQz5`2;o8@~Pk0F92RdKok8YwfhnrtI2Fe>m0JY#CWy22rw^2PoHZLA|rm;IlcCad8Igq9t=!$>2>9h_zQwgdw zNhKK6VcIPCgY{T(FUgVSBIYkI2k`{2fTU{(XmNt!mhvNRE3EFoV8RWoE}7JVz4)%- z?}{i}*y|H<|lZ#oM`o+cpW&-Z!eM@J9=r*~pA+1YME)DFZzYN4gxA z?(ApI)vQWRY#!vj!}JPnE|0AI^#HY5shnzj~$RnhKHE*snsvR52|2iRaSCV3Xq-GbWpCD7lfkQRs)-~4gIx48ZK z)+;fk;kNE34J%RJZ&%LcM(Q~7{qOJh-UF^l9_WhPGhN;B!_hcRvTk~r8zRl!lhp{H zEzN2bw{^11v8YJtp$qexaB9Lq{%K*?9j%ZoEf{&YqeCBL;}&*VU0%|{8WdJ(r1jIB z)xjwuJFp)$q7R(1phLGtm0DXb<&I4+Fs5(IcI~)ckVrux$JozQ}9jA(6>3pn8V=;x( zTxl~R!(*-kmK9ka_*-J7(w@2!%*3uy`faInXDju6@k;A^rYp|Egq+m-E{w2+cH0T& zhgjX?ch4=3oZtaMc|C+8;R4lUH4&K1{ zi-i_RDwX$PnY9zVsSv!W5WJ}&c$4&6Vm{9UQpT2s+}mxK`JB@}J6{|2`gg8jl_M&Z zPgKgZ3W!RXWbDw2T%LxsRgScwxFc;+SUPPsYnF=#b;AM0Znasq^S4@-vr{hYYQGuR zJZGbUSITT#O#qH)s=mBFYNToJ%x2Q(+$5M-3Sea?+ z$Ag<^^TAv+&>Av#K1Qze)W`H;6SA#$2t8`oyMe?{u|sbu6g#2fA6S(f;8`vi`AX+( zPW8y!NaENjyQ%ydz*t2ixP->v`;brqzb!-(0t^RD^Zv*jvF~$@Y{CKY|Ln|p?fbua z@=na1{A`=SXK6o%4%XCUpC~<}jI1Ve3diVwZCXV{uTQJz@uVlqDE}JDirmMKA+n!~ zO44nl*hj?)?`ak!v@!TD+GcXSRtA$>)Fu}UR30CF57YoqDJ{%`6%&Q7pQVCe0qD#` zaO}qHR5(K9^Td`{L1*$fsa$t+vDbDGmGEPr+!2h@u)WB2xwA^(-+r?Xe)}z|uiM)1 zH2g`Ymau?Pl=zC}u+Es*I^$gF{WO{b1L7 z&gF{{Htsff1C7+^27NwlSqUs1!7G=J+_}a9mddXCS24o&P3zWlf zL7F8Cl?t$(t=vV@cGNWX(ETnRI41-hODVRqrKmb>LysfeW#j$Y!rL`bm|)!~`>jj38%QoXks(`GYZU*JXU8WF?R zUl4OB+LYGF4YSzt0-nN&N%BkMR+vZ^ZzB5X)p2iSxf@_>j?;TxCQ!6?&!PyJ^V3{XQfAmN0U6Ixi;iBZ|(pnX;i zX9?W~5Ql%hl{nnaJLgUXb#XBlpQ%oueHuFDlhqy6Fv-pH2$5<5u%LMoh{=8lglq~T zd}NSNfqA5NlA5>}cAaN3w@2Bu5h4{R7u<*n*q2G}ocm}`n&eL}FJm}cPRFWKI-cI4 z*Oq$0O_*S@=?f{yJ&2>)q@|bFfJiz9yt`J$IW1w0rw%slxGDO3Acii+XK10az4o%p zmkc4(Seui<=_Of^3BSm=HYZ6>1Bza6gCWv|Cq|tZyADT4ymQ8Rcg}P&o->n$=M0J6 z&bHNV{(E&Me(YzG?d^=S$eoq>LM}KP5^~ze)%`~P;wV0|36#!E&ZIM!U+K){R6;t> z)s#LY=@xeP`EtHTT2HQ*{%+uMfeWg>+HTjhh$Ooz9U%8%lFX(}Bh-F4^-M#H&=)R= zHQX8ou`oHwbQX;~Zou!((82d)`fxqQq`OYTz6iiY$i1gDNTxlR&O)c46T4qEUa7r+ zoHQnHol80R;hZtntV7>8@s@q(x<#NEdY_M;0Jq-WpsE37|0F^eH2+W(#{Q#9uoA;7 z;v_hZ^e->J|7+Vy$9*Z$!6C~w6#n`1i|0+Vxhfa{K67%8GmppK-eJhig2NTQ{Qkuc zsBcS$cOg}Oy`=vPSgh_JYtF4avTin%POop{JL8~k^<+po9Tv^w;4g|89yjdU6#RSA1kZ5LnbAU91cCq+xB4 zo`O4g^A}CSO9pySa7n;8Q4#&?SQY8A5NBq(6|g%ZYFL-fZQ(7y&;oWM0Tz5NxWy*8 zfYs>XM?$%#E?pO~kvxq~J?gTfEwRqXa8+8s8O{!&M2ViUi!>q_j5AgN=!x1hKBNoZ zS$9{|9cwu17Lv=l@Zr2>@_cTiXc3IgZU%guWE`tL3heLtm78>?9db^&tK1w8lf$7J z-HeBm;dy7C-k6Z%1n7a;J_B#Rvt7F(E&>0*g)~-k%X9dJ;Us)GoLB4^2ed29iie6= z)LD{MFT2U!zO43pYTK?iS!8?dFv?l)tz&Q=J64>eDVp_8g@DT}f%>te@rz@4)8V1R z>+$fsXwcxo?&om1FHYeFYDM3U0U&JWG-pIzqIsI5tMi@=jc1uEHMrz*zSL;dOqTv$ z?Kae@>YFMn=~Rj|Ug+g3$z6l>SjN^VX9k9J#(p{YS5&hAYMtBJsNz9RTkjSL+|U+! ztB5DuEODi`l@(&Ind`(n)Hgaj>1ZVJ1EOdb55~~NRJz6~Lm5^yZyZ2nCVDdLOwOg_ z$P#ZFg1iHxi3?Y^!~3Sb$UUfDBpLUIR_`tzhuhKhICeSgyavZNqiJ2S8gyt%R3yDM zlcMKqg2CjZwmbu0&#OhbqZWg5j9TuGkx&t7JUD48n)o$)*0JW_zri^WO&4PJ8&}@v zIp+qil?f)srqiH%7)FsTCOkRQ^ky{Y$(X=VnRX9fs`IH^6F0mjU^8%>7ZPJ=>T`}Dlp!wI%qVJ&z2!CKlCgx>Wus_6+pZsW(Kyc2 zDKIp+V>m11w>}{j7Ef}w2Tv0m!;1eT)8QhC{<=PsDa0vwHld>*i;}vsUt<(VUI^>i z-E#~`q%ibJe)J%6GFo|WNUl>z{PAL4&L?vc(P*(%`wZUd$|p9-W8>9AWQH-&bAD=-#H;C=y3v)QikaZZu}lCpcZDsZ7GhNPH=&(d7Z%&WJT4-XHQ zG>gki`dqatNWv=z;J&v&Joxd|*O&hlDanuloSP?6vY{p+zl1P8B)KDFC*!fp9b880 zEq(`IS}xR?<(1UUj*;EG0=T0~AdQ;_WImd<)7Mca-oBF0@t3ZoCVu-0>TiF$$SaY# zNgPz9+iq?FdeAh++yhG~-_;uz{pN;6iLiab^qp}}F}(5yK3)c&-@dxX!O)Si+i23HLZMLFr zZ}+&l);DXPZ8~HNvGLtV;UJ_VforM2h?Lp}-EVa0#)1fu?(*SqtCsF3RlV?%DJ*(7 zoB|6l4-1=eTdjkr8MUFGUu25~zLdQwW1jbp~!?(`m14=VkRmGZw8Dch*!H)+5c#yKZvY$CjhRo<7^h zHn=T|?;ymF?${@g{t>R??2U=1$VPAUoY%5{DiU?Jt#;qT;oho;fX~1Mp?pH*l3QcI0ldNoa5um*KGnaTg2|alT?>7n95tJTA|J9F@ z=#x)AChilVnpC=0jB5+Z@z>G8i;LTycM>|wmlZBdIXxmNQ|HVK5C$2VKP(XVWvU(M zrMe|{@lC8`ryGo&!JI}&4q#-u>T__f9Q55TGx$Ny)rbipyhPnx%AXKIl%B!#qtmRM z-|7Qjio!jO2R|sqn@e{~KkufUvM}uVFuI_j8b)zf_fq@1$S)Gc6OO3H5QXChkymCq-J9=dHmA3qzTUbdvW`h0;x#_wZy9Jh0mC; ziu|q0;>)ehNwyn>PsYQOPr90M4{r}6MNI0Z=thIHyR_fyUq>))C$1U}5o1C2285Lc zLQa#orM)-GK1Q@AOG7lh}Ab2QXhowjqz2 ziYC{Z7r$0E=Ii-{@3|8k=o=Ro%i0Fy^c5p&*%(Wc*<_6R zUYwGVEs-Rac2o<~>PQ!c%F(_Vi&kn+Uxk^5nEtJLYebF*VGsBui&;uJdlH6Zxd9vVK5EvBV*3f*s1N4FZ7nPd*$-d9Bwcaw;pZljh|TN z4Ku3{hE7=xR>o~m(W$~BRLt+ZWclxuHBH{pW01S_J%RNO7HSo3*1LTZL9%WG3WI(^ z-76dV5rm0gEW49pEq!%Zgv&%u&y^v+NhKP9LK#w4)`FEBE1=xisoKE%Qy5IHa+ItI zk6E{Iwb^xaV@&p-Rx@I=gQAGAoiqb)R1Mcw89{VRYhobN#Ozk)I6O%D4?a}cPJJRfRg6qn{n*PCWQv7epFxH}T?Bb)YMJXKgoT6cMF`9! z(p#%GXk1le*c!}1f+wi z+Ozc!9{TFD;(f0Cx7E98wTiFij%oS4u4ivRo@#;^gp(01F6g){3?l20789UvQCl|} zMcqzADOC+~@V4D(9H~l-)?SimVhr>P=QhDKeY<3eSqnPOWBZwkt3#3osZk4xX^ZN_ zq$jIkA*3~=fnnY;E>1aIoan?D8TX%YDjL`y|Nbpb0HQ@ms7GBQL8|3R9v_3umAUr* z_qBfzK?56>FWGvSIvG3Ei>^(>Bm4l>u8pKO)0FuhFtT5IEBJIeUo$)K9=;EaXrUO# z2b1fd7H-z_Rfj}Ir|@_nKA;=b4-BHa^K3Eyy)}J}mfi)V(-f3`A*wRGb&5K)(r_I= zCRCx)#(U>T2A^K%sHFX6i6CgOgVrN~*Iq#FR_tPWZivl%*NGD6p>=WbncfD0XCN{M z{%-e&by+nD?ayDCM<7r^-~1qG-wQIt04+q@=vGJ(55?KBXmNIYL?4Lo^Ge`FkhpxN z?`THhb-?Ud(o(c&FDgm-gsui_`0Z@#2b4nO@}PWwe)0U9{eGimS1o*HNOH>5@C&%g ze#u@1U(vsPj+DKA`t)PE_~fX*F6vfa`dSA8qn|P3`?Kr_+E;Miq3=iH1r_=G!Y15h zi1Z@GsQAozJ^#~;_9EezMFM652(C>Y!%8k%}H=;y%Mp)W2by}Amy?Ow;df^T|VL|AzA^QE5ks=VX` zlbi&g?S2}C%R~d?Xp{7$sr;ZLg`p-|Qv-Ta4nV6Qg*&7M&=f<%-f1Jy=^Bw-|#imiQAGgt-5>CH>SMJE9lK0*#v(ss*9Ls7q!#U5IE(X2*u=p2s=| zf$*{DJxh}{TZ)jP?UT;9sLlfW8G%#`|T@N+_)2Na9t=2gwC$o4^9 zMH@pOq;dr;!O>AeYtHLfAuON_HlMZVbva&DG`tdA8>;n9_&LP8*nYzioz`!#=Ym&{ z^RhEG@qp^Ehz(gH`c!IrM#ny%uaC}BEtY2l#Bz}@9`NFO@dBH{B`^mRF~2X&UqfFx)G$+3kI{sdWR1Yyd2!_`nXzsDA3G~7#2YhxBgRb5 zwgD))kxfBR4o7*-ffQNw;&H-IrD580+I}qA2%eJ3VZNf#aOBA393LS?m#0C{V*0lN zO4I{s8`!J1F=)f~`nE7*@a+j$qJif}C?L~tCiEg1-8Orcay+WB8)w=94#|ZWO^|=} zIoVKMyw&5Zb)-q&g6AhGgQ6e4fB6b4*E-u9Sv62NR-Up^T#K}0Ra}F`MTCx9(NryG zuP=u^K_UM ztxA^scUsRo=B2ehuC!8$BU`@b%lBazcn&`%-J(w78Nua;L-zn|K8V1i&I=8TZIV~@ zOzAY*R*MYM7Q6`4g>FSh;QVyeY-vz)cLRBf>CR2saT!DL0U#I6-x8I?56;!so%;t9 zl2XLaF0Wz5Legi`3KoeB6;zL8l73#R3Gy9=DG=wl)z}uSnaK?VX(2mDqR@Jb zy#~|EdLzFsXZW`}bw%)PE`tmGn@Zsij*OObwu1Z9ewSMh`( z{9{`EBK^8VHhR49iq;1=zLNQnQa{&9A?L>tY0lK9C1o@9&sY--IGQT#7?rg#GA{FW zRqLb^{&pMY67<*Bxul~tQ8HIQXtlV?{kwc#IiMW0L+EeJ9_^Ve2EO%_2FtOW_p>iz*kqtn8omQ!c%8wyXpv@ZtM`gRZB4zW;$bG#w{XA

tNu3)rPm)`nLCveit1@R$2|Y-W?w7|6Co z$ENT-mV62VT`=V(IvJK$UuGAli3dffY>O!8fi&n7nS}b6)nX#-Kdph!Ax}l-O%HOK z1o{;*Y4W$1ElYDvB0igL0NLA;w{oS=-sj*LZ~lgo>IJSpauc;$f3aRoh_m-1iajQ@ y_Yue#Gix8EFA6~p)^*t@`sZ_cygDkw?x%9x+Q+D2g%a`L-~K-=1nSlmNdW+}SyX5M literal 0 HcmV?d00001 From 7b23a46ce959805ce71487a6edffa787e92c2e80 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 1 Jan 2019 13:52:57 +0100 Subject: [PATCH 040/248] Inject core/scope.js into the browser --- core/scope.js | 2 +- lib/browser.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/core/scope.js b/core/scope.js index 8e4b230ca..7726c54ae 100644 --- a/core/scope.js +++ b/core/scope.js @@ -1,7 +1,7 @@ /** * phantomas browser "scope" with helper code * - * Code below is executed in page's "scope" (injected by onInitialized() in core/phantomas.js) + * Code below is executed in page's "scope" (injected by lib/browser.js) */ (function(scope) { 'use strict'; diff --git a/lib/browser.js b/lib/browser.js index dd43f11c4..064eb0748 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -46,6 +46,14 @@ Browser.prototype.init = async () => { this.events.emit('consoleLog', msg); }); + /** + * Inject helper code into the browser's scope + * + * https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args + */ + const preloadFile = require('fs').readFileSync(__dirname + '/../core/scope.js', 'utf8'); + await this.page.evaluateOnNewDocument(preloadFile); + // storage for requests metadata var responses = {}; From 971cbe555d032a3e0d90399aa1888b8254529b00 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 1 Jan 2019 14:15:24 +0100 Subject: [PATCH 041/248] Browser: handle connection errors within a promise --- lib/browser.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 064eb0748..f76b02997 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -136,15 +136,24 @@ Browser.prototype.init = async () => { * * @param {string} url */ -Browser.prototype.visit = async url => { - debug('Go to URL: <%s>', url); - await this.page.goto(url); - debug('URL opened: <%s>', url); - - const metrics = await this.page.metrics(); - debug('Metrics: %s', JSON.stringify(metrics)); - - this.events.emit('metrics', metrics); +Browser.prototype.visit = url => { + return new Promise(async (resolve, reject) => { + debug('Go to URL: <%s>', url); + try { + await this.page.goto(url); + } + catch(ex) { + debug('Opening URL failed: ' + ex); + return reject(ex); + } + debug('URL opened: <%s>', url); + + const metrics = await this.page.metrics(); + debug('Metrics: %s', JSON.stringify(metrics)); + + this.events.emit('metrics', metrics); + resolve(); + }); } // we're done From 081e8be436fc45a66ecdc3ca2001e1058edc938f Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 1 Jan 2019 14:20:32 +0100 Subject: [PATCH 042/248] requestsMonitor: lowercase response headers --- .../requestsMonitor/requestsMonitor.js | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 78fe44e12..b40a22503 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -6,6 +6,22 @@ const assert = require('assert'), debug = require('debug')('phantomas:modules:requestsMonitor'); +/** + * Given key-value set of HTTP headers returns the set with lowercased header names + * + * @param {object} headers + * @returns {object} + */ +function lowerCaseHeaders(headers) { + var res = {}; + + Object.keys(headers).forEach(headerName => { + res[headerName.toLowerCase()] = headers[headerName]; + }); + + return res; +} + // parse given URL to get protocol and domain function parseEntryUrl(entry) { const parseUrl = require('url').parse; @@ -169,7 +185,7 @@ module.exports = function(phantomas) { id: resId, url: resp.url, method: request.method, - headers: resp.headers, + headers: lowerCaseHeaders(resp.headers), // All header names are lower-case bodySize: resp.dataLength, transferedSize: resp.encodedDataLength, }; @@ -212,7 +228,7 @@ module.exports = function(phantomas) { Object.keys(entry.headers).forEach(headerName => { const headerValue = entry.headers[headerName]; - switch(headerName.toLowerCase()) { + switch(headerName) { // detect content type case 'content-type': entry = addContentType(headerValue, entry); From 23f889f0466f551be629e7cccd364b04c3c1590d Mon Sep 17 00:00:00 2001 From: macbre Date: Wed, 2 Jan 2019 21:46:38 +0100 Subject: [PATCH 043/248] Resolve "Protocol error (Runtime.callFunctionOn): Target closed." Call page.evaludate() when the page is fully loaded (after page.goto promise is resolved), but before page.close() is called. --- lib/browser.js | 23 +++++++++++++++---- lib/index.js | 62 ++++++++++++++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index f76b02997..24b648508 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -4,7 +4,10 @@ const debug = require('debug')('phantomas:browser'), puppeteer = require("puppeteer"); -function Browser() {} +function Browser() { + this.browser = null; + this.page = null; +} /** * Use the provided events emitter @@ -17,7 +20,13 @@ Browser.prototype.init = async () => { const networkDebug = require('debug')('phantomas:network'), env = require('process').env; - var options = {}; + var options = { + args: [ + // page.evaluate throw "Protocol error (Runtime.callFunctionOn): Target closed." without the following + // https://github.com/GoogleChrome/puppeteer/issues/1175#issuecomment-369728215 + '--disable-dev-shm-usage' + ] + }; // customize path to Chromium binary if (env['PHANTOMAS_CHROMIUM_EXECUTABLE']) { @@ -32,8 +41,6 @@ Browser.prototype.init = async () => { // A Chrome Devtools Protocol session attached to the target this.cdp = this.page._client; - this.events.emit('init', this.browser, this.page); - debug('Using binary from: %s', this.browser.process().spawnfile); // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion @@ -129,6 +136,8 @@ Browser.prototype.init = async () => { // Actual bytes received (might be less than dataLength for compressed encodings) responses[data.requestId]._encodedDataLength += data.encodedDataLength; }); + + return this.page; }; /** @@ -148,6 +157,11 @@ Browser.prototype.visit = url => { } debug('URL opened: <%s>', url); + // https://github.com/GoogleChrome/puppeteer/issues/1325#issuecomment-382003386 + // bind to this event when getting "Protocol error (Runtime.callFunctionOn): Target closed." + // while calling page.evaluate() + this.events.emit('loaded', this.page); + const metrics = await this.page.metrics(); debug('Metrics: %s', JSON.stringify(metrics)); @@ -160,6 +174,7 @@ Browser.prototype.visit = url => { Browser.prototype.close = async () => { await this.browser.close(); this.events.emit('close'); + debug('Browser closed'); }; module.exports = Browser; diff --git a/lib/index.js b/lib/index.js index da04c62d2..f392238cf 100644 --- a/lib/index.js +++ b/lib/index.js @@ -49,44 +49,52 @@ function phantomas(url, opts) { results.setUrl(url); results.setGenerator('phantomas v' + VERSION); - // TODO: prepare a small instance object that will be passed to modules and extensions on init - const scope = { - getParam: () => false, // TODO - getVersion: () => VERSION, - - emit: events.emit.bind(events), - on: events.on.bind(events), - once: events.once.bind(events), - - addOffender: results.addOffender.bind(results), - incrMetric: results.incrMetric.bind(results), - setMetric: results.setMetric - }; - - // load modules and extensions - debug('Loading core modules...'); - loader.loadCoreModules(scope); - - debug('Loading extensions...'); - loader.loadExtensions(scope); - - debug('Loading modules...'); - loader.loadModules(scope); - // set up and run Puppeteer browser = new Browser(); browser.bind(events); var promise = new Promise(async (resolve, reject) => { try { - await browser.init(); + const page = await browser.init(); + + // TODO: prepare a small instance object that will be passed to modules and extensions on init + const scope = { + getParam: () => false, // TODO + getVersion: () => VERSION, + + emit: events.emit.bind(events), + on: events.on.bind(events), + once: events.once.bind(events), + + addOffender: results.addOffender.bind(results), + incrMetric: results.incrMetric.bind(results), + setMetric: results.setMetric, + + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args + evaluate: page.evaluate.bind(page) + }; + + // load modules and extensions + debug('Loading core modules...'); + loader.loadCoreModules(scope); + + debug('Loading extensions...'); + loader.loadExtensions(scope); + + debug('Loading modules...'); + loader.loadModules(scope); + + // browser's scope and modules are set up, you can now use it in your modules + events.emit('init', browser); + await browser.visit(url); + // resolve our run + await browser.close(); + // your last chance to add metrics events.emit('report'); - // resolve our run - await browser.close(); resolve(results); } catch(ex) { From 3bd046086283f003ba992bc001d69e9957bb1116 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 6 Jan 2019 22:25:55 +0100 Subject: [PATCH 044/248] Inject phantomas page scope JS code via page.evaluateOnNewDocument when "init" event is fired --- core/scope.js | 2 +- lib/browser.js | 8 -------- lib/index.js | 17 ++++++++++++++++- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/core/scope.js b/core/scope.js index 7726c54ae..76e30d4b3 100644 --- a/core/scope.js +++ b/core/scope.js @@ -320,5 +320,5 @@ phantomas.getDOMPath = getDOMPath; phantomas.nodeRunner = nodeRunner; - phantomas.log('phantomas scope injected'); + phantomas.log('phantomas page scope initialized'); })(window); diff --git a/lib/browser.js b/lib/browser.js index 24b648508..b85ec6437 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -53,14 +53,6 @@ Browser.prototype.init = async () => { this.events.emit('consoleLog', msg); }); - /** - * Inject helper code into the browser's scope - * - * https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args - */ - const preloadFile = require('fs').readFileSync(__dirname + '/../core/scope.js', 'utf8'); - await this.page.evaluateOnNewDocument(preloadFile); - // storage for requests metadata var responses = {}; diff --git a/lib/index.js b/lib/index.js index f392238cf..8e15d8c75 100644 --- a/lib/index.js +++ b/lib/index.js @@ -71,9 +71,24 @@ function phantomas(url, opts) { setMetric: results.setMetric, // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args - evaluate: page.evaluate.bind(page) + evaluate: page.evaluate.bind(page), + + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args + injectJs: async (script) => { + const debug = require('debug')('phantomas:injectJs'), + preloadFile = require('fs').readFileSync(script, 'utf8'); + + await page.evaluateOnNewDocument(preloadFile); + + debug(script + ' JavaScript file has been injected into page scope'); + }, }; + // Inject helper code into the browser's scope + events.on('init', () => { + scope.injectJs(__dirname + '/../core/scope.js'); + }); + // load modules and extensions debug('Loading core modules...'); loader.loadCoreModules(scope); From 79e29fbfe005c80c4d2818ced6a5b3ddc412b9a7 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 6 Jan 2019 22:26:44 +0100 Subject: [PATCH 045/248] domQueries: move page scope code into a separate file and run it via phantomas.injectJs() --- examples/promise.js | 4 +- modules/domQueries/domQueries.js | 98 ++------------------------------ modules/domQueries/scope.js | 93 ++++++++++++++++++++++++++++++ test/integration-spec.yaml | 1 - 4 files changed, 100 insertions(+), 96 deletions(-) create mode 100644 modules/domQueries/scope.js diff --git a/examples/promise.js b/examples/promise.js index 9e6b5079c..750ed220b 100755 --- a/examples/promise.js +++ b/examples/promise.js @@ -20,8 +20,8 @@ const promise = phantomas('http://127.0.0.1:8888/dom-operations.html', { // handle the promise promise. then(results => { - console.log('Metrics', results.getMetrics()); - console.log('Offenders', results.getAllOffenders()); + //console.log('Metrics', results.getMetrics()); + //console.log('Offenders', results.getAllOffenders()); console.log('Number of requests: %d', results.getMetric('requests')); console.log('Failed asserts: %j', results.getFailedAsserts()); }). diff --git a/modules/domQueries/domQueries.js b/modules/domQueries/domQueries.js index 2c859d000..dd7656920 100644 --- a/modules/domQueries/domQueries.js +++ b/modules/domQueries/domQueries.js @@ -4,7 +4,7 @@ /* global Element: true, Document: true, Node: true, window: true */ 'use strict'; -module.exports = function(phantomas) { +module.exports = phantomas => { phantomas.setMetric('DOMqueries'); // @desc number of all DOM queries @offenders phantomas.setMetric('DOMqueriesWithoutResults'); // @desc number of DOM queries that returned nothing @offenders phantomas.setMetric('DOMqueriesById'); // @desc number of document.getElementById calls @@ -15,99 +15,11 @@ module.exports = function(phantomas) { phantomas.setMetric('DOMqueriesDuplicated'); // @desc number of DOM queries called more than once phantomas.setMetric('DOMqueriesAvoidable'); // @desc number of repeated uses of a duplicated query - return; // TODO + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); - // fake native DOM functions - phantomas.on('init', function() { - phantomas.evaluate(function() { - (function(phantomas) { - function querySpy(type, query, fnName, context, hasNoResults) { - phantomas.emit('domQuery', type, query, fnName, context, hasNoResults); // @desc DOM query has been made - } - - phantomas.spy(Document.prototype, 'getElementById', function(results, id) { - phantomas.incrMetric('DOMqueriesById'); - phantomas.addOffender('DOMqueriesById', '#%s (in %s)', id, '#document'); - querySpy('id', '#' + id, 'getElementById', '#document', (results === null)); - }, true); - - // selectors by class name - function selectorClassNameSpy(results, className) { - /* jshint validthis: true */ - var context = phantomas.getDOMPath(this); - - phantomas.incrMetric('DOMqueriesByClassName'); - phantomas.addOffender('DOMqueriesByClassName', '.%s (in %s)', className, context); - querySpy('class', '.' + className, 'getElementsByClassName', context, (results.length === 0)); - } - - phantomas.spy(Document.prototype, 'getElementsByClassName', selectorClassNameSpy, true); - phantomas.spy(Element.prototype, 'getElementsByClassName', selectorClassNameSpy, true); - - // selectors by tag name - function selectorTagNameSpy(results, tagName) { - /* jshint validthis: true */ - var context = phantomas.getDOMPath(this); - - // querying by BODY and body is the same (issue #419) - tagName = tagName.toLowerCase(); - - phantomas.incrMetric('DOMqueriesByTagName'); - phantomas.addOffender('DOMqueriesByTagName', '%s (in %s)', tagName, context); - querySpy('tag name', tagName, 'getElementsByTagName', context, (results.length === 0)); - } - - phantomas.spy(Document.prototype, 'getElementsByTagName', selectorTagNameSpy, true); - phantomas.spy(Element.prototype, 'getElementsByTagName', selectorTagNameSpy, true); - - // selector queries - function selectorQuerySpy(results, selector) { - /* jshint validthis: true */ - var context = phantomas.getDOMPath(this); - - phantomas.incrMetric('DOMqueriesByQuerySelectorAll'); - phantomas.addOffender('DOMqueriesByQuerySelectorAll', '%s (in %s)', selector, context); - querySpy('selector', selector, 'querySelectorAll', context, (results === null || results.length === 0)); - } - - phantomas.spy(Document.prototype, 'querySelector', selectorQuerySpy, true); - phantomas.spy(Document.prototype, 'querySelectorAll', selectorQuerySpy, true); - phantomas.spy(Element.prototype, 'querySelector', selectorQuerySpy, true); - phantomas.spy(Element.prototype, 'querySelectorAll', selectorQuerySpy, true); - - // count DOM inserts - function appendSpy(child) { - /* jshint validthis: true */ - // ignore appending to the node that's not yet added to DOM tree - if (!this.parentNode) { - return; - } - - var destNodePath = phantomas.getDOMPath(this), - appendedNodePath = phantomas.getDOMPath(child); - - // skip undefined nodes (issue #560) - if (destNodePath === false) { - return; - } - - // don't count elements added to fragments as a DOM inserts (issue #350) - // DocumentFragment > div[0] - if (destNodePath.indexOf('DocumentFragment') === 0) { - return; - } - - phantomas.incrMetric('DOMinserts'); - phantomas.addOffender('DOMinserts', '"%s" appended to "%s"', appendedNodePath, destNodePath); - - phantomas.log('DOM insert: node "%s" appended to "%s"', appendedNodePath, destNodePath); - } - - phantomas.spy(Node.prototype, 'appendChild', appendSpy); - phantomas.spy(Node.prototype, 'insertBefore', appendSpy); - })(window.__phantomas); - }); - }); + // + // TODO: pass events fired by page scoped code + // // report DOM queries that return no results (issue #420) phantomas.on('domQuery', function(type, query, fnName, context, hasNoResults) { diff --git a/modules/domQueries/scope.js b/modules/domQueries/scope.js new file mode 100644 index 000000000..79163edb1 --- /dev/null +++ b/modules/domQueries/scope.js @@ -0,0 +1,93 @@ +(function(phantomas) { + phantomas.log('domQueries: initializing page scope code'); + + function querySpy(type, query, fnName, context, hasNoResults) { + phantomas.emit('domQuery', type, query, fnName, context, hasNoResults); // @desc DOM query has been made + + phantomas.log('querySpy: ' + query); + } + + // selectors by element ID + phantomas.spy(Document.prototype, 'getElementById', function(results, id) { + phantomas.incrMetric('DOMqueriesById'); + phantomas.addOffender('DOMqueriesById', '#%s (in %s)', id, '#document'); + querySpy('id', '#' + id, 'getElementById', '#document', (results === null)); + }, true); + + // selectors by class name + function selectorClassNameSpy(results, className) { + /* jshint validthis: true */ + var context = phantomas.getDOMPath(this); + + phantomas.incrMetric('DOMqueriesByClassName'); + phantomas.addOffender('DOMqueriesByClassName', '.%s (in %s)', className, context); + querySpy('class', '.' + className, 'getElementsByClassName', context, (results.length === 0)); + } + + phantomas.spy(Document.prototype, 'getElementsByClassName', selectorClassNameSpy, true); + phantomas.spy(Element.prototype, 'getElementsByClassName', selectorClassNameSpy, true); + + // selectors by tag name + function selectorTagNameSpy(results, tagName) { + /* jshint validthis: true */ + var context = phantomas.getDOMPath(this); + + // querying by BODY and body is the same (issue #419) + tagName = tagName.toLowerCase(); + + phantomas.incrMetric('DOMqueriesByTagName'); + phantomas.addOffender('DOMqueriesByTagName', '%s (in %s)', tagName, context); + querySpy('tag name', tagName, 'getElementsByTagName', context, (results.length === 0)); + } + + phantomas.spy(Document.prototype, 'getElementsByTagName', selectorTagNameSpy, true); + phantomas.spy(Element.prototype, 'getElementsByTagName', selectorTagNameSpy, true); + + // selector queries + function selectorQuerySpy(results, selector) { + /* jshint validthis: true */ + var context = phantomas.getDOMPath(this); + + phantomas.incrMetric('DOMqueriesByQuerySelectorAll'); + phantomas.addOffender('DOMqueriesByQuerySelectorAll', '%s (in %s)', selector, context); + querySpy('selector', selector, 'querySelectorAll', context, (results === null || results.length === 0)); + } + + phantomas.spy(Document.prototype, 'querySelector', selectorQuerySpy, true); + phantomas.spy(Document.prototype, 'querySelectorAll', selectorQuerySpy, true); + phantomas.spy(Element.prototype, 'querySelector', selectorQuerySpy, true); + phantomas.spy(Element.prototype, 'querySelectorAll', selectorQuerySpy, true); + + // count DOM inserts + function appendSpy(child) { + /* jshint validthis: true */ + // ignore appending to the node that's not yet added to DOM tree + if (!this.parentNode) { + return; + } + + var destNodePath = phantomas.getDOMPath(this), + appendedNodePath = phantomas.getDOMPath(child); + + // skip undefined nodes (issue #560) + if (destNodePath === false) { + return; + } + + // don't count elements added to fragments as a DOM inserts (issue #350) + // DocumentFragment > div[0] + if (destNodePath.indexOf('DocumentFragment') === 0) { + return; + } + + phantomas.incrMetric('DOMinserts'); + phantomas.addOffender('DOMinserts', '"%s" appended to "%s"', appendedNodePath, destNodePath); + + phantomas.log('DOM insert: node "%s" appended to "%s"', appendedNodePath, destNodePath); + } + + phantomas.spy(Node.prototype, 'appendChild', appendSpy); + phantomas.spy(Node.prototype, 'insertBefore', appendSpy); + + phantomas.log('domQueries: page scope code initialized'); +})(window.__phantomas); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 541b49c39..00ec69f93 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -30,7 +30,6 @@ # DOM mutations - url: "/dom-operations.html" - skip: "webkit" # do not run using PhantomJS 1.9.x as it doesn't support DOM mutations API metrics: DOMmutationsInserts: 1 # DocumentFragment > b[0]" appended to "body > p#foo" From 1466493c3784aa186be98daa070153e955bf3a8e Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 6 Jan 2019 22:38:38 +0100 Subject: [PATCH 046/248] index.js: expose the function that will pass events from page scope code into Node.js layer @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageexposefunctionname-puppeteerfunction --- core/scope.js | 6 +----- lib/index.js | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/scope.js b/core/scope.js index 76e30d4b3..3d1d4880a 100644 --- a/core/scope.js +++ b/core/scope.js @@ -170,10 +170,6 @@ sendMsg('addOffender', Array.prototype.slice.apply(arguments)); } - function emit( /* eventName, arg1, arg2, ... */ ) { - sendMsg('emit', Array.prototype.slice.apply(arguments)); - } - // exports phantomas.log = log; phantomas.setMetric = setMetric; @@ -181,7 +177,7 @@ phantomas.addToAvgMetric = addToAvgMetric; phantomas.setMarkerMetric = setMarkerMetric; phantomas.addOffender = addOffender; - phantomas.emit = emit; + phantomas.emit = scope.__phantomas_emit.bind(scope); })(); /** diff --git a/lib/index.js b/lib/index.js index 8e15d8c75..2735000a3 100644 --- a/lib/index.js +++ b/lib/index.js @@ -84,6 +84,10 @@ function phantomas(url, opts) { }, }; + // expose the function that will pass events from page scope code into Node.js layer + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageexposefunctionname-puppeteerfunction + await page.exposeFunction('__phantomas_emit', scope.emit); + // Inject helper code into the browser's scope events.on('init', () => { scope.injectJs(__dirname + '/../core/scope.js'); From aa31f8c2d4289ac9154137669182b0067705936b Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 6 Jan 2019 22:53:34 +0100 Subject: [PATCH 047/248] Use __phantomas_emit to send messages from page scope code to Node.js layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fire "scopeMessage" events. /dom-operations.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "cssCount" metric properly set ✓ should have "cssSize" metric properly set ✓ should have "jsCount" metric properly set ✓ should have "jsSize" metric properly set ✓ should have "domains" metric properly set ✓ should have "DOMqueries" metric properly set ✓ should have "DOMqueriesById" metric properly set ✓ should have "DOMqueriesByClassName" metric properly set ✓ should have "DOMqueriesByTagName" metric properly set ✓ should have "DOMqueriesByQuerySelectorAll" metric properly set ✓ should have "DOMinserts" metric properly set ✓ should have "DOMqueriesDuplicated" metric properly set ✓ should have "DOMqueriesAvoidable" metric properly set ✓ should have "DOMqueriesWithoutResults" metric properly set --- core/scope.js | 26 ++------------------------ lib/index.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/core/scope.js b/core/scope.js index 3d1d4880a..3cd8b9780 100644 --- a/core/scope.js +++ b/core/scope.js @@ -112,26 +112,7 @@ }; function sendMsg(type, data) { - // @see https://github.com/ariya/phantomjs/wiki/API-Reference-WebPage#oncallback - // Stability: EXPERIMENTAL - see issue #62 - /** - if (typeof window.callPhantom === 'function') { - window.callPhantom({type: type, data: data}); - } - **/ - - try { - // Prototype 1.6 (and Mootools 1.2 too) creates an Array.prototype.toJSON - issue #482 - // @see http://stackoverflow.com/questions/710586/json-stringify-array-bizarreness-with-prototype-js - Array.prototype.toJSON = undefined; - - origConsoleLog.call(console, 'msg:' + stringify({ - type: type || false, - data: data || false - })); - } catch (e) { - throw new Error('phantomas: calling native console.log() failed ("' + e + '")!'); - } + scope.__phantomas_emit('scopeMessage', type, data); } function log() { @@ -147,10 +128,7 @@ } function incrMetric(name, incr /* =1 */ ) { - sendMsg('incrMetric', { - name: name, - incr: incr || 1 - }); + sendMsg('incrMetric', [name, incr || 1]); } function addToAvgMetric(name, value) { diff --git a/lib/index.js b/lib/index.js index 2735000a3..b30d0e2e2 100644 --- a/lib/index.js +++ b/lib/index.js @@ -93,6 +93,21 @@ function phantomas(url, opts) { scope.injectJs(__dirname + '/../core/scope.js'); }); + // bind to sendMsg calls from page scope code + events.on('scopeMessage', (type, args) => { + const debug = require('debug')('phantomas:core:scopeEvents'); + debug(type + ' [' + args + ']'); + + switch(type) { + case 'incrMetric': + scope.incrMetric.apply(scope, args); + break; + + default: + debug('Unrecognized event type: ' + type); + } + }); + // load modules and extensions debug('Loading core modules...'); loader.loadCoreModules(scope); From a140879d99fbc6306e02dd488bf1020df1134a9c Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 6 Jan 2019 23:21:42 +0100 Subject: [PATCH 048/248] setMetric: pass calls from page scope code + default to zero when no value is provided --- core/results.js | 3 ++- core/scope.js | 6 +----- lib/index.js | 3 ++- test/results-test.js | 4 ++++ 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/results.js b/core/results.js index 96487f1bc..003581344 100644 --- a/core/results.js +++ b/core/results.js @@ -24,7 +24,8 @@ module.exports = function (data) { return { // metrics setMetric: function(metricName, value) { - metrics[metricName] = value; + // default to zero when no value is provided + metrics[metricName] = (typeof value !== 'undefined') ? value : 0; // make the order of offenders match the order of metrics if (typeof offenders[metricName] === 'undefined') { diff --git a/core/scope.js b/core/scope.js index 3cd8b9780..5e5fd74b8 100644 --- a/core/scope.js +++ b/core/scope.js @@ -120,11 +120,7 @@ } function setMetric(name, value, isFinal) { - sendMsg('setMetric', { - name: name, - value: (typeof value !== 'undefined') ? value : 0, - isFinal: isFinal === true - }); + sendMsg('setMetric', [name, typeof value !== 'undefined' ? value : 0, isFinal === true]); } function incrMetric(name, incr /* =1 */ ) { diff --git a/lib/index.js b/lib/index.js index b30d0e2e2..db12b80e1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -100,7 +100,8 @@ function phantomas(url, opts) { switch(type) { case 'incrMetric': - scope.incrMetric.apply(scope, args); + case 'setMetric': + scope[type].apply(scope, args); break; default: diff --git a/test/results-test.js b/test/results-test.js index 3ee86732b..d7a149b75 100644 --- a/test/results-test.js +++ b/test/results-test.js @@ -13,6 +13,10 @@ var topic = function() { vows.describe('Results wrapper').addBatch({ 'Metrics': { topic: topic, + 'should be set to zero by default': function(results) { + results.setMetric('foo'); + assert.strictEqual(results.getMetric('foo'), 0); + }, 'should be correctly set': function(results) { results.setMetric('foo', 'bar'); assert.strictEqual(results.getMetric('foo'), 'bar'); From 34b49534cbcf2b2c40d167388728fe25a3800e72 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 6 Jan 2019 23:52:12 +0100 Subject: [PATCH 049/248] browser: pass pageerror event from Puppeteer to phantomas + test case for jserrors module --- lib/browser.js | 8 ++++++++ modules/jserrors/jserrors.js | 14 -------------- test/integration-spec.yaml | 5 +++++ test/webroot/js-errors.html | 12 ++++++++++++ 4 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 test/webroot/js-errors.html diff --git a/lib/browser.js b/lib/browser.js index b85ec6437..4d38ac6eb 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -53,6 +53,14 @@ Browser.prototype.init = async () => { this.events.emit('consoleLog', msg); }); + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-pageerror + this.page.on('pageerror', x => { + const lines = x.message.split('\n'); + + debug('Page error: ' + x); + this.events.emit('jserror', lines[0].trim(), lines.slice(1)); + }); + // storage for requests metadata var responses = {}; diff --git a/modules/jserrors/jserrors.js b/modules/jserrors/jserrors.js index 04d1fee5b..38456ca3c 100644 --- a/modules/jserrors/jserrors.js +++ b/modules/jserrors/jserrors.js @@ -6,21 +6,7 @@ module.exports = function(phantomas) { phantomas.setMetric('jsErrors'); // @desc number of JavaScript errors - function formatTrace(trace) { - var ret = []; - - if (Array.isArray(trace)) { - trace.forEach(function(entry) { - ret.push((entry.function ? entry.function+'(): ' : 'unknown fn: ') + (entry.sourceURL || entry.file) + ' @ ' + entry.line); - }); - } - - return ret; - } - phantomas.on('jserror', function(msg, trace) { - trace = formatTrace(trace); - phantomas.log(msg); phantomas.log('Backtrace: ' + trace.join(' / ')); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 00ec69f93..0c29437b2 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -240,3 +240,8 @@ - url: "/local-storage.html" metrics: localStorageEntries: 2 +# JS errors +- url: "/js-errors.html" + metrics: + requests: 2 + jsErrors: 1 \ No newline at end of file diff --git a/test/webroot/js-errors.html b/test/webroot/js-errors.html new file mode 100644 index 000000000..fc92bf1d3 --- /dev/null +++ b/test/webroot/js-errors.html @@ -0,0 +1,12 @@ + + . + + + From c4a8298fc5dbb27b2d7d074a26eeae1b6889850f Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 20:52:21 +0100 Subject: [PATCH 050/248] Remove old core/ files that were running in PhantomJS scope --- core/formatter.js | 23 -- core/ipc.js | 24 -- core/logger.js | 71 ---- core/pads.js | 23 -- core/phantomas.js | 793 ---------------------------------------- core/reporter.js | 74 ---- test/public-api-test.js | 196 ---------- 7 files changed, 1204 deletions(-) delete mode 100644 core/formatter.js delete mode 100644 core/ipc.js delete mode 100644 core/logger.js delete mode 100644 core/pads.js delete mode 100644 core/phantomas.js delete mode 100644 core/reporter.js delete mode 100644 test/public-api-test.js diff --git a/core/formatter.js b/core/formatter.js deleted file mode 100644 index 0426bc75f..000000000 --- a/core/formatter.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Generic JSON results formatter - */ -'use strict'; - -module.exports = function(results) { - var res = { - generator: results.getGenerator(), - url: results.getUrl(), - metrics: results.getMetrics(), - offenders: results.getAllOffenders(), - asserts: false - }; - - // add asserts - var asserts = results.getAsserts(); - - if (Object.keys(asserts).length > 0) { - res.asserts = asserts; - } - - return JSON.stringify(res); -}; diff --git a/core/ipc.js b/core/ipc.js deleted file mode 100644 index afd2605d1..000000000 --- a/core/ipc.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Simple IPC implementation using JSON-encoded messages sent over stdout stream - * - * Implements producer of the data (for PhantomJS environment) - */ -'use strict'; - -var stream = require('system').stdout, - SEPARATOR = "\xFF\xFF"; - -function ipc(event) { - /* jshint validthis: true */ - this.event = event; -} - -ipc.prototype.push = function() { - stream.writeLine(JSON.stringify({ - event: this.event, - data: Array.prototype.slice.apply(arguments) - })); - stream.writeLine(SEPARATOR); -}; - -module.exports = ipc; diff --git a/core/logger.js b/core/logger.js deleted file mode 100644 index e1d91d295..000000000 --- a/core/logger.js +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Simple logger (using both file and console) - */ -'use strict'; - -module.exports = function(logFile, params) { - var colors = require('../lib/ansicolors'), - styles = require('ansistyles'), - fs = require('fs'), - ipc = new(require('./ipc'))('log'), - beVerbose = params.beVerbose === true, - beSilent = params.beSilent === true, - stream; - - if (logFile !== '') { - // use an absolute path - logFile = fs.absolute(logFile); - log("Logging to " + logFile); - - // set up a stream to be used for logging - stream = fs.open(logFile, 'w'); - } - - function echo(msg) { - if (!beSilent) { - console.log(msg); - } - } - - function log(msg) { - var ts = (new Date()).toJSON().substr(11, 12); - - // format a message - msg = (typeof msg === 'object') ? JSON.stringify(msg) : msg.toString().trim(); - - // log to the console (--verbose) - if (beVerbose) { - var consoleMsg = msg; - - // error! - if (/!$/.test(consoleMsg) || /Error:/.test(consoleMsg)) { - consoleMsg = colors.brightRed(styles.bright(consoleMsg)); - } - // label: message - else if (/^(.*): /.test(consoleMsg)) { - var idx = consoleMsg.indexOf(': ') + 1; - consoleMsg = colors.brightGreen(consoleMsg.substr(0, idx)) + colors.brightBlack(consoleMsg.substr(idx)); - } - // the rest - else { - consoleMsg = colors.brightBlack(consoleMsg); - } - - if (!beSilent) { - ipc.push(ts + ' ' + consoleMsg); - } - } - - // log to the file (--log) - if (stream) { - stream.writeLine(ts + ': ' + msg); - stream.flush(); - } - } - - // public API - return { - echo: echo, - log: log - }; -}; diff --git a/core/pads.js b/core/pads.js deleted file mode 100644 index b6cda3d73..000000000 --- a/core/pads.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Helper functions for string formatting - */ -'use strict'; - -function lpad(str, len) { - var fill; - str = typeof str !== 'undefined' ? str : '-'; - - fill = new Array(Math.max(1, len - str.toString().length + 1)).join(' '); - return fill + str; -} - -function rpad(str, len) { - var fill; - str = typeof str !== 'undefined' ? str : '-'; - - fill = new Array(Math.max(1, len - str.toString().length + 1)).join(' '); - return str + fill; -} - -exports.lpad = lpad; -exports.rpad = rpad; diff --git a/core/phantomas.js b/core/phantomas.js deleted file mode 100644 index 168d0d199..000000000 --- a/core/phantomas.js +++ /dev/null @@ -1,793 +0,0 @@ -/** - * phantomas main file - */ -/* global phantom: true, window: true */ -'use strict'; - -/** - * Environment such PhantomJS 1.8.* does not provides the bind method on Function prototype. - * This shim will ensure that source-map will not break when running on PhantomJS. - * - * @see https://github.com/abe33/source-map/commit/61131e53ceb3b69d387da3c6daad6adbbaaae9b3 - * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/bind - */ -if (!Function.prototype.bind) { - Function.prototype.bind = function(scope) { - var self = this; - return function() { - return self.apply(scope, arguments); - }; - }; -} - -function getAverage(arr) { - var sum = arr.reduce(function(a, b) { - return a + b; - }); - return sum / arr.length; -} - -// exit codes -var EXIT_SUCCESS = 0, - EXIT_TIMED_OUT = 252, - EXIT_LOAD_FAILED = 254, - EXIT_ERROR = 255; - -// get phantomas version from package.json file -var VERSION = require('../package').version; - -var phantomas = function(params) { - var fs = require('fs'); - - // store script CLI parameters - this.params = params; - - // --url=http://example.com - this.url = params.url; - - // --format - this.format = params.format; - - // --verbose - this.verboseMode = params.verbose === true; - - // --silent - this.silentMode = params.silent === true; - - // --timeout (in seconds) - this.timeout = (params.timeout > 0 && parseInt(params.timeout, 10)) || 15; - - // --modules=localStorage,cookies - this.modules = (typeof params.modules === 'string') ? params.modules.split(',') : []; - - // --include-dirs=dirOne,dirTwo - this.includeDirs = (typeof params['include-dirs'] === 'string') ? params['include-dirs'].split(',') : []; - - // --skip-modules=jQuery,domQueries - this.skipModules = (typeof params['skip-modules'] === 'string') ? params['skip-modules'].split(',') : []; - - // disable JavaScript on the page that will be loaded - this.disableJs = params['disable-js'] === true; - - // setup the stuff - this.emitter = new(this.require('events').EventEmitter)(); - this.emitter.setMaxListeners(200); - this.ipc = require('./ipc'); - - this.util = this.require('util'); - - this.page = require('webpage').create(); - - // store the timestamp of responseEnd event - // should be bound before modules - this.on('responseEnd', this.proxy(function() { - this.responseEndTime = Date.now(); - })); - - // setup logger - var Logger = require('./logger'), - logFile = params.log || ''; - - this.logger = new Logger(logFile, { - beVerbose: this.verboseMode, - beSilent: this.silentMode - }); - - // detect phantomas working directory - if (typeof module.dirname !== 'undefined') { - this.dir = module.dirname.replace(/core$/, ''); - } else if (typeof slimer !== 'undefined') { - var args = require('system').args; - this.dir = fs.dirname(args[0]).replace(/scripts$/, ''); - } - - this.log('phantomas v%s: %s', this.getVersion(), this.dir); - this.log('Options: %j', this.params); - - // set up results wrapper - var Results = require('./results'); - this.results = new Results(); - - this.results.setGenerator('phantomas v' + this.getVersion()); - this.results.setUrl(this.url); - - this.metricsAvgStorage = {}; - - // allow asserts to be provided via command-line options (#128) - Object.keys(this.params).forEach(function(param) { - var value = parseFloat(this.params[param]), - name; - - if (!isNaN(value) && param.indexOf('assert-') === 0) { - name = param.substr(7); - - if (name.length > 0) { - this.results.setAssert(name, value); - } - } - }, this); - - // load core modules - this.log('Loading: core modules...'); - this.addCoreModule('navigationTiming'); - this.addCoreModule('requestsMonitor'); - this.addCoreModule('timeToFirstByte'); - - // load extensions - this.log('Loading: extensions...'); - var extensions = this.listExtensions(); - extensions.forEach(this.addExtension, this); - - // load modules - this.log('Loading: modules...'); - var modules = (this.modules.length > 0) ? this.modules : this.listModules(); - modules.forEach(this.addModule, this); - - // load 3rd party modules - this.log('Loading: 3rd party modules...'); - this.includeDirs.forEach(function(dirName) { - var dirPath = fs.absolute(dirName), - dirModules = this.listModulesInDir(dirPath); - - dirModules.forEach(function(moduleName) { - this.addModuleInDir(dirPath, moduleName); - }, this); - - }, this); -}; - -phantomas.version = VERSION; - -phantomas.prototype = { - // simple version of jQuery.proxy - proxy: function(fn, scope) { - scope = scope || this; - return function() { - return fn.apply(scope, arguments); - }; - }, - - // emit given event and pass it to CommonJS API via IPC - emit: function( /* eventName, arg1, arg2, ... */ ) { - this.emitInternal.apply(this, arguments); - - // pass it via IPC - var args = Array.prototype.slice.apply(arguments), - eventName = args.shift(), - ipc = new this.ipc(eventName); - - ipc.push.apply(ipc, args); - }, - - // emit given event "internally" - emitInternal: function( /* eventName, arg1, arg2, ... */ ) { - this.log('Event %s emitted', arguments[0]); - this.emitter.emit.apply(this.emitter, arguments); - }, - - // bind to a given event - on: function(ev, fn) { - this.emitter.on(ev, fn); - }, - - once: function(ev, fn) { - this.emitter.once(ev, fn); - }, - - getVersion: function() { - return VERSION; - }, - - getParam: function(key, defValue, typeCheck) { - var value = this.params[key]; - - // strict type check - if (typeof typeCheck === 'string' && typeof value !== typeCheck) { - value = undefined; - } - - return value || defValue; - }, - - // returns "wrapped" version of phantomas object with public methods / fields only - getPublicWrapper: function() { - function setParam(key, value) { - /* jshint validthis: true */ - this.log('setParam: %s set to %j', key, value); - this.params[key] = value; - } - - function setZoom(zoomFactor) { - /* jshint validthis: true */ - this.page.zoomFactor = zoomFactor; - } - - // modules API - return { - url: this.params.url, - getVersion: this.getVersion.bind(this), - getParam: this.getParam.bind(this), - setParam: setParam.bind(this), - - // events - on: this.on.bind(this), - once: this.once.bind(this), - emit: this.emit.bind(this), - emitInternal: this.emitInternal.bind(this), - - // metrics - setMetric: this.setMetric.bind(this), - setMetricEvaluate: this.setMetricEvaluate.bind(this), - setMarkerMetric: this.setMarkerMetric.bind(this), - incrMetric: this.incrMetric.bind(this), - addToAvgMetric: this.addToAvgMetric.bind(this), - getMetric: this.getMetric.bind(this), - - // offenders - addOffender: this.addOffender.bind(this), - - // debug - log: this.log.bind(this), - echo: this.echo.bind(this), - - // phantomJS - evaluate: this.page.evaluate.bind(this.page), - injectJs: this.page.injectJs.bind(this.page), - require: this.require.bind(this), - render: this.page.render.bind(this.page), - setZoom: setZoom.bind(this), - getSource: this.getSource.bind(this), - - // utils - runScript: this.runScript.bind(this), - tmpdir: this.tmpdir.bind(this) - }; - }, - - // initialize given core phantomas module - addCoreModule: function(name) { - var pkg = require('./modules/' + name + '/' + name); - - // init a module - pkg.module(this.getPublicWrapper()); - - this.log('Core module %s%s initialized', name, (pkg.version ? ' v' + pkg.version : '')); - }, - - // initialize given phantomas extension - addExtension: function(name) { - return this.addModuleInDir('./../extensions', name); - }, - - // initialize given phantomas module - addModule: function(name) { - return this.addModuleInDir('./../modules', name); - }, - - // initialize given phantomas module from dir - addModuleInDir: function(dir, name) { - var pkg; - if (this.skipModules.indexOf(name) > -1) { - this.log('Module %s skipped!', name); - return; - } - try { - pkg = require(dir + '/' + name + '/' + name); - } catch (e) { - this.log('Unable to load module "%s" from %s!', name, dir); - this.log('%s!', e); - return false; - } - - if (pkg.skip) { - this.log('Module %s skipped!', name); - return false; - } - - // init a module - pkg.module(this.getPublicWrapper()); - - this.log('Module %s%s initialized', name, (pkg.version ? ' v' + pkg.version : '')); - return true; - }, - - // returns list of extensions located in modules directory - listExtensions: function() { - return this.listModulesInDir(this.dir + 'extensions'); - }, - - // returns list of modules located in modules directory - listModules: function() { - return this.listModulesInDir(this.dir + 'modules'); - }, - - // returns list of 3rd party modules located in modules directory - listModulesInDir: function(modulesDir) { - this.log('Getting the list of all modules in %s...', modulesDir); - - var fs = require('fs'), - ls = fs.list(modulesDir) || [], - modules = []; - - ls.forEach(function(entry) { - /** - * README.md will be listed as an entry. That caused issue #409: - * SlimerJS raised: "Component returned failure code: 0x80520005 (NS_ERROR_FILE_DESTINATION_NOT_DIR) [nsILocalFile.isFile]" - * - * First check whether an entry is a directory, than check if it contains a module file - */ - if (fs.isDirectory(modulesDir + '/' + entry) && fs.isFile(modulesDir + '/' + entry + '/' + entry + '.js')) { - modules.push(entry); - } - }); - - // SlimerJS 'fs.list' does not order the returned array - // PhantomJS does that, so be consistent here :) - return modules.sort(); - }, - - // setup polling for loading progress (issue #204) - // pipe JSON messages over stderr - initLoadingProgress: function() { - var currentProgress = false; - - function pollFn() { - /* jshint validthis: true */ - var inc; - - if (currentProgress >= this.page.loadingProgress) { - return; - } - - // store the change and update the current progress - inc = this.page.loadingProgress - currentProgress; - currentProgress = this.page.loadingProgress; - - this.log('Loading progress: %d%', currentProgress); - - this.emit('progress', currentProgress, inc); // @desc loading progress has changed - } - - if (typeof this.page.loadingProgress !== 'undefined') { - setInterval(pollFn.bind(this), 50); - } else { - this.log('Loading progress: not available!'); - } - }, - - // runs phantomas - run: function() { - // check required params - if (!this.url) { - throw '--url argument must be provided!'; - } - - this.start = Date.now(); - - var self = this; - - // setup viewport / --viewport=1366x768 - var parsedViewport = this.getParam('viewport', '1366x768', 'string').split('x'); - - if (parsedViewport.length === 2) { - var viewportSize = { - width: parseInt(parsedViewport[0], 10) || 1366, - height: parseInt(parsedViewport[1], 10) || 768 - }; - - this.page.viewportSize = viewportSize; - - this.on('init', function() { - self.page.evaluate(function(viewportSize) { - try { - window.screen = { - width: viewportSize.width, - height: viewportSize.height, - availWidth: viewportSize.width, - availHeight: viewportSize.height, - availLeft: 0, - availTop: 0, - colorDepth: 24, - pixelDepth: 24 - }; - } catch (ex) { - // SlimmerJS complains: "Error: setting a property that has only a getter" - } - }, viewportSize); - }); - } - - // setup user agent / --user-agent=custom-agent - this.page.settings.userAgent = this.getParam('user-agent'); - - // disable JavaScript on the page that will be loaded - if (this.disableJs) { - this.page.settings.javascriptEnabled = false; - this.log('JavaScript execution disabled by --disable-js!'); - } - - // print out debug messages - this.log('Opening <%s>...', this.url); - this.log('Using %s as user agent', this.page.settings.userAgent); - this.log('Viewport set to %d x %d', this.page.viewportSize.width, this.page.viewportSize.height); - - if (typeof phantom !== 'undefined') { - this.log('phantom.version: %j', phantom.version); - this.log('phantom user agent: %s', phantom.defaultPageSettings.userAgent); - } - - // bind basic events - this.page.onInitialized = this.proxy(this.onInitialized); - this.page.onLoadStarted = this.proxy(this.onLoadStarted); - this.page.onLoadFinished = this.proxy(this.onLoadFinished); - this.page.onResourceRequested = this.proxy(this.onResourceRequested); - this.page.onResourceReceived = this.proxy(this.onResourceReceived); - - // debug - this.page.onAlert = this.proxy(this.onAlert); - this.page.onConfirm = this.proxy(this.onConfirm); - this.page.onPrompt = this.proxy(this.onPrompt); - this.page.onConsoleMessage = this.proxy(this.onConsoleMessage); - this.page.onCallback = this.proxy(this.onCallback); - this.page.onError = this.proxy(this.onError); - - this.initLoadingProgress(); - - // last time changes? - this.emitInternal('pageBeforeOpen', this.page); // @desc page.open is about to be called - - // open the page - this.page.open(this.url); - - this.emitInternal('pageOpen'); // @desc page.open has been called - - // fallback - always timeout after TIMEOUT seconds - this.log('Timeout set to %d sec', this.timeout); - setTimeout(function() { - this.log('Timeout of %d sec was reached!', this.timeout); - - this.emitInternal('timeout'); // @desc phantomas has timed out - this.timedOut = true; - - this.report(); - }.bind(this), this.timeout * 1000); - }, - - // called when all HTTP requests are completed - report: function() { - // restote the native JSON.parse (just in case, see #482) - this.page.evaluate(function() { - JSON.parse = window.__phantomas && window.__phantomas.JSON.parse; - }); - - this.emitInternal('report'); // @desc the report is about to be generated - - var time = Date.now() - this.start; - this.log('phantomas run for <%s> completed in %d ms', this.page.url, time); - - this.results.setUrl(this.page.url); - this.emitInternal('results', this.results); // @desc modify the results - - // count all metrics - this.log('Returning results with %d metric(s)...', this.results.getMetricsNames().length); - - // emit results in JSON - var formatter = require('./formatter'); - this.emit('json', formatter(this.results)); - - // handle timeouts (issue #129) - if (this.timedOut) { - this.tearDown(EXIT_TIMED_OUT, 'Timeout'); - return; - } - - // asserts handling - var failedAsserts = this.results.getFailedAsserts(), - failedAssertsCnt = failedAsserts.length; - - if (failedAssertsCnt > 0) { - this.log('Failed on %d assert(s) on the following metric(s): %s!', failedAssertsCnt, failedAsserts.join(', ')); - - // exit code should equal number of failed assertions - this.tearDown(failedAssertsCnt); - return; - } - - this.log('Done!'); - this.tearDown(); - }, - - tearDown: function(exitCode, msg) { - exitCode = exitCode || EXIT_SUCCESS; - - if (exitCode > 0) { - this.log('Exiting with code #%d%s!', exitCode, msg ? ' (' + msg + ')' : ''); - } - - this.emit('exit', exitCode, msg); - this.page.close(); - - phantom.exit(exitCode); - }, - - // core events - onInitialized: function() { - // SlimerJS triggers this event twice - // @see https://github.com/laurentj/slimerjs/blob/master/docs/api/webpage.rst#oninitialized - if (this.page.url === '') { - this.log('onInit: webpage.url is empty, waiting for the second trigger...'); - return; - } - - var currentUrl = this.page.url; - - // prevent multiple triggers in PhantomJS - // but only on the same page (issue #550) - if (this.initTriggered === currentUrl) { - this.log('onInit: was already triggered for <%s>', currentUrl); - return; - } - this.initTriggered = currentUrl; - - // Another multiple triggers case in PhantomJS (issue #606) - if (this.page.url === 'about:blank') { - this.log('onInit: webpage.url is about:blank, ignoring'); - return; - } - - // add helper tools into window.__phantomas "namespace" - if (!this.page.injectJs(this.dir + 'core/scope.js')) { - this.tearDown(EXIT_ERROR, 'Scope script injection failed'); - return; - } - - this.log('onInit: page object initialized'); - this.emitInternal('init'); // @desc page has been initialized, scripts can be injected - }, - - onLoadStarted: function(url, isFrame) { - if (this.onLoadStartedEmitted) { - return; - } - - // onLoadStarted is called for the page and each iframe - // tigger "loadStarted" event just once - this.onLoadStartedEmitted = true; - - this.log('Page loading started'); - this.emitInternal('loadStarted'); // @desc page loading has started - }, - - onResourceRequested: function(res, request /* added in PhantomJS v1.9 */ ) { - this.emitInternal('onResourceRequested', res, request); // @desc HTTP request has been sent - //this.log(JSON.stringify(res)); - }, - - onResourceReceived: function(res) { - this.emitInternal('onResourceReceived', res); // @desc HTTP response has been received - //this.log(JSON.stringify(res)); - }, - - onLoadFinished: function(status) { - // trigger this only once - if (this.onLoadFinishedEmitted) { - return; - } - this.onLoadFinishedEmitted = true; - - // we're done - this.log('Page loading finished ("%s")', status); - - switch (status) { - case 'success': - this.emitInternal('loadFinished', status); // @desc page has been fully loaded - break; - - default: - this.emitInternal('loadFailed', status); // @desc page loading failed - this.tearDown(EXIT_LOAD_FAILED, 'Page loading failed'); - break; - } - }, - - // debug - onAlert: function(msg) { - this.log('Alert: %s', msg); - this.emitInternal('alert', msg); // @desc the page called window.alert - }, - - onConfirm: function(msg) { - this.log('Confirm: %s', msg); - this.emitInternal('confirm', msg); // @desc the page called window.confirm - }, - - onPrompt: function(msg) { - this.log('Prompt: %s', msg); - this.emitInternal('prompt', msg); // @desc the page called window.prompt - }, - - onConsoleMessage: function(msg) { - var prefix, data; - - // split "foo:content" - prefix = msg.substr(0, 3); - data = msg.substr(4); - - try { - data = JSON.parse(data); - } catch (ex) { - // fallback to plain log - prefix = false; - } - - //console.log(JSON.stringify([prefix, data])); - - switch (prefix) { - // handle JSON-encoded messages from browser's scope sendMsg() - case 'msg': - this.onCallback(data); - break; - - // console.log arguments are passed as JSON-encoded array - case 'log': - msg = this.util.format.apply(this, data); - - this.log('console.log: %s', msg); - this.emitInternal('consoleLog', msg, data); // @desc the page called console.log - break; - - default: - this.log(msg); - } - }, - - // https://github.com/ariya/phantomjs/wiki/API-Reference-WebPage#oncallback - onCallback: function(msg) { - var type = msg && msg.type || '', - data = msg && msg.data || {}; - - switch (type) { - case 'log': - this.log.apply(this, data); - break; - - case 'setMetric': - this.setMetric(data.name, data.value, data.isFinal); - break; - - case 'incrMetric': - this.incrMetric(data.name, data.incr); - break; - - case 'addToAvgMetric': - this.addToAvgMetric(data.name, data.value); - break; - - case 'setMarkerMetric': - this.setMarkerMetric(data.name); - break; - - case 'addOffender': - this.addOffender.apply(this, data); - break; - - case 'emit': - this.emit.apply(this, data); - break; - - default: - this.log('Message "%s" from browser\'s scope: %j', type, data); - this.emitInternal('message', msg); // @desc the scope script sent a message - } - }, - - onError: function(msg, trace) { - this.log('JS error: %s', msg); - this.emitInternal('jserror', msg, trace); // @desc JS error occured - }, - - // metrics reporting - setMetric: function(name, value, isFinal) { - value = typeof value === 'string' ? value : (value || 0); // set to zero if undefined / null is provided - this.results.setMetric(name, value); - - // trigger an event when the metric value is said to be final (isse #240) - if (isFinal === true) { - this.emit('metric', name, value); // @desc the metric is given the final value - } - }, - - setMetricEvaluate: function(name, fn) { - this.setMetric(name, this.page.evaluate(fn), true /* isFinal */ ); - }, - - setMarkerMetric: function(name) { - var now = Date.now(), - value = now - this.responseEndTime; - - if (typeof this.responseEndTime === 'undefined') { - throw 'setMarkerMetric() called before responseEnd event!'; - } - - this.setMetric(name, value, true /* isFinal */ ); - return value; - }, - - // increements given metric by given number (default is one) - incrMetric: function(name, incr /* =1 */ ) { - var currVal = this.getMetric(name) || 0; - this.setMetric(name, currVal + (typeof incr === 'number' ? incr : 1)); - }, - - // push a value and update the metric if the current average value - addToAvgMetric: function(name, value) { - if (typeof this.metricsAvgStorage[name] === 'undefined') { - this.metricsAvgStorage[name] = []; - } - - this.metricsAvgStorage[name].push(value); - - this.setMetric(name, getAverage(this.metricsAvgStorage[name])); - }, - - getMetric: function(name) { - return this.results.getMetric(name); - }, - - getSource: function() { - return this.page.content; - }, - - addOffender: function( /**metricName, msg, ... */ ) { - var args = Array.prototype.slice.call(arguments), - metricName = args.shift(); - - this.results.addOffender(metricName, this.util.format.apply(this, args)); - }, - - // add log message - // will be printed out only when --verbose - // supports phantomas.log('foo: <%s>', url); - log: function() { - this.logger.log(this.util.format.apply(this, arguments)); - }, - - // console.log wrapper obeying --silent mode - echo: function(msg) { - this.logger.echo(msg); - }, - - // require CommonJS module from lib/modules - require: function(module) { - return require('../lib/modules/' + module); - }, - - // return temporary directory for the current phantomas run - // passed as PHANTOMAS_TMP_DIR environment variable by phantomas' node.js runner - tmpdir: function() { - // example: /tmp/phantomas/58aea8b5-2c97-48ee-9885-fcd81d38561f/ - return require('system').env.PHANTOMAS_TMP_DIR; - } -}; - -module.exports = phantomas; diff --git a/core/reporter.js b/core/reporter.js deleted file mode 100644 index 1aefec6bc..000000000 --- a/core/reporter.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Reporters factory - * - * All reporters are defined in /reporters directory - * and are provided with run results and command line options - */ -'use strict'; - -//convenience function to hide the dirty import logic -function _requireReporter(reporterName) { - var reporter, - reporterPath = 'phantomas-reporter-' + reporterName; - - try { - reporter = require(reporterPath); - } catch (ex) { - //external reporter doesn't exist yet, try as a "local" reporter - reporterPath = '../reporters/' + reporterName; - reporter = require(reporterPath); - } - - return reporter; -} - - -module.exports = function(results, options) { - var debug = require('debug')('phantomas:reporter'), - reporterName, - reporterOptions, - reporter, - inMultipleMode = false; - - // parse reporter options, examples: - // -R plain - // -R csv - // -R csv:no-header:url:timestamp - reporterOptions = (options.reporter + '').split(':'); - reporterName = reporterOptions.shift(); - - // allow access to options via object.key (for non-numeric values) - reporterOptions.forEach(function(option) { - if (!parseInt(option)) { - reporterOptions[option] = true; - } - }); - - debug('Setting up %s reporter (options: %j)...', reporterName, reporterOptions); - - if (Array.isArray(results)) { - debug('Multiple runs mode'); - inMultipleMode = true; - } - - try { - reporter = new(_requireReporter(reporterName))(results, reporterOptions, options); - } catch (ex) { - debug('Failed: %s', ex); - throw new Error('Reporter "' + reporterName + '" is not supported!'); - } - - // check handling of multiple runs results - if (inMultipleMode && reporter.handlesMultiple !== true) { - throw 'Reporter "' + reporterName + '" does not handle multiple runs!'; - } - - debug('Done'); - - // public interface - return { - render: function(doneFn) { - return reporter.render(doneFn); - } - }; -}; diff --git a/test/public-api-test.js b/test/public-api-test.js deleted file mode 100644 index 4a9994887..000000000 --- a/test/public-api-test.js +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Tests public phantomas API - */ -var vows = require('vows'), - assert = require('assert'), - phantomas = require('../core/phantomas'); - -// helper -function getPhantomasAPI(params) { - var instance = new phantomas(params || {}); - return instance.getPublicWrapper(); -} - -// run the test -/** -vows.describe('phantomas public API').addBatch({ - 'exposes values and methods': { - topic: function() { - return getPhantomasAPI({ - url: 'http://example.com' - }); - }, - 'url field is set correctly': function(api) { - assert.equal(api.url, 'http://example.com'); - }, - 'version is exposed via getVersion': function(api) { - assert.equal(api.getVersion(), require(__dirname + '/../package').version); - }, - 'methods are accessible': function(api) { - var methods = [ - 'getVersion', - 'getParam', - 'setParam', - 'on', - 'once', - 'emit', - 'emitInternal', - 'setMetric', - 'setMetricEvaluate', - 'setMarkerMetric', - 'incrMetric', - 'addToAvgMetric', - 'getMetric', - 'addOffender', - 'log', - 'echo', - 'evaluate', - 'injectJs', - 'require', - 'getSource' - ]; - - methods.forEach(function(method) { - assert.equal(typeof api[method], 'function'); - }); - } - }, - 'events are processed': { - topic: getPhantomasAPI, - 'event is triggered': function(api) { - var triggered; - - api.on('foo', function() { - triggered = true; - }); - api.emit('foo'); - - assert.isTrue(triggered); - }, - 'params are passed': function(api) { - var a, b; - - api.on('foo', function(valA, valB) { - a = valA; - b = valB; - }); - api.emit('foo', 123, 456); - - assert.equal(a, 123); - assert.equal(b, 456); - } - }, - 'metrics': { - topic: getPhantomasAPI, - 'have default value': function(api) { - assert.equal(typeof api.getMetric('undef'), 'undefined'); - }, - 'are set': function(api) { - api.setMetric('foo', 123); - assert.equal(api.getMetric('foo'), 123); - }, - 'are set with default value': function(api) { - api.setMetric('foo'); - assert.strictEqual(api.getMetric('foo'), 0); - }, - 'treat "undefined" value as 0': function(api) { - api.setMetric('foo', undefined); - assert.strictEqual(api.getMetric('foo'), 0); - }, - 'treat "null" value as 0': function(api) { - api.setMetric('foo', null); - assert.strictEqual(api.getMetric('foo'), 0); - }, - 'treat empty string as empty string': function(api) { - api.setMetric('foo', ''); - assert.strictEqual(api.getMetric('foo'), ''); - }, - 'are properly incremented': function(api) { - api.incrMetric('bar'); - assert.equal(api.getMetric('bar'), 1); - - api.incrMetric('bar', 3); - assert.equal(api.getMetric('bar'), 4); - }, - 'marker is properly calculated': function(api) { - var origDateNow = Date.now, - now = Date.now(), - diff = 5; - - // emit fake responseEnd event - Date.now = function() { - return now - diff; - }; - api.emit('responseEnd'); - - // set the marker - Date.now = function() { - return now; - }; - api.setMarkerMetric('marker'); - - assert.equal(api.getMetric('marker'), diff); - - // tearDown - Date.now = origDateNow; - }, - 'metric is correctly increased': function(results) { - results.setMetric('bar', 0); - - // default value = 1 - results.incrMetric('bar'); - assert.strictEqual(results.getMetric('bar'), 1); - - // no increase - results.incrMetric('bar', 0); - assert.strictEqual(results.getMetric('bar'), 1); - - results.incrMetric('bar', 41); - assert.strictEqual(results.getMetric('bar'), 42); - }, - 'average metric is correctly calculates': function(results) { - results.setMetric('bar', 0); - - results.addToAvgMetric('bar', 1); - assert.strictEqual(results.getMetric('bar'), 1); - - results.addToAvgMetric('bar', 0); - results.addToAvgMetric('bar', 5); - assert.strictEqual(results.getMetric('bar'), 2); - }, - }, - 'parameters': { - topic: function() { - return getPhantomasAPI({ - foo: 123, - bar: 'abc' - }); - }, - 'params are accessible': function(api) { - assert.strictEqual(api.getParam('foo'), 123); - assert.strictEqual(api.getParam('bar'), 'abc'); - }, - 'getParam() handles the default value': function(api) { - assert.strictEqual(api.getParam('test', 123), 123); - assert.strictEqual(api.getParam('test'), undefined); - }, - 'getParam() handles strict type check': function(api) { - assert.strictEqual(api.getParam('foo', 'default', 'number'), 123); - assert.strictEqual(api.getParam('foo', 'default', 'string'), 'default'); - }, - 'parameters can be altered': function(api) { - api.setParam('foo', 124); - api.setParam('test', true); - - assert.strictEqual(api.getParam('foo'), 124); - assert.strictEqual(api.getParam('test'), true); - }, - }, - 'page source': { - topic: getPhantomasAPI, - 'getSource() return the page source': function(api) { - assert.equal(api.getSource(), ''); - } - } -}).export(module); -**/ \ No newline at end of file From 9baa0929645459458ad7556b0a2de4aa6ac61be0 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 20:53:52 +0100 Subject: [PATCH 051/248] examples: add a short README --- examples/README.md | 14 ++++++++++++++ examples/{promise.js => index.js} | 0 2 files changed, 14 insertions(+) create mode 100644 examples/README.md rename examples/{promise.js => index.js} (100%) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..68ffb7d2d --- /dev/null +++ b/examples/README.md @@ -0,0 +1,14 @@ +Examples +======== + +Run: + +``` +./index.js +``` + +or in debug mode: + +``` +DEBUG=phantomas:* ./index.js +``` diff --git a/examples/promise.js b/examples/index.js similarity index 100% rename from examples/promise.js rename to examples/index.js From f3443f72b9b602b47b5bd3f1f5693f8343fc8920 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 20:57:19 +0100 Subject: [PATCH 052/248] scopeMessage: handle log and addOffender called from page scope code --- lib/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/index.js b/lib/index.js index db12b80e1..4871baa8c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -66,6 +66,8 @@ function phantomas(url, opts) { on: events.on.bind(events), once: events.once.bind(events), + log: debug.bind(debug), + addOffender: results.addOffender.bind(results), incrMetric: results.incrMetric.bind(results), setMetric: results.setMetric, @@ -96,10 +98,12 @@ function phantomas(url, opts) { // bind to sendMsg calls from page scope code events.on('scopeMessage', (type, args) => { const debug = require('debug')('phantomas:core:scopeEvents'); - debug(type + ' [' + args + ']'); + // debug(type + ' [' + args + ']'); switch(type) { + case 'addOffender': case 'incrMetric': + case 'log': case 'setMetric': scope[type].apply(scope, args); break; From d42d8917b1cb6bb60030db3c7f31c60fc093a986 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 21:01:23 +0100 Subject: [PATCH 053/248] A separate log label for phantomas.log calls from page scope code --- lib/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/index.js b/lib/index.js index 4871baa8c..e81a10eb3 100644 --- a/lib/index.js +++ b/lib/index.js @@ -55,7 +55,8 @@ function phantomas(url, opts) { var promise = new Promise(async (resolve, reject) => { try { - const page = await browser.init(); + const page = await browser.init(), + debugScope = require('debug')('phantomas:scope:log'); // TODO: prepare a small instance object that will be passed to modules and extensions on init const scope = { @@ -66,7 +67,7 @@ function phantomas(url, opts) { on: events.on.bind(events), once: events.once.bind(events), - log: debug.bind(debug), + log: debugScope.bind(debug), addOffender: results.addOffender.bind(results), incrMetric: results.incrMetric.bind(results), From 3163a6f722ce39d17c1c4dca3ff2fdbd5e5d258c Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 21:01:30 +0100 Subject: [PATCH 054/248] domMutations: module works again :) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /dom-operations.html ✓ should be generated ✓ should have "DOMmutationsInserts" metric properly set --- modules/domMutations/domMutations.js | 79 +--------------------------- modules/domMutations/scope.js | 69 ++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 77 deletions(-) create mode 100644 modules/domMutations/scope.js diff --git a/modules/domMutations/domMutations.js b/modules/domMutations/domMutations.js index fcc64bc47..3e8fff6f2 100644 --- a/modules/domMutations/domMutations.js +++ b/modules/domMutations/domMutations.js @@ -7,84 +7,9 @@ 'use strict'; module.exports = function(phantomas) { - // SlimerJS only metrics phantomas.setMetric('DOMmutationsInserts'); // @desc number of node inserts phantomas.setMetric('DOMmutationsRemoves'); // @desc number of node removes phantomas.setMetric('DOMmutationsAttributes'); // @desc number of DOM nodes attributes changes - return; // TODO - - phantomas.on('init', function() { - phantomas.evaluate(function() { - (function(phantomas) { - if ('MutationObserver' in window) { - // wait for DOM ready - document.addEventListener('readystatechange', function() { - if (document.readyState !== 'interactive') { - return; - } - - phantomas.log('DOM query: setting up MutationObserver...'); - - var observer = new MutationObserver(function(allmutations) { - allmutations.map(function(mutation) { - // @see https://developer.mozilla.org/en-US/docs/Web/API/MutationObserver#MutationRecord - var targetPath = phantomas.getDOMPath(mutation.target, true /* dontGoUpTheDom */ ); - - switch (mutation.type) { - case 'attributes': - phantomas.log('DOM mutation: "%s" attr (was "%s") set on %s', - mutation.attributeName, - mutation.oldValue || '', - targetPath - ); - - phantomas.incrMetric('DOMmutationsAttributes'); - phantomas.addOffender('DOMmutationsAttributes', '"%s" attr set on %s', mutation.attributeName, targetPath); - break; - - case 'childList': - var wereAdded = (mutation.addedNodes.length > 0), - nodes = wereAdded ? mutation.addedNodes : mutation.removedNodes, - nodePath; - - for (var n = 0, nodesLen = nodes.length; n < nodesLen; n++) { - nodePath = phantomas.getDOMPath(nodes[n], true /* dontGoUpTheDom */ ); - - phantomas.log('DOM mutation: node "%s" %s "%s"', - nodePath, - wereAdded ? 'added to' : 'removed from', - targetPath - ); - - if (wereAdded) { - phantomas.incrMetric('DOMmutationsInserts'); - phantomas.addOffender('DOMmutationsInserts', '"%s" added to "%s"', nodePath, targetPath); - } else { - phantomas.incrMetric('DOMmutationsRemoves'); - phantomas.addOffender('DOMmutationsRemoves', '"%s" removed from "%s"', nodePath, targetPath); - } - } - break; - - default: - phantomas.log('DOM mutation: %s', mutation.type); - } - }); - }); - - observer.observe(document.body, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: true - }); - }); - } else { - phantomas.log('DOM query: MutationObserver not available!'); - } - })(window.__phantomas); - }); - }); -}; + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); +}; \ No newline at end of file diff --git a/modules/domMutations/scope.js b/modules/domMutations/scope.js new file mode 100644 index 000000000..f72cc1a56 --- /dev/null +++ b/modules/domMutations/scope.js @@ -0,0 +1,69 @@ +(function(phantomas) { + if ('MutationObserver' in window) { + // wait for DOM ready + document.addEventListener('readystatechange', function() { + if (document.readyState !== 'interactive') { + return; + } + + phantomas.log('DOM query: setting up MutationObserver...'); + + var observer = new MutationObserver(function(allmutations) { + allmutations.map(function(mutation) { + // @see https://developer.mozilla.org/en-US/docs/Web/API/MutationObserver#MutationRecord + var targetPath = phantomas.getDOMPath(mutation.target, true /* dontGoUpTheDom */ ); + + switch (mutation.type) { + case 'attributes': + phantomas.log('DOM mutation: "%s" attr (was "%s") set on %s', + mutation.attributeName, + mutation.oldValue || '', + targetPath + ); + + phantomas.incrMetric('DOMmutationsAttributes'); + phantomas.addOffender('DOMmutationsAttributes', '"%s" attr set on %s', mutation.attributeName, targetPath); + break; + + case 'childList': + var wereAdded = (mutation.addedNodes.length > 0), + nodes = wereAdded ? mutation.addedNodes : mutation.removedNodes, + nodePath; + + for (var n = 0, nodesLen = nodes.length; n < nodesLen; n++) { + nodePath = phantomas.getDOMPath(nodes[n], true /* dontGoUpTheDom */ ); + + phantomas.log('DOM mutation: node "%s" %s "%s"', + nodePath, + wereAdded ? 'added to' : 'removed from', + targetPath + ); + + if (wereAdded) { + phantomas.incrMetric('DOMmutationsInserts'); + phantomas.addOffender('DOMmutationsInserts', '"%s" added to "%s"', nodePath, targetPath); + } else { + phantomas.incrMetric('DOMmutationsRemoves'); + phantomas.addOffender('DOMmutationsRemoves', '"%s" removed from "%s"', nodePath, targetPath); + } + } + break; + + default: + phantomas.log('DOM mutation: %s', mutation.type); + } + }); + }); + + observer.observe(document.body, { + childList: true, + attributes: true, + characterData: true, + subtree: true, + attributeOldValue: true + }); + }); + } else { + phantomas.log('DOM query: MutationObserver not available!'); + } +})(window.__phantomas); \ No newline at end of file From 6ea4cc320cca6dbec3f93e6afda2166508f02705 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 21:53:52 +0100 Subject: [PATCH 055/248] domComplexity: start a refactoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /dom-id.html ✓ should be generated ✓ should have "DOMidDuplicated" metric properly set --- modules/domComplexity/domComplexity.js | 23 +++++++++++++++++------ modules/domComplexity/scope.js | 15 +++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 modules/domComplexity/scope.js diff --git a/modules/domComplexity/domComplexity.js b/modules/domComplexity/domComplexity.js index d166a3747..cfce0d790 100644 --- a/modules/domComplexity/domComplexity.js +++ b/modules/domComplexity/domComplexity.js @@ -24,8 +24,6 @@ module.exports = function(phantomas) { phantomas.setMetric('imagesScaledDown'); // @desc number of nodes that have images scaled down in HTML @offenders phantomas.setMetric('imagesWithoutDimensions'); // @desc number of nodes without both width and height attribute @offenders - return; // TODO - // keep the track of SVG graphics (#479) var svgResources = []; phantomas.on('recv', function(entry) { @@ -35,16 +33,29 @@ module.exports = function(phantomas) { } }); + // inject JS code + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); + // duplicated ID (issue #392) phantomas.setMetric('DOMidDuplicated'); // @desc number of duplicated IDs found in DOM - var Collection = require('../../lib/collection'), - DOMids = new Collection(); + phantomas.on('DOMids', ids => { + var Collection = require('../../lib/collection'), + DOMids = new Collection(); - phantomas.on('domId', function(id) { - DOMids.push(id); + ids.forEach(id => DOMids.push(id)); + phantomas.log('Nodes with IDs: ' + ids.length); + + DOMids.sort().forEach((id, cnt) => { + if (cnt > 1) { + phantomas.incrMetric('DOMidDuplicated'); + phantomas.addOffender('DOMidDuplicated', {id: id, count: cnt}); + } + }); }); + return; // TODO + // HTML size phantomas.on('report', function() { phantomas.setMetricEvaluate('bodyHTMLSize', function() { // @desc the size of body tag content (document.body.innerHTML.length) diff --git a/modules/domComplexity/scope.js b/modules/domComplexity/scope.js new file mode 100644 index 000000000..682d80220 --- /dev/null +++ b/modules/domComplexity/scope.js @@ -0,0 +1,15 @@ +(function(phantomas) { + + // duplicated ID (issue #392) + document.addEventListener("DOMContentLoaded", () => { + console.log("DOM fully loaded and parsed"); + + phantomas.spyEnabled(false, 'counting nodes with id'); + const nodes = document.querySelectorAll('*[id]'), + ids = Array.prototype.slice.apply(nodes).map((node) => node.id); + phantomas.spyEnabled(true); + + phantomas.emit('DOMids', ids); + }); + +})(window.__phantomas); From a236813c6f32afbda37c9d709cc4e0a279d8fc8a Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 8 Jan 2019 22:12:16 +0100 Subject: [PATCH 056/248] domComplexity: set nodesWithInlineCSS metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✓ should have "nodesWithInlineCSS" metric properly set --- modules/domComplexity/domComplexity.js | 17 ---------------- modules/domComplexity/scope.js | 27 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/modules/domComplexity/domComplexity.js b/modules/domComplexity/domComplexity.js index cfce0d790..ca77955a1 100644 --- a/modules/domComplexity/domComplexity.js +++ b/modules/domComplexity/domComplexity.js @@ -100,11 +100,6 @@ module.exports = function(phantomas) { DOMelementMaxDepthNodes.push(path); } - // report duplicated ID (issue #392) - if (typeof node.id === 'string' && node.id !== '') { - phantomas.emit('domId', node.id); - } - // ignore inline + From 43abdfab9ae9908c8d35743266066f33fec7237d Mon Sep 17 00:00:00 2001 From: macbre Date: Wed, 9 Jan 2019 22:07:58 +0100 Subject: [PATCH 059/248] localStorageEntries: refactor a module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And allow to assert offenders in integration tests /local-storage.html ✓ should be generated ✓ should have "localStorageEntries" metric properly set ✓ should have "localStorageEntries" offender(s) properly set --- modules/localStorage/localStorage.js | 23 ++--------------------- modules/localStorage/scope.js | 15 +++++++++++++++ test/integration-spec.yaml | 2 ++ test/integration-test.js | 10 ++++++++++ 4 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 modules/localStorage/scope.js diff --git a/modules/localStorage/localStorage.js b/modules/localStorage/localStorage.js index 57d1d8cbf..b99a0730b 100644 --- a/modules/localStorage/localStorage.js +++ b/modules/localStorage/localStorage.js @@ -7,25 +7,6 @@ module.exports = function(phantomas) { phantomas.setMetric('localStorageEntries'); // @desc number of entries in local storage - return; // TODO - - phantomas.on('report', function() { - phantomas.evaluate(function() { - (function(phantomas) { - var entries; - - try { - entries = Object.keys(window.localStorage); - - phantomas.setMetric('localStorageEntries', entries.length); - - entries.forEach(function(entry) { - phantomas.addOffender('localStorageEntries', entry); - }); - } catch (ex) { - phantomas.log('localStorageEntries: not set because ' + ex + '!'); - } - }(window.__phantomas)); - }); - }); + // inject JS code + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/localStorage/scope.js b/modules/localStorage/scope.js new file mode 100644 index 000000000..df6bc5bcb --- /dev/null +++ b/modules/localStorage/scope.js @@ -0,0 +1,15 @@ +(function(phantomas) { + window.addEventListener("load", () => { + try { + var entries = Object.keys(window.localStorage); + + phantomas.setMetric('localStorageEntries', entries.length); + + entries.forEach(function(entry) { + phantomas.addOffender('localStorageEntries', entry); + }); + } catch (ex) { + phantomas.log('localStorageEntries: not set because ' + ex + '!'); + } + }); +}(window.__phantomas)); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 73bd3ba4f..e426a35d6 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -240,6 +240,8 @@ - url: "/local-storage.html" metrics: localStorageEntries: 2 + offenders: + localStorageEntries: ['foo', 'test'] # JS errors - url: "/js-errors.html" metrics: diff --git a/test/integration-test.js b/test/integration-test.js index afb77203b..f1b791a33 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -53,6 +53,7 @@ spec.forEach(function(test) { }, }; + // check metrics Object.keys(test.metrics || {}).forEach(function(name) { batch[batchName]['should have "' + name + '" metric properly set'] = function(err, results) { assert.ok(!(err instanceof Error), 'Error should not be thrown: ' + err); @@ -60,6 +61,15 @@ spec.forEach(function(test) { }; }); + // check offenders + Object.keys(test.offenders || {}).forEach(function(name) { + batch[batchName]['should have "' + name + '" offender(s) properly set'] = function(err, results) { + assert.ok(!(err instanceof Error), 'Error should not be thrown: ' + err); + assert.deepStrictEqual(results.getOffenders(name), test.offenders[name]); + }; + }); + + suite.addBatch(batch); }); From bd6ae87313626465110d2672830519556be1e972 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 11 Jan 2019 19:05:04 +0100 Subject: [PATCH 060/248] Browser: set phantomas-specific user agent --- lib/browser.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 3a2e42ee3..60de5174a 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -2,7 +2,8 @@ * Expose puppeteer API and events emitter object for lib/index.js */ const debug = require('debug')('phantomas:browser'), - puppeteer = require("puppeteer"); + puppeteer = require("puppeteer"), + VERSION = require('../package.json').version; function Browser() { this.browser = null; @@ -44,8 +45,13 @@ Browser.prototype.init = async () => { debug('Using binary from: %s', this.browser.process().spawnfile); + // set a custom user agent, e.g. "phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)" + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagesetuseragentuseragent + const browserVersion = await this.browser.version(); + await this.page.setUserAgent('phantomas/' + VERSION + ' (' + browserVersion + ')'); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserversion - debug('Browser: %s', await this.browser.userAgent()); + debug('Original browser: %s', await this.browser.userAgent()); debug('Viewport: %j', await this.page.viewport()); // bind events From dc4231b56212824d202eb7fdce53fb377dcbc113 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 11 Jan 2019 21:01:30 +0100 Subject: [PATCH 061/248] jQuery: port the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /jquery.html ✓ should be generated ✓ should have "jQueryVersion" metric properly set ✓ should have "jQueryOnDOMReadyFunctions" metric properly set ✓ should have "jQueryWindowOnLoadFunctions" metric properly set ✓ should have "jQuerySizzleCalls" metric properly set ✓ should have "jQueryEventTriggers" metric properly set ✗ should have "eventsBound" metric properly set » expected 7, got 0 (strictEqual) // /home/macbre/src/git/phantomas/node_modules/vows/lib/assert/macros.js:14 ✗ should have "eventsDispatched" metric properly set » expected 1, got 0 (strictEqual) // /home/macbre/src/git/phantomas/node_modules/vows/lib/assert/macros.js:14 ✗ should have "eventsScrollBound" metric properly set » expected 2, got 0 (strictEqual) // /home/macbre/src/git/phantomas/node_modules/vows/lib/assert/macros.js:14 ✓ should have "jQueryVersionsLoaded" offender(s) properly set ✓ should have "jQueryWindowOnLoadFunctions" offender(s) properly set ✓ should have "jQueryEventTriggers" offender(s) properly set ✓ should have "jQuerySizzleCalls" offender(s) properly set ✓ should have "jQueryDOMReads" offender(s) properly set ✓ should have "jQueryDOMWrites" offender(s) properly set ✓ should have "jQueryDOMWriteReadSwitches" offender(s) properly set /jquery-multiple.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "jsCount" metric properly set ✓ should have "domains" metric properly set ✓ should have "jQueryVersion" metric properly set ✓ should have "jQueryVersionsLoaded" metric properly set ✓ should have "jQueryOnDOMReadyFunctions" metric properly set ✓ should have "jQueryVersionsLoaded" offender(s) properly set --- modules/jQuery/jQuery.js | 155 +++---------------------------------- modules/jQuery/scope.js | 130 +++++++++++++++++++++++++++++++ test/integration-spec.yaml | 24 ++++++ test/webroot/jquery.html | 6 +- 4 files changed, 169 insertions(+), 146 deletions(-) create mode 100644 modules/jQuery/scope.js diff --git a/modules/jQuery/jQuery.js b/modules/jQuery/jQuery.js index cfd730986..655d39375 100644 --- a/modules/jQuery/jQuery.js +++ b/modules/jQuery/jQuery.js @@ -20,156 +20,23 @@ module.exports = function(phantomas) { phantomas.setMetric('jQueryDOMWrites'); // @desc number of DOM write operations phantomas.setMetric('jQueryDOMWriteReadSwitches'); // @desc number of read operations that follow a series of write operations (will cause repaint and can cause reflow) - return; // TODO - - // spy calls to jQuery functions - phantomas.on('init', function() { - phantomas.evaluate(function() { - (function(phantomas) { - // read & write DOM operations (issue #436) - function spyReadsAndWrites(jQuery) { - var TYPE_SET = 'write', - TYPE_GET = 'read'; - - function report(type, funcName, context, args) { - var caller = phantomas.getCaller(1), - contextPath = phantomas.getDOMPath(context); - - args = (typeof args !== 'undefined') ? Array.prototype.slice.apply(args) : undefined; - - phantomas.emit('jQueryOp', type, funcName, args, contextPath, caller); - } - - // "complex" getters and setters - [ - 'attr', - 'css', - 'prop', - ].forEach(function(funcName) { - phantomas.spy(jQuery.fn, funcName, function(propName, val) { - // setter when called with two arguments or provided with key/value set - var isSet = (typeof val !== 'undefined') || (propName.toString() === '[object Object]'); - report(isSet ? TYPE_SET : TYPE_GET, funcName, this[0], arguments); - }); - }); - - // simple getters and setters - [ - 'height', - 'innerHeight', - 'innerWidth', - 'offset', - 'outerHeight', - 'outerWidth', - 'text', - 'width', - 'scrollLeft', - 'scrollTop' - ].forEach(function(funcName) { - phantomas.spy(jQuery.fn, funcName, function(val) { - // setter when called with an argument - var isSet = (typeof val !== 'undefined'); - report(isSet ? TYPE_SET : TYPE_GET, funcName, this[0], arguments); - }); - }); - - // setters - [ - 'addClass', - 'removeAttr', - 'removeClass', - 'removeProp', - 'toggleClass', - ].forEach(function(funcName) { - phantomas.spy(jQuery.fn, funcName, function(val) { - report(TYPE_SET, funcName, this[0], [val]); - }); - }); - // getters - [ - 'hasClass', - 'position', - ].forEach(function(funcName) { - phantomas.spy(jQuery.fn, funcName, function(val) { - report(TYPE_GET, funcName, this[0], arguments); - }); - }); - } - - phantomas.spyGlobalVar('jQuery', function(jQuery) { - var version; - - if (!jQuery || !jQuery.fn) { - phantomas.log('jQuery: unable to detect version!'); - return; - } - - // Tag the current version of jQuery to avoid multiple reports of jQuery being loaded - // when it's actually only restored via $.noConflict(true) - see comments in #435 - if (jQuery.__phantomas === true) { - phantomas.log('jQuery: this instance has already been seen by phantomas'); - return; - } - jQuery.__phantomas = true; - - // report the version of jQuery - version = jQuery.fn.jquery; - phantomas.emit('jQueryLoaded', version); - - // jQuery.ready.promise - // works for jQuery 1.8.0+ (released Aug 09 2012) - phantomas.spy(jQuery.ready, 'promise', function() { - phantomas.incrMetric('jQueryOnDOMReadyFunctions'); - phantomas.addOffender('jQueryOnDOMReadyFunctions', phantomas.getCaller(3)); - }) || phantomas.log('jQuery: can not measure jQueryOnDOMReadyFunctions (jQuery used on the page is too old)!'); - - // Sizzle calls - jQuery.find - // works for jQuery 1.3+ (released Jan 13 2009) - phantomas.spy(jQuery, 'find', function(selector, context) { - phantomas.incrMetric('jQuerySizzleCalls'); - phantomas.addOffender('jQuerySizzleCalls', '%s (in %s)', selector, (phantomas.getDOMPath(context) || 'unknown')); - }) || phantomas.log('jQuery: can not measure jQuerySizzleCalls (jQuery used on the page is too old)!'); - - // jQuery events triggers (issue #440) - phantomas.spy(jQuery.event, 'trigger', function(ev, data, elem) { - var path = phantomas.getDOMPath(elem), - type = ev.type || ev; - - phantomas.log('Event: triggered "%s" on "%s"', type, path); - - phantomas.incrMetric('jQueryEventTriggers'); - phantomas.addOffender('jQueryEventTriggers', '"%s" on "%s"', type, path); - }) || phantomas.log('jQuery: can not measure jQueryEventTriggers (jQuery used on the page is too old)!'); - - // jQuery events bound to window' onLoad event (#451) - phantomas.spy(jQuery.fn, 'on', function(eventName) { - if ((eventName === 'load') && (this[0] === window)) { - phantomas.incrMetric('jQueryWindowOnLoadFunctions'); - phantomas.addOffender('jQueryWindowOnLoadFunctions', phantomas.getCaller(2)); - } - }) || phantomas.log('jQuery: can not measure jQueryWindowOnLoadFunctions (jQuery used on the page is too old)!'); - - spyReadsAndWrites(jQuery); - }); - })(window.__phantomas); - }); - }); + // inject JS code + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); // store the last resource that was received // try to report where given jQuery version was loaded from - phantomas.on('recv', function(entry) { + phantomas.on('recv', entry => { if (entry.isJS) { lastUrl = entry.url; } }); - phantomas.on('jQueryLoaded', function(version) { - phantomas.log('jQuery: loaded v' + version); + phantomas.on('jQueryLoaded', version => { phantomas.setMetric('jQueryVersion', version); // report multiple jQuery "instances" (issue #435) phantomas.incrMetric('jQueryVersionsLoaded'); - phantomas.addOffender('jQueryVersionsLoaded', 'v%s', version); + phantomas.addOffender('jQueryVersionsLoaded', {version: version, url: lastUrl}); phantomas.log('jQuery: v%s (probably loaded from <%s>)', version, lastUrl); }); @@ -177,23 +44,25 @@ module.exports = function(phantomas) { // jQuery read & write operations (issue #436) var lastOp; - phantomas.on('jQueryOp', function(type, funcName, args, contextPath, caller) { - phantomas.log('jQuery: %s op from $.%s(%j) on "%s" - %s', type, funcName, args, contextPath, caller); + phantomas.on('jQueryOp', (type, functionName, args, contextPath, caller) => { + const offenderDetails = {functionName, arguments: JSON.stringify(args), contextPath}; + + phantomas.log('jQuery: %s op from $.%s(%j) on "%s" - %s', type, functionName, args, contextPath, caller); if (type === 'read') { phantomas.incrMetric('jQueryDOMReads'); - phantomas.addOffender('jQueryDOMReads', '$.%s(%j) on "%s"', funcName, args, contextPath); + phantomas.addOffender('jQueryDOMReads', offenderDetails); // This read operation may follow a write operation // In this case browser needs to perform all buffered write operations // in order to update the DOM - this can cause repaints and reflows if (lastOp === 'write') { phantomas.incrMetric('jQueryDOMWriteReadSwitches'); - phantomas.addOffender('jQueryDOMWriteReadSwitches', 'before $.%s(%j) on "%s"', funcName, args, contextPath); + phantomas.addOffender('jQueryDOMWriteReadSwitches', offenderDetails); } } else { phantomas.incrMetric('jQueryDOMWrites'); - phantomas.addOffender('jQueryDOMWrites', '$.%s(%j) on "%s"', funcName, args, contextPath); + phantomas.addOffender('jQueryDOMWrites', offenderDetails); } lastOp = type; diff --git a/modules/jQuery/scope.js b/modules/jQuery/scope.js new file mode 100644 index 000000000..f8594e33e --- /dev/null +++ b/modules/jQuery/scope.js @@ -0,0 +1,130 @@ +(function(phantomas) { + // read & write DOM operations (issue #436) + function spyReadsAndWrites(jQuery) { + var TYPE_SET = 'write', + TYPE_GET = 'read'; + + function report(type, funcName, context, args) { + var caller = phantomas.getCaller(1), + contextPath = phantomas.getDOMPath(context); + + args = (typeof args !== 'undefined') ? Array.prototype.slice.apply(args) : undefined; + + phantomas.emit('jQueryOp', type, funcName, args, contextPath, caller); + } + + // "complex" getters and setters + [ + 'attr', + 'css', + 'prop', + ].forEach(function(funcName) { + phantomas.spy(jQuery.fn, funcName, function(propName, val) { + // setter when called with two arguments or provided with key/value set + var isSet = (typeof val !== 'undefined') || (propName.toString() === '[object Object]'); + report(isSet ? TYPE_SET : TYPE_GET, funcName, this[0], arguments); + }); + }); + + // simple getters and setters + [ + 'height', + 'innerHeight', + 'innerWidth', + 'offset', + 'outerHeight', + 'outerWidth', + 'text', + 'width', + 'scrollLeft', + 'scrollTop' + ].forEach(function(funcName) { + phantomas.spy(jQuery.fn, funcName, function(val) { + // setter when called with an argument + var isSet = (typeof val !== 'undefined'); + report(isSet ? TYPE_SET : TYPE_GET, funcName, this[0], arguments); + }); + }); + + // setters + [ + 'addClass', + 'removeAttr', + 'removeClass', + 'removeProp', + 'toggleClass', + ].forEach(function(funcName) { + phantomas.spy(jQuery.fn, funcName, function(val) { + report(TYPE_SET, funcName, this[0], [val]); + }); + }); + // getters + [ + 'hasClass', + 'position', + ].forEach(function(funcName) { + phantomas.spy(jQuery.fn, funcName, function(val) { + report(TYPE_GET, funcName, this[0], arguments); + }); + }); + } + + phantomas.spyGlobalVar('jQuery', function(jQuery) { + var version; + + if (!jQuery || !jQuery.fn) { + phantomas.log('jQuery: unable to detect version!'); + return; + } + + // Tag the current version of jQuery to avoid multiple reports of jQuery being loaded + // when it's actually only restored via $.noConflict(true) - see comments in #435 + if (jQuery.__phantomas === true) { + phantomas.log('jQuery: this instance has already been seen by phantomas'); + return; + } + jQuery.__phantomas = true; + + // report the version of jQuery + version = jQuery.fn.jquery; + phantomas.emit('jQueryLoaded', version); + + // jQuery.ready.promise - works for jQuery 1.8.0+ (released Aug 09 2012) + // jQuery.read - works for jQuery 3.0.0+ + phantomas.spy(jQuery.ready, 'promise', function() { + phantomas.incrMetric('jQueryOnDOMReadyFunctions'); + phantomas.addOffender('jQueryOnDOMReadyFunctions', phantomas.getCaller(3)); + }) || phantomas.spy(jQuery, 'ready', function() { + phantomas.incrMetric('jQueryOnDOMReadyFunctions'); + phantomas.addOffender('jQueryOnDOMReadyFunctions', phantomas.getCaller(0)); + }) || phantomas.log('jQuery: can not measure jQueryOnDOMReadyFunctions (jQuery used on the page is too old)!'); + + // Sizzle calls - jQuery.find + // works for jQuery 1.3+ (released Jan 13 2009) + phantomas.spy(jQuery, 'find', function(selector, context) { + phantomas.incrMetric('jQuerySizzleCalls'); + phantomas.addOffender('jQuerySizzleCalls', {selector, element: (phantomas.getDOMPath(context) || 'unknown')}); + }) || phantomas.log('jQuery: can not measure jQuerySizzleCalls (jQuery used on the page is too old)!'); + + // jQuery events triggers (issue #440) + phantomas.spy(jQuery.event, 'trigger', function(ev, data, elem) { + var path = phantomas.getDOMPath(elem), + type = ev.type || ev; + + phantomas.log('Event: triggered "%s" on "%s"', type, path); + + phantomas.incrMetric('jQueryEventTriggers'); + phantomas.addOffender('jQueryEventTriggers', {type, element: path}); + }) || phantomas.log('jQuery: can not measure jQueryEventTriggers (jQuery used on the page is too old)!'); + + // jQuery events bound to window' onLoad event (#451) + phantomas.spy(jQuery.fn, 'on', function(eventName) { + if ((eventName === 'load') && (this[0] === window)) { + phantomas.incrMetric('jQueryWindowOnLoadFunctions'); + phantomas.addOffender('jQueryWindowOnLoadFunctions', phantomas.getCaller(2) || phantomas.getCaller(0)); + } + }) || phantomas.log('jQuery: can not measure jQueryWindowOnLoadFunctions (jQuery used on the page is too old)!'); + + spyReadsAndWrites(jQuery); + }); +})(window.__phantomas); \ No newline at end of file diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index e426a35d6..c9cf05215 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -107,6 +107,25 @@ eventsBound: 7 eventsDispatched: 1 eventsScrollBound: 2 + offenders: + jQueryVersionsLoaded: + - version: "2.1.1" + url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js' + jQueryWindowOnLoadFunctions: + - http://127.0.0.1:8888/jquery.html:49:13 + jQueryEventTriggers: + - { type: 'click', element: 'body > div#foo > span.bar' } + - { element: '#document', type: 'ready' } + jQuerySizzleCalls: + - { selector: '#foo .bar', element: '#document' } + jQueryDOMReads: + - { functionName: 'css', arguments: '["color"]', contextPath: 'body > div#foo > span.bar' } + jQueryDOMWrites: + - { functionName: 'css', arguments: '[{"color":"red","background":"green"}]', contextPath: 'body > div#foo > span.bar' } + - { functionName: 'css', arguments: '[{"background-color":"blue"}]', contextPath: 'body > div#foo' } + - { functionName: 'css', arguments: '[{"background-color":"blue"}]', contextPath: 'body > div#foo' } + jQueryDOMWriteReadSwitches: + - { functionName: 'css', arguments: '["color"]', contextPath: 'body > div#foo > span.bar' } # multiple jQuery "instances" (issue #435) - url: "/jquery-multiple.html" metrics: @@ -116,6 +135,11 @@ jQueryVersion: "1.11.1" # the last loaded version jQueryVersionsLoaded: 3 jQueryOnDOMReadyFunctions: 1 + offenders: + jQueryVersionsLoaded: + - { version: "2.1.1", url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js' } + - { version: "2.1.1", url: 'http://code.jquery.com/jquery-2.1.1.js' } + - { version: "1.11.1", url: 'http://code.jquery.com/jquery-1.11.1.js' } # --no-externals handling (issue #535) - url: "/jquery-multiple.html" label: "/jquery-multiple.html (with --no-externals)" diff --git a/test/webroot/jquery.html b/test/webroot/jquery.html index 884ad039e..5a976f79d 100644 --- a/test/webroot/jquery.html +++ b/test/webroot/jquery.html @@ -24,7 +24,7 @@ click(function() { $(this).parent().css({'background-color': 'blue'}); }). - load(function() { + on('load', function() { // foo }); @@ -46,8 +46,8 @@ }); }); - $(window).load(function() { - console.log('window loaded'); + $(window).on('load', function() { + // console.log('window loaded'); }); From a04766fe68b11bda4a540cbf29d6fd6b0fad6589 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 11 Jan 2019 21:11:35 +0100 Subject: [PATCH 062/248] events: module ported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /jquery.html ✓ should be generated ✓ should have "jQueryVersion" metric properly set ✓ should have "jQueryOnDOMReadyFunctions" metric properly set ✓ should have "jQueryWindowOnLoadFunctions" metric properly set ✓ should have "jQuerySizzleCalls" metric properly set ✓ should have "jQueryEventTriggers" metric properly set ✓ should have "eventsBound" metric properly set ✓ should have "eventsDispatched" metric properly set ✓ should have "eventsScrollBound" metric properly set ✓ should have "eventsDispatched" offender(s) properly set ✓ should have "eventsBound" offender(s) properly set ✓ should have "eventsScrollBound" offender(s) properly set ✓ should have "jQueryVersionsLoaded" offender(s) properly set ✓ should have "jQueryWindowOnLoadFunctions" offender(s) properly set ✓ should have "jQueryEventTriggers" offender(s) properly set ✓ should have "jQuerySizzleCalls" offender(s) properly set ✓ should have "jQueryDOMReads" offender(s) properly set ✓ should have "jQueryDOMWrites" offender(s) properly set ✓ should have "jQueryDOMWriteReadSwitches" offender(s) properly set --- modules/events/events.js | 42 ++------------------------------------ modules/events/scope.js | 34 ++++++++++++++++++++++++++++++ test/integration-spec.yaml | 16 ++++++++++++++- 3 files changed, 51 insertions(+), 41 deletions(-) create mode 100644 modules/events/scope.js diff --git a/modules/events/events.js b/modules/events/events.js index f15f93b0d..327c7ce9f 100644 --- a/modules/events/events.js +++ b/modules/events/events.js @@ -9,44 +9,6 @@ module.exports = function(phantomas) { phantomas.setMetric('eventsDispatched'); // @desc number of EventTarget.dispatchEvent calls phantomas.setMetric('eventsScrollBound'); // @desc number of scroll event bounds - return; // TODO - - phantomas.on('init', function() { - phantomas.evaluate(function() { - (function(phantomas) { - // spy calls to EventTarget.addEventListener - // @see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget.addEventListener - function eventSpy(eventType) { - /* jshint validthis: true */ - var path = phantomas.getDOMPath(this); - phantomas.log('DOM event: "' + eventType + '" bound to "' + path + '"'); - - phantomas.incrMetric('eventsBound'); - phantomas.addOffender('eventsBound', '"%s" bound to "%s"', eventType, path); - - // count window.addEventListener('scroll', ...) - issue #508 - if (eventType === 'scroll' && (path === 'window' || path === '#document')) { - phantomas.incrMetric('eventsScrollBound'); - phantomas.addOffender('eventsScrollBound', 'bound by %s', phantomas.getBacktrace()); - } - } - - phantomas.spy(Element.prototype, 'addEventListener', eventSpy); - phantomas.spy(Document.prototype, 'addEventListener', eventSpy); - phantomas.spy(window, 'addEventListener', eventSpy); - - // spy calls to EventTarget.dispatchEvent - // @see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget.dispatchEvent - phantomas.spy(Element.prototype, 'dispatchEvent', function(ev) { - /* jshint validthis: true */ - var path = phantomas.getDOMPath(this); - - phantomas.log('Core JS event: triggered "%s" on "%s"', ev.type, path); - - phantomas.incrMetric('eventsDispatched'); - phantomas.addOffender('eventsDispatched', '"%s" on "%s"', ev.type, path); - }); - })(window.__phantomas); - }); - }); + // inject JS code + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/events/scope.js b/modules/events/scope.js new file mode 100644 index 000000000..32c48f429 --- /dev/null +++ b/modules/events/scope.js @@ -0,0 +1,34 @@ +(function(phantomas) { + // spy calls to EventTarget.addEventListener + // @see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget.addEventListener + function eventSpy(eventType) { + /* jshint validthis: true */ + var path = phantomas.getDOMPath(this); + phantomas.log('DOM event: "' + eventType + '" bound to "' + path + '"'); + + phantomas.incrMetric('eventsBound'); + phantomas.addOffender('eventsBound', {eventType, path}); + + // count window.addEventListener('scroll', ...) - issue #508 + if (eventType === 'scroll' && (path === 'window' || path === '#document')) { + phantomas.incrMetric('eventsScrollBound'); + phantomas.addOffender('eventsScrollBound', {element: path}); + } + } + + phantomas.spy(Element.prototype, 'addEventListener', eventSpy); + phantomas.spy(Document.prototype, 'addEventListener', eventSpy); + phantomas.spy(window, 'addEventListener', eventSpy); + + // spy calls to EventTarget.dispatchEvent + // @see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget.dispatchEvent + phantomas.spy(Element.prototype, 'dispatchEvent', function(ev) { + /* jshint validthis: true */ + var path = phantomas.getDOMPath(this); + + phantomas.log('Core JS event: triggered "%s" on "%s"', ev.type, path); + + phantomas.incrMetric('eventsDispatched'); + phantomas.addOffender('eventsDispatched', {eventType: ev.type, path}); + }); +})(window.__phantomas); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index c9cf05215..a54df35c3 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -104,10 +104,24 @@ jQueryWindowOnLoadFunctions: 1 jQuerySizzleCalls: 1 jQueryEventTriggers: 2 - eventsBound: 7 + eventsBound: 8 eventsDispatched: 1 eventsScrollBound: 2 offenders: + eventsDispatched: + - { eventType: 'click', path: 'body > div#foo > span.bar' } + eventsBound: + - { eventType: 'load', path: 'window' } + - { eventType: 'DOMContentLoaded', path: '#document' } + - { eventType: 'load', path: 'window' } + - { eventType: 'load', path: 'window' } + - { eventType: 'click', path: 'body > div#foo > span.bar' } + - { eventType: 'load', path: 'body > div#foo > span.bar' } + - { eventType: 'scroll', path: '#document' } + - { eventType: 'scroll', path: 'window' } + eventsScrollBound: + - { element: '#document' } + - { element: 'window' } jQueryVersionsLoaded: - version: "2.1.1" url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js' From e705c099e4faed2074bbeaac2c4abb228e318b0e Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 12:21:31 +0100 Subject: [PATCH 063/248] ajaxRequest: port the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /jquery-ajax.html ✓ should be generated ✓ should have "ajaxRequests" metric properly set ✓ should have "ajaxRequests" offender(s) properly set --- modules/ajaxRequests/ajaxRequests.js | 14 ++------------ modules/ajaxRequests/scope.js | 8 ++++++++ test/integration-spec.yaml | 5 ++++- test/webroot/jquery-ajax.html | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 modules/ajaxRequests/scope.js diff --git a/modules/ajaxRequests/ajaxRequests.js b/modules/ajaxRequests/ajaxRequests.js index ef530e28f..0f7e83426 100644 --- a/modules/ajaxRequests/ajaxRequests.js +++ b/modules/ajaxRequests/ajaxRequests.js @@ -7,16 +7,6 @@ module.exports = function(phantomas) { phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests - return; // TODO - - phantomas.on('init', function() { - phantomas.evaluate(function() { - (function(phantomas) { - phantomas.spy(window.XMLHttpRequest.prototype, 'open', function(result, method, url, async) { - phantomas.incrMetric('ajaxRequests'); - phantomas.addOffender('ajaxRequests', '<%s> [%s]', url, method); - }, true); - })(window.__phantomas); - }); - }); + // inject JS code + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/ajaxRequests/scope.js b/modules/ajaxRequests/scope.js new file mode 100644 index 000000000..dfd1a05a5 --- /dev/null +++ b/modules/ajaxRequests/scope.js @@ -0,0 +1,8 @@ +(function(phantomas) { + phantomas.spy(window.XMLHttpRequest.prototype, 'open', function(result, method, url, async) { + phantomas.incrMetric('ajaxRequests'); + phantomas.addOffender('ajaxRequests', {url, method}); + + phantomas.log('Ajax request: ' + url); + }, true); +})(window.__phantomas); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index a54df35c3..c479ee2e1 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -261,8 +261,11 @@ # AJAX requests (#622) - url: "/jquery-ajax.html" metrics: - requests: 4 ajaxRequests: 2 + offenders: + ajaxRequests: + - { url: '/static/style.css', method: 'GET' } + - { url: '/static/broken.css', method: 'GET' } # multiple cookies (#597) - url: "/cookies.html" options: diff --git a/test/webroot/jquery-ajax.html b/test/webroot/jquery-ajax.html index 1b9cd6162..a8d629160 100644 --- a/test/webroot/jquery-ajax.html +++ b/test/webroot/jquery-ajax.html @@ -9,7 +9,7 @@ console.log('XMLHttpRequest'); var xhr = new XMLHttpRequest(); - xhr.open("GET", "/static/style.css", true); + xhr.open("GET", "/static/broken.css", true); xhr.send(); From 65cefc09f2baab3c36aa5d26d34c3202dfb828bd Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 13:42:39 +0100 Subject: [PATCH 064/248] globalVariables: port the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /js-globals.html ✓ should be generated ✓ should have "globalVariables" metric properly set ✓ should have "globalVariablesFalsy" metric properly set ✓ should have "jsErrors" metric properly set ✓ should have "globalVariables" offender(s) properly set ✓ should have "globalVariablesFalsy" offender(s) properly set --- modules/globalVariables/globalVariables.js | 65 +--------------------- modules/globalVariables/scope.js | 38 +++++++++++++ test/integration-spec.yaml | 4 ++ test/webroot/js-globals.html | 4 +- 4 files changed, 47 insertions(+), 64 deletions(-) create mode 100644 modules/globalVariables/scope.js diff --git a/modules/globalVariables/globalVariables.js b/modules/globalVariables/globalVariables.js index 1056c5aca..146e495d8 100644 --- a/modules/globalVariables/globalVariables.js +++ b/modules/globalVariables/globalVariables.js @@ -4,69 +4,10 @@ /* global document: true, window: true */ 'use strict'; -module.exports = function(phantomas) { +module.exports = phantomas => { phantomas.setMetric('globalVariables'); // @desc number of JS globals variables @offenders phantomas.setMetric('globalVariablesFalsy'); // @desc number of JS globals variables with falsy value @offenders - return; // TODO - - phantomas.on('report', function() { - phantomas.evaluate(function() { - (function(phantomas) { - var globals = [], - allowed = ['Components', 'XPCNativeWrapper', 'XPCSafeJSObjectWrapper', 'getInterface', 'netscape', 'GetWeakReference', '_phantom', 'callPhantom', '__phantomas', 'performance'], - varName, - iframe, - cleanWindow; - - if (!document.body) { - return false; - } - - phantomas.spyEnabled(false, 'counting global variables (injecting an empty iframe)'); - - // create an empty iframe to get the list of core members - iframe = document.createElement('iframe'); - iframe.style.display = 'none'; - iframe.src = 'about:blank'; - document.body.appendChild(iframe); - - cleanWindow = iframe.contentWindow; - - for (varName in cleanWindow) { - allowed.push(varName); - } - - // get all members of window and filter them - for (varName in window) { - try { - if ((allowed.indexOf(varName) > -1) || (typeof window[varName] === 'undefined') /* ignore variables exposed by window.__defineGetter__ */ ) { - continue; - } - - // filter out 0, 1, 2, ... - if (/^\d+$/.test(varName)) { - continue; - } - - phantomas.incrMetric('globalVariables'); - phantomas.addOffender('globalVariables', varName); - - if ([false, null].indexOf(window[varName]) > -1) { - phantomas.incrMetric('globalVariablesFalsy'); - phantomas.addOffender('globalVariablesFalsy', '%s = %j', varName, window[varName]); - } - } catch (ex) { - // handle errors (issue #560) - phantomas.log('globalVariables: error when checking %s - %s!', varName, ex.message); - } - } - - // cleanup (issue #297) - document.body.removeChild(iframe); - - phantomas.spyEnabled(true); - })(window.__phantomas); - }); - }); + // inject JS code + phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/globalVariables/scope.js b/modules/globalVariables/scope.js new file mode 100644 index 000000000..e9e6793de --- /dev/null +++ b/modules/globalVariables/scope.js @@ -0,0 +1,38 @@ +(function(phantomas) { + // get the list of initial, built-in global variables + var allowed = [], + varName; + + for (varName in window) { + allowed.push(varName); + } + + window.addEventListener("load", () => { + var varName; + + // get all members of window and filter them + for (varName in window) { + try { + if ((allowed.indexOf(varName) > -1) || (typeof window[varName] === 'undefined') /* ignore variables exposed by window.__defineGetter__ */ ) { + continue; + } + + // filter out 0, 1, 2, ... + if (/^\d+$/.test(varName)) { + continue; + } + + phantomas.incrMetric('globalVariables'); + phantomas.addOffender('globalVariables', varName); + + if ([false, null].indexOf(window[varName]) > -1) { + phantomas.incrMetric('globalVariablesFalsy'); + phantomas.addOffender('globalVariablesFalsy', {name: varName, value: window[varName]}); + } + } catch (ex) { + // handle errors (issue #560) + phantomas.log('globalVariables: error when checking %s - %s!', varName, ex.message); + } + } + }); +})(window.__phantomas); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index c479ee2e1..340e6f4a1 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -252,6 +252,10 @@ globalVariables: 3 globalVariablesFalsy: 1 jsErrors: 0 + offenders: + globalVariables: [ 'a_global', 'falsy', 'foo' ] + globalVariablesFalsy: + - { name: 'falsy', value: false } # JS redirects (#550) - url: "/js-redirect.html" metrics: diff --git a/test/webroot/js-globals.html b/test/webroot/js-globals.html index 2809c5bc3..800b91d31 100644 --- a/test/webroot/js-globals.html +++ b/test/webroot/js-globals.html @@ -8,10 +8,10 @@ } // overwrite some core objects (#482) - window.JSON = { + window.navigator = { foo: 'bar' }; - Array.prototype.toJSON = foo; + Array.prototype.foo = foo; From 3f9b6eb9ee4ab4d53581e318175e656065731caf Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:06:39 +0100 Subject: [PATCH 065/248] Scope: expose getParam function + autoload scope.js for modules --- core/scope.js | 6 ++++++ lib/index.js | 14 +++++++++++--- lib/loader.js | 17 +++++++++-------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/core/scope.js b/core/scope.js index 5e5fd74b8..bb28c4818 100644 --- a/core/scope.js +++ b/core/scope.js @@ -144,6 +144,11 @@ sendMsg('addOffender', Array.prototype.slice.apply(arguments)); } + // see lib/index.js code that injects __phantomas_options into page scope + function getParam(param, _default) { + return scope.__phantomas_options[param] || _default; + } + // exports phantomas.log = log; phantomas.setMetric = setMetric; @@ -152,6 +157,7 @@ phantomas.setMarkerMetric = setMarkerMetric; phantomas.addOffender = addOffender; phantomas.emit = scope.__phantomas_emit.bind(scope); + phantomas.getParam = getParam; })(); /** diff --git a/lib/index.js b/lib/index.js index e81a10eb3..2656e0260 100644 --- a/lib/index.js +++ b/lib/index.js @@ -58,11 +58,13 @@ function phantomas(url, opts) { const page = await browser.init(), debugScope = require('debug')('phantomas:scope:log'); - // TODO: prepare a small instance object that will be passed to modules and extensions on init + // prepare a small instance object that will be passed to modules and extensions on init const scope = { - getParam: () => false, // TODO + getParam: (param, _default) => { + return options[param] || _default; + }, getVersion: () => VERSION, - + emit: events.emit.bind(events), on: events.on.bind(events), once: events.once.bind(events), @@ -87,6 +89,12 @@ function phantomas(url, opts) { }, }; + // pass phantomas options to page scope + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args + await page.evaluateOnNewDocument(options => { + window.__phantomas_options = options; + }, options); + // expose the function that will pass events from page scope code into Node.js layer // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageexposefunctionname-puppeteerfunction await page.exposeFunction('__phantomas_emit', scope.emit); diff --git a/lib/loader.js b/lib/loader.js index e3f87f36e..3ec76f1a7 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -2,7 +2,8 @@ * Handles loading of modules and extensions */ const debug = require('debug'), - extend = require('util')._extend; + extend = require('util')._extend, + fs = require('fs'); /** * Lists all modules / extensions in a given directory @@ -15,8 +16,7 @@ function listModulesInDirectory(modulesDir) { log('Getting the list of all modules in ' + modulesDir); // https://nodejs.org/api/fs.html#fs_fs_readdirsync_path_options - var fs = require('fs'), - ls = fs.readdirSync(modulesDir), + var ls = fs.readdirSync(modulesDir), modules = []; ls.forEach(function(entry) { @@ -42,7 +42,6 @@ function loadCoreModules(scope) { _scope = extend({}, scope); _scope.log = log; - //_scope.log('Loading...'); var module = require(__dirname + '/../core/modules/' + name + '/' + name); module(_scope); @@ -57,8 +56,6 @@ function loadExtensions(scope) { _scope = extend({}, scope); _scope.log = log; - - //_scope.log('Loading...'); var module = require(__dirname + '/../extensions/' + name + '/' + name); module(_scope); @@ -73,11 +70,15 @@ function loadModules(scope) { _scope = extend({}, scope); _scope.log = log; - - //_scope.log('Loading...'); var module = require(__dirname + '/../modules/' + name + '/' + name); module(_scope); + + // auto-inject scope.js from module's directory + var scopeFile = __dirname + '/../modules/' + name + '/scope.js'; + if (fs.existsSync(scopeFile)) { + scope.on('init', () => scope.injectJs(scopeFile)); + } }); } From 1cec8e07c8ee8b4dd280eeafaa4b6c438cc0c7d7 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:07:16 +0100 Subject: [PATCH 066/248] javaScriptBottlenecks: module ported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /bottlenecks.html (--spy-eval) ✓ should be generated ✓ should have "documentWriteCalls" metric properly set ✓ should have "evalCalls" metric properly set ✓ should have "documentWriteCalls" offender(s) properly set ✓ should have "evalCalls" offender(s) properly set /bottlenecks.html ✓ should be generated ✓ should have "documentWriteCalls" metric properly set ✓ should have "evalCalls" metric properly set ✓ should have "documentWriteCalls" offender(s) properly set ✓ should have "evalCalls" offender(s) properly set --- .../javaScriptBottlenecks.js | 52 ------------------ modules/javaScriptBottlenecks/scope.js | 54 +++++++++++++++++++ test/integration-spec.yaml | 12 +++++ 3 files changed, 66 insertions(+), 52 deletions(-) create mode 100644 modules/javaScriptBottlenecks/scope.js diff --git a/modules/javaScriptBottlenecks/javaScriptBottlenecks.js b/modules/javaScriptBottlenecks/javaScriptBottlenecks.js index 704e1183d..ec8c3ca8c 100644 --- a/modules/javaScriptBottlenecks/javaScriptBottlenecks.js +++ b/modules/javaScriptBottlenecks/javaScriptBottlenecks.js @@ -13,56 +13,4 @@ module.exports = function(phantomas) { phantomas.setMetric('documentWriteCalls'); //@desc number of calls to either document.write or document.writeln @offenders phantomas.setMetric('evalCalls'); // @desc number of calls to eval (either direct or via setTimeout / setInterval) @offenders - - return; // TODO - - // spy calls to eval only when requested (issue #467) - var spyEval = phantomas.getParam('spy-eval') === true; - if (!spyEval) { - phantomas.log('javaScriptBottlenecks: to spy calls to eval() run phantomas with --spy-eval option'); - } - - phantomas.on('init', function() { - phantomas.evaluate(function(spyEval) { - (function(phantomas) { - function report(msg, caller, backtrace, metric) { - phantomas.log(msg + ': from ' + caller + '!'); - phantomas.log('Backtrace: ' + backtrace); - - phantomas.incrMetric(metric); - phantomas.addOffender(metric, "%s from %s", msg, caller); - } - - // spy calls to eval() - if (spyEval) { - phantomas.spy(window, 'eval', function(code) { - report('eval() called directly', phantomas.getCaller(), phantomas.getBacktrace(), 'evalCalls'); - phantomas.log('eval\'ed code: ' + (code || '').substring(0, 150) + '(...)'); - }); - } - - // spy calls to setTimeout / setInterval with string passed instead of a function - phantomas.spy(window, 'setTimeout', function(fn, interval) { - if (typeof fn !== 'string') return; - - report('eval() called via setTimeout("' + fn + '")', phantomas.getCaller(), phantomas.getBacktrace(), 'evalCalls'); - }); - - phantomas.spy(window, 'setInterval', function(fn, interval) { - if (typeof fn !== 'string') return; - - report('eval() called via setInterval("' + fn + '")', phantomas.getCaller(), phantomas.getBacktrace(), 'evalCalls'); - }); - - // spy document.write(ln) - phantomas.spy(document, 'write', function(arg) { - report('document.write() used', phantomas.getCaller(), phantomas.getBacktrace(), 'documentWriteCalls'); - }); - - phantomas.spy(document, 'writeln', function(arg) { - report('document.writeln() used', phantomas.getCaller(), phantomas.getBacktrace(), 'documentWriteCalls'); - }); - })(window.__phantomas); - }, spyEval); - }); }; diff --git a/modules/javaScriptBottlenecks/scope.js b/modules/javaScriptBottlenecks/scope.js new file mode 100644 index 000000000..e372c412b --- /dev/null +++ b/modules/javaScriptBottlenecks/scope.js @@ -0,0 +1,54 @@ +(async phantomas => { + + // spy calls to eval only when requested (issue #467) + var spyEval = await phantomas.getParam('spy-eval') === true; + + console.log(await phantomas.getParam('url')) + console.log(await phantomas.getParam('spy-eval')) + console.log(spyEval); + + if (!spyEval) { + phantomas.log('javaScriptBottlenecks: to spy calls to eval() run phantomas with "spy-eval" option set to true'); + } + else { + phantomas.log('javaScriptBottlenecks: eval() calls will be checked'); + } + + function report(msg, caller, backtrace, metric) { + phantomas.log(msg + ': from ' + caller + '!'); + phantomas.log('Backtrace: ' + backtrace); + + phantomas.incrMetric(metric); + phantomas.addOffender(metric, {message: msg, caller}); + } + + // spy calls to eval() + if (spyEval) { + phantomas.spy(window, 'eval', function(code) { + report('eval() called directly', phantomas.getCaller(), phantomas.getBacktrace(), 'evalCalls'); + phantomas.log('eval\'ed code: ' + (code || '').substring(0, 150) + '(...)'); + }); + } + + // spy calls to setTimeout / setInterval with string passed instead of a function + phantomas.spy(window, 'setTimeout', function(fn, _) { + if (typeof fn !== 'string') return; + + report('eval() called via setTimeout("' + fn + '")', phantomas.getCaller(), phantomas.getBacktrace(), 'evalCalls'); + }); + + phantomas.spy(window, 'setInterval', function(fn, _) { + if (typeof fn !== 'string') return; + + report('eval() called via setInterval("' + fn + '")', phantomas.getCaller(), phantomas.getBacktrace(), 'evalCalls'); + }); + + // spy document.write(ln) + phantomas.spy(document, 'write', function(_) { + report('document.write() used', phantomas.getCaller(), phantomas.getBacktrace(), 'documentWriteCalls'); + }); + + phantomas.spy(document, 'writeln', function(_) { + report('document.writeln() used', phantomas.getCaller(), phantomas.getBacktrace(), 'documentWriteCalls'); + }); +})(window.__phantomas); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 340e6f4a1..253587e43 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -225,10 +225,22 @@ metrics: documentWriteCalls: 1 evalCalls: 2 # via eval() and setTimeout() + offenders: + documentWriteCalls: + - { message: 'document.write() used', caller: 'http://127.0.0.1:8888/bottlenecks.html:11:11' } + evalCalls: + - { message: 'eval() called directly', caller: 'http://127.0.0.1:8888/bottlenecks.html:8:2' } + - { message: 'eval() called via setTimeout("foo()")', caller: 'http://127.0.0.1:8888/bottlenecks.html:9:2' } + - url: "/bottlenecks.html" metrics: documentWriteCalls: 1 evalCalls: 1 # via setTimeout() only + offenders: + documentWriteCalls: + - { message: 'document.write() used', caller: 'http://127.0.0.1:8888/bottlenecks.html:11:11' } + evalCalls: + - { message: 'eval() called via setTimeout("foo()")', caller: 'http://127.0.0.1:8888/bottlenecks.html:9:2' } # stop-at-onload (#513) - url: "/after-onload.html" label: "/after-onload.html (--stop-at-onload)" From 225228df4eecfcb042d11404a66d45d3d9c983aa Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:07:48 +0100 Subject: [PATCH 067/248] modules: scope.js is now autoloaded if exists --- modules/ajaxRequests/ajaxRequests.js | 3 --- modules/analyzeCss/analyzeCss.js | 3 ++- modules/documentHeight/documentHeight.js | 3 --- modules/domComplexity/domComplexity.js | 3 --- modules/domMutations/domMutations.js | 2 -- modules/domQueries/domQueries.js | 2 -- modules/events/events.js | 3 --- modules/globalVariables/globalVariables.js | 3 --- modules/jQuery/jQuery.js | 3 --- modules/localStorage/localStorage.js | 3 --- 10 files changed, 2 insertions(+), 26 deletions(-) diff --git a/modules/ajaxRequests/ajaxRequests.js b/modules/ajaxRequests/ajaxRequests.js index 0f7e83426..a693c9590 100644 --- a/modules/ajaxRequests/ajaxRequests.js +++ b/modules/ajaxRequests/ajaxRequests.js @@ -6,7 +6,4 @@ module.exports = function(phantomas) { phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests - - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/analyzeCss/analyzeCss.js b/modules/analyzeCss/analyzeCss.js index ef0c50761..a76be87e2 100644 --- a/modules/analyzeCss/analyzeCss.js +++ b/modules/analyzeCss/analyzeCss.js @@ -49,7 +49,7 @@ module.exports = function(phantomas) { phantomas.log('To enable CSS in-depth metrics please run phantomas with --analyze-css option'); return; } - +/** phantomas.setMetric('cssParsingErrors'); // @desc number of CSS files (or embeded CSS) that failed to be parse by analyze-css @optional phantomas.setMetric('cssInlineStyles'); // @desc number of inline styles @optional @@ -227,4 +227,5 @@ module.exports = function(phantomas) { analyzeCss(['--file', path]); }); }); +**/ }; diff --git a/modules/documentHeight/documentHeight.js b/modules/documentHeight/documentHeight.js index 7355f0b4a..bae793a78 100644 --- a/modules/documentHeight/documentHeight.js +++ b/modules/documentHeight/documentHeight.js @@ -6,7 +6,4 @@ module.exports = function(phantomas) { phantomas.setMetric('documentHeight'); // @desc the page height [px] - - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/domComplexity/domComplexity.js b/modules/domComplexity/domComplexity.js index ca77955a1..5a85666ea 100644 --- a/modules/domComplexity/domComplexity.js +++ b/modules/domComplexity/domComplexity.js @@ -33,9 +33,6 @@ module.exports = function(phantomas) { } }); - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); - // duplicated ID (issue #392) phantomas.setMetric('DOMidDuplicated'); // @desc number of duplicated IDs found in DOM diff --git a/modules/domMutations/domMutations.js b/modules/domMutations/domMutations.js index 3e8fff6f2..9ecefed02 100644 --- a/modules/domMutations/domMutations.js +++ b/modules/domMutations/domMutations.js @@ -10,6 +10,4 @@ module.exports = function(phantomas) { phantomas.setMetric('DOMmutationsInserts'); // @desc number of node inserts phantomas.setMetric('DOMmutationsRemoves'); // @desc number of node removes phantomas.setMetric('DOMmutationsAttributes'); // @desc number of DOM nodes attributes changes - - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; \ No newline at end of file diff --git a/modules/domQueries/domQueries.js b/modules/domQueries/domQueries.js index dd7656920..13a9cea20 100644 --- a/modules/domQueries/domQueries.js +++ b/modules/domQueries/domQueries.js @@ -15,8 +15,6 @@ module.exports = phantomas => { phantomas.setMetric('DOMqueriesDuplicated'); // @desc number of DOM queries called more than once phantomas.setMetric('DOMqueriesAvoidable'); // @desc number of repeated uses of a duplicated query - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); - // // TODO: pass events fired by page scoped code // diff --git a/modules/events/events.js b/modules/events/events.js index 327c7ce9f..09cf3b007 100644 --- a/modules/events/events.js +++ b/modules/events/events.js @@ -8,7 +8,4 @@ module.exports = function(phantomas) { phantomas.setMetric('eventsBound'); // @desc number of EventTarget.addEventListener calls phantomas.setMetric('eventsDispatched'); // @desc number of EventTarget.dispatchEvent calls phantomas.setMetric('eventsScrollBound'); // @desc number of scroll event bounds - - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/globalVariables/globalVariables.js b/modules/globalVariables/globalVariables.js index 146e495d8..daed6bd08 100644 --- a/modules/globalVariables/globalVariables.js +++ b/modules/globalVariables/globalVariables.js @@ -7,7 +7,4 @@ module.exports = phantomas => { phantomas.setMetric('globalVariables'); // @desc number of JS globals variables @offenders phantomas.setMetric('globalVariablesFalsy'); // @desc number of JS globals variables with falsy value @offenders - - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; diff --git a/modules/jQuery/jQuery.js b/modules/jQuery/jQuery.js index 655d39375..f2956919d 100644 --- a/modules/jQuery/jQuery.js +++ b/modules/jQuery/jQuery.js @@ -20,9 +20,6 @@ module.exports = function(phantomas) { phantomas.setMetric('jQueryDOMWrites'); // @desc number of DOM write operations phantomas.setMetric('jQueryDOMWriteReadSwitches'); // @desc number of read operations that follow a series of write operations (will cause repaint and can cause reflow) - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); - // store the last resource that was received // try to report where given jQuery version was loaded from phantomas.on('recv', entry => { diff --git a/modules/localStorage/localStorage.js b/modules/localStorage/localStorage.js index b99a0730b..6696b8478 100644 --- a/modules/localStorage/localStorage.js +++ b/modules/localStorage/localStorage.js @@ -6,7 +6,4 @@ module.exports = function(phantomas) { phantomas.setMetric('localStorageEntries'); // @desc number of entries in local storage - - // inject JS code - phantomas.on('init', () => phantomas.injectJs(__dirname + '/scope.js')); }; From 4014a9778c9b436e33b874d7dc5613c557c6e33a Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:11:09 +0100 Subject: [PATCH 068/248] javaScriptBottlenecks: remove debug console.log() calls --- modules/javaScriptBottlenecks/scope.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/javaScriptBottlenecks/scope.js b/modules/javaScriptBottlenecks/scope.js index e372c412b..448ab19c5 100644 --- a/modules/javaScriptBottlenecks/scope.js +++ b/modules/javaScriptBottlenecks/scope.js @@ -3,10 +3,6 @@ // spy calls to eval only when requested (issue #467) var spyEval = await phantomas.getParam('spy-eval') === true; - console.log(await phantomas.getParam('url')) - console.log(await phantomas.getParam('spy-eval')) - console.log(spyEval); - if (!spyEval) { phantomas.log('javaScriptBottlenecks: to spy calls to eval() run phantomas with "spy-eval" option set to true'); } From 40c221fbf6ac4935c0a3d20972b09628d71d62f3 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:18:29 +0100 Subject: [PATCH 069/248] modules: call spyEnabled(false) to avoid incorrect reporting of events binds --- modules/domComplexity/scope.js | 4 ++++ modules/globalVariables/scope.js | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/modules/domComplexity/scope.js b/modules/domComplexity/scope.js index 81918a458..f8f34c8a3 100644 --- a/modules/domComplexity/scope.js +++ b/modules/domComplexity/scope.js @@ -1,5 +1,7 @@ (function(phantomas) { + phantomas.spyEnabled(false, 'initializing domComplexity metrics'); + document.addEventListener("DOMContentLoaded", () => { phantomas.log("DOM fully loaded and parsed"); phantomas.spyEnabled(false, 'running domComplexity metrics'); @@ -27,4 +29,6 @@ phantomas.spyEnabled(true); }); + phantomas.spyEnabled(true); + })(window.__phantomas); diff --git a/modules/globalVariables/scope.js b/modules/globalVariables/scope.js index e9e6793de..ae4087570 100644 --- a/modules/globalVariables/scope.js +++ b/modules/globalVariables/scope.js @@ -7,6 +7,8 @@ allowed.push(varName); } + phantomas.spyEnabled(false, 'initializing global variables metrics'); + window.addEventListener("load", () => { var varName; @@ -35,4 +37,6 @@ } } }); + + phantomas.spyEnabled(true); })(window.__phantomas); From b34e084f386f2b6883f71c65067ab13d7ada4f75 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:22:57 +0100 Subject: [PATCH 070/248] jQuery: check globalVariables offenders --- test/integration-spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 253587e43..2bb9a2602 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -154,6 +154,7 @@ - { version: "2.1.1", url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js' } - { version: "2.1.1", url: 'http://code.jquery.com/jquery-2.1.1.js' } - { version: "1.11.1", url: 'http://code.jquery.com/jquery-1.11.1.js' } + globalVariables: [ 'jQuery', '$' ] # --no-externals handling (issue #535) - url: "/jquery-multiple.html" label: "/jquery-multiple.html (with --no-externals)" From fa05081f3593b53edd55ebc60fff0c9d43d35d12 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:25:56 +0100 Subject: [PATCH 071/248] test: comment out wait-* test - vows hangs because of them --- test/integration-spec.yaml | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 2bb9a2602..d1abb6796 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -188,28 +188,28 @@ jQuerySizzleCalls: 0 jQueryEventTriggers: 0 # --wait-for-selector -- url: "/" - label: "/ (--wait-for-selector)" - options: - timeout: 1 - 'wait-for-selector': "#foo" - metrics: - DOMqueries: 0 # no queries should be reported despite DOM being polled by waitForSelector module - exitCode: 252 # timeout +#- url: "/" +# label: "/ (--wait-for-selector)" +# options: +# timeout: 1 +# 'wait-for-selector': "#foo" +# metrics: +# DOMqueries: 0 # no queries should be reported despite DOM being polled by waitForSelector module +# exitCode: 252 # timeout # --wait-for-selector (issue #512) -- url: "/wait-for-selector.html" - options: - timeout: 10 - 'wait-for-selector': ".bar:nth-child(4):last-child" - metrics: - DOMelementsCount: 4 +#- url: "/wait-for-selector.html" +# options: +# timeout: 10 +# 'wait-for-selector': ".bar:nth-child(4):last-child" +# metrics: +# DOMelementsCount: 4 # --wait-for-event (issue #453) -- url: "/wait-for-event.html" - options: - timeout: 4 - 'wait-for-event': "done" - metrics: - testFooBar: 123 # a metric set just before event trigger +#- url: "/wait-for-event.html" +# options: +# timeout: 4 +# 'wait-for-event': "done" +# metrics: +# testFooBar: 123 # a metric set just before event trigger # requests to (#438 / #486) - url: "/timing.html" metrics: From d6126ce64aafee993f5460dfa9ad8afe8a152248 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 17:29:59 +0100 Subject: [PATCH 072/248] jquery-multiple: make tests more deterministic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✗ Broken » 242 honored ∙ 38 broken (13.407s) --- test/integration-spec.yaml | 7 +++---- test/webroot/jquery-multiple.html | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index d1abb6796..df8f3b196 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -143,16 +143,15 @@ # multiple jQuery "instances" (issue #435) - url: "/jquery-multiple.html" metrics: - requests: 4 - jsCount: 3 + requests: 3 + jsCount: 2 domains: 2 jQueryVersion: "1.11.1" # the last loaded version - jQueryVersionsLoaded: 3 + jQueryVersionsLoaded: 2 jQueryOnDOMReadyFunctions: 1 offenders: jQueryVersionsLoaded: - { version: "2.1.1", url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js' } - - { version: "2.1.1", url: 'http://code.jquery.com/jquery-2.1.1.js' } - { version: "1.11.1", url: 'http://code.jquery.com/jquery-1.11.1.js' } globalVariables: [ 'jQuery', '$' ] # --no-externals handling (issue #535) diff --git a/test/webroot/jquery-multiple.html b/test/webroot/jquery-multiple.html index 4b31079c4..f308638dc 100644 --- a/test/webroot/jquery-multiple.html +++ b/test/webroot/jquery-multiple.html @@ -1,7 +1,6 @@ - + - From 2bdc6443a977c4ca133cca1914e94e3ee695e5fe Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 12 Jan 2019 20:12:39 +0100 Subject: [PATCH 073/248] imagesScaledDown: metric works again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /image-scaling.html ✓ should be generated ✓ should have "imagesScaledDown" metric properly set ✓ should have "imagesWithoutDimensions" metric properly set ✓ should have "imagesScaledDown" offender(s) properly set --- modules/domComplexity/domComplexity.js | 122 +++---------------------- modules/domComplexity/scope.js | 91 +++++++++++++++++- test/integration-spec.yaml | 4 + 3 files changed, 108 insertions(+), 109 deletions(-) diff --git a/modules/domComplexity/domComplexity.js b/modules/domComplexity/domComplexity.js index 5a85666ea..b7b738626 100644 --- a/modules/domComplexity/domComplexity.js +++ b/modules/domComplexity/domComplexity.js @@ -6,6 +6,8 @@ module.exports = function(phantomas) { + phantomas.setMetric('bodyHTMLSize'); // @desc the size of body tag content (document.body.innerHTML.length) + // total length of HTML comments (including brackets) phantomas.setMetric('commentsSize'); // @desc the size of HTML comments on the page @offenders @@ -26,13 +28,25 @@ module.exports = function(phantomas) { // keep the track of SVG graphics (#479) var svgResources = []; - phantomas.on('recv', function(entry) { + phantomas.on('recv', entry => { if (entry.isSVG) { svgResources.push(entry.url); phantomas.log('imagesScaledDown: will ignore <%s> [%s]', entry.url, entry.contentType); } }); + phantomas.on('imagesScaledDown', (image) => { + if (svgResources.indexOf(image.url) === -1) { + phantomas.log('Scaled down image: %j', image); + + phantomas.incrMetric('imagesScaledDown'); + phantomas.addOffender('imagesScaledDown', image); + } + else { + phantomas.log('imagesScaledDown: ignored <%s> (is SVG)', image.url); + } + }); + // duplicated ID (issue #392) phantomas.setMetric('DOMidDuplicated'); // @desc number of duplicated IDs found in DOM @@ -50,110 +64,4 @@ module.exports = function(phantomas) { } }); }); - - return; // TODO - - // HTML size - phantomas.on('report', function() { - phantomas.setMetricEvaluate('bodyHTMLSize', function() { // @desc the size of body tag content (document.body.innerHTML.length) - return document.body && document.body.innerHTML.length || 0; - }); - - var scope = { - svgResources: svgResources - }; - - phantomas.evaluate(function(scope) { - (function(phantomas) { - var runner = new phantomas.nodeRunner(), - whitespacesRegExp = /^\s+$/, - DOMelementMaxDepth = 0, - DOMelementMaxDepthNodes = [], // stores offenders for DOMelementMaxDepth (issue #414) - size = 0; - - runner.walk(document.body, function(node, depth) { - var path = ''; - - switch (node.nodeType) { - case Node.COMMENT_NODE: - size = node.textContent.length + 7; // ''.length - phantomas.incrMetric('commentsSize', size); - - // log HTML comments bigger than 64 characters - if (size > 64) { - phantomas.addOffender('commentsSize', '%s (%d characters)', phantomas.getDOMPath(node), size); - } - break; - - case Node.ELEMENT_NODE: - path = phantomas.getDOMPath(node); - - phantomas.incrMetric('DOMelementsCount'); - - if (depth > DOMelementMaxDepth) { - DOMelementMaxDepth = depth; - DOMelementMaxDepthNodes = [path]; - } else if (depth === DOMelementMaxDepth) { - DOMelementMaxDepthNodes.push(path); - } - - // ignore inline + + From 7b8a47e1b12e6302b5e4930f47f6282872299360 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 13 Jan 2019 14:40:18 +0100 Subject: [PATCH 077/248] Cookie: use page.setCookie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /cookies.html ✓ should be generated ✓ should have "cookiesRaw" metric properly set /cookies.html ✓ should be generated ✓ should have "cookiesRaw" metric properly set --- extensions/cookies/cookies.js | 61 +++++++++++++---------------------- lib/index.js | 5 +-- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/extensions/cookies/cookies.js b/extensions/cookies/cookies.js index bdbbfd17b..8d454c55b 100644 --- a/extensions/cookies/cookies.js +++ b/extensions/cookies/cookies.js @@ -6,24 +6,17 @@ module.exports = function(phantomas) { - var cookiesJar = phantomas.getParam('cookies', [], 'object'), - COOKIE_SEPARATOR = '|'; - - // setup cookies handling - function initCookies() { - // cookie handling via command line and config.json - //phantom.cookiesEnabled = true; - - // handles multiple cookies from config.json, and used for storing - // constructed cookies from command line. - + function parseCookies(domain) { // --cookie='bar=foo;domain=url' // for multiple cookies, please use pipe-separated string (issue #667) // --cookie='foo=42|test=123' - var cookieParam = phantomas.getParam('cookie', false, 'string'); + const COOKIE_SEPARATOR = '|', + cookieParam = phantomas.getParam('cookie', false), + cookiesJar = []; if (cookieParam !== false) { phantomas.log('Cookies: parsing "cookie" parameter'); // issue #667 + phantomas.log('Cookies: %j', cookieParam); cookieParam.split(COOKIE_SEPARATOR).forEach(function(cookieParam) { // Parse cookie. at minimum, need a key=value pair, and a domain. @@ -49,44 +42,34 @@ module.exports = function(phantomas) { } else { cookie[frag[0]] = frag[1]; } + + cookie.domain = domain; } // see injectCookies for validation cookiesJar.push(cookie); }); } - } - // add cookies, if any, providing a domain shim - function injectCookies() { - if (cookiesJar && cookiesJar.length > 0) { - // @see http://nodejs.org/docs/latest/api/url.html - var parseUrl = phantomas.require('url').parse; + return cookiesJar; + } - cookiesJar.forEach(function(cookie) { - // phantomjs required attrs: *name, *value, *domain - if (!cookie.name || !cookie.value) { - throw 'this cookie is missing a name or value property: ' + JSON.stringify(cookie); - } + phantomas.on('init', async (_, page) => { + const url = phantomas.getParam('url'), + // https://nodejs.org/docs/latest/api/url.html#url_legacy_url_api + domain = require('url').parse(url).hostname; - if (!cookie.domain) { - var parsed = parseUrl(phantomas.url), - root = (parsed.hostname || '').replace(/^www/, ''); // strip www + // domain field in cookies needs to be set + // https://github.com/miyakogi/pyppeteer/issues/94#issuecomment-403261859 + const cookies = parseCookies(domain); - cookie.domain = root; - phantomas.log('Cookies: domain fallback applied - %s', root); - } + phantomas.log('Cookies: %j', cookies); - if (!phantom.addCookie(cookie)) { - // In PhantomJS 2.1, the addCookie function always returns false (#597). - //throw 'PhantomJS could not add cookie: ' + JSON.stringify(cookie); - } + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagesetcookiecookies + if (cookies.length > 0) { + await page.setCookie(...cookies); - phantomas.log('Cookies: set ' + JSON.stringify(cookie)); - }); + phantomas.log('Cookies: set up'); } - } - - initCookies(); - phantomas.on('pageBeforeOpen', injectCookies); + }); }; diff --git a/lib/index.js b/lib/index.js index b76ec400a..1ff45503b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -29,12 +29,13 @@ function phantomas(url, opts) { debug('phantomas: %s', VERSION); debug('Puppeteer: preferred revision r%s', puppeteer._launcher._preferredRevision); debug('URL: <%s>', url); - debug('Options: %s', JSON.stringify(opts)); // options handling options = util._extend({}, opts || {}); // use util._extend to avoid #563 options.url = options.url || url || false; + debug('Options: %s', JSON.stringify(options)); + // log emitted events events._emit = events.emit; events.log = require('debug')('phantomas:events'); @@ -135,7 +136,7 @@ function phantomas(url, opts) { loader.loadModules(scope); // browser's scope and modules are set up, you can now use it in your modules - events.emit('init', browser); + events.emit('init', browser, page); await browser.visit(url); From e6c66ba387d6cc76bcf6e4c1caa41fcd41a59666 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 14 Jan 2019 20:51:27 +0100 Subject: [PATCH 078/248] Bring back --no-externals mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /jquery-multiple.html (with --no-externals) ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "jsCount" metric properly set ✓ should have "blockedRequests" metric properly set ✓ should have "domains" metric properly set ✓ should have "jsErrors" metric properly set ✓ should have "jQueryVersion" metric properly set ✓ should have "blockedRequests" offender(s) properly set --- modules/blockDomains/blockDomains.js | 44 +++++++++++++++++----------- test/integration-spec.yaml | 6 +++- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/modules/blockDomains/blockDomains.js b/modules/blockDomains/blockDomains.js index e0d35009a..2d06ed777 100644 --- a/modules/blockDomains/blockDomains.js +++ b/modules/blockDomains/blockDomains.js @@ -6,7 +6,9 @@ 'use strict'; module.exports = function(phantomas) { - var ourDomain = false, + const { parse } = require('url'); + + var ourDomain, // --no-externals noExternalsMode = (phantomas.getParam('no-externals') === true), @@ -17,6 +19,8 @@ module.exports = function(phantomas) { blockedDomains = phantomas.getParam('block-domain'), blockedDomainsRegExp; + ourDomain = parse(phantomas.getParam('url')).hostname; + function checkBlock(domain) { var blocked = false; @@ -55,7 +59,7 @@ module.exports = function(phantomas) { blockedDomains = (typeof blockedDomains === 'string') ? parseParameter(blockedDomains) : false; if (noExternalsMode) { - phantomas.log('Block domains: working in --no-externals mode'); + phantomas.log('Block domains: working in --no-externals mode ("%s" is our domain)', ourDomain); } if (allowedDomains !== false) { @@ -68,22 +72,28 @@ module.exports = function(phantomas) { blockedDomainsRegExp = new RegExp('(' + blockedDomains.join('|') + ')$'); } - // get the "main" domain from the first request (see issues #197 and #535) - phantomas.on('beforeSend', function(entry) { - if (!ourDomain) { - ourDomain = entry.domain; - phantomas.log('Block domains: assuming "%s" to be the main domain (from the first request sent)', ourDomain); - } - }); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagesetrequestinterceptionvalue + phantomas.on('init', async (_, page) => { + await page.setRequestInterception(true); - // check each request before sending - phantomas.on('beforeSend', function(entry) { - if (checkBlock(entry.domain)) { - entry.block(); + page.on('request', interceptedRequest => { + const url = interceptedRequest.url(), + domain = parse(url).hostname; - // stats - phantomas.incrMetric('blockedRequests'); // @desc number of requests blocked due to domain filtering @optional - phantomas.addOffender('blockedRequests', entry.url); - } + if (checkBlock(domain)) { + interceptedRequest.abort(); + + phantomas.log('Request has been blocked: <%s>', url); + + // stats + phantomas.incrMetric('blockedRequests'); // @desc number of requests blocked due to domain filtering @optional + phantomas.addOffender('blockedRequests', url); + } + else { + interceptedRequest.continue(); + } + }); + + phantomas.log('Requests intercepting enabled'); }); }; diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index b2440ffa0..21252e708 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -171,9 +171,13 @@ metrics: requests: 2 jsCount: 1 - blockedRequests: 2 + blockedRequests: 1 domains: 1 jsErrors: 0 + jQueryVersion: '2.1.1' + offenders: + blockedRequests: + - 'http://code.jquery.com/jquery-1.11.1.js' # accept main domain in allow-domains - url: "/jquery-multiple.html" label: "/jquery-multiple.html (with --allow-domain)" From 26edeaffa9d5787277d7043d3e7429bb9cfe4e47 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 14 Jan 2019 21:32:49 +0100 Subject: [PATCH 079/248] Support --allow-domain switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /jquery-multiple.html (with --allow-domain) ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "jsCount" metric properly set ✓ should have "blockedRequests" metric properly set ✓ should have "jQueryVersion" metric properly set --- modules/blockDomains/blockDomains.js | 4 +++- test/integration-spec.yaml | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/blockDomains/blockDomains.js b/modules/blockDomains/blockDomains.js index 2d06ed777..2b4cd90be 100644 --- a/modules/blockDomains/blockDomains.js +++ b/modules/blockDomains/blockDomains.js @@ -21,6 +21,8 @@ module.exports = function(phantomas) { ourDomain = parse(phantomas.getParam('url')).hostname; + phantomas.setMetric('blockedRequests'); // @desc number of requests blocked due to domain filtering @optional + function checkBlock(domain) { var blocked = false; @@ -86,7 +88,7 @@ module.exports = function(phantomas) { phantomas.log('Request has been blocked: <%s>', url); // stats - phantomas.incrMetric('blockedRequests'); // @desc number of requests blocked due to domain filtering @optional + phantomas.incrMetric('blockedRequests'); phantomas.addOffender('blockedRequests', url); } else { diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 21252e708..1098ba1b5 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -182,10 +182,13 @@ - url: "/jquery-multiple.html" label: "/jquery-multiple.html (with --allow-domain)" options: - "allow-domain": "foo.com" + "no-externals": true + "allow-domain": "code.jquery.com" metrics: - requests: 2 - blockedRequests: 2 + requests: 3 + jsCount: 2 + blockedRequests: 0 + jQueryVersion: '1.11.1' # jQuery read & write operations (issue #436) - url: "/jquery-reads-writes.html" metrics: From 14d34c7bf7a85a0f538f91bf8a371281c5dec90d Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 14 Jan 2019 21:37:45 +0100 Subject: [PATCH 080/248] Travis: run on Xenial --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0707ba6f2..f750331eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # https://docs.travis-ci.com/user/languages/javascript-with-nodejs/ language: node_js -dist: trusty +dist: xenial node_js: - 10 - 8 @@ -12,9 +12,6 @@ addons: before_install: # Enable user namespace cloning - "sysctl kernel.unprivileged_userns_clone=1" - # Launch XVFB - - "export DISPLAY=:99.0" - - "sh -e /etc/init.d/xvfb start" # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. - "export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true" # Point Phantomas to Chrome stable binary provided by Travis From fa6473377f16cf72969238168285419d219388c5 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 14 Jan 2019 23:08:27 +0100 Subject: [PATCH 081/248] requestsMonitor: properly set entry.transferedSize when requests interception is enabled --- core/modules/requestsMonitor/requestsMonitor.js | 6 ++++++ test/integration-spec.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index b40a22503..ec881dda0 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -190,6 +190,12 @@ module.exports = function(phantomas) { transferedSize: resp.encodedDataLength, }; + // this is set to zero when requests interception is enabled + // use Content-Length response header instead + if (entry.transferedSize == 0) { + entry.transferedSize = parseInt(entry.headers['content-length'] || '0', 10); + } + entry = parseEntryUrl(entry); /** diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 1098ba1b5..9d60470b5 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -13,6 +13,8 @@ cookie: "bar=foo;domain=url" metrics: requests: 3 + htmlCount: 1 + htmlSize: 2094 cssCount: 1 cssSize: 22 jsCount: 1 From fe224b19b9d61ba34a74736f971339939bf37458 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 12:27:04 +0100 Subject: [PATCH 082/248] Add handling of --wait-for-network-idle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagegotourl-options Pass either 'load' or 'networkidle0' to page.goto() /after-onload.html ✓ should be generated ✓ should have "imageCount" metric properly set ✓ should have "requests" metric properly set /after-onload.html (with --wait-for-network-idle) ✓ should be generated ✓ should have "imageCount" metric properly set ✓ should have "requests" metric properly set --- lib/browser.js | 12 +++++++++--- lib/index.js | 4 +++- test/integration-spec.yaml | 16 ++++------------ test/webroot/after-onload.html | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 60de5174a..98421d681 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -151,12 +151,18 @@ Browser.prototype.init = async () => { * Opens the provided URL and emits all necessary events * * @param {string} url + * @param {string?} waitUntil */ -Browser.prototype.visit = url => { +Browser.prototype.visit = (url, waitUntil) => { return new Promise(async (resolve, reject) => { - debug('Go to URL: <%s>', url); + waitUntil = waitUntil || 'load'; + + debug('Go to URL <%s> and wait for "%s"', url, waitUntil); try { - await this.page.goto(url); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagegotourl-options + await this.page.goto(url, { + waitUntil: waitUntil + }); } catch(ex) { debug('Opening URL failed: ' + ex); diff --git a/lib/index.js b/lib/index.js index 1ff45503b..a43aaa98b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -138,7 +138,9 @@ function phantomas(url, opts) { // browser's scope and modules are set up, you can now use it in your modules events.emit('init', browser, page); - await browser.visit(url); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagegotourl-options + const waitUntil = opts['wait-for-network-idle'] ? 'networkidle0' : undefined; + await browser.visit(url, waitUntil); // resolve our run await browser.close(); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 9d60470b5..f275df250 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -71,14 +71,6 @@ - url: "/dom-id.html" metrics: DOMidDuplicated: 2 # foo and bar -# assertions & exit codes (#382 / #528) -- url: "/dom-id.html" - label: "Assertions and exit code" - options: - "assert-requests": 0 - metrics: - requests: 1 - exitCode: 1 # one assert not met # inline styles (#397 / #694) - url: "/inline-css.html" options: @@ -269,14 +261,14 @@ - { message: 'eval() called via setTimeout("foo()")', caller: 'http://127.0.0.1:8888/bottlenecks.html:9:2' } # stop-at-onload (#513) - url: "/after-onload.html" - label: "/after-onload.html (--stop-at-onload)" - options: - "stop-at-onload": true + label: "/after-onload.html" metrics: imageCount: 1 requests: 2 - url: "/after-onload.html" - label: "/after-onload.html" + label: "/after-onload.html (with --wait-for-network-idle)" + options: + "wait-for-network-idle": true metrics: imageCount: 2 requests: 3 diff --git a/test/webroot/after-onload.html b/test/webroot/after-onload.html index 54e6a0349..38fed3db0 100644 --- a/test/webroot/after-onload.html +++ b/test/webroot/after-onload.html @@ -11,6 +11,6 @@ console.log('Done'); }; img.src = '/static/mdn.png'; - }, 500); + }, 250); From df4b22ca6f98194ce0a868cdbf63891b52c64ab3 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 12:30:12 +0100 Subject: [PATCH 083/248] Make js-redirect test pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /js-redirect.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "htmlCount" metric properly set ✓ should have "jsErrors" metric properly set --- test/integration-spec.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index f275df250..91a937f65 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -296,6 +296,8 @@ requests: 2 htmlCount: 2 jsErrors: 0 + options: + "wait-for-network-idle": true # AJAX requests (#622) - url: "/jquery-ajax.html" metrics: From f80363b0917181b3f52c026bea0b005e4dea2089 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 12:33:00 +0100 Subject: [PATCH 084/248] timing.html test passed as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /timing.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "requestsToDomContentLoaded" metric properly set ✓ should have "domainsToDomContentLoaded" metric properly set ✓ should have "requestsToDomComplete" metric properly set ✓ should have "domainsToDomComplete" metric properly set ✓ should have "domainsToDomContentLoaded" offender(s) properly set ✓ should have "domainsToDomComplete" offender(s) properly set --- test/integration-spec.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 91a937f65..d7a2e5368 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -219,6 +219,7 @@ # 'wait-for-event': "done" # metrics: # testFooBar: 123 # a metric set just before event trigger + # requests to (#438 / #486) - url: "/timing.html" metrics: @@ -234,6 +235,8 @@ domainsToDomComplete: - { domain: '127.0.0.1', requests: 2 } - { domain: 'code.jquery.com', requests: 1 } + options: + "wait-for-network-idle": true # JS bottlenecks (#467) - url: "/bottlenecks.html" From 9178969f96119ba06530a66c3597bb8d9fba4ba1 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 15:16:29 +0100 Subject: [PATCH 085/248] Add test case for "headersBiggerThanContent" metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /headers.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "headersCount" metric properly set ✓ should have "headersSentCount" metric properly set ✓ should have "headersRecvCount" metric properly set ✓ should have "headersBiggerThanContent" metric properly set ✓ should have "headersBiggerThanContent" offender(s) properly set --- modules/headers/headers.js | 14 +++++++------- test/integration-spec.yaml | 12 ++++++++++++ test/webroot/headers.html | 9 +++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 test/webroot/headers.html diff --git a/modules/headers/headers.js b/modules/headers/headers.js index 24eaabf0a..22d672082 100644 --- a/modules/headers/headers.js +++ b/modules/headers/headers.js @@ -14,8 +14,6 @@ module.exports = function(phantomas) { phantomas.setMetric('headersBiggerThanContent'); // @desc number of responses with headers part bigger than the response body - return; // TODO - function processHeaders(headers) { var res = { count: 0, @@ -23,16 +21,16 @@ module.exports = function(phantomas) { }; if (headers) { - headers.forEach(function(header) { + Object.keys(headers).forEach(function(key) { res.count++; - res.size += (header.name + ': ' + header.value + '\r\n').length; + res.size += (key + ': ' + headers[key] + '\r\n').length; }); } return res; } - phantomas.on('send', function(entry) { + phantomas.on('request', function(entry) { var headers = processHeaders(entry.headers); phantomas.incrMetric('headersCount', headers.count); @@ -45,6 +43,8 @@ module.exports = function(phantomas) { phantomas.on('recv', function(entry) { var headers = processHeaders(entry.headers); + // phantomas.log('Headers: <%s> %d bytes', entry.url, headers.size); + phantomas.incrMetric('headersCount', headers.count); phantomas.incrMetric('headersSize', headers.size); @@ -52,9 +52,9 @@ module.exports = function(phantomas) { phantomas.incrMetric('headersRecvSize', headers.size); // skip HTTP 204 No Content responses - if ((entry.status !== 204) && (headers.size > entry.contentLength)) { + if ((entry.status !== 204) && (headers.size > entry.transferedSize)) { phantomas.incrMetric('headersBiggerThanContent'); - phantomas.addOffender('headersBiggerThanContent', '%s (body: %s kB / headers: %s kB)', entry.url, (entry.contentLength / 1024).toFixed(2), (headers.size / 1024).toFixed(2)); + phantomas.addOffender('headersBiggerThanContent', {url: entry.url, contentSize: entry.transferedSize, headersSize: headers.size}); } }); }; diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index d7a2e5368..4202ceaea 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -335,3 +335,15 @@ - url: "/console-log.html" metrics: consoleMessages: 2 + +# headers +- url: "/headers.html" + metrics: + requests: 2 + headersCount: 20 + headersSentCount: 4 + headersRecvCount: 16 + headersBiggerThanContent: 1 + offenders: + headersBiggerThanContent: + - { url: 'http://127.0.0.1:8888/static/blank.gif', contentSize: 43, headersSize: 268 } diff --git a/test/webroot/headers.html b/test/webroot/headers.html new file mode 100644 index 000000000..04bbb446b --- /dev/null +++ b/test/webroot/headers.html @@ -0,0 +1,9 @@ + + Foo + + + +

+ Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? +

+ \ No newline at end of file From af42456ff0bd058b221029ca0e1c524810065ca7 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 15:24:03 +0100 Subject: [PATCH 086/248] domains: structured offenders --- modules/domains/domains.js | 6 +++--- test/integration-spec.yaml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/domains/domains.js b/modules/domains/domains.js index 2cef05395..abdb3a689 100644 --- a/modules/domains/domains.js +++ b/modules/domains/domains.js @@ -25,10 +25,10 @@ module.exports = function(phantomas) { phantomas.on('report', function() { var domainsRequests = new Stats(); - domains.sort().forEach(function(name, cnt) { - phantomas.addOffender('domains', '%s: %d request(s)', name, cnt); + domains.sort().forEach(function(domain, requests) { + phantomas.addOffender('domains', {domain, requests}); - domainsRequests.push(cnt); + domainsRequests.push(requests); }); if (domains.getLength() > 0) { diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 4202ceaea..956c5b9b1 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -143,12 +143,14 @@ - { functionName: 'css', arguments: '[{"background-color":"blue"}]', contextPath: 'body > div#foo' } jQueryDOMWriteReadSwitches: - { functionName: 'css', arguments: '["color"]', contextPath: 'body > div#foo > span.bar' } + # multiple jQuery "instances" (issue #435) - url: "/jquery-multiple.html" metrics: requests: 3 jsCount: 2 domains: 2 + maxRequestsPerDomain: 2 jQueryVersion: "1.11.1" # the last loaded version jQueryVersionsLoaded: 2 jQueryOnDOMReadyFunctions: 1 @@ -157,6 +159,10 @@ - { version: "2.1.1", url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js' } - { version: "1.11.1", url: 'http://code.jquery.com/jquery-1.11.1.js' } globalVariables: [ 'jQuery', '$' ] + domains: + - { domain: '127.0.0.1', requests: 2 } + - { domain: 'code.jquery.com', requests: 1 } + # --no-externals handling (issue #535) - url: "/jquery-multiple.html" label: "/jquery-multiple.html (with --no-externals)" From 8215220ea7d93172e6fd6cef1a2a1a55f365b6a3 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 15:34:47 +0100 Subject: [PATCH 087/248] Structured offenders for domQueries metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /dom-operations.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "htmlCount" metric properly set ✓ should have "htmlSize" metric properly set ✓ should have "cssCount" metric properly set ✓ should have "cssSize" metric properly set ✓ should have "jsCount" metric properly set ✓ should have "jsSize" metric properly set ✓ should have "domains" metric properly set ✓ should have "DOMqueries" metric properly set ✓ should have "DOMqueriesById" metric properly set ✓ should have "DOMqueriesByClassName" metric properly set ✓ should have "DOMqueriesByTagName" metric properly set ✓ should have "DOMqueriesByQuerySelectorAll" metric properly set ✓ should have "DOMinserts" metric properly set ✓ should have "DOMqueriesDuplicated" metric properly set ✓ should have "DOMqueriesAvoidable" metric properly set ✓ should have "DOMqueriesWithoutResults" metric properly set ✓ should have "DOMqueriesById" offender(s) properly set ✓ should have "DOMqueriesByClassName" offender(s) properly set ✓ should have "DOMqueriesByTagName" offender(s) properly set ✓ should have "DOMinserts" offender(s) properly set ✓ should have "DOMqueriesDuplicated" offender(s) properly set --- modules/domQueries/domQueries.js | 12 ++++++------ modules/domQueries/scope.js | 10 +++++----- test/integration-spec.yaml | 28 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/modules/domQueries/domQueries.js b/modules/domQueries/domQueries.js index 13a9cea20..70a48abd1 100644 --- a/modules/domQueries/domQueries.js +++ b/modules/domQueries/domQueries.js @@ -28,7 +28,7 @@ module.exports = phantomas => { if (hasNoResults === true) { phantomas.incrMetric('DOMqueriesWithoutResults'); - phantomas.addOffender('DOMqueriesWithoutResults', '%s (in %s) using %s', query, context, fnName); + phantomas.addOffender('DOMqueriesWithoutResults', {query, node: context, 'function': fnName}); } }); @@ -52,12 +52,12 @@ module.exports = phantomas => { } }); - phantomas.on('report', function() { - DOMqueries.sort().forEach(function(query, cnt) { - if (cnt > 1) { + phantomas.on('report', () => { + DOMqueries.sort().forEach((query, count) => { + if (count > 1) { phantomas.incrMetric('DOMqueriesDuplicated'); - phantomas.incrMetric('DOMqueriesAvoidable', cnt - 1); - phantomas.addOffender('DOMqueriesDuplicated', '%s: %d queries', query, cnt); + phantomas.incrMetric('DOMqueriesAvoidable', count - 1); + phantomas.addOffender('DOMqueriesDuplicated', {query, count}); } }); }); diff --git a/modules/domQueries/scope.js b/modules/domQueries/scope.js index 79163edb1..d0d8f29a4 100644 --- a/modules/domQueries/scope.js +++ b/modules/domQueries/scope.js @@ -10,7 +10,7 @@ // selectors by element ID phantomas.spy(Document.prototype, 'getElementById', function(results, id) { phantomas.incrMetric('DOMqueriesById'); - phantomas.addOffender('DOMqueriesById', '#%s (in %s)', id, '#document'); + phantomas.addOffender('DOMqueriesById', {id, node: phantomas.getDOMPath(this)}); querySpy('id', '#' + id, 'getElementById', '#document', (results === null)); }, true); @@ -20,7 +20,7 @@ var context = phantomas.getDOMPath(this); phantomas.incrMetric('DOMqueriesByClassName'); - phantomas.addOffender('DOMqueriesByClassName', '.%s (in %s)', className, context); + phantomas.addOffender('DOMqueriesByClassName', {class: className, node: context}); querySpy('class', '.' + className, 'getElementsByClassName', context, (results.length === 0)); } @@ -36,7 +36,7 @@ tagName = tagName.toLowerCase(); phantomas.incrMetric('DOMqueriesByTagName'); - phantomas.addOffender('DOMqueriesByTagName', '%s (in %s)', tagName, context); + phantomas.addOffender('DOMqueriesByTagName', {tag: tagName, node: phantomas.getDOMPath(this)}); querySpy('tag name', tagName, 'getElementsByTagName', context, (results.length === 0)); } @@ -49,7 +49,7 @@ var context = phantomas.getDOMPath(this); phantomas.incrMetric('DOMqueriesByQuerySelectorAll'); - phantomas.addOffender('DOMqueriesByQuerySelectorAll', '%s (in %s)', selector, context); + phantomas.addOffender('DOMqueriesByQuerySelectorAll', {selector, node: context}); querySpy('selector', selector, 'querySelectorAll', context, (results === null || results.length === 0)); } @@ -81,7 +81,7 @@ } phantomas.incrMetric('DOMinserts'); - phantomas.addOffender('DOMinserts', '"%s" appended to "%s"', appendedNodePath, destNodePath); + phantomas.addOffender('DOMinserts', {append: appendedNodePath, node: destNodePath}); phantomas.log('DOM insert: node "%s" appended to "%s"', appendedNodePath, destNodePath); } diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 956c5b9b1..fee22b0b8 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -30,6 +30,34 @@ DOMqueriesAvoidable: 3 DOMqueriesWithoutResults: 4 + offenders: + DOMqueriesById: + - { id: 'foo', node: '#document' } + - { id: 'foo', node: '#document' } + - { id: 'delete', node: '#document' } + - { id: 'foo', node: '#document' } + - { id: 'list1', node: '#document' } + - { id: 'list2', node: '#document' } + - { id: 'foobar', node: '#document' } + DOMqueriesByClassName: + - { class: 'barr', node: 'body' } + DOMqueriesByTagName: + - { tag: '*', node: 'div' } + - { tag: 'script', node: 'DocumentFragment > strong[0]' } + - { tag: 'script', node: 'DocumentFragment > strong[0]' } + - { tag: 'strong', node: '#document' } + - { tag: 'strong', node: '#document' } + - { tag: 'li', node: 'body > ul#list1' } + - { tag: 'li', node: 'body > ul#list2' } + - { tag: 'a', node: 'div' } + - { tag: 'a', node: 'div' } + DOMinserts: + - { append: 'html > div[2]', node: 'html' } + - { append: 'body > p#foo > strong[0]', node: 'body > p#foo' } + DOMqueriesDuplicated: + - { query: 'id "#foo" (in #document)', count: 3 } + - { query: 'tag name "strong" (in #document)', count: 2 } + # DOM mutations - url: "/dom-operations.html" metrics: From 438f37f4ca7b681c40fa2a671531eea8c8fe6897 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 15:38:15 +0100 Subject: [PATCH 088/248] nodesWithInlineCSS: structured offender --- modules/domComplexity/scope.js | 2 +- test/integration-spec.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/domComplexity/scope.js b/modules/domComplexity/scope.js index d121b585b..5791b0e11 100644 --- a/modules/domComplexity/scope.js +++ b/modules/domComplexity/scope.js @@ -24,7 +24,7 @@ const path = phantomas.getDOMPath(node, true /* dontGoUpTheDom */ ); phantomas.incrMetric('nodesWithInlineCSS'); - phantomas.addOffender('nodesWithInlineCSS', {path: path, css: node.getAttribute('style')}); + phantomas.addOffender('nodesWithInlineCSS', {node: path, css: node.getAttribute('style')}); }); })(); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index fee22b0b8..8c091c383 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -82,6 +82,9 @@ - 'http://127.0.0.1:8888/static/mdn.png' - 'http://127.0.0.1:8888/static/blank.gif' - 'http://127.0.0.1:8888/static/example.svg' + nodesWithInlineCSS: + - { css: 'color: blue', node: 'p#foo' } + # document height - url: "/document-height.html" metrics: From 500ce11ed26e07995757acc93d93d9081c406a58 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 15:42:40 +0100 Subject: [PATCH 089/248] gzipRequests: structured offender --- core/modules/requestsMonitor/requestsMonitor.js | 4 ++-- test/integration-spec.yaml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index ec881dda0..4283c134a 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -279,12 +279,12 @@ module.exports = function(phantomas) { phantomas.incrMetric('requests'); phantomas.incrMetric('bodySize', entry.bodySize); - phantomas.incrMetric('contentLength', entry.contentLength); + phantomas.incrMetric('contentLength', entry.transferedSize); } if (entry.gzip) { phantomas.incrMetric('gzipRequests'); - phantomas.addOffender('gzipRequests', '%s (gzip: %s kB / uncompressed: %s kB)', entry.url, (entry.contentLength/1024).toFixed(2), (entry.bodySize/1024).toFixed(2)); + phantomas.addOffender('gzipRequests', {url: entry.url, transferedSize: entry.transferedSize, bodySize: entry.bodySize}); } if (entry.isSSL) { diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 8c091c383..786704eeb 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -130,9 +130,11 @@ cssParsingErrors: 0 cssLength: 22 cssRules: 1 + # jQuery and events (#440 / #451) - url: "/jquery.html" metrics: + gzipRequests: 1 jQueryVersion: "2.1.1" jQueryOnDOMReadyFunctions: 1 jQueryWindowOnLoadFunctions: 1 @@ -142,6 +144,8 @@ eventsDispatched: 1 eventsScrollBound: 2 offenders: + gzipRequests: + - { url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js', transferedSize: 29415, bodySize: 84245 } eventsDispatched: - { eventType: 'click', path: 'body > div#foo > span.bar' } eventsBound: From ecdfa634785891b6ab6ce41185d113d86d5cc7db Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 15:49:03 +0100 Subject: [PATCH 090/248] base64Size: add a test case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /dom-complexity.html ✓ should be generated ✓ should have "base64Count" metric properly set ✓ should have "base64Size" metric properly set --- modules/assetsTypes/assetsTypes.js | 6 +++--- test/integration-spec.yaml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/assetsTypes/assetsTypes.js b/modules/assetsTypes/assetsTypes.js index c281fab0f..0bc482b17 100644 --- a/modules/assetsTypes/assetsTypes.js +++ b/modules/assetsTypes/assetsTypes.js @@ -28,7 +28,7 @@ module.exports = function(phantomas) { phantomas.setMetric(key + 'Size'); }); - phantomas.on('recv', function(entry, res) { + phantomas.on('recv', (entry, _) => { var size = entry.transferedSize; phantomas.incrMetric(entry.type + 'Count'); @@ -41,8 +41,8 @@ module.exports = function(phantomas) { }); }); - phantomas.on('base64recv', function(entry, res) { + phantomas.on('base64recv', (entry, _) => { phantomas.incrMetric('base64Count'); - phantomas.incrMetric('base64Size', entry.contentLength); + phantomas.incrMetric('base64Size', entry.bodySize); }); }; diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 786704eeb..3830c293d 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -66,6 +66,8 @@ # DOM complexity - url: "/dom-complexity.html" metrics: + base64Count: 1 + base64Size: 43 bodyHTMLSize: 709 commentsSize: 85 DOMelementsCount: 11 From 154679defc77603e9129992e1ac6d93d259f0c44 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 20:02:35 +0100 Subject: [PATCH 091/248] cachingTooShort: add a test case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /caching.html ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "cachingTooShort" metric properly set ✓ should have "cachingTooShort" offender(s) properly set --- modules/caching/caching.js | 6 +++--- test/integration-spec.yaml | 9 +++++++++ test/webroot/caching.html | 3 +++ 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 test/webroot/caching.html diff --git a/modules/caching/caching.js b/modules/caching/caching.js index a84f4a563..6974bec37 100644 --- a/modules/caching/caching.js +++ b/modules/caching/caching.js @@ -38,7 +38,7 @@ module.exports = function(phantomas) { case 'x-pass-expires': case 'x-pass-cache-control': phantomas.incrMetric('oldCachingHeaders'); // @desc number of responses with old, HTTP 1.0 caching headers (Expires and Pragma) - phantomas.addOffender('oldCachingHeaders', url + ' - ' + headerName + ': ' + value); + phantomas.addOffender('oldCachingHeaders', {url, headerName, value}); if (ttl === false) { headerDate = Date.parse(value); if (headerDate) ttl = Math.round((headerDate - now) / 1000); @@ -73,7 +73,7 @@ module.exports = function(phantomas) { phantomas.addOffender('cachingDisabled', entry.url); } else if (ttl < 7 * 86400) { phantomas.incrMetric('cachingTooShort'); - phantomas.addOffender('cachingTooShort', entry.url + ' cached for ' + ttl + ' s'); + phantomas.addOffender('cachingTooShort', {url: entry.url, ttl}); } else { // long TTL, suggest the use of Cache-Control: immutable (issue #683) for (headerName in entry.headers) { @@ -82,7 +82,7 @@ module.exports = function(phantomas) { if (headerName.toLowerCase() === 'cache-control') { if (/,\s?immutable/.test(value) === false) { phantomas.incrMetric('cachingUseImmutable'); - phantomas.addOffender('cachingUseImmutable', entry.url + ' cached for ' + ttl + ' s'); + phantomas.addOffender('cachingUseImmutable', {url: entry.url, ttl}); } else { phantomas.log('caching: Cache-Control: immutable used for <%s>', entry.url); } diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 3830c293d..8c7de406e 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -390,3 +390,12 @@ offenders: headersBiggerThanContent: - { url: 'http://127.0.0.1:8888/static/blank.gif', contentSize: 43, headersSize: 268 } + +# caching +- url: "/caching.html" + metrics: + requests: 2 + cachingTooShort: 1 + offenders: + cachingTooShort: + - { url: 'http://127.0.0.1:8888/static/mdn.png', ttl: 84600 } diff --git a/test/webroot/caching.html b/test/webroot/caching.html new file mode 100644 index 000000000..c24d5609d --- /dev/null +++ b/test/webroot/caching.html @@ -0,0 +1,3 @@ + + + \ No newline at end of file From 4915aa7d85d72d6faacd951ed7a024160cb2429b Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 19 Jan 2019 20:32:53 +0100 Subject: [PATCH 092/248] Allow modules tests to assert offenders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ♢ caching caching not specified ✓ sets "cachingNotSpecified" metric correctly ✓ sets "cachingNotSpecified" offender(s) correctly old caching header used (Expires) ✓ sets "oldCachingHeaders" metric correctly ✓ sets "cachingNotSpecified" metric correctly ✓ sets "cachingDisabled" metric correctly ✓ sets "cachingDisabled" offender(s) correctly ... --- test/modules/caching-test.js | 23 ++++++++++++++++++++--- test/modules/mock.js | 26 ++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/test/modules/caching-test.js b/test/modules/caching-test.js index d046fa8d5..3892c306d 100644 --- a/test/modules/caching-test.js +++ b/test/modules/caching-test.js @@ -1,14 +1,15 @@ /** * Test caching module */ -var vows = require('vows'), - assert = require('assert'), - mock = require('./mock'); +const vows = require('vows'), + mock = require('./mock'), + URL = 'http://127.0.0.1:8888/static/mdn.png'; vows.describe('caching'). addBatch({ 'caching not specified': mock.getContext('caching', function(phantomas) { return phantomas.recv({ + url: URL, isImage: true, headers: { 'X-Foo': 'Bar' @@ -16,9 +17,12 @@ addBatch({ }).report(); }, { 'cachingNotSpecified': 1 + }, { + 'cachingNotSpecified': [URL] }), 'old caching header used (Expires)': mock.getContext('caching', function(phantomas) { return phantomas.recv({ + url: URL, isImage: true, headers: { 'Expires': 'Wed, 21 Oct 2015 07:28:00 GMT' @@ -28,9 +32,12 @@ addBatch({ 'oldCachingHeaders': 1, 'cachingNotSpecified': 0, 'cachingDisabled': 1, + }, { + 'cachingDisabled': [URL] }), 'old caching header used (Pragma)': mock.getContext('caching', function(phantomas) { return phantomas.recv({ + url: URL, isImage: true, headers: { 'Pragma': 'no-cache' @@ -39,9 +46,13 @@ addBatch({ }, { 'oldCachingHeaders': 1, 'cachingNotSpecified': 1 + }, { + 'oldCachingHeaders': [{headerName: 'Pragma', url: 'http://127.0.0.1:8888/static/mdn.png', value: 'no-cache'}], + 'cachingNotSpecified': [URL] }), 'caching too short': mock.getContext('caching', function(phantomas) { return phantomas.recv({ + url: URL, isImage: true, headers: { 'Cache-Control': 'max-age=600' @@ -50,9 +61,12 @@ addBatch({ }, { 'cachingTooShort': 1, 'cachingUseImmutable': 0, + }, { + 'cachingTooShort': [{url: URL, ttl: 600}], }), 'caching not too short (but without immutable)': mock.getContext('caching', function(phantomas) { return phantomas.recv({ + url: URL, isImage: true, headers: { 'Cache-Control': 'max-age=2592000' // 30 days @@ -61,9 +75,12 @@ addBatch({ }, { 'cachingTooShort': 0, 'cachingUseImmutable': 1, + }, { + 'cachingUseImmutable': [{ url: URL, ttl: 2592000 }] }), 'caching not too short (and use immutable)': mock.getContext('caching', function(phantomas) { return phantomas.recv({ + url: URL, isImage: true, headers: { 'Cache-Control': 'max-age=2592000, immutable' // 30 days diff --git a/test/modules/mock.js b/test/modules/mock.js index 77309b33a..2e16c011c 100644 --- a/test/modules/mock.js +++ b/test/modules/mock.js @@ -9,6 +9,7 @@ var phantomas = function(name) { this.emitter = new(require('events').EventEmitter)(); this.wasEmitted = {}; this.metrics = {}; + this.offenders = {}; this.log = debug('phantomas:test:' + name); }; @@ -61,6 +62,15 @@ phantomas.prototype = { return this.getMetric(name) === val; }, + addOffender: function(name, value) { + this.offenders[name] = this.offenders[name] || []; + this.offenders[name].push(value); + }, + + getOffenders: function(name) { + return this.offenders[name]; + }, + // mock core PhantomJS events sendRequest: function(req) { req = req || {}; @@ -122,9 +132,6 @@ phantomas.prototype = { }, // noop mocks - addOffender: function() { - this.log('addOffenders: %j', Array.prototype.slice.apply(arguments)); - }, log: noop, echo: noop, evaluate: noop, @@ -152,6 +159,12 @@ function assertMetric(name, value) { }; } +function assertOffender(name, value) { + return function(phantomas) { + assert.deepStrictEqual(phantomas.getOffenders(name), value); + }; +} + module.exports = { initModule: function(name) { return initModule(name); @@ -162,7 +175,7 @@ module.exports = { assertMetric: assertMetric, - getContext: function(moduleName, topic, metricsCheck) { + getContext: function(moduleName, topic, metricsCheck, offendersCheck) { var phantomas = initModule(moduleName), context = {}; @@ -175,6 +188,11 @@ module.exports = { context[check] = assertMetric(name, metricsCheck[name]); }); + Object.keys(offendersCheck || {}).forEach(function(name) { + var check = 'sets "' + name + '" offender(s) correctly'; + context[check] = assertOffender(name, offendersCheck[name]); + }); + return context; } }; From a2654a664033b3740687313fe22b9f820b6575e1 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 11:55:21 +0100 Subject: [PATCH 093/248] browser: add response.getContent() helper --- lib/browser.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/browser.js b/lib/browser.js index 98421d681..e2c2e2bb9 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -124,6 +124,17 @@ Browser.prototype.init = async () => { response.chunks = meta._chunks; response._timestamp = data.timestamp; + // https://chromedevtools.github.io/devtools-protocol/tot/Network#method-getResponseBody + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#cdpsessionsendmethod-params + response.getContent = (async () => { + networkDebug('Getting content for #%s', data.requestId); + const resp = await this.cdp.send('Network.getResponseBody', {requestId: data.requestId}); + networkDebug('Content for #%s received (%d bytes)', data.requestId, resp.body.length); + + //console.log('getContent()', resp); + return resp.body; + }).bind(this); + //networkDebug('Network.loadingFinished', data); networkDebug('Network.loadingFinished < %s %s %s %s (%f kB fetched, %f kB uncompressed)', response.protocol, response.status, response.statusText, response.url, From be4691a810ecb7e03a7ed575c8e567a6fd11babb Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 11:59:36 +0100 Subject: [PATCH 094/248] browser.js: log how many bytes where fetched via netrwork --- lib/browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/browser.js b/lib/browser.js index e2c2e2bb9..04038399b 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -138,7 +138,7 @@ Browser.prototype.init = async () => { //networkDebug('Network.loadingFinished', data); networkDebug('Network.loadingFinished < %s %s %s %s (%f kB fetched, %f kB uncompressed)', response.protocol, response.status, response.statusText, response.url, - 1.0 * response.encodedDataLength / 1024, + 1.0 * (response.encodedDataLength || response.headers['content-length']) / 1024, 1.0 * response.dataLength / 1024); this.events.emit('response', response); From a1ac85f3a769e6409c9cf75e1da1fbafb7eccc72 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 14:29:15 +0100 Subject: [PATCH 095/248] Install analyze-css v0.12.7 --- package-lock.json | 134 +++++++++++++++++++++++++++++++++++++++++++--- package.json | 1 + 2 files changed, 129 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index b30d560e8..b7cafbe38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,25 @@ "es6-promisify": "^5.0.0" } }, + "analyze-css": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/analyze-css/-/analyze-css-0.12.7.tgz", + "integrity": "sha512-UAuqBMSxqoXyWXvEsrjLBD1U1sylyl8S2EBTWgRq7Od3odN6Wu51RvEci731ln5szOZ6yJijWtE2H6eQQk54Dg==", + "requires": { + "cli": "^1.0.1", + "css": "^2.2.4", + "css-shorthand-properties": "^1.1.1", + "debug": "^4.1.1", + "fast-stats": "0.0.5", + "glob": "^7.1.3", + "http-proxy-agent": "^2.1.0", + "node-fetch": "^2.3.0", + "onecolor": "^3.1.0", + "optimist": "0.6.x", + "slick": "~1.12.1", + "specificity": "^0.4.1" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -50,6 +69,11 @@ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, + "atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -73,7 +97,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/cli/-/cli-1.0.1.tgz", "integrity": "sha1-IoF1NPJL+klQw01TLUjsvGIbjBQ=", - "dev": true, "requires": { "exit": "0.1.2", "glob": "^7.1.1" @@ -137,6 +160,22 @@ "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=", "dev": true }, + "css": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz", + "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==", + "requires": { + "inherits": "^2.0.3", + "source-map": "^0.6.1", + "source-map-resolve": "^0.5.2", + "urix": "^0.1.0" + } + }, + "css-shorthand-properties": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/css-shorthand-properties/-/css-shorthand-properties-1.1.1.tgz", + "integrity": "sha512-Md+Juc7M3uOdbAFwOYlTrccIZ7oCFuzrhKYQjdeUEW/sE1hv17Jp/Bws+ReOPpGVBTYCBoYo+G17V5Qo8QQ75A==" + }, "date-now": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", @@ -151,6 +190,11 @@ "ms": "^2.1.1" } }, + "decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" + }, "diff": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.8.tgz", @@ -280,8 +324,7 @@ "exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=", - "dev": true + "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" }, "extract-zip": { "version": "1.6.7", @@ -315,6 +358,11 @@ "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=", "dev": true }, + "fast-stats": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/fast-stats/-/fast-stats-0.0.5.tgz", + "integrity": "sha512-HtS5uSqMiwfxFFyukKP/F0f3o8/8oqHtbInsaq2s0+V2J2MEHGyukWajWqzKS57sWLTOgJ7bKMRhA4fG5cTQ3Q==" + }, "fd-slicer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", @@ -423,6 +471,30 @@ "requires-port": "^1.0.0" } }, + "http-proxy-agent": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", + "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", + "requires": { + "agent-base": "4", + "debug": "3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "http-serve": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/http-serve/-/http-serve-1.0.1.tgz", @@ -575,6 +647,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, + "node-fetch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz", + "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==" + }, "nopt": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", @@ -593,6 +670,11 @@ "wrappy": "1" } }, + "onecolor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/onecolor/-/onecolor-3.1.0.tgz", + "integrity": "sha512-YZSypViXzu3ul5LMu/m6XjJ9ol8qAy9S2VjHl5E6UlhUH1KGKWabyEJifn0Jjpw23bYDzC2ucKMPGiH5kfwSGQ==" + }, "opener": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/opener/-/opener-1.4.3.tgz", @@ -603,7 +685,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "dev": true, "requires": { "minimist": "~0.0.1", "wordwrap": "~0.0.2" @@ -719,6 +800,11 @@ "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", "dev": true }, + "resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" + }, "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", @@ -750,6 +836,38 @@ "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=", "dev": true }, + "slick": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/slick/-/slick-1.12.2.tgz", + "integrity": "sha1-vQSN23TefRymkV+qSldXCzVQwtc=" + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "source-map-resolve": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", + "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", + "requires": { + "atob": "^2.1.1", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "source-map-url": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", + "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=" + }, + "specificity": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/specificity/-/specificity-0.4.1.tgz", + "integrity": "sha512-1klA3Gi5PD1Wv9Q0wUoOQN1IWAuPu0D1U03ThXTr0cJ20+/iq2tHSDnK7Kk/0LXJ1ztUB2/1Os0wKmfyNgUQfg==" + }, "sprintf-js": { "version": "1.0.3", "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -784,6 +902,11 @@ "qs": "~2.3.3" } }, + "urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" + }, "url-join": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/url-join/-/url-join-2.0.5.tgz", @@ -809,8 +932,7 @@ "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" }, "wrappy": { "version": "1.0.2", diff --git a/package.json b/package.json index 626cf1ffe..a26754951 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "node": ">=8.0" }, "dependencies": { + "analyze-css": "^0.12.7", "debug": "^4.1.1", "puppeteer": "^1.11.0" }, From 8973a2a3ac5e0af8217aa23e6e5a0bc9063d9c5c Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 14:36:21 +0100 Subject: [PATCH 096/248] Add addToAvgMetric function to Results object --- core/results.js | 18 ++++++++++++++++++ test/results-test.js | 8 ++++++++ 2 files changed, 26 insertions(+) diff --git a/core/results.js b/core/results.js index 003581344..e393e92c7 100644 --- a/core/results.js +++ b/core/results.js @@ -5,10 +5,18 @@ */ 'use strict'; +function getAverage(arr) { + var sum = arr.reduce(function(a, b) { + return a + b; + }); + return sum / arr.length; +} + module.exports = function (data) { var asserts = {}, generator = '', metrics = {}, + metricsAvgStorage = {}, offenders = {}, url; @@ -37,6 +45,16 @@ module.exports = function (data) { var currVal = this.getMetric(name) || 0; this.setMetric(name, currVal + (typeof incr === 'number' ? incr : 1)); }, + // push a value and update the metric if the current average value + addToAvgMetric: function(name, value) { + if (typeof metricsAvgStorage[name] === 'undefined') { + metricsAvgStorage[name] = []; + } + + metricsAvgStorage[name].push(value); + + this.setMetric(name, getAverage(metricsAvgStorage[name])); + }, getMetric: function(name) { return metrics[name]; }, diff --git a/test/results-test.js b/test/results-test.js index d7a149b75..0c165ec85 100644 --- a/test/results-test.js +++ b/test/results-test.js @@ -21,6 +21,14 @@ vows.describe('Results wrapper').addBatch({ results.setMetric('foo', 'bar'); assert.strictEqual(results.getMetric('foo'), 'bar'); }, + 'avarage should be correctly calculated': () => { + const results = new Results(); + results.addToAvgMetric('foo', 2); + results.addToAvgMetric('foo', 1); + results.addToAvgMetric('bar', 4); + assert.strictEqual(results.getMetric('foo'), 1.5); + assert.strictEqual(results.getMetric('bar'), 4); + }, 'should be correctly set (with no casting)': function(results) { results.setMetric('bar', null); assert.strictEqual(results.getMetric('bar'), null); From f9d5b2d3d84c568546608dabc2f071e3b25b9139 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 14:51:38 +0100 Subject: [PATCH 097/248] scope: expose addToAvgMetric and querySelectorAll helpers --- lib/index.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/index.js b/lib/index.js index a43aaa98b..4e89b315d 100644 --- a/lib/index.js +++ b/lib/index.js @@ -77,10 +77,17 @@ function phantomas(url, opts) { addOffender: results.addOffender.bind(results), incrMetric: results.incrMetric.bind(results), setMetric: results.setMetric, + addToAvgMetric: results.addToAvgMetric, // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args evaluate: page.evaluate.bind(page), + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageselector-1 + querySelectorAll: async (selector) => { + debug('querySelectorAll("%s")', selector) + return page.$$(selector); + }, + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args injectJs: async (script) => { const debug = require('debug')('phantomas:injectJs'), From 0712b4045c34e9bba802de302f15a5afcaf18f50 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 14:51:50 +0100 Subject: [PATCH 098/248] analyze-css: use a npm module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /broken-css.html ✓ should be generated ✓ should have "cssParsingErrors" metric properly set ✓ should have "cssLength" metric properly set ✓ should have "cssRules" metric properly set /dom-operations.html ✓ should be generated ✓ should have "cssParsingErrors" metric properly set ✓ should have "cssLength" metric properly set ✓ should have "cssRules" metric properly set ✗ Broken » 315 honored ∙ 6 broken (15.264s) --- modules/analyzeCss/analyzeCss.js | 66 ++++++++++++++++++-------------- test/integration-spec.yaml | 3 ++ 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/modules/analyzeCss/analyzeCss.js b/modules/analyzeCss/analyzeCss.js index a76be87e2..b4fbe1613 100644 --- a/modules/analyzeCss/analyzeCss.js +++ b/modules/analyzeCss/analyzeCss.js @@ -45,11 +45,16 @@ 'use strict'; module.exports = function(phantomas) { - if (!phantomas.getParam('analyze-css')) { + if (phantomas.getParam('analyze-css') !== true) { phantomas.log('To enable CSS in-depth metrics please run phantomas with --analyze-css option'); return; } -/** + + // load analyze-css module + // https://www.npmjs.com/package/analyze-css + const analyzer = require('analyze-css'); + phantomas.log('Using version %s', analyzer.version); + phantomas.setMetric('cssParsingErrors'); // @desc number of CSS files (or embeded CSS) that failed to be parse by analyze-css @optional phantomas.setMetric('cssInlineStyles'); // @desc number of inline styles @optional @@ -65,13 +70,10 @@ module.exports = function(phantomas) { return f + str.substr(1); } - // run analyze-css "binary" installed by npm - function analyzeCss(options) { - var system = require('system'), - isWindows = (system.os.name === 'windows'), - binary = system.env.ANALYZE_CSS_BIN, - proxy; + function analyzeCss(css, context) { + var proxy; + /** // force JSON output format options.push('--json'); @@ -91,9 +93,13 @@ module.exports = function(phantomas) { options.push('--proxy', proxy); } + **/ - phantomas.runScript(binary, options, function(err, results) { - var offenderSrc = (options[0] === '--url') ? '<' + options[1] + '>' : '[inline CSS]'; + // https://www.npmjs.com/package/analyze-css#commonjs-module + var options = {}; + + new analyzer(css, options, function(err, results) { + var offenderSrc = context || '[inline CSS]'; if (err !== null) { phantomas.log('analyzeCss: sub-process failed! - %s', err); @@ -117,7 +123,7 @@ module.exports = function(phantomas) { return; } - phantomas.log('analyzeCss: using ' + results.generator); + phantomas.log('Got results for %s from %s', context, results.generator); var metrics = results.metrics || {}, offenders = results.offenders || {}; @@ -136,19 +142,7 @@ module.exports = function(phantomas) { // and add offenders if (typeof offenders[metric] !== 'undefined') { offenders[metric].forEach(function(msg) { - var re = / @ \d+:\d+$/; - - // add the file name to offenders (issue #442) - // the message ends with something similar to " @ 25:1" - if (re.test(msg)) { - msg = msg.replace(re, ' ' + offenderSrc + '$&'); - } - // add file url in offenders for cssDuplicatedSelectors (issue #693) - else if (metricPrefixed === 'cssDuplicatedSelectors') { - msg += ' ' + offenderSrc; - } - - phantomas.addOffender(metricPrefixed, msg); + phantomas.addOffender(metricPrefixed, {url: offenderSrc, value: msg}); }); } // add more offenders (#578) @@ -163,7 +157,7 @@ module.exports = function(phantomas) { case 'cssSpecificityIdAvg': case 'cssSpecificityClassAvg': case 'cssSpecificityTagAvg': - phantomas.addOffender(metricPrefixed, offenderSrc + ': ' + metrics[metric]); + phantomas.addOffender(metricPrefixed, {url: offenderSrc, value: metrics[metric]}); break; } } @@ -171,13 +165,29 @@ module.exports = function(phantomas) { }); } - phantomas.on('recv', function(entry, res) { + phantomas.on('recv', async (entry, res) => { if (entry.isCSS) { phantomas.log('CSS: analyzing <%s>...', entry.url); - analyzeCss(['--url', entry.url]); + + // get the response content and pass it to the analyze-css module + var content = await res.getContent(); + analyzeCss(content, entry.url); } }); + /** + phantomas.on('milestone', async (milestone) => { + if (milestone !== 'domComplete') { + return; + } + + phantomas.log('domComplete: checking inline styles'); + + const nodes = await phantomas.querySelectorAll('[style]'); + console.log(nodes); + }); + + /** phantomas.on('loadFinished', function() { var fs = require('fs'); @@ -227,5 +237,5 @@ module.exports = function(phantomas) { analyzeCss(['--file', path]); }); }); -**/ + **/ }; diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 8c7de406e..9d53b6523 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -104,6 +104,7 @@ - url: "/dom-id.html" metrics: DOMidDuplicated: 2 # foo and bar + # inline styles (#397 / #694) - url: "/inline-css.html" options: @@ -117,6 +118,7 @@ cssRules: 4 cssImportants: 1 DOMqueries: 0 + # analyze-css fails to parse a file (#404) - url: "/broken-css.html" options: @@ -125,6 +127,7 @@ cssParsingErrors: 3 cssLength: 39 cssRules: 1 + - url: "/dom-operations.html" options: "analyze-css": true From 118ce1b02b7838bed4eccdbfa264c4640df9616e Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 20 Jan 2019 21:11:31 +0100 Subject: [PATCH 099/248] analyzeCss: handle inline CSS + all tests now pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✓ OK » 323 honored (18.380s) --- modules/analyzeCss/analyzeCss.js | 67 ++------------------------------ modules/analyzeCss/scope.js | 24 ++++++++++++ test/integration-spec.yaml | 13 +++++++ 3 files changed, 41 insertions(+), 63 deletions(-) create mode 100644 modules/analyzeCss/scope.js diff --git a/modules/analyzeCss/analyzeCss.js b/modules/analyzeCss/analyzeCss.js index b4fbe1613..112c61daa 100644 --- a/modules/analyzeCss/analyzeCss.js +++ b/modules/analyzeCss/analyzeCss.js @@ -123,7 +123,7 @@ module.exports = function(phantomas) { return; } - phantomas.log('Got results for %s from %s', context, results.generator); + phantomas.log('Got results for %s from %s', offenderSrc, results.generator); var metrics = results.metrics || {}, offenders = results.offenders || {}; @@ -175,67 +175,8 @@ module.exports = function(phantomas) { } }); - /** - phantomas.on('milestone', async (milestone) => { - if (milestone !== 'domComplete') { - return; - } - - phantomas.log('domComplete: checking inline styles'); - - const nodes = await phantomas.querySelectorAll('[style]'); - console.log(nodes); - }); - - /** - phantomas.on('loadFinished', function() { - var fs = require('fs'); - - // get the content of inline CSS (issue #397) - var inlineCss = phantomas.evaluate(function() { - return (function(phantomas) { - phantomas.spyEnabled(false, 'looking for inline styles'); - - var styles = document.getElementsByTagName('style'), - content = [], - type; - - for (var i = 0, len = styles.length; i < len; i++) { - type = styles[i].getAttribute('type') || 'text/css'; // ignore inline + + +

Foo

+

test

+ From f17b04a40928e095a554fc438669675319e4a0c4 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 12:21:06 +0100 Subject: [PATCH 155/248] Detect brotli compression https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#Directives Add an assert for "assetsNotGzipped" metric. Metrics tests coverage: from 55.81% to 56.40% --- core/modules/requestsMonitor/requestsMonitor.js | 8 +++++++- lib/metadata/metadata.json | 2 +- test/integration-spec.yaml | 3 +++ test/modules/requestsMonitor-test.js | 9 +++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 5af113c06..a0c87fd94 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -246,13 +246,19 @@ module.exports = function(phantomas) { break; // content compression + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#Directives case 'content-encoding': - if (headerValue === 'gzip') { + if (headerValue === 'gzip' || headerValue === 'br') { entry.gzip = true; phantomas.log('Response compressed with %s, %f kB -> %f kB (x%f)', headerValue, entry.bodySize / 1024, entry.transferedSize / 1024, entry.bodySize / entry.transferedSize); } + + // A format using the Brotli algorithm. + if (headerValue === 'br') { + entry.brotli = true; + } break; // detect cookies (issue #92) diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 7a6b100bf..5cc8ab9b6 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -1091,7 +1091,7 @@ "offenders": true, "unit": "number", "module": "staticAssets", - "testsCovered": false + "testsCovered": true }, "assetsWithQueryString": { "desc": "number of static assets requested with query string (e.g. ?foo) in URL", diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index b00922624..c830eef4a 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -632,6 +632,9 @@ - { domain: '127.0.0.1', requests: 1 } - { domain: 'fonts.googleapis.com', requests: 1 } - { domain: 'fonts.gstatic.com', requests: 1 } + assetsNotGzipped: + # CSS response is compressed using "brotli" + - 'http://127.0.0.1:8888/https-fonts.html (HTML)' # # integration-test-extra section diff --git a/test/modules/requestsMonitor-test.js b/test/modules/requestsMonitor-test.js index bb873d584..8ed7ee230 100644 --- a/test/modules/requestsMonitor-test.js +++ b/test/modules/requestsMonitor-test.js @@ -205,5 +205,14 @@ vows.describe('requestMonitor').addBatch({ headers: {'Content-Encoding': 'gzip'} }), 'gzip is set': assertField('gzip', true) + }, + 'Brotli responses are detected': { + topic: sendReq(undefined, { + // A format using the Brotli algorithm. + // https://fonts.googleapis.com/css?family=IBM+Plex+Sans+Condensed + headers: {'content-encoding': 'br'} + }), + 'gzip is set': assertField('gzip', true), + 'brotli is set': assertField('brotli', true) } }).export(module); From d5494edc1c3cdb63c6ec9bb934d87481d257e1fa Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 14:30:59 +0100 Subject: [PATCH 156/248] staticAssets: add an integration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /assets.html ✓ should be generated ✓ should have "assetsWithQueryString" offender(s) properly set ✓ should have "smallImages" offender(s) properly set ✓ should have "smallCssFiles" offender(s) properly set Metrics tests coverage: 58.14% --- lib/metadata/metadata.json | 10 +++------- modules/staticAssets/staticAssets.js | 30 +++++++++++++--------------- test/integration-spec.yaml | 12 ++++++++++- test/modules/staticAssets-test.js | 8 ++++---- test/webroot/assets.html | 8 ++++++++ 5 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 test/webroot/assets.html diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 5cc8ab9b6..28eb3ec49 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -1087,7 +1087,6 @@ }, "assetsNotGzipped": { "desc": "number of static assets that were not gzipped", - "unreliable": true, "offenders": true, "unit": "number", "module": "staticAssets", @@ -1098,7 +1097,7 @@ "offenders": true, "unit": "number", "module": "staticAssets", - "testsCovered": false + "testsCovered": true }, "assetsWithCookies": { "desc": "number of static assets requested from domains with cookie set", @@ -1109,23 +1108,20 @@ }, "smallImages": { "desc": "number of images smaller than 2 KiB that can be base64 encoded", - "unreliable": true, "offenders": true, "unit": "number", "module": "staticAssets", - "testsCovered": false + "testsCovered": true }, "smallCssFiles": { "desc": "number of CSS assets smaller than 2 KiB that can be inlined or merged", - "unreliable": true, "offenders": true, "unit": "number", "module": "staticAssets", - "testsCovered": false + "testsCovered": true }, "smallJsFiles": { "desc": "number of JS assets smaller than 2 KiB that can be inlined or merged", - "unreliable": true, "offenders": true, "unit": "number", "module": "staticAssets", diff --git a/modules/staticAssets/staticAssets.js b/modules/staticAssets/staticAssets.js index edcb36caf..d04ea9894 100644 --- a/modules/staticAssets/staticAssets.js +++ b/modules/staticAssets/staticAssets.js @@ -13,17 +13,16 @@ module.exports = function(phantomas) { // TODO: use 3pc database with tracking services trackingUrls = /google-analytics.com\/__utm.gif|pixel.quantserve.com\/pixel/; - phantomas.setMetric('assetsNotGzipped'); // @desc number of static assets that were not gzipped @unreliable + phantomas.setMetric('assetsNotGzipped'); // @desc number of static assets that were not gzipped phantomas.setMetric('assetsWithQueryString'); // @desc number of static assets requested with query string (e.g. ?foo) in URL phantomas.setMetric('assetsWithCookies'); // @desc number of static assets requested from domains with cookie set - phantomas.setMetric('smallImages'); // @desc number of images smaller than 2 KiB that can be base64 encoded @unreliable - phantomas.setMetric('smallCssFiles'); // @desc number of CSS assets smaller than 2 KiB that can be inlined or merged @unreliable - phantomas.setMetric('smallJsFiles'); // @desc number of JS assets smaller than 2 KiB that can be inlined or merged @unreliable + phantomas.setMetric('smallImages'); // @desc number of images smaller than 2 KiB that can be base64 encoded + phantomas.setMetric('smallCssFiles'); // @desc number of CSS assets smaller than 2 KiB that can be inlined or merged + phantomas.setMetric('smallJsFiles'); // @desc number of JS assets smaller than 2 KiB that can be inlined or merged phantomas.setMetric('multipleRequests'); // @desc number of static assets that are requested more than once phantomas.on('recv', entry => { - var isContent = (entry.status === 200), - sizeFormatted; + var isContent = (entry.status === 200); // mark domains with cookie set if (entry.hasCookies) { @@ -39,7 +38,7 @@ module.exports = function(phantomas) { if (entry.isImage || entry.isJS || entry.isCSS) { if (entry.url.indexOf('?') > -1) { phantomas.incrMetric('assetsWithQueryString'); - phantomas.addOffender('assetsWithQueryString', entry.url + ' (' + entry.type.toUpperCase() + ')'); + phantomas.addOffender('assetsWithQueryString', {url: entry.url, contentType: entry.contentType}); } } @@ -47,26 +46,25 @@ module.exports = function(phantomas) { if (entry.isJS || entry.isCSS || entry.isHTML || entry.isJSON || entry.isSVG || entry.isTTF || entry.isXML || entry.isFavicon) { if (!entry.gzip && isContent) { phantomas.incrMetric('assetsNotGzipped'); - phantomas.addOffender('assetsNotGzipped', entry.url + ' (' + entry.type.toUpperCase() + ')'); + phantomas.addOffender('assetsNotGzipped', {url: entry.url, contentType: entry.contentType}); } } // small assets can be inlined - if (entry.contentLength < SIZE_THRESHOLD) { - sizeFormatted = (entry.contentLength / 1024).toFixed(2); - + // responseSize - that's the response size as reported by Chrome's dev tools (headers + compressed body) + if (entry.responseSize < SIZE_THRESHOLD) { // check small images that can be base64 encoded if (entry.isImage) { phantomas.incrMetric('smallImages'); - phantomas.addOffender('smallImages', '%s (%s kB)', entry.url, sizeFormatted); + phantomas.addOffender('smallImages', {url: entry.url, size: entry.responseSize}); } // CSS / JS that can be inlined else if (entry.isCSS) { phantomas.incrMetric('smallCssFiles'); - phantomas.addOffender('smallCssFiles', '%s (%s kB)', entry.url, sizeFormatted); + phantomas.addOffender('smallCssFiles', {url: entry.url, size: entry.responseSize}); } else if (entry.isJS) { phantomas.incrMetric('smallJsFiles'); - phantomas.addOffender('smallJsFiles', '%s (%s kB)', entry.url, sizeFormatted); + phantomas.addOffender('smallJsFiles', {url: entry.url, size: entry.responseSize}); } } @@ -77,7 +75,7 @@ module.exports = function(phantomas) { // count static assets requested from domains with cookie set if (cookieDomains.has(entry.domain)) { phantomas.incrMetric('assetsWithCookies'); - phantomas.addOffender('assetsWithCookies', '%s (%s)', entry.url, entry.type.toUpperCase()); + phantomas.addOffender('assetsWithCookies', {url: entry.url, contentType: entry.contentType}); } } }); @@ -86,7 +84,7 @@ module.exports = function(phantomas) { assetsReqCounter.forEach((asset, cnt) => { if (cnt > 1) { phantomas.incrMetric('multipleRequests'); - phantomas.addOffender('multipleRequests', asset); + phantomas.addOffender('multipleRequests', {url: asset, count: cnt}); } }); }); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index c830eef4a..82f62eed6 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -634,7 +634,17 @@ - { domain: 'fonts.gstatic.com', requests: 1 } assetsNotGzipped: # CSS response is compressed using "brotli" - - 'http://127.0.0.1:8888/https-fonts.html (HTML)' + - { url: 'http://127.0.0.1:8888/https-fonts.html', contentType: 'text/html' } + +# staticAssets +- url: "/assets.html" + offenders: + assetsWithQueryString: + - { url: 'http://127.0.0.1:8888/static/mdn.png?cb=123', contentType: 'image/png' } + smallImages: + - { url: 'http://127.0.0.1:8888/static/blank.gif', size: 330 } + smallCssFiles: + - { url: 'http://127.0.0.1:8888/static/style.css', size: 308 } # # integration-test-extra section diff --git a/test/modules/staticAssets-test.js b/test/modules/staticAssets-test.js index dffefa0f6..57ed722b2 100644 --- a/test/modules/staticAssets-test.js +++ b/test/modules/staticAssets-test.js @@ -58,7 +58,7 @@ addBatch({ status: 200, isImage: true, type: 'image', - contentLength: 32 * 1024 + responseSize: 32 * 1024 }).report(); }, { 'smallImages': 0, @@ -69,7 +69,7 @@ addBatch({ status: 200, isImage: true, type: 'image', - contentLength: 1024 + responseSize: 1024 }).report(); }, { 'smallImages': 1, @@ -80,7 +80,7 @@ addBatch({ status: 200, isCSS: true, type: 'css', - contentLength: 1024 + responseSize: 1024 }).report(); }, { 'smallCssFiles': 1, @@ -91,7 +91,7 @@ addBatch({ status: 200, isJS: true, type: 'js', - contentLength: 1024 + responseSize: 1024 }).report(); }, { 'smallJsFiles': 1, diff --git a/test/webroot/assets.html b/test/webroot/assets.html new file mode 100644 index 000000000..2f0b7ce17 --- /dev/null +++ b/test/webroot/assets.html @@ -0,0 +1,8 @@ + + + + + + + + From 96554bcf7fb7dacd64219c3ee1ee25907f66bfdf Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 18:08:47 +0100 Subject: [PATCH 157/248] httpAuth: handle HTTP authentication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit httpbin's /basic-auth ✓ should reject a promise httpbin's /basic-auth (with --auth-user and --auth-pass) ✓ should be generated ✓ should have "requests" metric properly set ✓ should have "httpsRequests" offender(s) properly set ✓ OK » 432 honored (47.745s) --- extensions/httpAuth/httpAuth.js | 14 +++++++------- lib/index.js | 16 ++++++++++++++++ test/integration-spec.yaml | 17 +++++++++++++++++ test/integration-test.js | 24 ++++++++++++++++-------- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/extensions/httpAuth/httpAuth.js b/extensions/httpAuth/httpAuth.js index dec8c8005..c4686dbe8 100644 --- a/extensions/httpAuth/httpAuth.js +++ b/extensions/httpAuth/httpAuth.js @@ -4,17 +4,17 @@ 'use strict'; module.exports = function(phantomas) { - var userName = phantomas.getParam('auth-user', '', 'string'), - password = phantomas.getParam('auth-pass', '', 'string'); + var username = phantomas.getParam('auth-user') || '', + password = phantomas.getParam('auth-pass') || ''; - if (userName === '' || password === '') { + if (username === '' || password === '') { return; } - phantomas.on('pageBeforeOpen', function(page) { - phantomas.log('Using HTTP auth: %s (pass: %s)', userName, new Array(password.length + 1).join('*')); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageauthenticatecredentials + phantomas.on('init', async (_, page) => { + await page.authenticate({username, password}); - page.settings.userName = userName; - page.settings.password = password; + phantomas.log('Set HTTP authentication: %s (pass: %s)', username, new Array(password.length + 1).join('*')); }); }; diff --git a/lib/index.js b/lib/index.js index 2cca87833..adf599f9d 100644 --- a/lib/index.js +++ b/lib/index.js @@ -126,6 +126,22 @@ function phantomas(url, opts) { } }); + // bind to a first response + // and reject a promise if the first response is 4xx / 5xx HTTP error + var firstResponseReceived = false; + + events.once('recv', async entry => { + if (!firstResponseReceived && entry.status >= 400) { + debug('<%s> response code is HTTP %d %s', entry.url, entry.status, entry.statusText); + + // close the browser before leaving here, otherwise subsequent instances will have problems + await browser.close(); + reject(new Error('HTTP response code from <' + entry.url + '> is ' + entry.status)); + } + + firstResponseReceived = true; + }); + // load modules and extensions debug('Loading core modules...'); loader.loadCoreModules(scope); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 82f62eed6..c9f4c87c3 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -646,6 +646,23 @@ smallCssFiles: - { url: 'http://127.0.0.1:8888/static/style.css', size: 308 } +# authentication +- url: "https://httpbin.org/basic-auth/foo/bar" + label: "httpbin's /basic-auth" + # the following promise rejection is expected + error: 'HTTP response code from is 401' + +- url: "https://httpbin.org/basic-auth/foo/bar" + label: "httpbin's /basic-auth (with --auth-user and --auth-pass)" + options: + auth-user: 'foo' + auth-pass: 'bar' + metrics: + requests: 1 + offenders: + httpsRequests: + - 'https://httpbin.org/basic-auth/foo/bar' + # # integration-test-extra section # diff --git a/test/integration-test.js b/test/integration-test.js index 99b326842..1b4251f0a 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -40,26 +40,34 @@ spec.forEach(function(test) { batch[batchName] = { topic: function() { - var promise = phantomas(WEBROOT + test.url, test.options || {}); + const url = test.url[0] == '/' ? WEBROOT + test.url : test.url, + promise = phantomas(url, test.options || {}); promise. then(res => this.callback(null, res)). - catch(err => this.callback(err)) + catch(err => this.callback(null, err)) if (test.assertFunction) { extras[test.assertFunction](promise, batch[batchName]); } }, 'should be generated': (err, res) => { - if (test.exitCode) { - assert.ok(err instanceof Error); - } else { - assert.equal(err, null, 'No error should be thrown: got ' + err); - assert.ok(res.getMetric instanceof Function, 'Results wrapper should be passed'); - } + assert.ok(!(res instanceof Error), 'No error should be thrown: got ' + res); + assert.ok(res.getMetric instanceof Function, 'Results wrapper should be passed'); }, }; + // check for errors + if (test.error) { + delete batch[batchName]['should be generated']; + + batch[batchName]['should reject a promise'] = (_, err) => { + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error + assert.ok(err instanceof Error); + assert.ok(err.message.indexOf(test.error) === 0, test.error + ' should be raised'); + }; + } + // check metrics Object.keys(test.metrics || {}).forEach(function(name) { batch[batchName]['should have "' + name + '" metric properly set'] = function(err, results) { From ffe551207208ee98d60105d5b623574ca5041a0d Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 20:39:02 +0100 Subject: [PATCH 158/248] init: switch event's arguments order "browser" is not used currently by functions bound to this event --- extensions/cookies/cookies.js | 3 +-- extensions/devices/devices.js | 2 +- extensions/httpAuth/httpAuth.js | 2 +- lib/index.js | 3 +-- modules/blockDomains/blockDomains.js | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/extensions/cookies/cookies.js b/extensions/cookies/cookies.js index 8d454c55b..e1631d2d6 100644 --- a/extensions/cookies/cookies.js +++ b/extensions/cookies/cookies.js @@ -1,7 +1,6 @@ /** * Support for cookies */ -/* global phantom: true */ 'use strict'; module.exports = function(phantomas) { @@ -54,7 +53,7 @@ module.exports = function(phantomas) { return cookiesJar; } - phantomas.on('init', async (_, page) => { + phantomas.on('init', async page => { const url = phantomas.getParam('url'), // https://nodejs.org/docs/latest/api/url.html#url_legacy_url_api domain = require('url').parse(url).hostname; diff --git a/extensions/devices/devices.js b/extensions/devices/devices.js index 939af2886..765fc8124 100644 --- a/extensions/devices/devices.js +++ b/extensions/devices/devices.js @@ -37,7 +37,7 @@ module.exports = function(phantomas) { phantomas.log('Devices: %s provided - using "%s" profile: %j', device, profileName, devices[profileName]); - phantomas.on('init', async (_, page) => { + phantomas.on('init', async page => { await page.emulate(devices[profileName]); phantomas.log('page.emulate() called'); }); diff --git a/extensions/httpAuth/httpAuth.js b/extensions/httpAuth/httpAuth.js index c4686dbe8..80c1d6113 100644 --- a/extensions/httpAuth/httpAuth.js +++ b/extensions/httpAuth/httpAuth.js @@ -12,7 +12,7 @@ module.exports = function(phantomas) { } // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageauthenticatecredentials - phantomas.on('init', async (_, page) => { + phantomas.on('init', async page => { await page.authenticate({username, password}); phantomas.log('Set HTTP authentication: %s (pass: %s)', username, new Array(password.length + 1).join('*')); diff --git a/lib/index.js b/lib/index.js index adf599f9d..8af309d91 100644 --- a/lib/index.js +++ b/lib/index.js @@ -152,8 +152,7 @@ function phantomas(url, opts) { debug('Loading modules...'); loader.loadModules(scope); - // browser's scope and modules are set up, you can now use it in your modules - await events.emit('init', browser, page); + await events.emit('init', page, browser); // @desc browser's scope and modules are set up, the page is about to be loaded // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagegotourl-options const waitUntil = options['wait-for-network-idle'] ? 'networkidle0' : undefined, diff --git a/modules/blockDomains/blockDomains.js b/modules/blockDomains/blockDomains.js index cac8eb807..ab532a470 100644 --- a/modules/blockDomains/blockDomains.js +++ b/modules/blockDomains/blockDomains.js @@ -75,7 +75,7 @@ module.exports = function(phantomas) { } // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagesetrequestinterceptionvalue - phantomas.on('init', async (_, page) => { + phantomas.on('init', async page => { await page.setRequestInterception(true); page.on('request', interceptedRequest => { From 6e78b8e3bbef3cac2dc82d20071bc4909967f324 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 20:47:45 +0100 Subject: [PATCH 159/248] lib: document "core" events --- lib/browser.js | 16 ++++++++-------- lib/index.js | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 5fdbe7c48..43f209810 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -57,7 +57,7 @@ Browser.prototype.init = async () => { // bind events this.page.on('console', msg => { debug('console.log:', msg.text()); - this.events.emit('consoleLog', msg); + this.events.emit('consoleLog', msg); // @desc `console.log` has been called in page's scope }); // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-dialog @@ -71,7 +71,7 @@ Browser.prototype.init = async () => { case 'alert': case 'confirm': case 'prompt': - this.events.emit(dialog._type, message); + this.events.emit(dialog._type, message); // @desc Emitted when a JavaScript dialog appears: alert, prompt or confirm break; } @@ -83,7 +83,7 @@ Browser.prototype.init = async () => { const lines = x.message.split('\n'); debug('Page error: ' + x); - this.events.emit('jserror', lines[0].trim(), lines.slice(1)); + this.events.emit('jserror', lines[0].trim(), lines.slice(1)); // @desc Emitted when an uncaught exception happens within the page }); // storage for requests metadata @@ -107,7 +107,7 @@ Browser.prototype.init = async () => { networkDebug('Network.requestWillBeSent > %s %s [%s]', request.method, request.url, request._initiator.type); - this.events.emit('request', request); + this.events.emit('request', request); // @desc Emitted when page is about to send HTTP request responses[data.requestId] = { _chunks: 0, @@ -168,7 +168,7 @@ Browser.prototype.init = async () => { 1.0 * (response.encodedDataLength || response.headers['content-length'] || 0) / 1024, 1.0 * response.dataLength / 1024); - this.events.emit('response', response); + this.events.emit('response', response); // @desc Emitted when page received a HTTP response }; // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-loadingFinished @@ -221,12 +221,12 @@ Browser.prototype.visit = (url, waitUntil, timeout) => { // https://github.com/GoogleChrome/puppeteer/issues/1325#issuecomment-382003386 // bind to this event when getting "Protocol error (Runtime.callFunctionOn): Target closed." // while calling page.evaluate() - this.events.emit('loaded', this.page); + this.events.emit('loaded', this.page); // @desc Emitted when the page has been fully loaded const metrics = await this.page.metrics(); debug('Metrics: %s', JSON.stringify(metrics)); - this.events.emit('metrics', metrics); + this.events.emit('metrics', metrics); // @desc Emitted when Chromuim's page.metrics() has been called resolve(); }); } @@ -234,7 +234,7 @@ Browser.prototype.visit = (url, waitUntil, timeout) => { // we're done Browser.prototype.close = async () => { await this.browser.close(); - this.events.emit('close'); + this.events.emit('close'); // @desc Chromium has been closed debug('Browser closed'); }; diff --git a/lib/index.js b/lib/index.js index 8af309d91..6fdfe8dee 100644 --- a/lib/index.js +++ b/lib/index.js @@ -152,7 +152,7 @@ function phantomas(url, opts) { debug('Loading modules...'); loader.loadModules(scope); - await events.emit('init', page, browser); // @desc browser's scope and modules are set up, the page is about to be loaded + await events.emit('init', page, browser); // @desc Browser's scope and modules are set up, the page is about to be loaded // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagegotourl-options const waitUntil = options['wait-for-network-idle'] ? 'networkidle0' : undefined, @@ -161,11 +161,12 @@ function phantomas(url, opts) { await browser.visit(url, waitUntil, timeout); // resolve our run - await events.emit('beforeClose', page); + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserclose + await events.emit('beforeClose', page); // @desc Called before the Chromium (and all of its pages) is closed await browser.close(); // your last chance to add metrics - await events.emit('report'); + await events.emit('report'); // @desc Called just before the phantomas results are returned to the caller resolve(results); } From 9174a3432e54c9942fd68658f9d036d0b09bef6f Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 20:55:55 +0100 Subject: [PATCH 160/248] core/modules: milestone and responseEnd event documented --- core/modules/navigationTiming/scope.js | 2 +- core/modules/timeToFirstByte/timeToFirstByte.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core/modules/navigationTiming/scope.js b/core/modules/navigationTiming/scope.js index c569764b6..67575f375 100644 --- a/core/modules/navigationTiming/scope.js +++ b/core/modules/navigationTiming/scope.js @@ -1,7 +1,7 @@ (function(phantomas) { function emit(eventName) { phantomas.log('Navigation Timing milestone: %s', eventName); - phantomas.emit('milestone', eventName); + phantomas.emit('milestone', eventName); // @desc Page loading milestone has been reached: domInteractive, domReady and domComplete } document.addEventListener("DOMContentLoaded", function() { diff --git a/core/modules/timeToFirstByte/timeToFirstByte.js b/core/modules/timeToFirstByte/timeToFirstByte.js index da2613549..0501f3b35 100644 --- a/core/modules/timeToFirstByte/timeToFirstByte.js +++ b/core/modules/timeToFirstByte/timeToFirstByte.js @@ -34,8 +34,7 @@ module.exports = function(phantomas) { phantomas.log('Time to first byte: set to %d ms for #%d request to <%s> (HTTP %d)', entry.timeToFirstByte, entry.id, entry.url, entry.status); phantomas.log('Time to last byte: set to %d ms', entry.timeToLastByte); - phantomas.emitInternal('responseEnd', entry, res); // @desc the first response (that was not a redirect) fully received - phantomas.emit('milestone', 'responseEnd'); + phantomas.emit('responseEnd', entry, res); // @desc The first response (that was not a redirect) fully received } }); }; From 6dcff7adb22bfd7ed7184cfdeba9b0c1ac1b04d4 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 2 Feb 2019 21:00:53 +0100 Subject: [PATCH 161/248] jserror: pass variables with an event --- lib/browser.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index 43f209810..d045774dc 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -80,10 +80,12 @@ Browser.prototype.init = async () => { // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#event-pageerror this.page.on('pageerror', x => { - const lines = x.message.split('\n'); + const lines = x.message.split('\n'), + message = lines[0].trim(), + trace = lines.slice(1); debug('Page error: ' + x); - this.events.emit('jserror', lines[0].trim(), lines.slice(1)); // @desc Emitted when an uncaught exception happens within the page + this.events.emit('jserror', message, trace); // @desc Emitted when an uncaught exception happens within the page }); // storage for requests metadata From 2435974ae925ff8b3892785245a4d6db2a56f4a5 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 11:16:10 +0100 Subject: [PATCH 162/248] metadata: add events from core modules --- lib/metadata/generate.js | 32 ++++++++++++++++++++++++++++---- lib/metadata/metadata.json | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/lib/metadata/generate.js b/lib/metadata/generate.js index 72c8171df..a86a007ad 100755 --- a/lib/metadata/generate.js +++ b/lib/metadata/generate.js @@ -11,6 +11,7 @@ var debug = require('debug')('generate'), phantomas = require('../../'), yaml = require('js-yaml'), metadata = { + events: {}, metrics: {}, metricsCount: 0, modulesCount: 0, @@ -52,16 +53,24 @@ function getOptionsCoveredByTests(spec) { function getModuleMetadata(moduleFile) { var content = fs.readFileSync(moduleFile).toString(), data = { - metrics: {} + name: '', + description: '', + metrics: {}, + events: {}, }, matches, moduleName, - re = /(setMetric|setMetricEvaluate|incrMetric)\(['"]([^'"]+)['"](\)|,)(.*@desc.*$)?/mg; + metricRegEx = /(setMetric|setMetricEvaluate|incrMetric)\(['"]([^'"]+)['"](\)|,)(.*@desc.*$)?/mg, + eventsRegEx = /emit\(['"]([^'"]+)['"]([^)]+).*@desc(.*)$/mg; - moduleName = moduleFile.split('/').pop().replace(/\.js$/, ''); + data.name = moduleName = moduleFile.split('/').pop().replace(/\.js$/, ''); + // // scan the source code - while ((matches = re.exec(content)) !== null) { + // + + // look for metrics metadata + while ((matches = metricRegEx.exec(content)) !== null) { var entry = {}, metricName = matches[2], metricComment = matches[4], @@ -114,6 +123,17 @@ function getModuleMetadata(moduleFile) { data.metrics[metricName] = entry; } + // look for events metadata + while ((matches = eventsRegEx.exec(content)) !== null) { + // console.log([moduleName, matches[1], matches[2], matches[3]]); + + data.events[matches[1]] = { + 'module': moduleName, + 'desc': matches[3].trim(), + 'arguments': matches[2].replace(/^[,\s]+/g, ''), + }; + } + return data; } @@ -152,6 +172,10 @@ debug('Looking for modules in %s...', dir); metadata.metrics[metricName].testsCovered = (coveredMetrics.indexOf(metricName) > -1); }); + Object.keys(data.events).forEach(function(eventName) { + metadata.events[eventName] = data.events[eventName]; + }); + metadata.modulesCount++; }); diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 28eb3ec49..8864c90ed 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -1,4 +1,26 @@ { + "events": { + "send": { + "module": "requestsMonitor", + "desc": "request has been sent", + "arguments": "request" + }, + "base64recv": { + "module": "requestsMonitor", + "desc": "base64-encoded \"response\" has been received", + "arguments": "entry, resp" + }, + "recv": { + "module": "requestsMonitor", + "desc": "response has been received", + "arguments": "entry, resp" + }, + "responseEnd": { + "module": "timeToFirstByte", + "desc": "The first response (that was not a redirect) fully received", + "arguments": "entry, res" + } + }, "metrics": { "requests": { "desc": "total number of HTTP requests made", From c3b804a9aaeb4efa291ad19d7b0654aca4320a4d Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 11:26:14 +0100 Subject: [PATCH 163/248] Add events from /lib files --- lib/metadata/generate.js | 32 ++++++++++----------- lib/metadata/metadata.json | 58 +++++++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/lib/metadata/generate.js b/lib/metadata/generate.js index a86a007ad..8642db986 100755 --- a/lib/metadata/generate.js +++ b/lib/metadata/generate.js @@ -65,6 +65,8 @@ function getModuleMetadata(moduleFile) { data.name = moduleName = moduleFile.split('/').pop().replace(/\.js$/, ''); + const localPath = moduleFile.replace(fs.realpathSync(__dirname + '/../../'), ''); + // // scan the source code // @@ -113,7 +115,7 @@ function getModuleMetadata(moduleFile) { entry.desc = metricComment; entry.unit = metricUnit; - } else { + } else if (moduleName !== 'scope') { debug('Metadata missing for %s metric in %s module', metricName, moduleName); } @@ -128,7 +130,7 @@ function getModuleMetadata(moduleFile) { // console.log([moduleName, matches[1], matches[2], matches[3]]); data.events[matches[1]] = { - 'module': moduleName, + 'file': localPath, 'desc': matches[3].trim(), 'arguments': matches[2].replace(/^[,\s]+/g, ''), }; @@ -157,26 +159,24 @@ debug('Looking for modules in %s...', dir); [].concat( glob.sync(dir + '/core/modules/**/*.js'), - glob.sync(dir + '/modules/**/*.js') + glob.sync(dir + '/modules/**/*.js'), + glob.sync(dir + '/lib/*.js') ).forEach(function(moduleFile) { - // skip scope.js files - if (moduleFile.indexOf('/scope.js') > -1) { - return; + const data = getModuleMetadata(moduleFile); + + // skip scope.js files when listing metrics + if (moduleFile.indexOf('/scope.js') === -1 && moduleFile.indexOf('/lib/') === -1) { + Object.keys(data.metrics).forEach(function(metricName) { + metadata.metricsCount++; + metadata.metrics[metricName] = data.metrics[metricName]; + metadata.metrics[metricName].testsCovered = (coveredMetrics.indexOf(metricName) > -1); + }); + metadata.modulesCount++; } - var data = getModuleMetadata(moduleFile); - - Object.keys(data.metrics).forEach(function(metricName) { - metadata.metricsCount++; - metadata.metrics[metricName] = data.metrics[metricName]; - metadata.metrics[metricName].testsCovered = (coveredMetrics.indexOf(metricName) > -1); - }); - Object.keys(data.events).forEach(function(eventName) { metadata.events[eventName] = data.events[eventName]; }); - - metadata.modulesCount++; }); // store metadata diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 8864c90ed..5c79f3efe 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -1,24 +1,74 @@ { "events": { + "milestone": { + "file": "/core/modules/navigationTiming/scope.js", + "desc": "Page loading milestone has been reached: domInteractive, domReady and domComplete", + "arguments": "eventName" + }, "send": { - "module": "requestsMonitor", + "file": "/core/modules/requestsMonitor/requestsMonitor.js", "desc": "request has been sent", "arguments": "request" }, "base64recv": { - "module": "requestsMonitor", + "file": "/core/modules/requestsMonitor/requestsMonitor.js", "desc": "base64-encoded \"response\" has been received", "arguments": "entry, resp" }, "recv": { - "module": "requestsMonitor", + "file": "/core/modules/requestsMonitor/requestsMonitor.js", "desc": "response has been received", "arguments": "entry, resp" }, "responseEnd": { - "module": "timeToFirstByte", + "file": "/core/modules/timeToFirstByte/timeToFirstByte.js", "desc": "The first response (that was not a redirect) fully received", "arguments": "entry, res" + }, + "domQuery": { + "file": "/modules/domQueries/scope.js", + "desc": "DOM query has been made", + "arguments": "type, query, fnName, context, hasNoResults" + }, + "consoleLog": { + "file": "/lib/browser.js", + "desc": "`console.log` has been called in page's scope", + "arguments": "msg" + }, + "jserror": { + "file": "/lib/browser.js", + "desc": "Emitted when an uncaught exception happens within the page", + "arguments": "message, trace" + }, + "request": { + "file": "/lib/browser.js", + "desc": "Emitted when page is about to send HTTP request", + "arguments": "request" + }, + "response": { + "file": "/lib/browser.js", + "desc": "Emitted when page received a HTTP response", + "arguments": "response" + }, + "loaded": { + "file": "/lib/browser.js", + "desc": "Emitted when the page has been fully loaded", + "arguments": "this.page" + }, + "metrics": { + "file": "/lib/browser.js", + "desc": "Emitted when Chromuim's page.metrics() has been called", + "arguments": "metrics" + }, + "init": { + "file": "/lib/index.js", + "desc": "Browser's scope and modules are set up, the page is about to be loaded", + "arguments": "page, browser" + }, + "beforeClose": { + "file": "/lib/index.js", + "desc": "Called before the Chromium (and all of its pages) is closed", + "arguments": "page" } }, "metrics": { From 5ef561f33b95ea3cdbd13dbeadc251334b694a1c Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 11:44:54 +0100 Subject: [PATCH 164/248] metadata: add modules metadata --- lib/metadata/generate.js | 26 +++ lib/metadata/metadata.json | 384 +++++++++++++++++++++++++++++++++++++ 2 files changed, 410 insertions(+) diff --git a/lib/metadata/generate.js b/lib/metadata/generate.js index 8642db986..946812973 100755 --- a/lib/metadata/generate.js +++ b/lib/metadata/generate.js @@ -12,6 +12,7 @@ var debug = require('debug')('generate'), yaml = require('js-yaml'), metadata = { events: {}, + modules: {}, metrics: {}, metricsCount: 0, modulesCount: 0, @@ -66,6 +67,7 @@ function getModuleMetadata(moduleFile) { data.name = moduleName = moduleFile.split('/').pop().replace(/\.js$/, ''); const localPath = moduleFile.replace(fs.realpathSync(__dirname + '/../../'), ''); + data.localPath = localPath; // // scan the source code @@ -136,6 +138,21 @@ function getModuleMetadata(moduleFile) { }; } + // parse modules and extensions initial commit block + matches = /^\/\*\*([^/]+)\*\//m.exec(content); + if (matches) { + var desc = matches[0]. + replace(/^[\s/]?\*+\s?/mg, ''). // remove the beginning of block comment + replace(/\/$/mg, ''). // remove the ending of block comment + replace(/^\s?setMetric.*$/mg, ''). // remove setMetric annotations + trim(); + + //console.log(localPath, moduleName, matches[0]); + + data.description = desc; + } + //console.log(data); throw new Error('foo'); + return data; } @@ -174,6 +191,15 @@ debug('Looking for modules in %s...', dir); metadata.modulesCount++; } + // modules and extensions + if (moduleFile.indexOf('/modules/') > -1) { + metadata.modules[data.name] = { + 'file': data.localPath, + 'desc': data.description, + 'metrics': Object.keys(data.metrics), + } + } + Object.keys(data.events).forEach(function(eventName) { metadata.events[eventName] = data.events[eventName]; }); diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 5c79f3efe..c85d951f1 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -71,6 +71,390 @@ "arguments": "page" } }, + "modules": { + "navigationTiming": { + "file": "/core/modules/navigationTiming/navigationTiming.js", + "desc": "Emits \"milestone\" event on Navigation Timing related events:\n- domContentLoaded\n- domInteractive\n- domComplete\nCode taken from windowPerformance module", + "metrics": [] + }, + "scope": { + "file": "/modules/localStorage/scope.js", + "desc": "", + "metrics": [ + "localStorageEntries" + ] + }, + "requestsMonitor": { + "file": "/core/modules/requestsMonitor/requestsMonitor.js", + "desc": "Simple HTTP requests monitor and analyzer", + "metrics": [ + "requests", + "gzipRequests", + "postRequests", + "httpsRequests", + "notFound", + "bodySize", + "contentLength", + "httpTrafficCompleted" + ] + }, + "timeToFirstByte": { + "file": "/core/modules/timeToFirstByte/timeToFirstByte.js", + "desc": "", + "metrics": [ + "timeToFirstByte", + "timeToLastByte" + ] + }, + "ajaxRequests": { + "file": "/modules/ajaxRequests/ajaxRequests.js", + "desc": "Analyzes AJAX requests", + "metrics": [ + "ajaxRequests" + ] + }, + "alerts": { + "file": "/modules/alerts/alerts.js", + "desc": "Meters number of invocations of window.alert, window.confirm, and\nwindow.prompt.", + "metrics": [ + "windowAlerts", + "windowConfirms", + "windowPrompts" + ] + }, + "analyzeCss": { + "file": "/modules/analyzeCss/analyzeCss.js", + "desc": "", + "metrics": [ + "cssBase64Length", + "cssRedundantBodySelectors", + "redundantChildNodesSelectors", + "cssColors", + "cssComments", + "cssCommentsLength", + "cssComplexSelectorsByAttribute", + "cssDuplicatedSelectors", + "cssDuplicatedProperties", + "cssEmptyRules", + "cssExpressions", + "cssOldIEFixes", + "cssImports", + "cssImportants", + "cssMediaQueries", + "cssMultiClassesSelectors", + "cssOldPropertyPrefixes", + "cssQualifiedSelectors", + "cssSpecificityIdAvg", + "cssSpecificityIdTotal", + "cssSpecificityClassAvg", + "cssSpecificityClassTotal", + "cssSpecificityTagAvg", + "cssSpecificityTagTotal", + "cssSelectorsByAttribute", + "cssSelectorsByClass", + "cssSelectorsById", + "cssSelectorsByPseudo", + "cssSelectorsByTag", + "cssLength", + "cssRules", + "cssSelectors", + "cssDeclarations", + "cssNotMinified", + "cssSelectorLengthAvg", + "cssParsingErrors", + "cssInlineStyles" + ] + }, + "assetsTypes": { + "file": "/modules/assetsTypes/assetsTypes.js", + "desc": "Analyzes number of requests and sizes of different types of assets", + "metrics": [ + "htmlCount", + "htmlSize", + "cssCount", + "cssSize", + "jsCount", + "jsSize", + "jsonCount", + "jsonSize", + "imageCount", + "imageSize", + "webfontCount", + "webfontSize", + "videoCount", + "videoSize", + "base64Count", + "base64Size", + "otherCount", + "otherSize" + ] + }, + "blockDomains": { + "file": "/modules/blockDomains/blockDomains.js", + "desc": "Aborts requests to external resources or given domains\nDoes not emit any metrics", + "metrics": [ + "blockedRequests" + ] + }, + "cacheHits": { + "file": "/modules/cacheHits/cacheHits.js", + "desc": "Analyzes Age and X-Cache headers from caching servers like Squid or Varnish", + "metrics": [ + "cacheHits", + "cacheMisses", + "cachePasses" + ] + }, + "caching": { + "file": "/modules/caching/caching.js", + "desc": "", + "metrics": [ + "oldCachingHeaders", + "cachingNotSpecified", + "cachingTooShort", + "cachingDisabled", + "cachingUseImmutable" + ] + }, + "console": { + "file": "/modules/console/console.js", + "desc": "Meters number of console logs", + "metrics": [ + "consoleMessages" + ] + }, + "cookies": { + "file": "/modules/cookies/cookies.js", + "desc": "cookies metrics", + "metrics": [ + "cookiesSent", + "cookiesRecv", + "domainsWithCookies", + "documentCookiesLength", + "documentCookiesCount" + ] + }, + "documentHeight": { + "file": "/modules/documentHeight/documentHeight.js", + "desc": "Measure document height", + "metrics": [ + "documentHeight" + ] + }, + "domains": { + "file": "/modules/domains/domains.js", + "desc": "Domains monitor", + "metrics": [ + "domains", + "maxRequestsPerDomain", + "medianRequestsPerDomain" + ] + }, + "domComplexity": { + "file": "/modules/domComplexity/domComplexity.js", + "desc": "Analyzes DOM complexity", + "metrics": [ + "bodyHTMLSize", + "commentsSize", + "whiteSpacesSize", + "DOMelementsCount", + "DOMelementMaxDepth", + "nodesWithInlineCSS", + "iframesCount", + "imagesScaledDown", + "imagesWithoutDimensions", + "DOMidDuplicated" + ] + }, + "domHiddenContent": { + "file": "/modules/domHiddenContent/domHiddenContent.js", + "desc": "Analyzes DOM hidden content", + "metrics": [ + "hiddenContentSize", + "hiddenImages" + ] + }, + "domMutations": { + "file": "/modules/domMutations/domMutations.js", + "desc": "", + "metrics": [ + "DOMmutationsInserts", + "DOMmutationsRemoves", + "DOMmutationsAttributes" + ] + }, + "domQueries": { + "file": "/modules/domQueries/domQueries.js", + "desc": "Analyzes DOM queries done via native DOM methods", + "metrics": [ + "DOMqueries", + "DOMqueriesWithoutResults", + "DOMqueriesById", + "DOMqueriesByClassName", + "DOMqueriesByTagName", + "DOMqueriesByQuerySelectorAll", + "DOMinserts", + "DOMqueriesDuplicated", + "DOMqueriesAvoidable" + ] + }, + "events": { + "file": "/modules/events/events.js", + "desc": "Analyzes events bound to DOM elements", + "metrics": [ + "eventsBound", + "eventsDispatched", + "eventsScrollBound" + ] + }, + "globalVariables": { + "file": "/modules/globalVariables/globalVariables.js", + "desc": "Counts global JavaScript variables", + "metrics": [ + "globalVariables", + "globalVariablesFalsy" + ] + }, + "headers": { + "file": "/modules/headers/headers.js", + "desc": "Analyzes HTTP headers in both requests and responses", + "metrics": [ + "headersCount", + "headersSentCount", + "headersRecvCount", + "headersSize", + "headersSentSize", + "headersRecvSize", + "headersBiggerThanContent" + ] + }, + "javaScriptBottlenecks": { + "file": "/modules/javaScriptBottlenecks/javaScriptBottlenecks.js", + "desc": "", + "metrics": [ + "documentWriteCalls", + "evalCalls" + ] + }, + "jQuery": { + "file": "/modules/jQuery/jQuery.js", + "desc": "", + "metrics": [ + "jQueryVersion", + "jQueryVersionsLoaded", + "jQueryOnDOMReadyFunctions", + "jQueryWindowOnLoadFunctions", + "jQuerySizzleCalls", + "jQueryEventTriggers", + "jQueryDOMReads", + "jQueryDOMWrites", + "jQueryDOMWriteReadSwitches" + ] + }, + "jserrors": { + "file": "/modules/jserrors/jserrors.js", + "desc": "Meters the number of page errors, and provides traces as offenders for \"jsErrors\" metric", + "metrics": [ + "jsErrors" + ] + }, + "keepAlive": { + "file": "/modules/keepAlive/keepAlive.js", + "desc": "Analyzes if HTTP responses keep the connections alive.", + "metrics": [ + "closedConnections" + ] + }, + "lazyLoadableImages": { + "file": "/modules/lazyLoadableImages/lazyLoadableImages.js", + "desc": "", + "metrics": [ + "lazyLoadableImagesBelowTheFold" + ] + }, + "localStorage": { + "file": "/modules/localStorage/localStorage.js", + "desc": "localStorage metrics", + "metrics": [ + "localStorageEntries" + ] + }, + "mainRequest": { + "file": "/modules/mainRequest/mainRequest.js", + "desc": "Analyzes bits of data pertaining to the main request only", + "metrics": [ + "statusCodesTrail" + ] + }, + "redirects": { + "file": "/modules/redirects/redirects.js", + "desc": "Analyzes HTTP redirects", + "metrics": [ + "redirects", + "redirectsTime" + ] + }, + "requestsStats": { + "file": "/modules/requestsStats/requestsStats.js", + "desc": "Analyzes HTTP requests and generates stats metrics", + "metrics": [ + "smallestResponse", + "biggestResponse", + "fastestResponse", + "slowestResponse", + "smallestLatency", + "biggestLatency", + "medianResponse", + "medianLatency" + ] + }, + "requestsTo": { + "file": "/modules/requestsTo/requestsTo.js", + "desc": "Number of requests it took to make the page enter DomContentLoaded and DomComplete states accordingly", + "metrics": [ + "requestsToFirstPaint", + "domainsToFirstPaint", + "requestsToDomContentLoaded", + "domainsToDomContentLoaded", + "requestsToDomComplete", + "domainsToDomComplete" + ] + }, + "staticAssets": { + "file": "/modules/staticAssets/staticAssets.js", + "desc": "Analyzes static assets (CSS, JS and images)", + "metrics": [ + "assetsNotGzipped", + "assetsWithQueryString", + "assetsWithCookies", + "smallImages", + "smallCssFiles", + "smallJsFiles", + "multipleRequests" + ] + }, + "timeToFirst": { + "file": "/modules/timeToFirst/timeToFirst.js", + "desc": "Provides metrics for time to first image, CSS and JS file", + "metrics": [ + "timeToFirstCss", + "timeToFirstJs", + "timeToFirstImage" + ] + }, + "windowPerformance": { + "file": "/modules/windowPerformance/windowPerformance.js", + "desc": "", + "metrics": [ + "domInteractive", + "domContentLoaded", + "domContentLoadedEnd", + "domComplete", + "timeBackend", + "timeFrontend" + ] + } + }, "metrics": { "requests": { "desc": "total number of HTTP requests made", From e3ed7e6058b5ada95debd998089734a2089aa6ef Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 11:59:12 +0100 Subject: [PATCH 165/248] More modules metadata --- core/modules/timeToFirstByte/timeToFirstByte.js | 2 +- lib/metadata/generate.js | 2 +- lib/metadata/metadata.json | 11 ++--------- modules/analyzeCss/analyzeCss.js | 4 +--- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/core/modules/timeToFirstByte/timeToFirstByte.js b/core/modules/timeToFirstByte/timeToFirstByte.js index 0501f3b35..4001681d2 100644 --- a/core/modules/timeToFirstByte/timeToFirstByte.js +++ b/core/modules/timeToFirstByte/timeToFirstByte.js @@ -1,5 +1,5 @@ /** - * TTFB / TTLB metrics + * Takes a look at "time to first (last) byte" metrics */ 'use strict'; diff --git a/lib/metadata/generate.js b/lib/metadata/generate.js index 946812973..a307ae9d2 100755 --- a/lib/metadata/generate.js +++ b/lib/metadata/generate.js @@ -192,7 +192,7 @@ debug('Looking for modules in %s...', dir); } // modules and extensions - if (moduleFile.indexOf('/modules/') > -1) { + if (moduleFile.indexOf('/scope.js') === -1 && moduleFile.indexOf('/modules/') > -1) { metadata.modules[data.name] = { 'file': data.localPath, 'desc': data.description, diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index c85d951f1..9c6e2e71c 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -77,13 +77,6 @@ "desc": "Emits \"milestone\" event on Navigation Timing related events:\n- domContentLoaded\n- domInteractive\n- domComplete\nCode taken from windowPerformance module", "metrics": [] }, - "scope": { - "file": "/modules/localStorage/scope.js", - "desc": "", - "metrics": [ - "localStorageEntries" - ] - }, "requestsMonitor": { "file": "/core/modules/requestsMonitor/requestsMonitor.js", "desc": "Simple HTTP requests monitor and analyzer", @@ -100,7 +93,7 @@ }, "timeToFirstByte": { "file": "/core/modules/timeToFirstByte/timeToFirstByte.js", - "desc": "", + "desc": "Takes a look at \"time to first (last) byte\" metrics", "metrics": [ "timeToFirstByte", "timeToLastByte" @@ -124,7 +117,7 @@ }, "analyzeCss": { "file": "/modules/analyzeCss/analyzeCss.js", - "desc": "", + "desc": "Adds CSS complexity metrics using analyze-css npm module.\n\nRun phantomas with --analyze-css option to use this module", "metrics": [ "cssBase64Length", "cssRedundantBodySelectors", diff --git a/modules/analyzeCss/analyzeCss.js b/modules/analyzeCss/analyzeCss.js index da5096cf2..e5ac893da 100644 --- a/modules/analyzeCss/analyzeCss.js +++ b/modules/analyzeCss/analyzeCss.js @@ -1,7 +1,5 @@ /** - * Adds CSS related metrics using analyze-css NPM module - * - * @see https://github.com/macbre/analyze-css + * Adds CSS complexity metrics using analyze-css npm module. * * Run phantomas with --analyze-css option to use this module * From 89ba5f91c9b1c3cc8c74b6cc9c828ec811bcd3f9 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 12:02:30 +0100 Subject: [PATCH 166/248] More module descriptions --- lib/metadata/metadata.json | 14 +++++++------- modules/caching/caching.js | 3 +-- modules/domMutations/domMutations.js | 3 +-- modules/jQuery/jQuery.js | 3 --- .../javaScriptBottlenecks/javaScriptBottlenecks.js | 12 ++++++------ modules/lazyLoadableImages/lazyLoadableImages.js | 2 -- modules/windowPerformance/windowPerformance.js | 6 +++--- 7 files changed, 18 insertions(+), 25 deletions(-) diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 9c6e2e71c..b7e7d3a64 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -117,7 +117,7 @@ }, "analyzeCss": { "file": "/modules/analyzeCss/analyzeCss.js", - "desc": "Adds CSS complexity metrics using analyze-css npm module.\n\nRun phantomas with --analyze-css option to use this module", + "desc": "Adds CSS complexity metrics using analyze-css npm module.\nRun phantomas with --analyze-css option to use this module", "metrics": [ "cssBase64Length", "cssRedundantBodySelectors", @@ -200,7 +200,7 @@ }, "caching": { "file": "/modules/caching/caching.js", - "desc": "", + "desc": "Analyzes HTTP caching headers", "metrics": [ "oldCachingHeaders", "cachingNotSpecified", @@ -269,7 +269,7 @@ }, "domMutations": { "file": "/modules/domMutations/domMutations.js", - "desc": "", + "desc": "Analyzes DOM changes via MutationObserver API", "metrics": [ "DOMmutationsInserts", "DOMmutationsRemoves", @@ -323,7 +323,7 @@ }, "javaScriptBottlenecks": { "file": "/modules/javaScriptBottlenecks/javaScriptBottlenecks.js", - "desc": "", + "desc": "Reports the use of functions known to be serious performance bottlenecks in JS\n\nRun phantomas with --spy-eval to count eval() calls (see issue #467)", "metrics": [ "documentWriteCalls", "evalCalls" @@ -331,7 +331,7 @@ }, "jQuery": { "file": "/modules/jQuery/jQuery.js", - "desc": "", + "desc": "Analyzes jQuery activity", "metrics": [ "jQueryVersion", "jQueryVersionsLoaded", @@ -360,7 +360,7 @@ }, "lazyLoadableImages": { "file": "/modules/lazyLoadableImages/lazyLoadableImages.js", - "desc": "", + "desc": "Analyzes images and detects which one can be lazy-loaded (are below the fold)", "metrics": [ "lazyLoadableImagesBelowTheFold" ] @@ -437,7 +437,7 @@ }, "windowPerformance": { "file": "/modules/windowPerformance/windowPerformance.js", - "desc": "", + "desc": "Measure when the page reaches certain states", "metrics": [ "domInteractive", "domContentLoaded", diff --git a/modules/caching/caching.js b/modules/caching/caching.js index b3daaa03d..b06dc1891 100644 --- a/modules/caching/caching.js +++ b/modules/caching/caching.js @@ -1,13 +1,12 @@ /** * Analyzes HTTP caching headers - * - * @see https://developers.google.com/speed/docs/best-practices/caching */ 'use strict'; module.exports = function(phantomas) { var cacheControlRegExp = /max-age=(\d+)/; + // @see https://developers.google.com/speed/docs/best-practices/caching function getCachingTime(url, headers) { // false means "no caching" var ttl = false, diff --git a/modules/domMutations/domMutations.js b/modules/domMutations/domMutations.js index 4a06c2ddd..a8a3a904f 100644 --- a/modules/domMutations/domMutations.js +++ b/modules/domMutations/domMutations.js @@ -1,12 +1,11 @@ /** * Analyzes DOM changes via MutationObserver API - * - * @see http://dev.opera.com/articles/mutation-observers-tutorial/ */ /* global window: true, document: true, MutationObserver: true */ 'use strict'; module.exports = phantomas => { + // @see http://dev.opera.com/articles/mutation-observers-tutorial/ phantomas.setMetric('DOMmutationsInserts'); // @desc number of node inserts @offenders phantomas.setMetric('DOMmutationsRemoves'); // @desc number of node removes @offenders phantomas.setMetric('DOMmutationsAttributes'); // @desc number of DOM nodes attributes changes @offenders diff --git a/modules/jQuery/jQuery.js b/modules/jQuery/jQuery.js index 3fa690a39..3050aeddb 100644 --- a/modules/jQuery/jQuery.js +++ b/modules/jQuery/jQuery.js @@ -1,8 +1,5 @@ /** * Analyzes jQuery activity - * - * @see http://code.jquery.com/jquery-1.10.2.js - * @see http://code.jquery.com/jquery-2.1.1.js */ /* global document: true, window: true */ 'use strict'; diff --git a/modules/javaScriptBottlenecks/javaScriptBottlenecks.js b/modules/javaScriptBottlenecks/javaScriptBottlenecks.js index ec8c3ca8c..9f745f078 100644 --- a/modules/javaScriptBottlenecks/javaScriptBottlenecks.js +++ b/modules/javaScriptBottlenecks/javaScriptBottlenecks.js @@ -1,15 +1,15 @@ /** * Reports the use of functions known to be serious performance bottlenecks in JS - * - * @see http://www.nczonline.net/blog/2013/06/25/eval-isnt-evil-just-misunderstood/ - * @see http://www.quirksmode.org/blog/archives/2005/06/three_javascrip_1.html - * @see http://www.stevesouders.com/blog/2012/04/10/dont-docwrite-scripts/ - * + * * Run phantomas with --spy-eval to count eval() calls (see issue #467) */ -/* global document: true, window: true */ 'use strict'; +/** + * @see http://www.nczonline.net/blog/2013/06/25/eval-isnt-evil-just-misunderstood/ + * @see http://www.quirksmode.org/blog/archives/2005/06/three_javascrip_1.html + * @see http://www.stevesouders.com/blog/2012/04/10/dont-docwrite-scripts/ + */ module.exports = function(phantomas) { phantomas.setMetric('documentWriteCalls'); //@desc number of calls to either document.write or document.writeln @offenders phantomas.setMetric('evalCalls'); // @desc number of calls to eval (either direct or via setTimeout / setInterval) @offenders diff --git a/modules/lazyLoadableImages/lazyLoadableImages.js b/modules/lazyLoadableImages/lazyLoadableImages.js index c2061632c..181d8d307 100644 --- a/modules/lazyLoadableImages/lazyLoadableImages.js +++ b/modules/lazyLoadableImages/lazyLoadableImages.js @@ -1,7 +1,5 @@ /** * Analyzes images and detects which one can be lazy-loaded (are below the fold) - * - * @see https://github.com/macbre/phantomas/issues/494 */ /* global document: true, window: true */ 'use strict'; diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index e1688804a..93173e48b 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -1,13 +1,13 @@ /** * Measure when the page reaches certain states - * - * @see http://w3c-test.org/webperf/specs/NavigationTiming/#dom-performancetiming-domloading - * @see https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp */ /* global document: true, window: true */ 'use strict'; module.exports = function(phantomas) { + // @see http://w3c-test.org/webperf/specs/NavigationTiming/#dom-performancetiming-domloading + // @see https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp + // times below are calculated relative to performance.timing.responseEnd (#117) phantomas.setMetric('domInteractive'); // @desc time it took to parse the HTML and construct the DOM phantomas.setMetric('domContentLoaded'); // @desc time it took to construct both DOM and CSSOM, no stylesheets that are blocking JavaScript execution (i.e. onDOMReady) From 2bfd9f6bfa974325b953e952f0a862f35e8aaa1e Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 12:15:53 +0100 Subject: [PATCH 167/248] Metadata: add extensions --- extensions/devices/devices.js | 4 +- extensions/filmStrip/filmStrip.js | 5 +- extensions/har/har.js | 3 +- lib/metadata/generate.js | 20 ++++++- lib/metadata/metadata.json | 98 +++++++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 9 deletions(-) diff --git a/extensions/devices/devices.js b/extensions/devices/devices.js index 765fc8124..6d95333e8 100644 --- a/extensions/devices/devices.js +++ b/extensions/devices/devices.js @@ -1,8 +1,5 @@ /** * Provides --phone, --phone-landscape, --tablet and --table-landscape options to force given device viewport and user agent. - * - * @see https://github.com/macbre/phantomas/issues/213 - * @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageemulateoptions */ 'use strict'; @@ -38,6 +35,7 @@ module.exports = function(phantomas) { phantomas.log('Devices: %s provided - using "%s" profile: %j', device, profileName, devices[profileName]); phantomas.on('init', async page => { + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageemulateoptions await page.emulate(devices[profileName]); phantomas.log('page.emulate() called'); }); diff --git a/extensions/filmStrip/filmStrip.js b/extensions/filmStrip/filmStrip.js index 17e618d58..c0defec39 100644 --- a/extensions/filmStrip/filmStrip.js +++ b/extensions/filmStrip/filmStrip.js @@ -1,6 +1,7 @@ /** * Renders a serie of screenshots of page being loaded - * + */ +/** * Please note that rendering each screenshot takes * several hundreds ms. Consider increasing default timeout. * @@ -9,7 +10,7 @@ * --film-strip-dir folder path to output film strip (default is ./filmstrip directory) * --film-strip-prefix film strip files name prefix (defaults to 'screenshot') * - * Youcan pass a comma separated list of milliseconds when to trigger a screenshot. + * You can pass a comma separated list of milliseconds when to trigger a screenshot. * The time will be calculated relative to "responseEnd" event (issue #174) */ 'use strict'; diff --git a/extensions/har/har.js b/extensions/har/har.js index cbe5e44c5..aba573963 100644 --- a/extensions/har/har.js +++ b/extensions/har/har.js @@ -2,8 +2,6 @@ * Log requests for build HAR output * * Depends on windowPerformance module! - * - * @see: https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html */ 'use strict'; @@ -17,6 +15,7 @@ var fs = require('fs'); */ function createHAR(page, creator) { + // @see: https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html var address = page.address; var title = page.title; var startTime = page.startTime; diff --git a/lib/metadata/generate.js b/lib/metadata/generate.js index a307ae9d2..d7d20d4b5 100755 --- a/lib/metadata/generate.js +++ b/lib/metadata/generate.js @@ -12,10 +12,12 @@ var debug = require('debug')('generate'), yaml = require('js-yaml'), metadata = { events: {}, + extensions: {}, modules: {}, metrics: {}, metricsCount: 0, modulesCount: 0, + extensionsCount: 0, version: phantomas.version }; @@ -177,6 +179,7 @@ debug('Looking for modules in %s...', dir); [].concat( glob.sync(dir + '/core/modules/**/*.js'), glob.sync(dir + '/modules/**/*.js'), + glob.sync(dir + '/extensions/**/*.js'), glob.sync(dir + '/lib/*.js') ).forEach(function(moduleFile) { const data = getModuleMetadata(moduleFile); @@ -188,16 +191,29 @@ debug('Looking for modules in %s...', dir); metadata.metrics[metricName] = data.metrics[metricName]; metadata.metrics[metricName].testsCovered = (coveredMetrics.indexOf(metricName) > -1); }); - metadata.modulesCount++; } - // modules and extensions + // modules ... if (moduleFile.indexOf('/scope.js') === -1 && moduleFile.indexOf('/modules/') > -1) { metadata.modules[data.name] = { 'file': data.localPath, 'desc': data.description, + 'events': Object.keys(data.events), 'metrics': Object.keys(data.metrics), } + + metadata.modulesCount++; + } + + // ... and extensions + if (moduleFile.indexOf('/extensions/') > -1) { + metadata.extensions[data.name] = { + 'file': data.localPath, + 'desc': data.description, + 'options': [] + } + + metadata.extensionsCount++; } Object.keys(data.events).forEach(function(eventName) { diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index b7e7d3a64..56e37ebd8 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -71,15 +71,78 @@ "arguments": "page" } }, + "extensions": { + "cookies": { + "file": "/extensions/cookies/cookies.js", + "desc": "Support for cookies", + "options": [] + }, + "devices": { + "file": "/extensions/devices/devices.js", + "desc": "Provides --phone, --phone-landscape, --tablet and --table-landscape options to force given device viewport and user agent.", + "options": [] + }, + "filmStrip": { + "file": "/extensions/filmStrip/filmStrip.js", + "desc": "Renders a serie of screenshots of page being loaded", + "options": [] + }, + "har": { + "file": "/extensions/har/har.js", + "desc": "Log requests for build HAR output\nDepends on windowPerformance module!", + "options": [] + }, + "httpAuth": { + "file": "/extensions/httpAuth/httpAuth.js", + "desc": "Support for HTTP authentication", + "options": [] + }, + "pageSource": { + "file": "/extensions/pageSource/pageSource.js", + "desc": "Saves the source of page being loaded to a file\nPlease note that saving each file takes a few ms.\nConsider increasing default timeout.\nRun phantomas with --page-source option to use this module.", + "options": [] + }, + "postLoadDelay": { + "file": "/extensions/postLoadDelay/postLoadDelay.js", + "desc": "Delays report generation for a given time", + "options": [] + }, + "screenshot": { + "file": "/extensions/screenshot/screenshot.js", + "desc": "Renders a screenshot of the full page when it's fully loaded", + "options": [] + }, + "scroll": { + "file": "/extensions/scroll/scroll.js", + "desc": "Allow page to be scrolled after it is loaded\nPass --scroll as an option in CLI mode", + "options": [] + }, + "waitForEvent": { + "file": "/extensions/waitForEvent/waitForEvent.js", + "desc": "Delays report generation until given phantomas event is emitted (issue #453)", + "options": [] + }, + "waitForSelector": { + "file": "/extensions/waitForSelector/waitForSelector.js", + "desc": "Delays report generation until given CSS selector can be resolved (i.e. given element exists)", + "options": [] + } + }, "modules": { "navigationTiming": { "file": "/core/modules/navigationTiming/navigationTiming.js", "desc": "Emits \"milestone\" event on Navigation Timing related events:\n- domContentLoaded\n- domInteractive\n- domComplete\nCode taken from windowPerformance module", + "events": [], "metrics": [] }, "requestsMonitor": { "file": "/core/modules/requestsMonitor/requestsMonitor.js", "desc": "Simple HTTP requests monitor and analyzer", + "events": [ + "send", + "base64recv", + "recv" + ], "metrics": [ "requests", "gzipRequests", @@ -94,6 +157,9 @@ "timeToFirstByte": { "file": "/core/modules/timeToFirstByte/timeToFirstByte.js", "desc": "Takes a look at \"time to first (last) byte\" metrics", + "events": [ + "responseEnd" + ], "metrics": [ "timeToFirstByte", "timeToLastByte" @@ -102,6 +168,7 @@ "ajaxRequests": { "file": "/modules/ajaxRequests/ajaxRequests.js", "desc": "Analyzes AJAX requests", + "events": [], "metrics": [ "ajaxRequests" ] @@ -109,6 +176,7 @@ "alerts": { "file": "/modules/alerts/alerts.js", "desc": "Meters number of invocations of window.alert, window.confirm, and\nwindow.prompt.", + "events": [], "metrics": [ "windowAlerts", "windowConfirms", @@ -118,6 +186,7 @@ "analyzeCss": { "file": "/modules/analyzeCss/analyzeCss.js", "desc": "Adds CSS complexity metrics using analyze-css npm module.\nRun phantomas with --analyze-css option to use this module", + "events": [], "metrics": [ "cssBase64Length", "cssRedundantBodySelectors", @@ -161,6 +230,7 @@ "assetsTypes": { "file": "/modules/assetsTypes/assetsTypes.js", "desc": "Analyzes number of requests and sizes of different types of assets", + "events": [], "metrics": [ "htmlCount", "htmlSize", @@ -185,6 +255,7 @@ "blockDomains": { "file": "/modules/blockDomains/blockDomains.js", "desc": "Aborts requests to external resources or given domains\nDoes not emit any metrics", + "events": [], "metrics": [ "blockedRequests" ] @@ -192,6 +263,7 @@ "cacheHits": { "file": "/modules/cacheHits/cacheHits.js", "desc": "Analyzes Age and X-Cache headers from caching servers like Squid or Varnish", + "events": [], "metrics": [ "cacheHits", "cacheMisses", @@ -201,6 +273,7 @@ "caching": { "file": "/modules/caching/caching.js", "desc": "Analyzes HTTP caching headers", + "events": [], "metrics": [ "oldCachingHeaders", "cachingNotSpecified", @@ -212,6 +285,7 @@ "console": { "file": "/modules/console/console.js", "desc": "Meters number of console logs", + "events": [], "metrics": [ "consoleMessages" ] @@ -219,6 +293,7 @@ "cookies": { "file": "/modules/cookies/cookies.js", "desc": "cookies metrics", + "events": [], "metrics": [ "cookiesSent", "cookiesRecv", @@ -230,6 +305,7 @@ "documentHeight": { "file": "/modules/documentHeight/documentHeight.js", "desc": "Measure document height", + "events": [], "metrics": [ "documentHeight" ] @@ -237,6 +313,7 @@ "domains": { "file": "/modules/domains/domains.js", "desc": "Domains monitor", + "events": [], "metrics": [ "domains", "maxRequestsPerDomain", @@ -246,6 +323,7 @@ "domComplexity": { "file": "/modules/domComplexity/domComplexity.js", "desc": "Analyzes DOM complexity", + "events": [], "metrics": [ "bodyHTMLSize", "commentsSize", @@ -262,6 +340,7 @@ "domHiddenContent": { "file": "/modules/domHiddenContent/domHiddenContent.js", "desc": "Analyzes DOM hidden content", + "events": [], "metrics": [ "hiddenContentSize", "hiddenImages" @@ -270,6 +349,7 @@ "domMutations": { "file": "/modules/domMutations/domMutations.js", "desc": "Analyzes DOM changes via MutationObserver API", + "events": [], "metrics": [ "DOMmutationsInserts", "DOMmutationsRemoves", @@ -279,6 +359,7 @@ "domQueries": { "file": "/modules/domQueries/domQueries.js", "desc": "Analyzes DOM queries done via native DOM methods", + "events": [], "metrics": [ "DOMqueries", "DOMqueriesWithoutResults", @@ -294,6 +375,7 @@ "events": { "file": "/modules/events/events.js", "desc": "Analyzes events bound to DOM elements", + "events": [], "metrics": [ "eventsBound", "eventsDispatched", @@ -303,6 +385,7 @@ "globalVariables": { "file": "/modules/globalVariables/globalVariables.js", "desc": "Counts global JavaScript variables", + "events": [], "metrics": [ "globalVariables", "globalVariablesFalsy" @@ -311,6 +394,7 @@ "headers": { "file": "/modules/headers/headers.js", "desc": "Analyzes HTTP headers in both requests and responses", + "events": [], "metrics": [ "headersCount", "headersSentCount", @@ -324,6 +408,7 @@ "javaScriptBottlenecks": { "file": "/modules/javaScriptBottlenecks/javaScriptBottlenecks.js", "desc": "Reports the use of functions known to be serious performance bottlenecks in JS\n\nRun phantomas with --spy-eval to count eval() calls (see issue #467)", + "events": [], "metrics": [ "documentWriteCalls", "evalCalls" @@ -332,6 +417,7 @@ "jQuery": { "file": "/modules/jQuery/jQuery.js", "desc": "Analyzes jQuery activity", + "events": [], "metrics": [ "jQueryVersion", "jQueryVersionsLoaded", @@ -347,6 +433,7 @@ "jserrors": { "file": "/modules/jserrors/jserrors.js", "desc": "Meters the number of page errors, and provides traces as offenders for \"jsErrors\" metric", + "events": [], "metrics": [ "jsErrors" ] @@ -354,6 +441,7 @@ "keepAlive": { "file": "/modules/keepAlive/keepAlive.js", "desc": "Analyzes if HTTP responses keep the connections alive.", + "events": [], "metrics": [ "closedConnections" ] @@ -361,6 +449,7 @@ "lazyLoadableImages": { "file": "/modules/lazyLoadableImages/lazyLoadableImages.js", "desc": "Analyzes images and detects which one can be lazy-loaded (are below the fold)", + "events": [], "metrics": [ "lazyLoadableImagesBelowTheFold" ] @@ -368,6 +457,7 @@ "localStorage": { "file": "/modules/localStorage/localStorage.js", "desc": "localStorage metrics", + "events": [], "metrics": [ "localStorageEntries" ] @@ -375,6 +465,7 @@ "mainRequest": { "file": "/modules/mainRequest/mainRequest.js", "desc": "Analyzes bits of data pertaining to the main request only", + "events": [], "metrics": [ "statusCodesTrail" ] @@ -382,6 +473,7 @@ "redirects": { "file": "/modules/redirects/redirects.js", "desc": "Analyzes HTTP redirects", + "events": [], "metrics": [ "redirects", "redirectsTime" @@ -390,6 +482,7 @@ "requestsStats": { "file": "/modules/requestsStats/requestsStats.js", "desc": "Analyzes HTTP requests and generates stats metrics", + "events": [], "metrics": [ "smallestResponse", "biggestResponse", @@ -404,6 +497,7 @@ "requestsTo": { "file": "/modules/requestsTo/requestsTo.js", "desc": "Number of requests it took to make the page enter DomContentLoaded and DomComplete states accordingly", + "events": [], "metrics": [ "requestsToFirstPaint", "domainsToFirstPaint", @@ -416,6 +510,7 @@ "staticAssets": { "file": "/modules/staticAssets/staticAssets.js", "desc": "Analyzes static assets (CSS, JS and images)", + "events": [], "metrics": [ "assetsNotGzipped", "assetsWithQueryString", @@ -429,6 +524,7 @@ "timeToFirst": { "file": "/modules/timeToFirst/timeToFirst.js", "desc": "Provides metrics for time to first image, CSS and JS file", + "events": [], "metrics": [ "timeToFirstCss", "timeToFirstJs", @@ -438,6 +534,7 @@ "windowPerformance": { "file": "/modules/windowPerformance/windowPerformance.js", "desc": "Measure when the page reaches certain states", + "events": [], "metrics": [ "domInteractive", "domContentLoaded", @@ -1644,5 +1741,6 @@ }, "metricsCount": 172, "modulesCount": 34, + "extensionsCount": 11, "version": "2.0.0-beta" } \ No newline at end of file From ca0b6069d10942388d9ca93f026ea28a8e05a572 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 14:26:36 +0100 Subject: [PATCH 168/248] lib/metadata/generate.js: update docs --- lib/metadata/generate.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/metadata/generate.js b/lib/metadata/generate.js index d7d20d4b5..8ce908393 100755 --- a/lib/metadata/generate.js +++ b/lib/metadata/generate.js @@ -1,7 +1,12 @@ #!/usr/bin/env node /** - * Generates metrics.json file that stores metrics metadata + * Generates metadata.json file that stores metadata about: + * + * - events + * - metrics + * - modules + * - extensions */ 'use strict'; From 24e9220c96bd94b9970c17a4eec2c2a4ea4e010c Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 14:45:14 +0100 Subject: [PATCH 169/248] Introduce "npm run make-docs" (#729) --- docs/events.md | 130 ++++++++++++++++++++++++++++++++++++++ lib/metadata/make_docs.js | 50 +++++++++++++++ package.json | 3 +- 3 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 docs/events.md create mode 100644 lib/metadata/make_docs.js diff --git a/docs/events.md b/docs/events.md new file mode 100644 index 000000000..5f3b7f5e9 --- /dev/null +++ b/docs/events.md @@ -0,0 +1,130 @@ +Events +====== + +## base64recv + +**Description**: base64-encoded "response" has been received + +**Arguments**: entry, resp + +[View source](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) + + +## beforeClose + +**Description**: Called before the Chromium (and all of its pages) is closed + +**Arguments**: page + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/index.js) + + +## consoleLog + +**Description**: `console.log` has been called in page's scope + +**Arguments**: msg + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) + + +## domQuery + +**Description**: DOM query has been made + +**Arguments**: type, query, fnName, context, hasNoResults + +[View source](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/scope.js) + + +## init + +**Description**: Browser's scope and modules are set up, the page is about to be loaded + +**Arguments**: page, browser + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/index.js) + + +## jserror + +**Description**: Emitted when an uncaught exception happens within the page + +**Arguments**: message, trace + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) + + +## loaded + +**Description**: Emitted when the page has been fully loaded + +**Arguments**: this.page + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) + + +## metrics + +**Description**: Emitted when Chromuim's page.metrics() has been called + +**Arguments**: metrics + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) + + +## milestone + +**Description**: Page loading milestone has been reached: domInteractive, domReady and domComplete + +**Arguments**: eventName + +[View source](https://github.com/macbre/phantomas/tree/devel/core/modules/navigationTiming/scope.js) + + +## recv + +**Description**: response has been received + +**Arguments**: entry, resp + +[View source](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) + + +## request + +**Description**: Emitted when page is about to send HTTP request + +**Arguments**: request + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) + + +## response + +**Description**: Emitted when page received a HTTP response + +**Arguments**: response + +[View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) + + +## responseEnd + +**Description**: The first response (that was not a redirect) fully received + +**Arguments**: entry, res + +[View source](https://github.com/macbre/phantomas/tree/devel/core/modules/timeToFirstByte/timeToFirstByte.js) + + +## send + +**Description**: request has been sent + +**Arguments**: request + +[View source](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) + +--- +> This file is auto-generated from code comments. Please run `npm run make-docs` to update it. \ No newline at end of file diff --git a/lib/metadata/make_docs.js b/lib/metadata/make_docs.js new file mode 100644 index 000000000..b458c1615 --- /dev/null +++ b/lib/metadata/make_docs.js @@ -0,0 +1,50 @@ +/** + * Generates Markdown files in /docs directory + */ +const debug = require('debug')('docs'), + fs = require('fs'), + metadata_file = __dirname + '/metadata.json', + docs_dir = fs.realpathSync(__dirname + '/../../docs'), + github_root = "https://github.com/macbre/phantomas/tree/devel"; + +debug('Generating docs into %s ...', docs_dir); +debug('Reading %s ...', metadata_file); + +const metadata = JSON.parse(fs.readFileSync(metadata_file)); + +// console.log(metadata); + +const docs_notice = "\n\n---\n> This file is auto-generated from code comments. Please run `npm run make-docs` to update it." + +/** + * events.md + * + * https://github.com/macbre/phantomas/issues/729 + */ +var events = ` +Events +====== + +` + +Object.keys(metadata.events).sort().forEach(eventName => { + const entry = metadata.events[eventName]; + + events += ` +## ${eventName} + +**Description**: ${entry.desc} + +**Arguments**: ${entry.arguments} + +[View source](${github_root}${entry.file}) + +`.trim() + + events += "\n\n\n"; +}); + +fs.writeFileSync(docs_dir + '/events.md', events.trim() + docs_notice); + +// --- +debug('Done'); diff --git a/package.json b/package.json index 568b8fac4..65e9ac58b 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "unit-test": "vows test/modules/*-test.js --spec", "lint": "eslint core/ modules/ test/*.js test/*/*-test.js lib/*.js lib/metadata/*.js reporters/ examples/", "beautify": "js-beautify -r bin/phantomas.js core/*.js examples/*.js extensions/*/*.js lib/*.js lib/engines/*.js lib/metadata/*.js modules/*/*.js reporters/*.js scripts/*.js test/*.js test/*/*.js", - "metadata": "DEBUG=generate node lib/metadata/generate.js" + "metadata": "DEBUG=generate node lib/metadata/generate.js", + "make-docs": "DEBUG=docs node lib/metadata/make_docs.js" } } From b1b5ac13a25f234dbebb780bc4440e6478f6c777 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 15:03:24 +0100 Subject: [PATCH 170/248] events.md: add real-life examples of data passed to events --- docs/events.md | 269 ++++++++++++++++++++++++++++++++++++-- lib/metadata/make_docs.js | 35 ++++- 2 files changed, 286 insertions(+), 18 deletions(-) diff --git a/docs/events.md b/docs/events.md index 5f3b7f5e9..aa8bc6716 100644 --- a/docs/events.md +++ b/docs/events.md @@ -1,7 +1,7 @@ Events ====== -## base64recv +### base64recv **Description**: base64-encoded "response" has been received @@ -10,7 +10,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) -## beforeClose +### beforeClose **Description**: Called before the Chromium (and all of its pages) is closed @@ -19,7 +19,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/index.js) -## consoleLog +### consoleLog **Description**: `console.log` has been called in page's scope @@ -28,7 +28,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) -## domQuery +### domQuery **Description**: DOM query has been made @@ -37,7 +37,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/scope.js) -## init +### init **Description**: Browser's scope and modules are set up, the page is about to be loaded @@ -46,7 +46,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/index.js) -## jserror +### jserror **Description**: Emitted when an uncaught exception happens within the page @@ -55,7 +55,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) -## loaded +### loaded **Description**: Emitted when the page has been fully loaded @@ -64,7 +64,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) -## metrics +### metrics **Description**: Emitted when Chromuim's page.metrics() has been called @@ -73,7 +73,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) -## milestone +### milestone **Description**: Page loading milestone has been reached: domInteractive, domReady and domComplete @@ -82,7 +82,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/core/modules/navigationTiming/scope.js) -## recv +### recv **Description**: response has been received @@ -91,7 +91,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) -## request +### request **Description**: Emitted when page is about to send HTTP request @@ -100,7 +100,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) -## response +### response **Description**: Emitted when page received a HTTP response @@ -109,7 +109,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/lib/browser.js) -## responseEnd +### responseEnd **Description**: The first response (that was not a redirect) fully received @@ -118,7 +118,7 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/core/modules/timeToFirstByte/timeToFirstByte.js) -## send +### send **Description**: request has been sent @@ -126,5 +126,246 @@ Events [View source](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) + +## Examples + +### request + +Arguments passed to the event: + +```json +[ + { + "url": "http://0.0.0.0:8888/lazy-load-scroll.html", + "method": "GET", + "headers": { + "Upgrade-Insecure-Requests": "1", + "User-Agent": "phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)" + }, + "mixedContentType": "none", + "initialPriority": "VeryHigh", + "referrerPolicy": "no-referrer-when-downgrade", + "_requestId": "0DF55D2959E58FA968527228D27B52FB", + "_timestamp": 10762.711753, + "_type": "Document", + "_initiator": { + "type": "other" + } + } +] +``` + +### send + +Arguments passed to the event: + +```json +[ + { + "url": "http://0.0.0.0:8888/lazy-load-scroll.html", + "method": "GET", + "headers": { + "Upgrade-Insecure-Requests": "1", + "User-Agent": "phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)" + }, + "mixedContentType": "none", + "initialPriority": "VeryHigh", + "referrerPolicy": "no-referrer-when-downgrade", + "_requestId": "0DF55D2959E58FA968527228D27B52FB", + "_timestamp": 10762.711753, + "_type": "Document", + "_initiator": { + "type": "other" + } + } +] +``` + +### response + +Arguments passed to the event: + +```json +[ + { + "url": "http://0.0.0.0:8888/lazy-load-scroll.html", + "status": 200, + "statusText": "OK", + "headers": { + "server": "ecstatic-2.2.1", + "last-modified": "Sat, 02 Feb 2019 17:21:13 GMT", + "etag": "\"3112379-981-\"2019-02-02T17:21:13.659Z\"\"", + "cache-control": "max-age=84600", + "content-length": "981", + "content-type": "text/html; charset=UTF-8", + "Date": "Sun, 03 Feb 2019 14:01:24 GMT", + "Connection": "keep-alive" + }, + "headersText": "HTTP/1.1 200 OK\r\nserver: ecstatic-2.2.1\r\nlast-modified: Sat, 02 Feb 2019 17:21:13 GMT\r\netag: \"3112379-981-\"2019-02-02T17:21:13.659Z\"\"\r\ncache-control: max-age=84600\r\ncontent-length: 981\r\ncontent-type: text/html; charset=UTF-8\r\nDate: Sun, 03 Feb 2019 14:01:24 GMT\r\nConnection: keep-alive\r\n\r\n", + "mimeType": "text/html", + "requestHeaders": { + "Host": "0.0.0.0:8888", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate" + }, + "requestHeadersText": "GET /lazy-load-scroll.html HTTP/1.1\r\nHost: 0.0.0.0:8888\r\nConnection: keep-alive\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n", + "connectionReused": false, + "connectionId": 11, + "remoteIPAddress": "0.0.0.0", + "remotePort": 8888, + "fromDiskCache": false, + "fromServiceWorker": false, + "encodedDataLength": 0, + "timing": { + "requestTime": 10762.712068, + "proxyStart": -1, + "proxyEnd": -1, + "dnsStart": 35.381, + "dnsEnd": 35.417, + "connectStart": 35.417, + "connectEnd": 35.592, + "sslStart": -1, + "sslEnd": -1, + "workerStart": -1, + "workerReady": -1, + "sendStart": 35.652, + "sendEnd": 35.683, + "pushStart": 0, + "pushEnd": 0, + "receiveHeadersEnd": 36.835 + }, + "protocol": "http/1.1", + "securityState": "neutral", + "_requestId": "0DF55D2959E58FA968527228D27B52FB", + "dataLength": 981, + "chunks": 1, + "_timestamp": 10762.749706 + } +] +``` + +### recv + +Arguments passed to the event: + +```json +[ + { + "id": "0DF55D2959E58FA968527228D27B52FB", + "url": "http://0.0.0.0:8888/lazy-load-scroll.html", + "method": "GET", + "headers": { + "server": "ecstatic-2.2.1", + "last-modified": "Sat, 02 Feb 2019 17:21:13 GMT", + "etag": "\"3112379-981-\"2019-02-02T17:21:13.659Z\"\"", + "cache-control": "max-age=84600", + "content-length": "981", + "content-type": "text/html; charset=UTF-8", + "date": "Sun, 03 Feb 2019 14:01:24 GMT", + "connection": "keep-alive" + }, + "bodySize": 981, + "transferedSize": 981, + "headersSize": 289, + "responseSize": 1270, + "type": "html", + "protocol": "http", + "domain": "0.0.0.0", + "query": null, + "stalled": 35.652, + "timeToFirstByte": 1.152000000000001, + "timeToLastByte": 0.037953000000925385, + "contentType": "text/html", + "isHTML": true, + "status": 200, + "statusText": "OK" + }, + { + "url": "http://0.0.0.0:8888/lazy-load-scroll.html", + "status": 200, + "statusText": "OK", + "headers": { + "server": "ecstatic-2.2.1", + "last-modified": "Sat, 02 Feb 2019 17:21:13 GMT", + "etag": "\"3112379-981-\"2019-02-02T17:21:13.659Z\"\"", + "cache-control": "max-age=84600", + "content-length": "981", + "content-type": "text/html; charset=UTF-8", + "Date": "Sun, 03 Feb 2019 14:01:24 GMT", + "Connection": "keep-alive" + }, + "headersText": "HTTP/1.1 200 OK\r\nserver: ecstatic-2.2.1\r\nlast-modified: Sat, 02 Feb 2019 17:21:13 GMT\r\netag: \"3112379-981-\"2019-02-02T17:21:13.659Z\"\"\r\ncache-control: max-age=84600\r\ncontent-length: 981\r\ncontent-type: text/html; charset=UTF-8\r\nDate: Sun, 03 Feb 2019 14:01:24 GMT\r\nConnection: keep-alive\r\n\r\n", + "mimeType": "text/html", + "requestHeaders": { + "Host": "0.0.0.0:8888", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate" + }, + "requestHeadersText": "GET /lazy-load-scroll.html HTTP/1.1\r\nHost: 0.0.0.0:8888\r\nConnection: keep-alive\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: phantomas/2.0.0-beta (HeadlessChrome/72.0.3617.0)\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n", + "connectionReused": false, + "connectionId": 11, + "remoteIPAddress": "0.0.0.0", + "remotePort": 8888, + "fromDiskCache": false, + "fromServiceWorker": false, + "encodedDataLength": 0, + "timing": { + "requestTime": 10762.712068, + "proxyStart": -1, + "proxyEnd": -1, + "dnsStart": 35.381, + "dnsEnd": 35.417, + "connectStart": 35.417, + "connectEnd": 35.592, + "sslStart": -1, + "sslEnd": -1, + "workerStart": -1, + "workerReady": -1, + "sendStart": 35.652, + "sendEnd": 35.683, + "pushStart": 0, + "pushEnd": 0, + "receiveHeadersEnd": 36.835 + }, + "protocol": "http/1.1", + "securityState": "neutral", + "_requestId": "0DF55D2959E58FA968527228D27B52FB", + "dataLength": 981, + "chunks": 1, + "_timestamp": 10762.749706 + } +] +``` + +### metrics + +Arguments passed to the event: + +```json +[ + { + "Timestamp": 10762.802943, + "Documents": 2, + "Frames": 1, + "JSEventListeners": 11, + "Nodes": 19, + "LayoutCount": 1, + "RecalcStyleCount": 1, + "LayoutDuration": 0.009849, + "RecalcStyleDuration": 0.000208, + "ScriptDuration": 0.016547, + "TaskDuration": 0.053504, + "JSHeapUsedSize": 2470408, + "JSHeapTotalSize": 4546560 + } +] +``` + --- > This file is auto-generated from code comments. Please run `npm run make-docs` to update it. \ No newline at end of file diff --git a/lib/metadata/make_docs.js b/lib/metadata/make_docs.js index b458c1615..7536846d2 100644 --- a/lib/metadata/make_docs.js +++ b/lib/metadata/make_docs.js @@ -2,11 +2,12 @@ * Generates Markdown files in /docs directory */ const debug = require('debug')('docs'), + phantomas = require('../..'), fs = require('fs'), metadata_file = __dirname + '/metadata.json', docs_dir = fs.realpathSync(__dirname + '/../../docs'), github_root = "https://github.com/macbre/phantomas/tree/devel"; - + debug('Generating docs into %s ...', docs_dir); debug('Reading %s ...', metadata_file); @@ -18,7 +19,7 @@ const docs_notice = "\n\n---\n> This file is auto-generated from code comments. /** * events.md - * + * * https://github.com/macbre/phantomas/issues/729 */ var events = ` @@ -31,7 +32,7 @@ Object.keys(metadata.events).sort().forEach(eventName => { const entry = metadata.events[eventName]; events += ` -## ${eventName} +### ${eventName} **Description**: ${entry.desc} @@ -44,7 +45,33 @@ Object.keys(metadata.events).sort().forEach(eventName => { events += "\n\n\n"; }); -fs.writeFileSync(docs_dir + '/events.md', events.trim() + docs_notice); +// now add some real life examples from loading an example URL from our tests +events += `## Examples` + +const promise = phantomas('http://0.0.0.0:8888/lazy-load-scroll.html'); + +['recv', 'send', 'request', 'response', 'metrics'].forEach(eventName => { + promise.on(eventName, (...args) => { + const dumped = JSON.stringify(args, null, ' '); + debug('events.md: %s event triggered ...', eventName); + + events += ` + +### ${eventName} + +Arguments passed to the event: + +\`\`\`json +${dumped} +\`\`\` +`.trimRight() + }) +}); + +promise.on('beforeClose', () => { + debug('Saving events.md ...'); + fs.writeFileSync(docs_dir + '/events.md', events.trim() + docs_notice); +}); // --- debug('Done'); From d2772cea5247142fe9a79fa570285f519e4ef968 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 15:24:33 +0100 Subject: [PATCH 171/248] Add docs/metrics.md: see #728 --- docs/metrics.md | 840 ++++++++++++++++++++++++++++++++++++++ lib/metadata/make_docs.js | 38 ++ 2 files changed, 878 insertions(+) create mode 100644 docs/metrics.md diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 000000000..7473a46c9 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,840 @@ +Modules and metrics +=================== + +This fille describes all phantomas modules and metrics that they emit. + +### [ajaxRequests](https://github.com/macbre/phantomas/tree/devel/modules/ajaxRequests/ajaxRequests.js) + +Analyzes AJAX requests + +#### ``ajaxRequests`` + +number of AJAX requests (number) +**offenders** + +### [alerts](https://github.com/macbre/phantomas/tree/devel/modules/alerts/alerts.js) + +Meters number of invocations of window.alert, window.confirm, and +window.prompt. + +#### ``windowAlerts`` + +number of calls to window.alert (number) +**offenders** + +#### ``windowConfirms`` + +number of calls to window.confirm (number) +**offenders** + +#### ``windowPrompts`` + +number of calls to window.prompt (number) +**offenders** + +### [analyzeCss](https://github.com/macbre/phantomas/tree/devel/modules/analyzeCss/analyzeCss.js) + +Adds CSS complexity metrics using analyze-css npm module. +Run phantomas with --analyze-css option to use this module + +#### ``cssBase64Length`` + +total length of base64-encoded data in CSS source (will warn about base64-encoded data bigger than 4 kB) (bytes) +**offenders** + +#### ``cssRedundantBodySelectors`` + +number of redundant body selectors (e.g. body .foo, section body h2, but not body > h1) (number) +**offenders** + +#### ``redundantChildNodesSelectors`` + +number of redundant child nodes selectors (number) +**offenders** + +#### ``cssColors`` + +number of unique colors used in CSS (number) +**offenders** + +#### ``cssComments`` + +number of comments in CSS source (number) +**offenders** + +#### ``cssCommentsLength`` + +length of comments content in CSS source (bytes) + +#### ``cssComplexSelectorsByAttribute`` + +number of selectors with complex matching by attribute (e.g. [class$="foo"]) (number) +**offenders** + +#### ``cssDuplicatedSelectors`` + +number of CSS selectors defined more than once in CSS source (number) +**offenders** + +#### ``cssDuplicatedProperties`` + +number of CSS property definitions duplicated within a selector (number) +**offenders** + +#### ``cssEmptyRules`` + +number of rules with no properties (e.g. .foo { }) (number) +**offenders** + +#### ``cssExpressions`` + +number of rules with CSS expressions (e.g. expression( document.body.clientWidth > 600 ? "600px" : "auto" )) (number) +**offenders** + +#### ``cssOldIEFixes`` + +number of fixes for old versions of Internet Explorer (e.g. * html .foo {} and .foo { *zoom: 1 }) (number) +**offenders** + +#### ``cssImports`` + +number of @import rules (number) +**offenders** + +#### ``cssImportants`` + +number of properties with value forced by !important (number) +**offenders** + +#### ``cssMediaQueries`` + +number of media queries (e.g. @media screen and (min-width: 1370px)) (number) +**offenders** + +#### ``cssMultiClassesSelectors`` + +number of selectors with multiple classes (e.g. span.foo.bar) (number) +**offenders** + +#### ``cssOldPropertyPrefixes`` + +number of properties with no longer needed vendor prefix, powered by data provided by autoprefixer (e.g. --moz-border-radius) (number) +**offenders** + +#### ``cssQualifiedSelectors`` + +number of qualified selectors (e.g. header#nav, .foo#bar, h1.title) (number) +**offenders** + +#### ``cssSpecificityIdAvg`` + +average specificity for ID (number) +**offenders** + +#### ``cssSpecificityIdTotal`` + +total specificity for ID (number) + +#### ``cssSpecificityClassAvg`` + +average specificity for class, pseudo-class or attribute (number) +**offenders** + +#### ``cssSpecificityClassTotal`` + +total specificity for class, pseudo-class or attribute (number) + +#### ``cssSpecificityTagAvg`` + +average specificity for element (number) +**offenders** + +#### ``cssSpecificityTagTotal`` + +total specificity for element (number) + +#### ``cssSelectorsByAttribute`` + +number of selectors by attribute (e.g. .foo[value=bar]) (number) + +#### ``cssSelectorsByClass`` + +number of selectors by class (number) + +#### ``cssSelectorsById`` + +number of selectors by ID (number) + +#### ``cssSelectorsByPseudo`` + +number of pseudo-selectors (e,g. :hover) (number) + +#### ``cssSelectorsByTag`` + +number of selectors by tag name (number) + +#### ``cssLength`` + +length of CSS source (in bytes) (bytes) +**offenders** + +#### ``cssRules`` + +number of rules (e.g. .foo, .bar { color: red } is counted as one rule) (number) +**offenders** + +#### ``cssSelectors`` + +number of selectors (e.g. .foo, .bar { color: red } is counted as two selectors - .foo and .bar) (number) +**offenders** + +#### ``cssDeclarations`` + +number of declarations (e.g. .foo, .bar { color: red } is counted as one declaration - color: red) (number) +**offenders** + +#### ``cssNotMinified`` + +set to 1 if the provided CSS is not minified (number) +**offenders** + +#### ``cssSelectorLengthAvg`` + +average length of selector (e.g. for ``.foo .bar, #test div > span { color: red }`` will be set as 2.5) (number) +**offenders** + +#### ``cssParsingErrors`` + +number of CSS files (or embeded CSS) that failed to be parse by analyze-css (number) +**offenders** + +#### ``cssInlineStyles`` + +number of inline styles (number) + +### [assetsTypes](https://github.com/macbre/phantomas/tree/devel/modules/assetsTypes/assetsTypes.js) + +Analyzes number of requests and sizes of different types of assets + +#### ``htmlCount`` + +number of HTML responses (number) +**offenders** + +#### ``htmlSize`` + +size of HTML responses (with compression) (bytes) + +#### ``cssCount`` + +number of CSS responses (number) +**offenders** + +#### ``cssSize`` + +size of CSS responses (with compression) (bytes) + +#### ``jsCount`` + +number of JS responses (number) +**offenders** + +#### ``jsSize`` + +size of JS responses (with compression) (bytes) + +#### ``jsonCount`` + +number of JSON responses (number) +**offenders** + +#### ``jsonSize`` + +size of JSON responses (with compression) (bytes) + +#### ``imageCount`` + +number of image responses (number) +**offenders** + +#### ``imageSize`` + +size of image responses (with compression) (bytes) + +#### ``webfontCount`` + +number of web font responses (number) +**offenders** + +#### ``webfontSize`` + +size of web font responses (with compression) (bytes) + +#### ``videoCount`` + +number of video responses (number) +**offenders** + +#### ``videoSize`` + +size of video responses (with compression) (bytes) + +#### ``base64Count`` + +number of base64 encoded "responses" (no HTTP request was actually made) (number) +**offenders** + +#### ``base64Size`` + +size of base64 encoded responses (bytes) + +#### ``otherCount`` + +number of other responses (number) +**offenders** + +#### ``otherSize`` + +size of other responses (with compression) (bytes) + +### [blockDomains](https://github.com/macbre/phantomas/tree/devel/modules/blockDomains/blockDomains.js) + +Aborts requests to external resources or given domains +Does not emit any metrics + +#### ``blockedRequests`` + +number of requests blocked due to domain filtering (number) +**offenders** + +### [cacheHits](https://github.com/macbre/phantomas/tree/devel/modules/cacheHits/cacheHits.js) + +Analyzes Age and X-Cache headers from caching servers like Squid or Varnish + +#### ``cacheHits`` + +number of cache hits (number) +**offenders** + +#### ``cacheMisses`` + +number of cache misses (number) +**offenders** + +#### ``cachePasses`` + +number of cache passes (number) +**offenders** + +### [caching](https://github.com/macbre/phantomas/tree/devel/modules/caching/caching.js) + +Analyzes HTTP caching headers + +#### ``oldCachingHeaders`` + +number of responses with old, HTTP 1.0 caching headers (Expires and Pragma) (number) +**offenders** + +#### ``cachingNotSpecified`` + +number of responses with no caching header sent (no Cache-Control header) (number) +**offenders** + +#### ``cachingTooShort`` + +number of responses with too short (less than a week) caching time (number) +**offenders** + +#### ``cachingDisabled`` + +number of responses with caching disabled (max-age=0) (number) +**offenders** + +#### ``cachingUseImmutable`` + +number of responses with a long TTL that can benefit from Cache-Control: immutable (number) +**offenders** + +### [console](https://github.com/macbre/phantomas/tree/devel/modules/console/console.js) + +Meters number of console logs + +#### ``consoleMessages`` + +number of calls to console.* functions (number) +**offenders** + +### [cookies](https://github.com/macbre/phantomas/tree/devel/modules/cookies/cookies.js) + +cookies metrics + +#### ``cookiesSent`` + +length of cookies sent in HTTP requests (bytes) + +#### ``cookiesRecv`` + +length of cookies received in HTTP responses (bytes) + +#### ``domainsWithCookies`` + +number of domains with cookies set (number) +**offenders** + +#### ``documentCookiesLength`` + +length of document.cookie (bytes) + +#### ``documentCookiesCount`` + +number of cookies in document.cookie (number) + +### [documentHeight](https://github.com/macbre/phantomas/tree/devel/modules/documentHeight/documentHeight.js) + +Measure document height + +#### ``documentHeight`` + +the page height (px) + +### [domComplexity](https://github.com/macbre/phantomas/tree/devel/modules/domComplexity/domComplexity.js) + +Analyzes DOM complexity + +#### ``bodyHTMLSize`` + +the size of body tag content (document.body.innerHTML.length) (bytes) + +#### ``commentsSize`` + +the size of HTML comments on the page (bytes) +**offenders** + +#### ``whiteSpacesSize`` + +the size of text nodes with whitespaces only (bytes) + +#### ``DOMelementsCount`` + +total number of HTML element nodes (number) + +#### ``DOMelementMaxDepth`` + +maximum level on nesting of HTML element node (number) +**offenders** + +#### ``nodesWithInlineCSS`` + +number of nodes with inline CSS styling (with style attribute) (number) +**offenders** + +#### ``iframesCount`` + +number of iframe nodes (number) +**offenders** + +#### ``imagesScaledDown`` + +number of nodes that have images scaled down in HTML (number) +**offenders** + +#### ``imagesWithoutDimensions`` + +number of nodes without both width and height attribute (number) +**offenders** + +#### ``DOMidDuplicated`` + +number of duplicated IDs found in DOM (number) +**offenders** + +### [domHiddenContent](https://github.com/macbre/phantomas/tree/devel/modules/domHiddenContent/domHiddenContent.js) + +Analyzes DOM hidden content + +#### ``hiddenContentSize`` + +the size of content of hidden elements on the page (with CSS display: none) (bytes) +**offenders** + +#### ``hiddenImages`` + +number of hidden images that can be lazy-loaded (number) +**offenders** + +### [domMutations](https://github.com/macbre/phantomas/tree/devel/modules/domMutations/domMutations.js) + +Analyzes DOM changes via MutationObserver API + +#### ``DOMmutationsInserts`` + +number of node inserts (number) +**offenders** + +#### ``DOMmutationsRemoves`` + +number of node removes (number) +**offenders** + +#### ``DOMmutationsAttributes`` + +number of DOM nodes attributes changes (number) +**offenders** + +### [domQueries](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/domQueries.js) + +Analyzes DOM queries done via native DOM methods + +#### ``DOMqueries`` + +number of all DOM queries (number) +**offenders** + +#### ``DOMqueriesWithoutResults`` + +number of DOM queries that returned nothing (number) +**offenders** + +#### ``DOMqueriesById`` + +number of document.getElementById calls (number) +**offenders** + +#### ``DOMqueriesByClassName`` + +number of document.getElementsByClassName calls (number) +**offenders** + +#### ``DOMqueriesByTagName`` + +number of document.getElementsByTagName calls (number) +**offenders** + +#### ``DOMqueriesByQuerySelectorAll`` + +number of document.querySelector(All) calls (number) +**offenders** + +#### ``DOMinserts`` + +number of DOM nodes inserts (number) +**offenders** + +#### ``DOMqueriesDuplicated`` + +number of DOM queries called more than once (number) +**offenders** + +#### ``DOMqueriesAvoidable`` + +number of repeated uses of a duplicated query (number) + +### [domains](https://github.com/macbre/phantomas/tree/devel/modules/domains/domains.js) + +Domains monitor + +#### ``domains`` + +number of domains used to fetch the page (number) +**offenders** + +#### ``maxRequestsPerDomain`` + +maximum number of requests fetched from a single domain (number) + +#### ``medianRequestsPerDomain`` + +median of number of requests fetched from each domain (number) + +### [events](https://github.com/macbre/phantomas/tree/devel/modules/events/events.js) + +Analyzes events bound to DOM elements + +#### ``eventsBound`` + +number of EventTarget.addEventListener calls (number) +**offenders** + +#### ``eventsDispatched`` + +number of EventTarget.dispatchEvent calls (number) +**offenders** + +#### ``eventsScrollBound`` + +number of scroll event bounds (number) +**offenders** + +### [globalVariables](https://github.com/macbre/phantomas/tree/devel/modules/globalVariables/globalVariables.js) + +Counts global JavaScript variables + +#### ``globalVariables`` + +number of JS globals variables (number) +**offenders** + +#### ``globalVariablesFalsy`` + +number of JS globals variables with falsy value (number) +**offenders** + +### [headers](https://github.com/macbre/phantomas/tree/devel/modules/headers/headers.js) + +Analyzes HTTP headers in both requests and responses + +#### ``headersCount`` + +number of requests and responses headers (number) + +#### ``headersSentCount`` + +number of headers sent in requests (number) + +#### ``headersRecvCount`` + +number of headers received in responses (number) + +#### ``headersSize`` + +size of all headers (bytes) + +#### ``headersSentSize`` + +size of sent headers (bytes) + +#### ``headersRecvSize`` + +size of received headers (bytes) + +#### ``headersBiggerThanContent`` + +number of responses with headers part bigger than the response body (number) +**offenders** + +### [jQuery](https://github.com/macbre/phantomas/tree/devel/modules/jQuery/jQuery.js) + +Analyzes jQuery activity + +#### ``jQueryVersion`` + +version of jQuery framework (if loaded) (string) + +#### ``jQueryVersionsLoaded`` + +number of loaded jQuery "instances" (even in the same version) (number) +**offenders** + +#### ``jQueryOnDOMReadyFunctions`` + +number of functions bound to onDOMReady event (number) +**offenders** + +#### ``jQueryWindowOnLoadFunctions`` + +number of functions bound to windowOnLoad event (number) +**offenders** + +#### ``jQuerySizzleCalls`` + +number of calls to Sizzle (including those that will be resolved using querySelectorAll) (number) +**offenders** + +#### ``jQueryEventTriggers`` + +number of jQuery event triggers (number) +**offenders** + +#### ``jQueryDOMReads`` + +number of DOM read operations (number) +**offenders** + +#### ``jQueryDOMWrites`` + +number of DOM write operations (number) +**offenders** + +#### ``jQueryDOMWriteReadSwitches`` + +number of read operations that follow a series of write operations (will cause repaint and can cause reflow) (number) +**offenders** + +### [javaScriptBottlenecks](https://github.com/macbre/phantomas/tree/devel/modules/javaScriptBottlenecks/javaScriptBottlenecks.js) + +Reports the use of functions known to be serious performance bottlenecks in JS + +Run phantomas with --spy-eval to count eval() calls (see issue #467) + +#### ``documentWriteCalls`` + +number of calls to either document.write or document.writeln (number) +**offenders** + +#### ``evalCalls`` + +number of calls to eval (either direct or via setTimeout / setInterval) (number) +**offenders** + +### [jserrors](https://github.com/macbre/phantomas/tree/devel/modules/jserrors/jserrors.js) + +Meters the number of page errors, and provides traces as offenders for "jsErrors" metric + +#### ``jsErrors`` + +number of JavaScript errors (number) +**offenders** + +### [keepAlive](https://github.com/macbre/phantomas/tree/devel/modules/keepAlive/keepAlive.js) + +Analyzes if HTTP responses keep the connections alive. + +#### ``closedConnections`` + +number of requests not keeping the connection alive and slowing down the next request (number) +**offenders** + +### [lazyLoadableImages](https://github.com/macbre/phantomas/tree/devel/modules/lazyLoadableImages/lazyLoadableImages.js) + +Analyzes images and detects which one can be lazy-loaded (are below the fold) + +#### ``lazyLoadableImagesBelowTheFold`` + +number of images displayed below the fold that can be lazy-loaded (number) +**offenders** + +### [localStorage](https://github.com/macbre/phantomas/tree/devel/modules/localStorage/localStorage.js) + +localStorage metrics + +#### ``localStorageEntries`` + +number of entries in local storage (number) +**offenders** + +### [mainRequest](https://github.com/macbre/phantomas/tree/devel/modules/mainRequest/mainRequest.js) + +Analyzes bits of data pertaining to the main request only + +#### ``statusCodesTrail`` + +comma-separated list of HTTP status codes that main request followed through (could contain a single element if the main request is a terminal one) (string) + +### [navigationTiming](https://github.com/macbre/phantomas/tree/devel/core/modules/navigationTiming/navigationTiming.js) + +Emits "milestone" event on Navigation Timing related events: +- domContentLoaded +- domInteractive +- domComplete +Code taken from windowPerformance module + +### [redirects](https://github.com/macbre/phantomas/tree/devel/modules/redirects/redirects.js) + +Analyzes HTTP redirects + +#### ``redirects`` + +number of HTTP redirects (either 301, 302 or 303) (number) +**offenders** + +#### ``redirectsTime`` + +time it took to send and receive redirects (ms) + +### [requestsMonitor](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) + +Simple HTTP requests monitor and analyzer + +#### ``requests`` + +total number of HTTP requests made (number) +**offenders** + +#### ``gzipRequests`` + +number of gzipped HTTP responses (number) +**offenders** + +#### ``postRequests`` + +number of POST requests (number) +**offenders** + +#### ``httpsRequests`` + +number of HTTPS requests (number) +**offenders** + +#### ``notFound`` + +number of HTTP 404 responses (number) +**offenders** + +#### ``bodySize`` + +size of the uncompressed content of all responses (bytes) + +#### ``contentLength`` + +size of the compressed content of all responses, i.e. what was transfered in packets (bytes) + +#### ``httpTrafficCompleted`` + +time it took to receive the last byte of the last HTTP response (ms) + +### [requestsStats](https://github.com/macbre/phantomas/tree/devel/modules/requestsStats/requestsStats.js) + +Analyzes HTTP requests and generates stats metrics + +#### ``smallestResponse`` + +the size of the smallest response (bytes) +**offenders** + +#### ``biggestResponse`` + +the size of the biggest response (bytes) +**offenders** + +#### ``fastestResponse`` + +the time to the last byte of the fastest response (ms) +**offenders** + +#### ``slowestResponse`` + +the time to the last byte of the slowest response (ms) +**offenders** + +#### ``smallestLatency`` + +the time to the first byte of the fastest response (ms) +**offenders** + +#### ``biggestLatency`` + +the time to the first byte of the slowest response (ms) +**offenders** + +#### ``medianResponse`` + +median value of time to the last byte for all responses (ms) +**offenders** + +#### ``medianLatency`` + +median value of time to the first byte for all responses (ms) +**offenders** + +### [requestsTo](https://github.com/macbre/phantomas/tree/devel/modules/requestsTo/requestsTo.js) + +Number of requests it took to make the page enter DomContentLoaded and DomComplete states accordingly + +#### ``requestsToFirstPaint`` + +number of HTTP requests it took to make the first paint (number) + +#### ``domainsToFirstPaint`` + +number of domains used to make the first paint (number) +**offenders** + +#### ``requestsToDomContentLoaded`` + +number of HTTP requests it took to make the page reach DomContentLoaded state (number) + +#### ``domainsToDomContentLoaded`` + +number of domains used to make the page reach DomContentLoaded state (number) +**offenders** + +#### ``requestsToDomComplete`` + +number of HTTP requests it took to make the page reach DomComplete state (number) + +#### ``domainsToDomComplete`` + +number of domains used to make the page reach DomComplete state (number) +**offenders** + +### [staticAssets](https://github.com/macbre/phantomas/tree/devel/modules/staticAssets/staticAssets.js) + +Analyzes static assets (CSS, JS and images) + +#### ``assetsNotGzipped`` + +number of static assets that were not gzipped (number) +**offenders** + +#### ``assetsWithQueryString`` + +number of static assets requested with query string (e.g. ?foo) in URL (number) +**offenders** + +#### ``assetsWithCookies`` + +number of static assets requested from domains with cookie set (number) +**offenders** + +#### ``smallImages`` + +number of images smaller than 2 KiB that can be base64 encoded (number) +**offenders** + +#### ``smallCssFiles`` + +number of CSS assets smaller than 2 KiB that can be inlined or merged (number) +**offenders** + +#### ``smallJsFiles`` + +number of JS assets smaller than 2 KiB that can be inlined or merged (number) +**offenders** + +#### ``multipleRequests`` + +number of static assets that are requested more than once (number) +**offenders** + +### [timeToFirst](https://github.com/macbre/phantomas/tree/devel/modules/timeToFirst/timeToFirst.js) + +Provides metrics for time to first image, CSS and JS file + +#### ``timeToFirstCss`` + +time it took to receive the last byte of the first CSS (ms) +**offenders** + +#### ``timeToFirstJs`` + +time it took to receive the last byte of the first JS (ms) +**offenders** + +#### ``timeToFirstImage`` + +time it took to receive the last byte of the first image (ms) +**offenders** + +### [timeToFirstByte](https://github.com/macbre/phantomas/tree/devel/core/modules/timeToFirstByte/timeToFirstByte.js) + +Takes a look at "time to first (last) byte" metrics + +#### ``timeToFirstByte`` + +time it took to receive the first byte of the first response (that was not a redirect) (ms) + +#### ``timeToLastByte`` + +time it took to receive the last byte of the first response (that was not a redirect) (ms) + +### [windowPerformance](https://github.com/macbre/phantomas/tree/devel/modules/windowPerformance/windowPerformance.js) + +Measure when the page reaches certain states + +#### ``domInteractive`` + +time it took to parse the HTML and construct the DOM (ms) + +#### ``domContentLoaded`` + +time it took to construct both DOM and CSSOM, no stylesheets that are blocking JavaScript execution (i.e. onDOMReady) (ms) + +#### ``domContentLoadedEnd`` + +time it took to finish handling of onDOMReady event (ms) + +#### ``domComplete`` + +time it took to load all page resources, the loading spinner has stopped spinning (ms) + +#### ``timeBackend`` + +time to the first byte compared to the total loading time (%) + +#### ``timeFrontend`` + +time to window.load compared to the total loading time (%) + +--- +> This file is auto-generated from code comments. Please run `npm run make-docs` to update it. \ No newline at end of file diff --git a/lib/metadata/make_docs.js b/lib/metadata/make_docs.js index 7536846d2..034ead144 100644 --- a/lib/metadata/make_docs.js +++ b/lib/metadata/make_docs.js @@ -73,5 +73,43 @@ promise.on('beforeClose', () => { fs.writeFileSync(docs_dir + '/events.md', events.trim() + docs_notice); }); + +/** + * metrics.md + * + * https://github.com/macbre/phantomas/issues/729 + */ +var metrics = ` +Modules and metrics +=================== + +This fille describes all phantomas modules and metrics that they emit. +` + +Object.keys(metadata.modules).sort().forEach(moduleName => { + const entry = metadata.modules[moduleName]; + + metrics += ` +### [${moduleName}](${github_root}${entry.file}) + +${entry.desc} +` + + entry.metrics.forEach(metricName => { + const metric = metadata.metrics[metricName], + offenders = metric.offenders ? " +**offenders**" : ''; + + metrics += ` +#### \`\`${metricName}\`\` + +${metric.desc} (${metric.unit})${offenders} +` + }); + +}); + +debug('Saving metrics.md ...'); +fs.writeFileSync(docs_dir + '/metrics.md', metrics.trim() + docs_notice); + // --- debug('Done'); From e80463786a043f9c823f2505b26dc09456853fda Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 15:26:48 +0100 Subject: [PATCH 172/248] metrics.md: minor style tweaks --- docs/metrics.md | 790 +++++++++++++++++++------------------- lib/metadata/make_docs.js | 12 +- 2 files changed, 401 insertions(+), 401 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 7473a46c9..ef0487d40 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -3,836 +3,836 @@ Modules and metrics This fille describes all phantomas modules and metrics that they emit. -### [ajaxRequests](https://github.com/macbre/phantomas/tree/devel/modules/ajaxRequests/ajaxRequests.js) +## [ajaxRequests](https://github.com/macbre/phantomas/tree/devel/modules/ajaxRequests/ajaxRequests.js) -Analyzes AJAX requests +> Analyzes AJAX requests -#### ``ajaxRequests`` +##### ``ajaxRequests`` -number of AJAX requests (number) +**offenders** +number of AJAX requests (number, with offenders) -### [alerts](https://github.com/macbre/phantomas/tree/devel/modules/alerts/alerts.js) +## [alerts](https://github.com/macbre/phantomas/tree/devel/modules/alerts/alerts.js) -Meters number of invocations of window.alert, window.confirm, and +> Meters number of invocations of window.alert, window.confirm, and window.prompt. -#### ``windowAlerts`` +##### ``windowAlerts`` -number of calls to window.alert (number) +**offenders** +number of calls to window.alert (number, with offenders) -#### ``windowConfirms`` +##### ``windowConfirms`` -number of calls to window.confirm (number) +**offenders** +number of calls to window.confirm (number, with offenders) -#### ``windowPrompts`` +##### ``windowPrompts`` -number of calls to window.prompt (number) +**offenders** +number of calls to window.prompt (number, with offenders) -### [analyzeCss](https://github.com/macbre/phantomas/tree/devel/modules/analyzeCss/analyzeCss.js) +## [analyzeCss](https://github.com/macbre/phantomas/tree/devel/modules/analyzeCss/analyzeCss.js) -Adds CSS complexity metrics using analyze-css npm module. +> Adds CSS complexity metrics using analyze-css npm module. Run phantomas with --analyze-css option to use this module -#### ``cssBase64Length`` +##### ``cssBase64Length`` -total length of base64-encoded data in CSS source (will warn about base64-encoded data bigger than 4 kB) (bytes) +**offenders** +total length of base64-encoded data in CSS source (will warn about base64-encoded data bigger than 4 kB) (bytes, with offenders) -#### ``cssRedundantBodySelectors`` +##### ``cssColors`` -number of redundant body selectors (e.g. body .foo, section body h2, but not body > h1) (number) +**offenders** +number of unique colors used in CSS (number, with offenders) -#### ``redundantChildNodesSelectors`` +##### ``cssComments`` -number of redundant child nodes selectors (number) +**offenders** +number of comments in CSS source (number, with offenders) -#### ``cssColors`` +##### ``cssCommentsLength`` -number of unique colors used in CSS (number) +**offenders** +length of comments content in CSS source (bytes) -#### ``cssComments`` +##### ``cssComplexSelectorsByAttribute`` -number of comments in CSS source (number) +**offenders** +number of selectors with complex matching by attribute (e.g. [class$="foo"]) (number, with offenders) -#### ``cssCommentsLength`` +##### ``cssDeclarations`` -length of comments content in CSS source (bytes) +number of declarations (e.g. .foo, .bar { color: red } is counted as one declaration - color: red) (number, with offenders) -#### ``cssComplexSelectorsByAttribute`` +##### ``cssDuplicatedProperties`` -number of selectors with complex matching by attribute (e.g. [class$="foo"]) (number) +**offenders** +number of CSS property definitions duplicated within a selector (number, with offenders) -#### ``cssDuplicatedSelectors`` +##### ``cssDuplicatedSelectors`` -number of CSS selectors defined more than once in CSS source (number) +**offenders** +number of CSS selectors defined more than once in CSS source (number, with offenders) -#### ``cssDuplicatedProperties`` +##### ``cssEmptyRules`` -number of CSS property definitions duplicated within a selector (number) +**offenders** +number of rules with no properties (e.g. .foo { }) (number, with offenders) -#### ``cssEmptyRules`` +##### ``cssExpressions`` -number of rules with no properties (e.g. .foo { }) (number) +**offenders** +number of rules with CSS expressions (e.g. expression( document.body.clientWidth > 600 ? "600px" : "auto" )) (number, with offenders) -#### ``cssExpressions`` +##### ``cssImportants`` -number of rules with CSS expressions (e.g. expression( document.body.clientWidth > 600 ? "600px" : "auto" )) (number) +**offenders** +number of properties with value forced by !important (number, with offenders) -#### ``cssOldIEFixes`` +##### ``cssImports`` -number of fixes for old versions of Internet Explorer (e.g. * html .foo {} and .foo { *zoom: 1 }) (number) +**offenders** +number of @import rules (number, with offenders) -#### ``cssImports`` +##### ``cssInlineStyles`` -number of @import rules (number) +**offenders** +number of inline styles (number) -#### ``cssImportants`` +##### ``cssLength`` -number of properties with value forced by !important (number) +**offenders** +length of CSS source (in bytes) (bytes, with offenders) -#### ``cssMediaQueries`` +##### ``cssMediaQueries`` -number of media queries (e.g. @media screen and (min-width: 1370px)) (number) +**offenders** +number of media queries (e.g. @media screen and (min-width: 1370px)) (number, with offenders) -#### ``cssMultiClassesSelectors`` +##### ``cssMultiClassesSelectors`` -number of selectors with multiple classes (e.g. span.foo.bar) (number) +**offenders** +number of selectors with multiple classes (e.g. span.foo.bar) (number, with offenders) -#### ``cssOldPropertyPrefixes`` +##### ``cssNotMinified`` -number of properties with no longer needed vendor prefix, powered by data provided by autoprefixer (e.g. --moz-border-radius) (number) +**offenders** +set to 1 if the provided CSS is not minified (number, with offenders) -#### ``cssQualifiedSelectors`` +##### ``cssOldIEFixes`` -number of qualified selectors (e.g. header#nav, .foo#bar, h1.title) (number) +**offenders** +number of fixes for old versions of Internet Explorer (e.g. * html .foo {} and .foo { *zoom: 1 }) (number, with offenders) -#### ``cssSpecificityIdAvg`` +##### ``cssOldPropertyPrefixes`` -average specificity for ID (number) +**offenders** +number of properties with no longer needed vendor prefix, powered by data provided by autoprefixer (e.g. --moz-border-radius) (number, with offenders) -#### ``cssSpecificityIdTotal`` +##### ``cssParsingErrors`` -total specificity for ID (number) +number of CSS files (or embeded CSS) that failed to be parse by analyze-css (number, with offenders) -#### ``cssSpecificityClassAvg`` +##### ``cssQualifiedSelectors`` -average specificity for class, pseudo-class or attribute (number) +**offenders** +number of qualified selectors (e.g. header#nav, .foo#bar, h1.title) (number, with offenders) -#### ``cssSpecificityClassTotal`` +##### ``cssRedundantBodySelectors`` -total specificity for class, pseudo-class or attribute (number) +number of redundant body selectors (e.g. body .foo, section body h2, but not body > h1) (number, with offenders) -#### ``cssSpecificityTagAvg`` +##### ``cssRules`` -average specificity for element (number) +**offenders** +number of rules (e.g. .foo, .bar { color: red } is counted as one rule) (number, with offenders) -#### ``cssSpecificityTagTotal`` +##### ``cssSelectorLengthAvg`` -total specificity for element (number) +average length of selector (e.g. for ``.foo .bar, #test div > span { color: red }`` will be set as 2.5) (number, with offenders) + +##### ``cssSelectors`` -#### ``cssSelectorsByAttribute`` +number of selectors (e.g. .foo, .bar { color: red } is counted as two selectors - .foo and .bar) (number, with offenders) + +##### ``cssSelectorsByAttribute`` number of selectors by attribute (e.g. .foo[value=bar]) (number) -#### ``cssSelectorsByClass`` +##### ``cssSelectorsByClass`` number of selectors by class (number) -#### ``cssSelectorsById`` +##### ``cssSelectorsById`` number of selectors by ID (number) -#### ``cssSelectorsByPseudo`` +##### ``cssSelectorsByPseudo`` number of pseudo-selectors (e,g. :hover) (number) -#### ``cssSelectorsByTag`` +##### ``cssSelectorsByTag`` number of selectors by tag name (number) -#### ``cssLength`` - -length of CSS source (in bytes) (bytes) +**offenders** +##### ``cssSpecificityClassAvg`` -#### ``cssRules`` +average specificity for class, pseudo-class or attribute (number, with offenders) -number of rules (e.g. .foo, .bar { color: red } is counted as one rule) (number) +**offenders** +##### ``cssSpecificityClassTotal`` -#### ``cssSelectors`` - -number of selectors (e.g. .foo, .bar { color: red } is counted as two selectors - .foo and .bar) (number) +**offenders** +total specificity for class, pseudo-class or attribute (number) -#### ``cssDeclarations`` +##### ``cssSpecificityIdAvg`` -number of declarations (e.g. .foo, .bar { color: red } is counted as one declaration - color: red) (number) +**offenders** +average specificity for ID (number, with offenders) -#### ``cssNotMinified`` +##### ``cssSpecificityIdTotal`` -set to 1 if the provided CSS is not minified (number) +**offenders** +total specificity for ID (number) -#### ``cssSelectorLengthAvg`` +##### ``cssSpecificityTagAvg`` -average length of selector (e.g. for ``.foo .bar, #test div > span { color: red }`` will be set as 2.5) (number) +**offenders** +average specificity for element (number, with offenders) -#### ``cssParsingErrors`` +##### ``cssSpecificityTagTotal`` -number of CSS files (or embeded CSS) that failed to be parse by analyze-css (number) +**offenders** +total specificity for element (number) -#### ``cssInlineStyles`` +##### ``redundantChildNodesSelectors`` -number of inline styles (number) +number of redundant child nodes selectors (number, with offenders) -### [assetsTypes](https://github.com/macbre/phantomas/tree/devel/modules/assetsTypes/assetsTypes.js) +## [assetsTypes](https://github.com/macbre/phantomas/tree/devel/modules/assetsTypes/assetsTypes.js) -Analyzes number of requests and sizes of different types of assets +> Analyzes number of requests and sizes of different types of assets -#### ``htmlCount`` +##### ``base64Count`` -number of HTML responses (number) +**offenders** +number of base64 encoded "responses" (no HTTP request was actually made) (number, with offenders) -#### ``htmlSize`` +##### ``base64Size`` -size of HTML responses (with compression) (bytes) +size of base64 encoded responses (bytes) -#### ``cssCount`` +##### ``cssCount`` -number of CSS responses (number) +**offenders** +number of CSS responses (number, with offenders) -#### ``cssSize`` +##### ``cssSize`` size of CSS responses (with compression) (bytes) -#### ``jsCount`` +##### ``htmlCount`` -number of JS responses (number) +**offenders** +number of HTML responses (number, with offenders) -#### ``jsSize`` +##### ``htmlSize`` -size of JS responses (with compression) (bytes) +size of HTML responses (with compression) (bytes) -#### ``jsonCount`` +##### ``imageCount`` -number of JSON responses (number) +**offenders** +number of image responses (number, with offenders) -#### ``jsonSize`` +##### ``imageSize`` -size of JSON responses (with compression) (bytes) +size of image responses (with compression) (bytes) -#### ``imageCount`` +##### ``jsCount`` -number of image responses (number) +**offenders** +number of JS responses (number, with offenders) -#### ``imageSize`` +##### ``jsSize`` -size of image responses (with compression) (bytes) +size of JS responses (with compression) (bytes) -#### ``webfontCount`` +##### ``jsonCount`` -number of web font responses (number) +**offenders** +number of JSON responses (number, with offenders) -#### ``webfontSize`` +##### ``jsonSize`` -size of web font responses (with compression) (bytes) +size of JSON responses (with compression) (bytes) -#### ``videoCount`` +##### ``otherCount`` -number of video responses (number) +**offenders** +number of other responses (number, with offenders) -#### ``videoSize`` +##### ``otherSize`` -size of video responses (with compression) (bytes) +size of other responses (with compression) (bytes) -#### ``base64Count`` +##### ``videoCount`` -number of base64 encoded "responses" (no HTTP request was actually made) (number) +**offenders** +number of video responses (number, with offenders) -#### ``base64Size`` +##### ``videoSize`` -size of base64 encoded responses (bytes) +size of video responses (with compression) (bytes) -#### ``otherCount`` +##### ``webfontCount`` -number of other responses (number) +**offenders** +number of web font responses (number, with offenders) -#### ``otherSize`` +##### ``webfontSize`` -size of other responses (with compression) (bytes) +size of web font responses (with compression) (bytes) -### [blockDomains](https://github.com/macbre/phantomas/tree/devel/modules/blockDomains/blockDomains.js) +## [blockDomains](https://github.com/macbre/phantomas/tree/devel/modules/blockDomains/blockDomains.js) -Aborts requests to external resources or given domains +> Aborts requests to external resources or given domains Does not emit any metrics -#### ``blockedRequests`` +##### ``blockedRequests`` -number of requests blocked due to domain filtering (number) +**offenders** +number of requests blocked due to domain filtering (number, with offenders) -### [cacheHits](https://github.com/macbre/phantomas/tree/devel/modules/cacheHits/cacheHits.js) +## [cacheHits](https://github.com/macbre/phantomas/tree/devel/modules/cacheHits/cacheHits.js) -Analyzes Age and X-Cache headers from caching servers like Squid or Varnish +> Analyzes Age and X-Cache headers from caching servers like Squid or Varnish -#### ``cacheHits`` +##### ``cacheHits`` -number of cache hits (number) +**offenders** +number of cache hits (number, with offenders) -#### ``cacheMisses`` +##### ``cacheMisses`` -number of cache misses (number) +**offenders** +number of cache misses (number, with offenders) -#### ``cachePasses`` +##### ``cachePasses`` -number of cache passes (number) +**offenders** +number of cache passes (number, with offenders) -### [caching](https://github.com/macbre/phantomas/tree/devel/modules/caching/caching.js) +## [caching](https://github.com/macbre/phantomas/tree/devel/modules/caching/caching.js) -Analyzes HTTP caching headers +> Analyzes HTTP caching headers -#### ``oldCachingHeaders`` +##### ``cachingDisabled`` -number of responses with old, HTTP 1.0 caching headers (Expires and Pragma) (number) +**offenders** +number of responses with caching disabled (max-age=0) (number, with offenders) -#### ``cachingNotSpecified`` +##### ``cachingNotSpecified`` -number of responses with no caching header sent (no Cache-Control header) (number) +**offenders** +number of responses with no caching header sent (no Cache-Control header) (number, with offenders) -#### ``cachingTooShort`` +##### ``cachingTooShort`` -number of responses with too short (less than a week) caching time (number) +**offenders** +number of responses with too short (less than a week) caching time (number, with offenders) -#### ``cachingDisabled`` +##### ``cachingUseImmutable`` -number of responses with caching disabled (max-age=0) (number) +**offenders** +number of responses with a long TTL that can benefit from Cache-Control: immutable (number, with offenders) -#### ``cachingUseImmutable`` +##### ``oldCachingHeaders`` -number of responses with a long TTL that can benefit from Cache-Control: immutable (number) +**offenders** +number of responses with old, HTTP 1.0 caching headers (Expires and Pragma) (number, with offenders) -### [console](https://github.com/macbre/phantomas/tree/devel/modules/console/console.js) +## [console](https://github.com/macbre/phantomas/tree/devel/modules/console/console.js) -Meters number of console logs +> Meters number of console logs -#### ``consoleMessages`` +##### ``consoleMessages`` -number of calls to console.* functions (number) +**offenders** +number of calls to console.* functions (number, with offenders) -### [cookies](https://github.com/macbre/phantomas/tree/devel/modules/cookies/cookies.js) +## [cookies](https://github.com/macbre/phantomas/tree/devel/modules/cookies/cookies.js) -cookies metrics +> cookies metrics -#### ``cookiesSent`` +##### ``cookiesRecv`` -length of cookies sent in HTTP requests (bytes) +length of cookies received in HTTP responses (bytes) -#### ``cookiesRecv`` +##### ``cookiesSent`` -length of cookies received in HTTP responses (bytes) +length of cookies sent in HTTP requests (bytes) -#### ``domainsWithCookies`` +##### ``documentCookiesCount`` -number of domains with cookies set (number) +**offenders** +number of cookies in document.cookie (number) -#### ``documentCookiesLength`` +##### ``documentCookiesLength`` length of document.cookie (bytes) -#### ``documentCookiesCount`` +##### ``domainsWithCookies`` -number of cookies in document.cookie (number) +number of domains with cookies set (number, with offenders) -### [documentHeight](https://github.com/macbre/phantomas/tree/devel/modules/documentHeight/documentHeight.js) +## [documentHeight](https://github.com/macbre/phantomas/tree/devel/modules/documentHeight/documentHeight.js) -Measure document height +> Measure document height -#### ``documentHeight`` +##### ``documentHeight`` the page height (px) -### [domComplexity](https://github.com/macbre/phantomas/tree/devel/modules/domComplexity/domComplexity.js) +## [domComplexity](https://github.com/macbre/phantomas/tree/devel/modules/domComplexity/domComplexity.js) -Analyzes DOM complexity +> Analyzes DOM complexity -#### ``bodyHTMLSize`` +##### ``DOMelementMaxDepth`` -the size of body tag content (document.body.innerHTML.length) (bytes) +maximum level on nesting of HTML element node (number, with offenders) -#### ``commentsSize`` +##### ``DOMelementsCount`` -the size of HTML comments on the page (bytes) +**offenders** +total number of HTML element nodes (number) -#### ``whiteSpacesSize`` +##### ``DOMidDuplicated`` -the size of text nodes with whitespaces only (bytes) +number of duplicated IDs found in DOM (number, with offenders) -#### ``DOMelementsCount`` +##### ``bodyHTMLSize`` -total number of HTML element nodes (number) +the size of body tag content (document.body.innerHTML.length) (bytes) -#### ``DOMelementMaxDepth`` +##### ``commentsSize`` -maximum level on nesting of HTML element node (number) +**offenders** +the size of HTML comments on the page (bytes, with offenders) -#### ``nodesWithInlineCSS`` +##### ``iframesCount`` -number of nodes with inline CSS styling (with style attribute) (number) +**offenders** +number of iframe nodes (number, with offenders) -#### ``iframesCount`` +##### ``imagesScaledDown`` -number of iframe nodes (number) +**offenders** +number of nodes that have images scaled down in HTML (number, with offenders) -#### ``imagesScaledDown`` +##### ``imagesWithoutDimensions`` -number of nodes that have images scaled down in HTML (number) +**offenders** +number of nodes without both width and height attribute (number, with offenders) -#### ``imagesWithoutDimensions`` +##### ``nodesWithInlineCSS`` -number of nodes without both width and height attribute (number) +**offenders** +number of nodes with inline CSS styling (with style attribute) (number, with offenders) -#### ``DOMidDuplicated`` +##### ``whiteSpacesSize`` -number of duplicated IDs found in DOM (number) +**offenders** +the size of text nodes with whitespaces only (bytes) -### [domHiddenContent](https://github.com/macbre/phantomas/tree/devel/modules/domHiddenContent/domHiddenContent.js) +## [domHiddenContent](https://github.com/macbre/phantomas/tree/devel/modules/domHiddenContent/domHiddenContent.js) -Analyzes DOM hidden content +> Analyzes DOM hidden content -#### ``hiddenContentSize`` +##### ``hiddenContentSize`` -the size of content of hidden elements on the page (with CSS display: none) (bytes) +**offenders** +the size of content of hidden elements on the page (with CSS display: none) (bytes, with offenders) -#### ``hiddenImages`` +##### ``hiddenImages`` -number of hidden images that can be lazy-loaded (number) +**offenders** +number of hidden images that can be lazy-loaded (number, with offenders) -### [domMutations](https://github.com/macbre/phantomas/tree/devel/modules/domMutations/domMutations.js) +## [domMutations](https://github.com/macbre/phantomas/tree/devel/modules/domMutations/domMutations.js) -Analyzes DOM changes via MutationObserver API +> Analyzes DOM changes via MutationObserver API -#### ``DOMmutationsInserts`` +##### ``DOMmutationsAttributes`` -number of node inserts (number) +**offenders** +number of DOM nodes attributes changes (number, with offenders) -#### ``DOMmutationsRemoves`` +##### ``DOMmutationsInserts`` -number of node removes (number) +**offenders** +number of node inserts (number, with offenders) -#### ``DOMmutationsAttributes`` +##### ``DOMmutationsRemoves`` -number of DOM nodes attributes changes (number) +**offenders** +number of node removes (number, with offenders) -### [domQueries](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/domQueries.js) +## [domQueries](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/domQueries.js) -Analyzes DOM queries done via native DOM methods +> Analyzes DOM queries done via native DOM methods -#### ``DOMqueries`` +##### ``DOMinserts`` -number of all DOM queries (number) +**offenders** +number of DOM nodes inserts (number, with offenders) -#### ``DOMqueriesWithoutResults`` +##### ``DOMqueries`` -number of DOM queries that returned nothing (number) +**offenders** +number of all DOM queries (number, with offenders) -#### ``DOMqueriesById`` +##### ``DOMqueriesAvoidable`` -number of document.getElementById calls (number) +**offenders** +number of repeated uses of a duplicated query (number) -#### ``DOMqueriesByClassName`` +##### ``DOMqueriesByClassName`` -number of document.getElementsByClassName calls (number) +**offenders** +number of document.getElementsByClassName calls (number, with offenders) -#### ``DOMqueriesByTagName`` +##### ``DOMqueriesById`` -number of document.getElementsByTagName calls (number) +**offenders** +number of document.getElementById calls (number, with offenders) -#### ``DOMqueriesByQuerySelectorAll`` +##### ``DOMqueriesByQuerySelectorAll`` -number of document.querySelector(All) calls (number) +**offenders** +number of document.querySelector(All) calls (number, with offenders) -#### ``DOMinserts`` +##### ``DOMqueriesByTagName`` -number of DOM nodes inserts (number) +**offenders** +number of document.getElementsByTagName calls (number, with offenders) -#### ``DOMqueriesDuplicated`` +##### ``DOMqueriesDuplicated`` -number of DOM queries called more than once (number) +**offenders** +number of DOM queries called more than once (number, with offenders) -#### ``DOMqueriesAvoidable`` +##### ``DOMqueriesWithoutResults`` -number of repeated uses of a duplicated query (number) +number of DOM queries that returned nothing (number, with offenders) -### [domains](https://github.com/macbre/phantomas/tree/devel/modules/domains/domains.js) +## [domains](https://github.com/macbre/phantomas/tree/devel/modules/domains/domains.js) -Domains monitor +> Domains monitor -#### ``domains`` +##### ``domains`` -number of domains used to fetch the page (number) +**offenders** +number of domains used to fetch the page (number, with offenders) -#### ``maxRequestsPerDomain`` +##### ``maxRequestsPerDomain`` maximum number of requests fetched from a single domain (number) -#### ``medianRequestsPerDomain`` +##### ``medianRequestsPerDomain`` median of number of requests fetched from each domain (number) -### [events](https://github.com/macbre/phantomas/tree/devel/modules/events/events.js) +## [events](https://github.com/macbre/phantomas/tree/devel/modules/events/events.js) -Analyzes events bound to DOM elements +> Analyzes events bound to DOM elements -#### ``eventsBound`` +##### ``eventsBound`` -number of EventTarget.addEventListener calls (number) +**offenders** +number of EventTarget.addEventListener calls (number, with offenders) -#### ``eventsDispatched`` +##### ``eventsDispatched`` -number of EventTarget.dispatchEvent calls (number) +**offenders** +number of EventTarget.dispatchEvent calls (number, with offenders) -#### ``eventsScrollBound`` +##### ``eventsScrollBound`` -number of scroll event bounds (number) +**offenders** +number of scroll event bounds (number, with offenders) -### [globalVariables](https://github.com/macbre/phantomas/tree/devel/modules/globalVariables/globalVariables.js) +## [globalVariables](https://github.com/macbre/phantomas/tree/devel/modules/globalVariables/globalVariables.js) -Counts global JavaScript variables +> Counts global JavaScript variables -#### ``globalVariables`` +##### ``globalVariables`` -number of JS globals variables (number) +**offenders** +number of JS globals variables (number, with offenders) -#### ``globalVariablesFalsy`` +##### ``globalVariablesFalsy`` -number of JS globals variables with falsy value (number) +**offenders** +number of JS globals variables with falsy value (number, with offenders) -### [headers](https://github.com/macbre/phantomas/tree/devel/modules/headers/headers.js) +## [headers](https://github.com/macbre/phantomas/tree/devel/modules/headers/headers.js) -Analyzes HTTP headers in both requests and responses +> Analyzes HTTP headers in both requests and responses -#### ``headersCount`` +##### ``headersBiggerThanContent`` -number of requests and responses headers (number) +number of responses with headers part bigger than the response body (number, with offenders) -#### ``headersSentCount`` +##### ``headersCount`` -number of headers sent in requests (number) +number of requests and responses headers (number) -#### ``headersRecvCount`` +##### ``headersRecvCount`` number of headers received in responses (number) -#### ``headersSize`` +##### ``headersRecvSize`` -size of all headers (bytes) +size of received headers (bytes) -#### ``headersSentSize`` +##### ``headersSentCount`` -size of sent headers (bytes) +number of headers sent in requests (number) -#### ``headersRecvSize`` +##### ``headersSentSize`` -size of received headers (bytes) +size of sent headers (bytes) -#### ``headersBiggerThanContent`` +##### ``headersSize`` -number of responses with headers part bigger than the response body (number) +**offenders** +size of all headers (bytes) -### [jQuery](https://github.com/macbre/phantomas/tree/devel/modules/jQuery/jQuery.js) +## [jQuery](https://github.com/macbre/phantomas/tree/devel/modules/jQuery/jQuery.js) -Analyzes jQuery activity +> Analyzes jQuery activity -#### ``jQueryVersion`` +##### ``jQueryDOMReads`` -version of jQuery framework (if loaded) (string) +number of DOM read operations (number, with offenders) -#### ``jQueryVersionsLoaded`` +##### ``jQueryDOMWriteReadSwitches`` -number of loaded jQuery "instances" (even in the same version) (number) +**offenders** +number of read operations that follow a series of write operations (will cause repaint and can cause reflow) (number, with offenders) -#### ``jQueryOnDOMReadyFunctions`` +##### ``jQueryDOMWrites`` -number of functions bound to onDOMReady event (number) +**offenders** +number of DOM write operations (number, with offenders) -#### ``jQueryWindowOnLoadFunctions`` +##### ``jQueryEventTriggers`` -number of functions bound to windowOnLoad event (number) +**offenders** +number of jQuery event triggers (number, with offenders) -#### ``jQuerySizzleCalls`` +##### ``jQueryOnDOMReadyFunctions`` -number of calls to Sizzle (including those that will be resolved using querySelectorAll) (number) +**offenders** +number of functions bound to onDOMReady event (number, with offenders) -#### ``jQueryEventTriggers`` +##### ``jQuerySizzleCalls`` -number of jQuery event triggers (number) +**offenders** +number of calls to Sizzle (including those that will be resolved using querySelectorAll) (number, with offenders) -#### ``jQueryDOMReads`` +##### ``jQueryVersion`` -number of DOM read operations (number) +**offenders** +version of jQuery framework (if loaded) (string) -#### ``jQueryDOMWrites`` +##### ``jQueryVersionsLoaded`` -number of DOM write operations (number) +**offenders** +number of loaded jQuery "instances" (even in the same version) (number, with offenders) -#### ``jQueryDOMWriteReadSwitches`` +##### ``jQueryWindowOnLoadFunctions`` -number of read operations that follow a series of write operations (will cause repaint and can cause reflow) (number) +**offenders** +number of functions bound to windowOnLoad event (number, with offenders) -### [javaScriptBottlenecks](https://github.com/macbre/phantomas/tree/devel/modules/javaScriptBottlenecks/javaScriptBottlenecks.js) +## [javaScriptBottlenecks](https://github.com/macbre/phantomas/tree/devel/modules/javaScriptBottlenecks/javaScriptBottlenecks.js) -Reports the use of functions known to be serious performance bottlenecks in JS +> Reports the use of functions known to be serious performance bottlenecks in JS Run phantomas with --spy-eval to count eval() calls (see issue #467) -#### ``documentWriteCalls`` +##### ``documentWriteCalls`` -number of calls to either document.write or document.writeln (number) +**offenders** +number of calls to either document.write or document.writeln (number, with offenders) -#### ``evalCalls`` +##### ``evalCalls`` -number of calls to eval (either direct or via setTimeout / setInterval) (number) +**offenders** +number of calls to eval (either direct or via setTimeout / setInterval) (number, with offenders) -### [jserrors](https://github.com/macbre/phantomas/tree/devel/modules/jserrors/jserrors.js) +## [jserrors](https://github.com/macbre/phantomas/tree/devel/modules/jserrors/jserrors.js) -Meters the number of page errors, and provides traces as offenders for "jsErrors" metric +> Meters the number of page errors, and provides traces as offenders for "jsErrors" metric -#### ``jsErrors`` +##### ``jsErrors`` -number of JavaScript errors (number) +**offenders** +number of JavaScript errors (number, with offenders) -### [keepAlive](https://github.com/macbre/phantomas/tree/devel/modules/keepAlive/keepAlive.js) +## [keepAlive](https://github.com/macbre/phantomas/tree/devel/modules/keepAlive/keepAlive.js) -Analyzes if HTTP responses keep the connections alive. +> Analyzes if HTTP responses keep the connections alive. -#### ``closedConnections`` +##### ``closedConnections`` -number of requests not keeping the connection alive and slowing down the next request (number) +**offenders** +number of requests not keeping the connection alive and slowing down the next request (number, with offenders) -### [lazyLoadableImages](https://github.com/macbre/phantomas/tree/devel/modules/lazyLoadableImages/lazyLoadableImages.js) +## [lazyLoadableImages](https://github.com/macbre/phantomas/tree/devel/modules/lazyLoadableImages/lazyLoadableImages.js) -Analyzes images and detects which one can be lazy-loaded (are below the fold) +> Analyzes images and detects which one can be lazy-loaded (are below the fold) -#### ``lazyLoadableImagesBelowTheFold`` +##### ``lazyLoadableImagesBelowTheFold`` -number of images displayed below the fold that can be lazy-loaded (number) +**offenders** +number of images displayed below the fold that can be lazy-loaded (number, with offenders) -### [localStorage](https://github.com/macbre/phantomas/tree/devel/modules/localStorage/localStorage.js) +## [localStorage](https://github.com/macbre/phantomas/tree/devel/modules/localStorage/localStorage.js) -localStorage metrics +> localStorage metrics -#### ``localStorageEntries`` +##### ``localStorageEntries`` -number of entries in local storage (number) +**offenders** +number of entries in local storage (number, with offenders) -### [mainRequest](https://github.com/macbre/phantomas/tree/devel/modules/mainRequest/mainRequest.js) +## [mainRequest](https://github.com/macbre/phantomas/tree/devel/modules/mainRequest/mainRequest.js) -Analyzes bits of data pertaining to the main request only +> Analyzes bits of data pertaining to the main request only -#### ``statusCodesTrail`` +##### ``statusCodesTrail`` comma-separated list of HTTP status codes that main request followed through (could contain a single element if the main request is a terminal one) (string) -### [navigationTiming](https://github.com/macbre/phantomas/tree/devel/core/modules/navigationTiming/navigationTiming.js) +## [navigationTiming](https://github.com/macbre/phantomas/tree/devel/core/modules/navigationTiming/navigationTiming.js) -Emits "milestone" event on Navigation Timing related events: +> Emits "milestone" event on Navigation Timing related events: - domContentLoaded - domInteractive - domComplete Code taken from windowPerformance module -### [redirects](https://github.com/macbre/phantomas/tree/devel/modules/redirects/redirects.js) +## [redirects](https://github.com/macbre/phantomas/tree/devel/modules/redirects/redirects.js) -Analyzes HTTP redirects +> Analyzes HTTP redirects -#### ``redirects`` +##### ``redirects`` -number of HTTP redirects (either 301, 302 or 303) (number) +**offenders** +number of HTTP redirects (either 301, 302 or 303) (number, with offenders) -#### ``redirectsTime`` +##### ``redirectsTime`` time it took to send and receive redirects (ms) -### [requestsMonitor](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) +## [requestsMonitor](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) -Simple HTTP requests monitor and analyzer +> Simple HTTP requests monitor and analyzer -#### ``requests`` +##### ``bodySize`` -total number of HTTP requests made (number) +**offenders** +size of the uncompressed content of all responses (bytes) -#### ``gzipRequests`` +##### ``contentLength`` -number of gzipped HTTP responses (number) +**offenders** +size of the compressed content of all responses, i.e. what was transfered in packets (bytes) -#### ``postRequests`` +##### ``gzipRequests`` -number of POST requests (number) +**offenders** +number of gzipped HTTP responses (number, with offenders) -#### ``httpsRequests`` +##### ``httpTrafficCompleted`` -number of HTTPS requests (number) +**offenders** +time it took to receive the last byte of the last HTTP response (ms) -#### ``notFound`` +##### ``httpsRequests`` -number of HTTP 404 responses (number) +**offenders** +number of HTTPS requests (number, with offenders) -#### ``bodySize`` +##### ``notFound`` -size of the uncompressed content of all responses (bytes) +number of HTTP 404 responses (number, with offenders) -#### ``contentLength`` +##### ``postRequests`` -size of the compressed content of all responses, i.e. what was transfered in packets (bytes) +number of POST requests (number, with offenders) -#### ``httpTrafficCompleted`` +##### ``requests`` -time it took to receive the last byte of the last HTTP response (ms) +total number of HTTP requests made (number, with offenders) -### [requestsStats](https://github.com/macbre/phantomas/tree/devel/modules/requestsStats/requestsStats.js) +## [requestsStats](https://github.com/macbre/phantomas/tree/devel/modules/requestsStats/requestsStats.js) -Analyzes HTTP requests and generates stats metrics +> Analyzes HTTP requests and generates stats metrics -#### ``smallestResponse`` +##### ``biggestLatency`` -the size of the smallest response (bytes) +**offenders** +the time to the first byte of the slowest response (ms, with offenders) -#### ``biggestResponse`` +##### ``biggestResponse`` -the size of the biggest response (bytes) +**offenders** +the size of the biggest response (bytes, with offenders) -#### ``fastestResponse`` +##### ``fastestResponse`` -the time to the last byte of the fastest response (ms) +**offenders** +the time to the last byte of the fastest response (ms, with offenders) -#### ``slowestResponse`` +##### ``medianLatency`` -the time to the last byte of the slowest response (ms) +**offenders** +median value of time to the first byte for all responses (ms, with offenders) -#### ``smallestLatency`` +##### ``medianResponse`` -the time to the first byte of the fastest response (ms) +**offenders** +median value of time to the last byte for all responses (ms, with offenders) -#### ``biggestLatency`` +##### ``slowestResponse`` -the time to the first byte of the slowest response (ms) +**offenders** +the time to the last byte of the slowest response (ms, with offenders) -#### ``medianResponse`` +##### ``smallestLatency`` -median value of time to the last byte for all responses (ms) +**offenders** +the time to the first byte of the fastest response (ms, with offenders) -#### ``medianLatency`` +##### ``smallestResponse`` -median value of time to the first byte for all responses (ms) +**offenders** +the size of the smallest response (bytes, with offenders) -### [requestsTo](https://github.com/macbre/phantomas/tree/devel/modules/requestsTo/requestsTo.js) +## [requestsTo](https://github.com/macbre/phantomas/tree/devel/modules/requestsTo/requestsTo.js) -Number of requests it took to make the page enter DomContentLoaded and DomComplete states accordingly +> Number of requests it took to make the page enter DomContentLoaded and DomComplete states accordingly -#### ``requestsToFirstPaint`` +##### ``domainsToDomComplete`` -number of HTTP requests it took to make the first paint (number) +number of domains used to make the page reach DomComplete state (number, with offenders) -#### ``domainsToFirstPaint`` +##### ``domainsToDomContentLoaded`` -number of domains used to make the first paint (number) +**offenders** +number of domains used to make the page reach DomContentLoaded state (number, with offenders) -#### ``requestsToDomContentLoaded`` +##### ``domainsToFirstPaint`` -number of HTTP requests it took to make the page reach DomContentLoaded state (number) +number of domains used to make the first paint (number, with offenders) -#### ``domainsToDomContentLoaded`` +##### ``requestsToDomComplete`` -number of domains used to make the page reach DomContentLoaded state (number) +**offenders** +number of HTTP requests it took to make the page reach DomComplete state (number) -#### ``requestsToDomComplete`` +##### ``requestsToDomContentLoaded`` -number of HTTP requests it took to make the page reach DomComplete state (number) +number of HTTP requests it took to make the page reach DomContentLoaded state (number) -#### ``domainsToDomComplete`` +##### ``requestsToFirstPaint`` -number of domains used to make the page reach DomComplete state (number) +**offenders** +number of HTTP requests it took to make the first paint (number) -### [staticAssets](https://github.com/macbre/phantomas/tree/devel/modules/staticAssets/staticAssets.js) +## [staticAssets](https://github.com/macbre/phantomas/tree/devel/modules/staticAssets/staticAssets.js) -Analyzes static assets (CSS, JS and images) +> Analyzes static assets (CSS, JS and images) -#### ``assetsNotGzipped`` +##### ``assetsNotGzipped`` -number of static assets that were not gzipped (number) +**offenders** +number of static assets that were not gzipped (number, with offenders) -#### ``assetsWithQueryString`` +##### ``assetsWithCookies`` -number of static assets requested with query string (e.g. ?foo) in URL (number) +**offenders** +number of static assets requested from domains with cookie set (number, with offenders) -#### ``assetsWithCookies`` +##### ``assetsWithQueryString`` -number of static assets requested from domains with cookie set (number) +**offenders** +number of static assets requested with query string (e.g. ?foo) in URL (number, with offenders) -#### ``smallImages`` +##### ``multipleRequests`` -number of images smaller than 2 KiB that can be base64 encoded (number) +**offenders** +number of static assets that are requested more than once (number, with offenders) -#### ``smallCssFiles`` +##### ``smallCssFiles`` -number of CSS assets smaller than 2 KiB that can be inlined or merged (number) +**offenders** +number of CSS assets smaller than 2 KiB that can be inlined or merged (number, with offenders) -#### ``smallJsFiles`` +##### ``smallImages`` -number of JS assets smaller than 2 KiB that can be inlined or merged (number) +**offenders** +number of images smaller than 2 KiB that can be base64 encoded (number, with offenders) -#### ``multipleRequests`` +##### ``smallJsFiles`` -number of static assets that are requested more than once (number) +**offenders** +number of JS assets smaller than 2 KiB that can be inlined or merged (number, with offenders) -### [timeToFirst](https://github.com/macbre/phantomas/tree/devel/modules/timeToFirst/timeToFirst.js) +## [timeToFirst](https://github.com/macbre/phantomas/tree/devel/modules/timeToFirst/timeToFirst.js) -Provides metrics for time to first image, CSS and JS file +> Provides metrics for time to first image, CSS and JS file -#### ``timeToFirstCss`` +##### ``timeToFirstCss`` -time it took to receive the last byte of the first CSS (ms) +**offenders** +time it took to receive the last byte of the first CSS (ms, with offenders) -#### ``timeToFirstJs`` +##### ``timeToFirstImage`` -time it took to receive the last byte of the first JS (ms) +**offenders** +time it took to receive the last byte of the first image (ms, with offenders) -#### ``timeToFirstImage`` +##### ``timeToFirstJs`` -time it took to receive the last byte of the first image (ms) +**offenders** +time it took to receive the last byte of the first JS (ms, with offenders) -### [timeToFirstByte](https://github.com/macbre/phantomas/tree/devel/core/modules/timeToFirstByte/timeToFirstByte.js) +## [timeToFirstByte](https://github.com/macbre/phantomas/tree/devel/core/modules/timeToFirstByte/timeToFirstByte.js) -Takes a look at "time to first (last) byte" metrics +> Takes a look at "time to first (last) byte" metrics -#### ``timeToFirstByte`` +##### ``timeToFirstByte`` time it took to receive the first byte of the first response (that was not a redirect) (ms) -#### ``timeToLastByte`` +##### ``timeToLastByte`` time it took to receive the last byte of the first response (that was not a redirect) (ms) -### [windowPerformance](https://github.com/macbre/phantomas/tree/devel/modules/windowPerformance/windowPerformance.js) +## [windowPerformance](https://github.com/macbre/phantomas/tree/devel/modules/windowPerformance/windowPerformance.js) -Measure when the page reaches certain states +> Measure when the page reaches certain states -#### ``domInteractive`` +##### ``domComplete`` -time it took to parse the HTML and construct the DOM (ms) +time it took to load all page resources, the loading spinner has stopped spinning (ms) -#### ``domContentLoaded`` +##### ``domContentLoaded`` time it took to construct both DOM and CSSOM, no stylesheets that are blocking JavaScript execution (i.e. onDOMReady) (ms) -#### ``domContentLoadedEnd`` +##### ``domContentLoadedEnd`` time it took to finish handling of onDOMReady event (ms) -#### ``domComplete`` +##### ``domInteractive`` -time it took to load all page resources, the loading spinner has stopped spinning (ms) +time it took to parse the HTML and construct the DOM (ms) -#### ``timeBackend`` +##### ``timeBackend`` time to the first byte compared to the total loading time (%) -#### ``timeFrontend`` +##### ``timeFrontend`` time to window.load compared to the total loading time (%) diff --git a/lib/metadata/make_docs.js b/lib/metadata/make_docs.js index 034ead144..19b02419c 100644 --- a/lib/metadata/make_docs.js +++ b/lib/metadata/make_docs.js @@ -90,19 +90,19 @@ Object.keys(metadata.modules).sort().forEach(moduleName => { const entry = metadata.modules[moduleName]; metrics += ` -### [${moduleName}](${github_root}${entry.file}) +## [${moduleName}](${github_root}${entry.file}) -${entry.desc} +> ${entry.desc} ` - entry.metrics.forEach(metricName => { + entry.metrics.sort().forEach(metricName => { const metric = metadata.metrics[metricName], - offenders = metric.offenders ? " +**offenders**" : ''; + hasOffenders = metric.offenders ? ", with offenders" : ''; metrics += ` -#### \`\`${metricName}\`\` +##### \`\`${metricName}\`\` -${metric.desc} (${metric.unit})${offenders} +${metric.desc} (${metric.unit}${hasOffenders}) ` }); From 4ba11a566a0d8882a96d622bbd74b244ef1304fd Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 15:34:10 +0100 Subject: [PATCH 173/248] metrics.md: add some stats --- docs/metrics.md | 380 +++++++++++++++++++++----------------- lib/metadata/make_docs.js | 5 +- 2 files changed, 210 insertions(+), 175 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index ef0487d40..cb1416a48 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,633 +1,659 @@ Modules and metrics =================== -This fille describes all phantomas modules and metrics that they emit. +This file describes all [`phantomas` modules](https://github.com/macbre/phantomas/tree/devel/modules) (34 of them) and 172 metrics that they emit. + ## [ajaxRequests](https://github.com/macbre/phantomas/tree/devel/modules/ajaxRequests/ajaxRequests.js) > Analyzes AJAX requests -##### ``ajaxRequests`` +##### `ajaxRequests` number of AJAX requests (number, with offenders) + ## [alerts](https://github.com/macbre/phantomas/tree/devel/modules/alerts/alerts.js) > Meters number of invocations of window.alert, window.confirm, and window.prompt. -##### ``windowAlerts`` +##### `windowAlerts` number of calls to window.alert (number, with offenders) -##### ``windowConfirms`` +##### `windowConfirms` number of calls to window.confirm (number, with offenders) -##### ``windowPrompts`` +##### `windowPrompts` number of calls to window.prompt (number, with offenders) + ## [analyzeCss](https://github.com/macbre/phantomas/tree/devel/modules/analyzeCss/analyzeCss.js) > Adds CSS complexity metrics using analyze-css npm module. Run phantomas with --analyze-css option to use this module -##### ``cssBase64Length`` +##### `cssBase64Length` total length of base64-encoded data in CSS source (will warn about base64-encoded data bigger than 4 kB) (bytes, with offenders) -##### ``cssColors`` +##### `cssColors` number of unique colors used in CSS (number, with offenders) -##### ``cssComments`` +##### `cssComments` number of comments in CSS source (number, with offenders) -##### ``cssCommentsLength`` +##### `cssCommentsLength` length of comments content in CSS source (bytes) -##### ``cssComplexSelectorsByAttribute`` +##### `cssComplexSelectorsByAttribute` number of selectors with complex matching by attribute (e.g. [class$="foo"]) (number, with offenders) -##### ``cssDeclarations`` +##### `cssDeclarations` number of declarations (e.g. .foo, .bar { color: red } is counted as one declaration - color: red) (number, with offenders) -##### ``cssDuplicatedProperties`` +##### `cssDuplicatedProperties` number of CSS property definitions duplicated within a selector (number, with offenders) -##### ``cssDuplicatedSelectors`` +##### `cssDuplicatedSelectors` number of CSS selectors defined more than once in CSS source (number, with offenders) -##### ``cssEmptyRules`` +##### `cssEmptyRules` number of rules with no properties (e.g. .foo { }) (number, with offenders) -##### ``cssExpressions`` +##### `cssExpressions` number of rules with CSS expressions (e.g. expression( document.body.clientWidth > 600 ? "600px" : "auto" )) (number, with offenders) -##### ``cssImportants`` +##### `cssImportants` number of properties with value forced by !important (number, with offenders) -##### ``cssImports`` +##### `cssImports` number of @import rules (number, with offenders) -##### ``cssInlineStyles`` +##### `cssInlineStyles` number of inline styles (number) -##### ``cssLength`` +##### `cssLength` length of CSS source (in bytes) (bytes, with offenders) -##### ``cssMediaQueries`` +##### `cssMediaQueries` number of media queries (e.g. @media screen and (min-width: 1370px)) (number, with offenders) -##### ``cssMultiClassesSelectors`` +##### `cssMultiClassesSelectors` number of selectors with multiple classes (e.g. span.foo.bar) (number, with offenders) -##### ``cssNotMinified`` +##### `cssNotMinified` set to 1 if the provided CSS is not minified (number, with offenders) -##### ``cssOldIEFixes`` +##### `cssOldIEFixes` number of fixes for old versions of Internet Explorer (e.g. * html .foo {} and .foo { *zoom: 1 }) (number, with offenders) -##### ``cssOldPropertyPrefixes`` +##### `cssOldPropertyPrefixes` number of properties with no longer needed vendor prefix, powered by data provided by autoprefixer (e.g. --moz-border-radius) (number, with offenders) -##### ``cssParsingErrors`` +##### `cssParsingErrors` number of CSS files (or embeded CSS) that failed to be parse by analyze-css (number, with offenders) -##### ``cssQualifiedSelectors`` +##### `cssQualifiedSelectors` number of qualified selectors (e.g. header#nav, .foo#bar, h1.title) (number, with offenders) -##### ``cssRedundantBodySelectors`` +##### `cssRedundantBodySelectors` number of redundant body selectors (e.g. body .foo, section body h2, but not body > h1) (number, with offenders) -##### ``cssRules`` +##### `cssRules` number of rules (e.g. .foo, .bar { color: red } is counted as one rule) (number, with offenders) -##### ``cssSelectorLengthAvg`` +##### `cssSelectorLengthAvg` average length of selector (e.g. for ``.foo .bar, #test div > span { color: red }`` will be set as 2.5) (number, with offenders) -##### ``cssSelectors`` +##### `cssSelectors` number of selectors (e.g. .foo, .bar { color: red } is counted as two selectors - .foo and .bar) (number, with offenders) -##### ``cssSelectorsByAttribute`` +##### `cssSelectorsByAttribute` number of selectors by attribute (e.g. .foo[value=bar]) (number) -##### ``cssSelectorsByClass`` +##### `cssSelectorsByClass` number of selectors by class (number) -##### ``cssSelectorsById`` +##### `cssSelectorsById` number of selectors by ID (number) -##### ``cssSelectorsByPseudo`` +##### `cssSelectorsByPseudo` number of pseudo-selectors (e,g. :hover) (number) -##### ``cssSelectorsByTag`` +##### `cssSelectorsByTag` number of selectors by tag name (number) -##### ``cssSpecificityClassAvg`` +##### `cssSpecificityClassAvg` average specificity for class, pseudo-class or attribute (number, with offenders) -##### ``cssSpecificityClassTotal`` +##### `cssSpecificityClassTotal` total specificity for class, pseudo-class or attribute (number) -##### ``cssSpecificityIdAvg`` +##### `cssSpecificityIdAvg` average specificity for ID (number, with offenders) -##### ``cssSpecificityIdTotal`` +##### `cssSpecificityIdTotal` total specificity for ID (number) -##### ``cssSpecificityTagAvg`` +##### `cssSpecificityTagAvg` average specificity for element (number, with offenders) -##### ``cssSpecificityTagTotal`` +##### `cssSpecificityTagTotal` total specificity for element (number) -##### ``redundantChildNodesSelectors`` +##### `redundantChildNodesSelectors` number of redundant child nodes selectors (number, with offenders) + ## [assetsTypes](https://github.com/macbre/phantomas/tree/devel/modules/assetsTypes/assetsTypes.js) > Analyzes number of requests and sizes of different types of assets -##### ``base64Count`` +##### `base64Count` number of base64 encoded "responses" (no HTTP request was actually made) (number, with offenders) -##### ``base64Size`` +##### `base64Size` size of base64 encoded responses (bytes) -##### ``cssCount`` +##### `cssCount` number of CSS responses (number, with offenders) -##### ``cssSize`` +##### `cssSize` size of CSS responses (with compression) (bytes) -##### ``htmlCount`` +##### `htmlCount` number of HTML responses (number, with offenders) -##### ``htmlSize`` +##### `htmlSize` size of HTML responses (with compression) (bytes) -##### ``imageCount`` +##### `imageCount` number of image responses (number, with offenders) -##### ``imageSize`` +##### `imageSize` size of image responses (with compression) (bytes) -##### ``jsCount`` +##### `jsCount` number of JS responses (number, with offenders) -##### ``jsSize`` +##### `jsSize` size of JS responses (with compression) (bytes) -##### ``jsonCount`` +##### `jsonCount` number of JSON responses (number, with offenders) -##### ``jsonSize`` +##### `jsonSize` size of JSON responses (with compression) (bytes) -##### ``otherCount`` +##### `otherCount` number of other responses (number, with offenders) -##### ``otherSize`` +##### `otherSize` size of other responses (with compression) (bytes) -##### ``videoCount`` +##### `videoCount` number of video responses (number, with offenders) -##### ``videoSize`` +##### `videoSize` size of video responses (with compression) (bytes) -##### ``webfontCount`` +##### `webfontCount` number of web font responses (number, with offenders) -##### ``webfontSize`` +##### `webfontSize` size of web font responses (with compression) (bytes) + ## [blockDomains](https://github.com/macbre/phantomas/tree/devel/modules/blockDomains/blockDomains.js) > Aborts requests to external resources or given domains Does not emit any metrics -##### ``blockedRequests`` +##### `blockedRequests` number of requests blocked due to domain filtering (number, with offenders) + ## [cacheHits](https://github.com/macbre/phantomas/tree/devel/modules/cacheHits/cacheHits.js) > Analyzes Age and X-Cache headers from caching servers like Squid or Varnish -##### ``cacheHits`` +##### `cacheHits` number of cache hits (number, with offenders) -##### ``cacheMisses`` +##### `cacheMisses` number of cache misses (number, with offenders) -##### ``cachePasses`` +##### `cachePasses` number of cache passes (number, with offenders) + ## [caching](https://github.com/macbre/phantomas/tree/devel/modules/caching/caching.js) > Analyzes HTTP caching headers -##### ``cachingDisabled`` +##### `cachingDisabled` number of responses with caching disabled (max-age=0) (number, with offenders) -##### ``cachingNotSpecified`` +##### `cachingNotSpecified` number of responses with no caching header sent (no Cache-Control header) (number, with offenders) -##### ``cachingTooShort`` +##### `cachingTooShort` number of responses with too short (less than a week) caching time (number, with offenders) -##### ``cachingUseImmutable`` +##### `cachingUseImmutable` number of responses with a long TTL that can benefit from Cache-Control: immutable (number, with offenders) -##### ``oldCachingHeaders`` +##### `oldCachingHeaders` number of responses with old, HTTP 1.0 caching headers (Expires and Pragma) (number, with offenders) + ## [console](https://github.com/macbre/phantomas/tree/devel/modules/console/console.js) > Meters number of console logs -##### ``consoleMessages`` +##### `consoleMessages` number of calls to console.* functions (number, with offenders) + ## [cookies](https://github.com/macbre/phantomas/tree/devel/modules/cookies/cookies.js) > cookies metrics -##### ``cookiesRecv`` +##### `cookiesRecv` length of cookies received in HTTP responses (bytes) -##### ``cookiesSent`` +##### `cookiesSent` length of cookies sent in HTTP requests (bytes) -##### ``documentCookiesCount`` +##### `documentCookiesCount` number of cookies in document.cookie (number) -##### ``documentCookiesLength`` +##### `documentCookiesLength` length of document.cookie (bytes) -##### ``domainsWithCookies`` +##### `domainsWithCookies` number of domains with cookies set (number, with offenders) + ## [documentHeight](https://github.com/macbre/phantomas/tree/devel/modules/documentHeight/documentHeight.js) > Measure document height -##### ``documentHeight`` +##### `documentHeight` the page height (px) + ## [domComplexity](https://github.com/macbre/phantomas/tree/devel/modules/domComplexity/domComplexity.js) > Analyzes DOM complexity -##### ``DOMelementMaxDepth`` +##### `DOMelementMaxDepth` maximum level on nesting of HTML element node (number, with offenders) -##### ``DOMelementsCount`` +##### `DOMelementsCount` total number of HTML element nodes (number) -##### ``DOMidDuplicated`` +##### `DOMidDuplicated` number of duplicated IDs found in DOM (number, with offenders) -##### ``bodyHTMLSize`` +##### `bodyHTMLSize` the size of body tag content (document.body.innerHTML.length) (bytes) -##### ``commentsSize`` +##### `commentsSize` the size of HTML comments on the page (bytes, with offenders) -##### ``iframesCount`` +##### `iframesCount` number of iframe nodes (number, with offenders) -##### ``imagesScaledDown`` +##### `imagesScaledDown` number of nodes that have images scaled down in HTML (number, with offenders) -##### ``imagesWithoutDimensions`` +##### `imagesWithoutDimensions` number of nodes without both width and height attribute (number, with offenders) -##### ``nodesWithInlineCSS`` +##### `nodesWithInlineCSS` number of nodes with inline CSS styling (with style attribute) (number, with offenders) -##### ``whiteSpacesSize`` +##### `whiteSpacesSize` the size of text nodes with whitespaces only (bytes) + ## [domHiddenContent](https://github.com/macbre/phantomas/tree/devel/modules/domHiddenContent/domHiddenContent.js) > Analyzes DOM hidden content -##### ``hiddenContentSize`` +##### `hiddenContentSize` the size of content of hidden elements on the page (with CSS display: none) (bytes, with offenders) -##### ``hiddenImages`` +##### `hiddenImages` number of hidden images that can be lazy-loaded (number, with offenders) + ## [domMutations](https://github.com/macbre/phantomas/tree/devel/modules/domMutations/domMutations.js) > Analyzes DOM changes via MutationObserver API -##### ``DOMmutationsAttributes`` +##### `DOMmutationsAttributes` number of DOM nodes attributes changes (number, with offenders) -##### ``DOMmutationsInserts`` +##### `DOMmutationsInserts` number of node inserts (number, with offenders) -##### ``DOMmutationsRemoves`` +##### `DOMmutationsRemoves` number of node removes (number, with offenders) + ## [domQueries](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/domQueries.js) > Analyzes DOM queries done via native DOM methods -##### ``DOMinserts`` +##### `DOMinserts` number of DOM nodes inserts (number, with offenders) -##### ``DOMqueries`` +##### `DOMqueries` number of all DOM queries (number, with offenders) -##### ``DOMqueriesAvoidable`` +##### `DOMqueriesAvoidable` number of repeated uses of a duplicated query (number) -##### ``DOMqueriesByClassName`` +##### `DOMqueriesByClassName` number of document.getElementsByClassName calls (number, with offenders) -##### ``DOMqueriesById`` +##### `DOMqueriesById` number of document.getElementById calls (number, with offenders) -##### ``DOMqueriesByQuerySelectorAll`` +##### `DOMqueriesByQuerySelectorAll` number of document.querySelector(All) calls (number, with offenders) -##### ``DOMqueriesByTagName`` +##### `DOMqueriesByTagName` number of document.getElementsByTagName calls (number, with offenders) -##### ``DOMqueriesDuplicated`` +##### `DOMqueriesDuplicated` number of DOM queries called more than once (number, with offenders) -##### ``DOMqueriesWithoutResults`` +##### `DOMqueriesWithoutResults` number of DOM queries that returned nothing (number, with offenders) + ## [domains](https://github.com/macbre/phantomas/tree/devel/modules/domains/domains.js) > Domains monitor -##### ``domains`` +##### `domains` number of domains used to fetch the page (number, with offenders) -##### ``maxRequestsPerDomain`` +##### `maxRequestsPerDomain` maximum number of requests fetched from a single domain (number) -##### ``medianRequestsPerDomain`` +##### `medianRequestsPerDomain` median of number of requests fetched from each domain (number) + ## [events](https://github.com/macbre/phantomas/tree/devel/modules/events/events.js) > Analyzes events bound to DOM elements -##### ``eventsBound`` +##### `eventsBound` number of EventTarget.addEventListener calls (number, with offenders) -##### ``eventsDispatched`` +##### `eventsDispatched` number of EventTarget.dispatchEvent calls (number, with offenders) -##### ``eventsScrollBound`` +##### `eventsScrollBound` number of scroll event bounds (number, with offenders) + ## [globalVariables](https://github.com/macbre/phantomas/tree/devel/modules/globalVariables/globalVariables.js) > Counts global JavaScript variables -##### ``globalVariables`` +##### `globalVariables` number of JS globals variables (number, with offenders) -##### ``globalVariablesFalsy`` +##### `globalVariablesFalsy` number of JS globals variables with falsy value (number, with offenders) + ## [headers](https://github.com/macbre/phantomas/tree/devel/modules/headers/headers.js) > Analyzes HTTP headers in both requests and responses -##### ``headersBiggerThanContent`` +##### `headersBiggerThanContent` number of responses with headers part bigger than the response body (number, with offenders) -##### ``headersCount`` +##### `headersCount` number of requests and responses headers (number) -##### ``headersRecvCount`` +##### `headersRecvCount` number of headers received in responses (number) -##### ``headersRecvSize`` +##### `headersRecvSize` size of received headers (bytes) -##### ``headersSentCount`` +##### `headersSentCount` number of headers sent in requests (number) -##### ``headersSentSize`` +##### `headersSentSize` size of sent headers (bytes) -##### ``headersSize`` +##### `headersSize` size of all headers (bytes) + ## [jQuery](https://github.com/macbre/phantomas/tree/devel/modules/jQuery/jQuery.js) > Analyzes jQuery activity -##### ``jQueryDOMReads`` +##### `jQueryDOMReads` number of DOM read operations (number, with offenders) -##### ``jQueryDOMWriteReadSwitches`` +##### `jQueryDOMWriteReadSwitches` number of read operations that follow a series of write operations (will cause repaint and can cause reflow) (number, with offenders) -##### ``jQueryDOMWrites`` +##### `jQueryDOMWrites` number of DOM write operations (number, with offenders) -##### ``jQueryEventTriggers`` +##### `jQueryEventTriggers` number of jQuery event triggers (number, with offenders) -##### ``jQueryOnDOMReadyFunctions`` +##### `jQueryOnDOMReadyFunctions` number of functions bound to onDOMReady event (number, with offenders) -##### ``jQuerySizzleCalls`` +##### `jQuerySizzleCalls` number of calls to Sizzle (including those that will be resolved using querySelectorAll) (number, with offenders) -##### ``jQueryVersion`` +##### `jQueryVersion` version of jQuery framework (if loaded) (string) -##### ``jQueryVersionsLoaded`` +##### `jQueryVersionsLoaded` number of loaded jQuery "instances" (even in the same version) (number, with offenders) -##### ``jQueryWindowOnLoadFunctions`` +##### `jQueryWindowOnLoadFunctions` number of functions bound to windowOnLoad event (number, with offenders) + ## [javaScriptBottlenecks](https://github.com/macbre/phantomas/tree/devel/modules/javaScriptBottlenecks/javaScriptBottlenecks.js) > Reports the use of functions known to be serious performance bottlenecks in JS Run phantomas with --spy-eval to count eval() calls (see issue #467) -##### ``documentWriteCalls`` +##### `documentWriteCalls` number of calls to either document.write or document.writeln (number, with offenders) -##### ``evalCalls`` +##### `evalCalls` number of calls to eval (either direct or via setTimeout / setInterval) (number, with offenders) + ## [jserrors](https://github.com/macbre/phantomas/tree/devel/modules/jserrors/jserrors.js) > Meters the number of page errors, and provides traces as offenders for "jsErrors" metric -##### ``jsErrors`` +##### `jsErrors` number of JavaScript errors (number, with offenders) + ## [keepAlive](https://github.com/macbre/phantomas/tree/devel/modules/keepAlive/keepAlive.js) > Analyzes if HTTP responses keep the connections alive. -##### ``closedConnections`` +##### `closedConnections` number of requests not keeping the connection alive and slowing down the next request (number, with offenders) + ## [lazyLoadableImages](https://github.com/macbre/phantomas/tree/devel/modules/lazyLoadableImages/lazyLoadableImages.js) > Analyzes images and detects which one can be lazy-loaded (are below the fold) -##### ``lazyLoadableImagesBelowTheFold`` +##### `lazyLoadableImagesBelowTheFold` number of images displayed below the fold that can be lazy-loaded (number, with offenders) + ## [localStorage](https://github.com/macbre/phantomas/tree/devel/modules/localStorage/localStorage.js) > localStorage metrics -##### ``localStorageEntries`` +##### `localStorageEntries` number of entries in local storage (number, with offenders) + ## [mainRequest](https://github.com/macbre/phantomas/tree/devel/modules/mainRequest/mainRequest.js) > Analyzes bits of data pertaining to the main request only -##### ``statusCodesTrail`` +##### `statusCodesTrail` comma-separated list of HTTP status codes that main request followed through (could contain a single element if the main request is a terminal one) (string) + ## [navigationTiming](https://github.com/macbre/phantomas/tree/devel/core/modules/navigationTiming/navigationTiming.js) > Emits "milestone" event on Navigation Timing related events: @@ -636,203 +662,211 @@ comma-separated list of HTTP status codes that main request followed through (co - domComplete Code taken from windowPerformance module + ## [redirects](https://github.com/macbre/phantomas/tree/devel/modules/redirects/redirects.js) > Analyzes HTTP redirects -##### ``redirects`` +##### `redirects` number of HTTP redirects (either 301, 302 or 303) (number, with offenders) -##### ``redirectsTime`` +##### `redirectsTime` time it took to send and receive redirects (ms) + ## [requestsMonitor](https://github.com/macbre/phantomas/tree/devel/core/modules/requestsMonitor/requestsMonitor.js) > Simple HTTP requests monitor and analyzer -##### ``bodySize`` +##### `bodySize` size of the uncompressed content of all responses (bytes) -##### ``contentLength`` +##### `contentLength` size of the compressed content of all responses, i.e. what was transfered in packets (bytes) -##### ``gzipRequests`` +##### `gzipRequests` number of gzipped HTTP responses (number, with offenders) -##### ``httpTrafficCompleted`` +##### `httpTrafficCompleted` time it took to receive the last byte of the last HTTP response (ms) -##### ``httpsRequests`` +##### `httpsRequests` number of HTTPS requests (number, with offenders) -##### ``notFound`` +##### `notFound` number of HTTP 404 responses (number, with offenders) -##### ``postRequests`` +##### `postRequests` number of POST requests (number, with offenders) -##### ``requests`` +##### `requests` total number of HTTP requests made (number, with offenders) + ## [requestsStats](https://github.com/macbre/phantomas/tree/devel/modules/requestsStats/requestsStats.js) > Analyzes HTTP requests and generates stats metrics -##### ``biggestLatency`` +##### `biggestLatency` the time to the first byte of the slowest response (ms, with offenders) -##### ``biggestResponse`` +##### `biggestResponse` the size of the biggest response (bytes, with offenders) -##### ``fastestResponse`` +##### `fastestResponse` the time to the last byte of the fastest response (ms, with offenders) -##### ``medianLatency`` +##### `medianLatency` median value of time to the first byte for all responses (ms, with offenders) -##### ``medianResponse`` +##### `medianResponse` median value of time to the last byte for all responses (ms, with offenders) -##### ``slowestResponse`` +##### `slowestResponse` the time to the last byte of the slowest response (ms, with offenders) -##### ``smallestLatency`` +##### `smallestLatency` the time to the first byte of the fastest response (ms, with offenders) -##### ``smallestResponse`` +##### `smallestResponse` the size of the smallest response (bytes, with offenders) + ## [requestsTo](https://github.com/macbre/phantomas/tree/devel/modules/requestsTo/requestsTo.js) > Number of requests it took to make the page enter DomContentLoaded and DomComplete states accordingly -##### ``domainsToDomComplete`` +##### `domainsToDomComplete` number of domains used to make the page reach DomComplete state (number, with offenders) -##### ``domainsToDomContentLoaded`` +##### `domainsToDomContentLoaded` number of domains used to make the page reach DomContentLoaded state (number, with offenders) -##### ``domainsToFirstPaint`` +##### `domainsToFirstPaint` number of domains used to make the first paint (number, with offenders) -##### ``requestsToDomComplete`` +##### `requestsToDomComplete` number of HTTP requests it took to make the page reach DomComplete state (number) -##### ``requestsToDomContentLoaded`` +##### `requestsToDomContentLoaded` number of HTTP requests it took to make the page reach DomContentLoaded state (number) -##### ``requestsToFirstPaint`` +##### `requestsToFirstPaint` number of HTTP requests it took to make the first paint (number) + ## [staticAssets](https://github.com/macbre/phantomas/tree/devel/modules/staticAssets/staticAssets.js) > Analyzes static assets (CSS, JS and images) -##### ``assetsNotGzipped`` +##### `assetsNotGzipped` number of static assets that were not gzipped (number, with offenders) -##### ``assetsWithCookies`` +##### `assetsWithCookies` number of static assets requested from domains with cookie set (number, with offenders) -##### ``assetsWithQueryString`` +##### `assetsWithQueryString` number of static assets requested with query string (e.g. ?foo) in URL (number, with offenders) -##### ``multipleRequests`` +##### `multipleRequests` number of static assets that are requested more than once (number, with offenders) -##### ``smallCssFiles`` +##### `smallCssFiles` number of CSS assets smaller than 2 KiB that can be inlined or merged (number, with offenders) -##### ``smallImages`` +##### `smallImages` number of images smaller than 2 KiB that can be base64 encoded (number, with offenders) -##### ``smallJsFiles`` +##### `smallJsFiles` number of JS assets smaller than 2 KiB that can be inlined or merged (number, with offenders) + ## [timeToFirst](https://github.com/macbre/phantomas/tree/devel/modules/timeToFirst/timeToFirst.js) > Provides metrics for time to first image, CSS and JS file -##### ``timeToFirstCss`` +##### `timeToFirstCss` time it took to receive the last byte of the first CSS (ms, with offenders) -##### ``timeToFirstImage`` +##### `timeToFirstImage` time it took to receive the last byte of the first image (ms, with offenders) -##### ``timeToFirstJs`` +##### `timeToFirstJs` time it took to receive the last byte of the first JS (ms, with offenders) + ## [timeToFirstByte](https://github.com/macbre/phantomas/tree/devel/core/modules/timeToFirstByte/timeToFirstByte.js) > Takes a look at "time to first (last) byte" metrics -##### ``timeToFirstByte`` +##### `timeToFirstByte` time it took to receive the first byte of the first response (that was not a redirect) (ms) -##### ``timeToLastByte`` +##### `timeToLastByte` time it took to receive the last byte of the first response (that was not a redirect) (ms) + ## [windowPerformance](https://github.com/macbre/phantomas/tree/devel/modules/windowPerformance/windowPerformance.js) > Measure when the page reaches certain states -##### ``domComplete`` +##### `domComplete` time it took to load all page resources, the loading spinner has stopped spinning (ms) -##### ``domContentLoaded`` +##### `domContentLoaded` time it took to construct both DOM and CSSOM, no stylesheets that are blocking JavaScript execution (i.e. onDOMReady) (ms) -##### ``domContentLoadedEnd`` +##### `domContentLoadedEnd` time it took to finish handling of onDOMReady event (ms) -##### ``domInteractive`` +##### `domInteractive` time it took to parse the HTML and construct the DOM (ms) -##### ``timeBackend`` +##### `timeBackend` time to the first byte compared to the total loading time (%) -##### ``timeFrontend`` +##### `timeFrontend` time to window.load compared to the total loading time (%) diff --git a/lib/metadata/make_docs.js b/lib/metadata/make_docs.js index 19b02419c..496a5356e 100644 --- a/lib/metadata/make_docs.js +++ b/lib/metadata/make_docs.js @@ -83,13 +83,14 @@ var metrics = ` Modules and metrics =================== -This fille describes all phantomas modules and metrics that they emit. +This file describes all [\`phantomas\` modules](https://github.com/macbre/phantomas/tree/devel/modules) (${metadata.modulesCount} of them) and ${metadata.metricsCount} metrics that they emit. ` Object.keys(metadata.modules).sort().forEach(moduleName => { const entry = metadata.modules[moduleName]; metrics += ` + ## [${moduleName}](${github_root}${entry.file}) > ${entry.desc} @@ -100,7 +101,7 @@ Object.keys(metadata.modules).sort().forEach(moduleName => { hasOffenders = metric.offenders ? ", with offenders" : ''; metrics += ` -##### \`\`${metricName}\`\` +##### \`${metricName}\` ${metric.desc} (${metric.unit}${hasOffenders}) ` From 91024e7bb29859caf8d928eafd36e404e4d6c75c Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 15:50:10 +0100 Subject: [PATCH 174/248] metrics.md: add offenders examples --- docs/metrics.md | 380 ++++++++++++++++++++++++++++++++++++++ lib/metadata/make_docs.js | 34 ++++ 2 files changed, 414 insertions(+) diff --git a/docs/metrics.md b/docs/metrics.md index cb1416a48..a7d717719 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -3,6 +3,9 @@ Modules and metrics This file describes all [`phantomas` modules](https://github.com/macbre/phantomas/tree/devel/modules) (34 of them) and 172 metrics that they emit. +When applicable, [offender](https://github.com/macbre/phantomas/issues/140) example is provided. + + ## [ajaxRequests](https://github.com/macbre/phantomas/tree/devel/modules/ajaxRequests/ajaxRequests.js) @@ -12,6 +15,13 @@ This file describes all [`phantomas` modules](https://github.com/macbre/phantoma number of AJAX requests (number, with offenders) +```json +{ + "url": "/static/style.css", + "method": "POST" +} +``` + ## [alerts](https://github.com/macbre/phantomas/tree/devel/modules/alerts/alerts.js) @@ -22,14 +32,26 @@ window.prompt. number of calls to window.alert (number, with offenders) +```json +"Oh dear!" +``` + ##### `windowConfirms` number of calls to window.confirm (number, with offenders) +```json +"Oh dear?" +``` + ##### `windowPrompts` number of calls to window.prompt (number, with offenders) +```json +"What's your name, dear?" +``` + ## [analyzeCss](https://github.com/macbre/phantomas/tree/devel/modules/analyzeCss/analyzeCss.js) @@ -116,6 +138,25 @@ number of properties with no longer needed vendor prefix, powered by data provid number of CSS files (or embeded CSS) that failed to be parse by analyze-css (number, with offenders) +```json +{ + "url": "[inline CSS]", + "value": { + "message": "missing '{'", + "position": { + "end": { + "column": 2, + "line": 3 + }, + "start": { + "column": 2, + "line": 3 + } + } + } +} +``` + ##### `cssQualifiedSelectors` number of qualified selectors (e.g. header#nav, .foo#bar, h1.title) (number, with offenders) @@ -201,6 +242,13 @@ size of base64 encoded responses (bytes) number of CSS responses (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/style.css", + "size": 308 +} +``` + ##### `cssSize` size of CSS responses (with compression) (bytes) @@ -209,6 +257,13 @@ size of CSS responses (with compression) (bytes) number of HTML responses (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/dom-operations.html", + "size": 2385 +} +``` + ##### `htmlSize` size of HTML responses (with compression) (bytes) @@ -217,6 +272,13 @@ size of HTML responses (with compression) (bytes) number of image responses (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/blank.gif", + "size": 330 +} +``` + ##### `imageSize` size of image responses (with compression) (bytes) @@ -225,6 +287,13 @@ size of image responses (with compression) (bytes) number of JS responses (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/jquery-2.1.1.min.js", + "size": 29730 +} +``` + ##### `jsSize` size of JS responses (with compression) (bytes) @@ -233,6 +302,13 @@ size of JS responses (with compression) (bytes) number of JSON responses (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/foo.json", + "size": 333 +} +``` + ##### `jsonSize` size of JSON responses (with compression) (bytes) @@ -271,6 +347,10 @@ Does not emit any metrics number of requests blocked due to domain filtering (number, with offenders) +```json +"http://code.jquery.com/jquery-1.11.1.js" +``` + ## [cacheHits](https://github.com/macbre/phantomas/tree/devel/modules/cacheHits/cacheHits.js) @@ -305,6 +385,13 @@ number of responses with no caching header sent (no Cache-Control header) (numbe number of responses with too short (less than a week) caching time (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/mdn.png", + "ttl": 84600 +} +``` + ##### `cachingUseImmutable` number of responses with a long TTL that can benefit from Cache-Control: immutable (number, with offenders) @@ -322,6 +409,10 @@ number of responses with old, HTTP 1.0 caching headers (Expires and Pragma) (num number of calls to console.* functions (number, with offenders) +```json +"log:[\"Hi!\"]" +``` + ## [cookies](https://github.com/macbre/phantomas/tree/devel/modules/cookies/cookies.js) @@ -385,10 +476,27 @@ the size of HTML comments on the page (bytes, with offenders) number of iframe nodes (number, with offenders) +```json +{ + "element": "body > iframe[1]", + "url": "http://127.0.0.1:8888/image-scaling.html" +} +``` + ##### `imagesScaledDown` number of nodes that have images scaled down in HTML (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/mdn.png", + "naturalWidth": 600, + "naturalHeight": 529, + "imgWidth": 300, + "imgHeight": 265 +} +``` + ##### `imagesWithoutDimensions` number of nodes without both width and height attribute (number, with offenders) @@ -397,6 +505,13 @@ number of nodes without both width and height attribute (number, with offe number of nodes with inline CSS styling (with style attribute) (number, with offenders) +```json +{ + "css": "color: blue", + "node": "p#foo" +} +``` + ##### `whiteSpacesSize` the size of text nodes with whitespaces only (bytes) @@ -414,6 +529,10 @@ the size of content of hidden elements on the page (with CSS display: none) (byt number of hidden images that can be lazy-loaded (number, with offenders) +```json +"http://127.0.0.1:8888/static/mdn.png" +``` + ## [domMutations](https://github.com/macbre/phantomas/tree/devel/modules/domMutations/domMutations.js) @@ -423,14 +542,35 @@ number of hidden images that can be lazy-loaded (number, with offenders) number of DOM nodes attributes changes (number, with offenders) +```json +{ + "attribute": "style", + "node": "body" +} +``` + ##### `DOMmutationsInserts` number of node inserts (number, with offenders) +```json +{ + "node": "strong[0]", + "target": "p#foo" +} +``` + ##### `DOMmutationsRemoves` number of node removes (number, with offenders) +```json +{ + "node": "span#delete", + "target": "body" +} +``` + ## [domQueries](https://github.com/macbre/phantomas/tree/devel/modules/domQueries/domQueries.js) @@ -440,6 +580,13 @@ number of node removes (number, with offenders) number of DOM nodes inserts (number, with offenders) +```json +{ + "append": "html > div[2]", + "node": "html" +} +``` + ##### `DOMqueries` number of all DOM queries (number, with offenders) @@ -452,10 +599,24 @@ number of repeated uses of a duplicated query (number) number of document.getElementsByClassName calls (number, with offenders) +```json +{ + "class": "barr", + "node": "body" +} +``` + ##### `DOMqueriesById` number of document.getElementById calls (number, with offenders) +```json +{ + "id": "foo", + "node": "#document" +} +``` + ##### `DOMqueriesByQuerySelectorAll` number of document.querySelector(All) calls (number, with offenders) @@ -464,10 +625,24 @@ number of document.querySelector(All) calls (number, with offenders) number of document.getElementsByTagName calls (number, with offenders) +```json +{ + "tag": "*", + "node": "div" +} +``` + ##### `DOMqueriesDuplicated` number of DOM queries called more than once (number, with offenders) +```json +{ + "query": "id \"#foo\" (in #document)", + "count": 2 +} +``` + ##### `DOMqueriesWithoutResults` number of DOM queries that returned nothing (number, with offenders) @@ -481,6 +656,13 @@ number of DOM queries that returned nothing (number, with offenders) number of domains used to fetch the page (number, with offenders) +```json +{ + "domain": "127.0.0.1", + "requests": 2 +} +``` + ##### `maxRequestsPerDomain` maximum number of requests fetched from a single domain (number) @@ -498,14 +680,34 @@ median of number of requests fetched from each domain (number) number of EventTarget.addEventListener calls (number, with offenders) +```json +{ + "eventType": "load", + "path": "window" +} +``` + ##### `eventsDispatched` number of EventTarget.dispatchEvent calls (number, with offenders) +```json +{ + "eventType": "click", + "path": "body > div#foo > span.bar" +} +``` + ##### `eventsScrollBound` number of scroll event bounds (number, with offenders) +```json +{ + "element": "#document" +} +``` + ## [globalVariables](https://github.com/macbre/phantomas/tree/devel/modules/globalVariables/globalVariables.js) @@ -515,10 +717,21 @@ number of scroll event bounds (number, with offenders) number of JS globals variables (number, with offenders) +```json +"jQuery" +``` + ##### `globalVariablesFalsy` number of JS globals variables with falsy value (number, with offenders) +```json +{ + "name": "falsy", + "value": false +} +``` + ## [headers](https://github.com/macbre/phantomas/tree/devel/modules/headers/headers.js) @@ -528,6 +741,14 @@ number of JS globals variables with falsy value (number, with offenders) number of responses with headers part bigger than the response body (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/blank.gif", + "contentSize": 43, + "headersSize": 287 +} +``` + ##### `headersCount` number of requests and responses headers (number) @@ -561,18 +782,49 @@ size of all headers (bytes) number of DOM read operations (number, with offenders) +```json +{ + "functionName": "css", + "arguments": "[\"color\"]", + "contextPath": "body > div#foo > span.bar" +} +``` + ##### `jQueryDOMWriteReadSwitches` number of read operations that follow a series of write operations (will cause repaint and can cause reflow) (number, with offenders) +```json +{ + "functionName": "css", + "arguments": "[\"color\"]", + "contextPath": "body > div#foo > span.bar" +} +``` + ##### `jQueryDOMWrites` number of DOM write operations (number, with offenders) +```json +{ + "functionName": "css", + "arguments": "[{\"color\":\"red\",\"background\":\"green\"}]", + "contextPath": "body > div#foo > span.bar" +} +``` + ##### `jQueryEventTriggers` number of jQuery event triggers (number, with offenders) +```json +{ + "type": "click", + "element": "body > div#foo > span.bar" +} +``` + ##### `jQueryOnDOMReadyFunctions` number of functions bound to onDOMReady event (number, with offenders) @@ -581,6 +833,13 @@ number of functions bound to onDOMReady event (number, with offenders) number of calls to Sizzle (including those that will be resolved using querySelectorAll) (number, with offenders) +```json +{ + "selector": "#foo .bar", + "element": "#document" +} +``` + ##### `jQueryVersion` version of jQuery framework (if loaded) (string) @@ -589,10 +848,21 @@ version of jQuery framework (if loaded) (string) number of loaded jQuery "instances" (even in the same version) (number, with offenders) +```json +{ + "version": "2.1.1", + "url": "http://127.0.0.1:8888/static/jquery-2.1.1.min.js" +} +``` + ##### `jQueryWindowOnLoadFunctions` number of functions bound to windowOnLoad event (number, with offenders) +```json +"http://127.0.0.1:8888/jquery.html:49:13" +``` + ## [javaScriptBottlenecks](https://github.com/macbre/phantomas/tree/devel/modules/javaScriptBottlenecks/javaScriptBottlenecks.js) @@ -604,10 +874,24 @@ Run phantomas with --spy-eval to count eval() calls (see issue #467) number of calls to either document.write or document.writeln (number, with offenders) +```json +{ + "message": "document.write() used", + "caller": "http://127.0.0.1:8888/bottlenecks.html:11:11" +} +``` + ##### `evalCalls` number of calls to eval (either direct or via setTimeout / setInterval) (number, with offenders) +```json +{ + "message": "eval() called directly", + "caller": "http://127.0.0.1:8888/bottlenecks.html:8:2" +} +``` + ## [jserrors](https://github.com/macbre/phantomas/tree/devel/modules/jserrors/jserrors.js) @@ -635,6 +919,14 @@ number of requests not keeping the connection alive and slowing down the next re number of images displayed below the fold that can be lazy-loaded (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/blank.gif", + "node": "body > img#dot", + "offset": 900 +} +``` + ## [localStorage](https://github.com/macbre/phantomas/tree/devel/modules/localStorage/localStorage.js) @@ -644,6 +936,10 @@ number of images displayed below the fold that can be lazy-loaded (number, with number of entries in local storage (number, with offenders) +```json +"foo" +``` + ## [mainRequest](https://github.com/macbre/phantomas/tree/devel/modules/mainRequest/mainRequest.js) @@ -692,6 +988,14 @@ size of the compressed content of all responses, i.e. what was transfered in pac number of gzipped HTTP responses (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/jquery-2.1.1.min.js", + "bodySize": 84245, + "transferedSize": 29415 +} +``` + ##### `httpTrafficCompleted` time it took to receive the last byte of the last HTTP response (ms) @@ -700,18 +1004,38 @@ time it took to receive the last byte of the last HTTP response (ms) number of HTTPS requests (number, with offenders) +```json +"https://httpbin.org/basic-auth/foo/bar" +``` + ##### `notFound` number of HTTP 404 responses (number, with offenders) +```json +"http://127.0.0.1:8888/not_found/foo.js" +``` + ##### `postRequests` number of POST requests (number, with offenders) +```json +"http://127.0.0.1:8888/static/style.css" +``` + ##### `requests` total number of HTTP requests made (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/headers.html", + "type": "html", + "size": 1313 +} +``` + ## [requestsStats](https://github.com/macbre/phantomas/tree/devel/modules/requestsStats/requestsStats.js) @@ -725,6 +1049,13 @@ the time to the first byte of the slowest response (ms, with offenders) the size of the biggest response (bytes, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/headers.html", + "size": 1313 +} +``` + ##### `fastestResponse` the time to the last byte of the fastest response (ms, with offenders) @@ -749,6 +1080,13 @@ the time to the first byte of the fastest response (ms, with offenders) the size of the smallest response (bytes, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/blank.gif", + "size": 330 +} +``` + ## [requestsTo](https://github.com/macbre/phantomas/tree/devel/modules/requestsTo/requestsTo.js) @@ -758,10 +1096,24 @@ the size of the smallest response (bytes, with offenders) number of domains used to make the page reach DomComplete state (number, with offenders) +```json +{ + "domain": "127.0.0.1", + "requests": 2 +} +``` + ##### `domainsToDomContentLoaded` number of domains used to make the page reach DomContentLoaded state (number, with offenders) +```json +{ + "domain": "127.0.0.1", + "requests": 2 +} +``` + ##### `domainsToFirstPaint` number of domains used to make the first paint (number, with offenders) @@ -787,6 +1139,13 @@ number of HTTP requests it took to make the first paint (number) number of static assets that were not gzipped (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/https-fonts.html", + "contentType": "text/html" +} +``` + ##### `assetsWithCookies` number of static assets requested from domains with cookie set (number, with offenders) @@ -795,6 +1154,13 @@ number of static assets requested from domains with cookie set (number, with off number of static assets requested with query string (e.g. ?foo) in URL (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/mdn.png?cb=123", + "contentType": "image/png" +} +``` + ##### `multipleRequests` number of static assets that are requested more than once (number, with offenders) @@ -803,10 +1169,24 @@ number of static assets that are requested more than once (number, with offender number of CSS assets smaller than 2 KiB that can be inlined or merged (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/style.css", + "size": 308 +} +``` + ##### `smallImages` number of images smaller than 2 KiB that can be base64 encoded (number, with offenders) +```json +{ + "url": "http://127.0.0.1:8888/static/blank.gif", + "size": 330 +} +``` + ##### `smallJsFiles` number of JS assets smaller than 2 KiB that can be inlined or merged (number, with offenders) diff --git a/lib/metadata/make_docs.js b/lib/metadata/make_docs.js index 496a5356e..927dcc2f6 100644 --- a/lib/metadata/make_docs.js +++ b/lib/metadata/make_docs.js @@ -79,11 +79,34 @@ promise.on('beforeClose', () => { * * https://github.com/macbre/phantomas/issues/729 */ + +// get offender examples from integration-spec.yaml +const offenders = (() => { + const yaml = require('js-yaml'), + spec = yaml.safeLoad(fs.readFileSync(__dirname + '/../../test/integration-spec.yaml').toString()); + + var offenders = {}; + + spec.forEach(testCase => { + Object.keys(testCase.offenders || {}).forEach(metric => { + if (!offenders[metric]) { + offenders[metric] = testCase.offenders[metric][0]; + } + }); + }); + + return offenders; +})(); + +// build Markdown file var metrics = ` Modules and metrics =================== This file describes all [\`phantomas\` modules](https://github.com/macbre/phantomas/tree/devel/modules) (${metadata.modulesCount} of them) and ${metadata.metricsCount} metrics that they emit. + +When applicable, [offender](https://github.com/macbre/phantomas/issues/140) example is provided. + ` Object.keys(metadata.modules).sort().forEach(moduleName => { @@ -105,6 +128,17 @@ Object.keys(metadata.modules).sort().forEach(moduleName => { ${metric.desc} (${metric.unit}${hasOffenders}) ` + + // add offender's example + if (hasOffenders && offenders[metricName]) { + const dumped = JSON.stringify(offenders[metricName], null, ' '); + + metrics += ` +\`\`\`json +${dumped} +\`\`\` +` + } }); }); From 655247e16b866ac93e13e68a8695cf767d6d8921 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 18:29:08 +0100 Subject: [PATCH 175/248] README.md: make this file a bit smaller, refer to /docs --- README.md | 479 +----------------------------------------------------- 1 file changed, 7 insertions(+), 472 deletions(-) diff --git a/README.md b/README.md index 97399ca45..f807d4417 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,19 @@ phantomas [![npm](https://img.shields.io/npm/dt/phantomas.svg)]() [![Build Status](https://api.travis-ci.org/macbre/phantomas.png?branch=devel)](http://travis-ci.org/macbre/phantomas) [![Known Vulnerabilities](https://snyk.io/test/github/macbre/phantomas/badge.svg)](https://snyk.io/test/github/macbre/phantomas) ========= -PhantomJS-based modular web performance metrics collector. And why phantomas? Well, [because](http://en.wikipedia.org/wiki/Fantômas) :) +[Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md)-based modular web performance metrics collector. And why phantomas? Well, [because](http://en.wikipedia.org/wiki/Fantômas) :) ## Requirements -* [NodeJS](http://nodejs.org) 4+ -* [NPM](https://www.npmjs.com/) 3+ +* [NodeJS](http://nodejs.org) 8+ ## Installation ``` -npm install --global --no-optional phantomas phantomjs-prebuilt-macbre +npm install phantomas ``` -> This will install [the latest version of PhantomJS](https://github.com/macbre/phantomjs-static-2.5/releases/tag/v2.5-beta) and add a symlink called ``phantomas`` (pointing to ``./bin/phantomas.js``) to your system's ``PATH`` - -You may need to install libfontconfig and libjpeg8 by running ``sudo apt-get install libfontconfig1 libjpeg8``. +> This will install [a recent version of Chromium](https://github.com/GoogleChrome/puppeteer#installation) supported by `puppeteer` module. ### Development version @@ -31,23 +28,10 @@ npm install Please refer to **[/Troubleshooting.md](https://github.com/macbre/phantomas/blob/devel/Troubleshooting.md)** -## Libraries - -phantomas is written in JavaScript, but you can experience it in different languages as well ;) - -### Python - -[![Latest Version](https://pypip.in/version/phantomas/badge.svg?style=flat)](https://pypi.python.org/pypi/phantomas/) -[![Supported Python versions](https://pypip.in/py_versions/phantomas/badge.svg?style=flat)](https://pypi.python.org/pypi/phantomas/) - -``` -pip install phantomas -``` - ## Features * modular approach - each metric is generated by a separate "module" -* phantomas "core" acts as an [events emitter](https://github.com/macbre/phantomas/wiki/Events) that each module can hook into +* phantomas "core" acts as an events emitter that each module can hook into * in-depth metrics such as: number of events bound via jQuery, calls to ``window.write``or [complex and duplicated CSS selectors (via analyze-css)](https://github.com/macbre/analyze-css) * JSON and CSV as available output formats for easy integration with automated reporting / monitoring tools * easy integration with Continuous Integration tools via TAP format and assertions handling @@ -55,7 +39,6 @@ pip install phantomas * easy integration with other nodejs projects via CommonJS module ([see API docs](https://github.com/macbre/phantomas/wiki/npm-module)) * metrics can be emitted from JavaScript code of the page phantomas is run against (thanks to [helper functions available in window.__phantomas](https://github.com/macbre/phantomas/wiki/Phantomas-scope)) * device profiles allow phantomas to emulate mobile or tablet (by setting a proper user agent and viewport) -* ability to run phantomas using WebKit (PhantomJS) or Gecko (SlimerJS) engine (experimental) ## Contributors @@ -75,449 +58,11 @@ pip install phantomas ## Usage -> phantomas comes as both command line tool and CommonJS module ([see API docs](https://github.com/macbre/phantomas/wiki/npm-module)) that you can use in your nodejs projects. - -### Single run - -``` bash -phantomas https://github.com/macbre/phantomas --verbose -``` - -You can measure the performance of your site without requests to 3rd party domains (but allowing CDN that serves your static assets): - -```bash -phantomas https://github.com/macbre/phantomas --verbose --no-externals --allow-domain .fastly.net -``` - -You can provide phantomas config via JSON or YAML file. We [support environment variables in YAML config files](https://github.com/macbre/phantomas/issues/685) ([the way Docker does](https://docs.docker.com/compose/environment-variables/)]. - -``` -./bin/phantomas.js --config examples/config.yaml -URL='https://www.google.is' ./bin/phantomas.js --config examples/config.yaml -``` - -An example config file with customizable test URL for various environments (production, staging, sandboxes, ...): - -```yaml -url: "https://${ENV:-prod}.super-fast-app.io" -``` - -#### Parameters - -* `--reporter=[json|csv|tap|plain|statsd|elasticsearch|cloudwatch]` results reporter aka format (``plain`` is the default one) -* `--timeout=[seconds]` timeout for phantomas run (defaults to 15 seconds) -* `--viewport=[width]x[height]` phantomJS viewport dimensions (1366x768 is the default) -* `--verbose` writes debug messages to the console -* `--debug` run PhantomJS in debug mode -* `--engine` select engine used to run the phantomas ``[webkit|gecko]`` -* `--colors` forces ANSI colors even when output is piped (e,g. via ``less -r``) -* `--silent` don't write anything to the console -* `--progress` shows page loading progress bar (disables verbose mode) -* `--log=[log file]` log to a given file -* `--modules=[moduleOne],[moduleTwo]` run only selected modules -* `--include-dirs=[dirOne],[dirTwo]` load modules from specified directories -* `--skip-modules=[moduleOne],[moduleTwo]` skip selected modules -* `--user-agent='Custom user agent'` provide a custom user agent (will default to something similar to ``phantomas/0.6.0 (PhantomJS/1.9.0; linux 64bit)``) -* `--config=[JSON/YAML config file]` uses JSON or YAML-formatted config file to set parameters -* `--cookie='bar=foo;domain=url'` document.cookie formatted string for setting a single cookie (separate multiple cookies using a pipe, e.g. `--cookie='foo=42;path=/foo|test=123'`) -* `--cookies-file=[JAR file]` specifies the file name to store the persistent Cookies -* `--no-externals` block requests to 3rd party domains -* `--allow-domain=[domain],[domain]` allow requests to given domain(s) - aka whitelist -* `--block-domain=[domain],[domain]` disallow requests to given domain(s) - aka blacklist -* `--auth-user` sets the user name used for HTTP authentication -* `--auth-pass` sets the password used for HTTP authentication -* `--disable-js` disable JavaScript on the page that will be loaded -* `--analyze-css` emit in-depth CSS metrics **experimental** -* `--film-strip` register film strip when page is loading **experimental** -* `--film-strip-dir=[dir path]` folder path to output film strip (default is ``./filmstrip`` directory) -* `--film-strip-prefix` film strip files name prefix (defaults to ``screenshot``) -* `--film-strip-zoom-factor` film strip zoom factor (defaults to ``0.5``) -* `--page-source` save page source to file **experimental** -* `--page-source-dir=[dir path]` folder path to output page source (default is ``./html`` directory) **experimental** -* `--assert-[metric-name]=value` assert that given metric should be less or equal the value -* `--screenshot=[file name]` render fully loaded page to a given file -* `--har=[file name]` save HAR to a given file -* `--wait-for-event=[phantomas event name]` wait for a given phantomas event before generating a report, timeout setting still applies (e.g. ``--wait-for-event "done"``) -* `--wait-for-selector=[CSS selector]` wait for an element matching given CSS selector before generating a report, timeout setting still applies (e.g. ``--wait-for-selector "body.loaded"``) -* `--stop-at-onload` stop phantomas **immediately after `onload` event** -* `--scroll` scroll down the page when it''s loaded -* `--post-load-delay=[seconds]` wait X seconds before generating a report, timeout setting still applies -* `--ignore-ssl-errors` ignores SSL errors, such as expired or self-signed certificate errors -* `--ssl-protocol` sets the SSL protocol for secure connections ``[sslv3|sslv2|tlsv1|any]`` -* `--proxy=[host:port]` specifies the proxy server to use -* `--proxy-auth=[username:password]` specifies the authentication information for the proxy -* `--proxy-type=[http|socks5|none]` specifies the type of the proxy server (default is http) -* `--phone` force viewport and user agent of a mobile phone -* `--tablet` force viewport and user agent of a tablet -* `--spy-eval` report calls to eval() - -### Multiple runs - -Simply provide ``--runs`` option: - -``` bash -phantomas https://github.com/macbre/phantomas --verbose --runs 5 -``` - -Only ``plain`` (the default one) and ``json`` reporters are currently supported in multiple runs mode. +> phantomas comes as a CommonJS module ([see API docs](https://github.com/macbre/phantomas/wiki/npm-module)) that you can use in your nodejs projects. ## Metrics -_Current number of metrics: 136_ - -Units: - -* ms for time -* bytes for size - -### Requests monitor (core module) - -> Due to [PhantomJS issue #10156](https://github.com/ariya/phantomjs/issues/10156) **body size related metrics are not reliable** - -* requests: total number of HTTP requests made -* gzipRequests: number of gzipped HTTP responses -* postRequests: number of POST requests -* httpsRequests: number of HTTPS requests -* notFound: number of HTTP 404 responses -* timeToFirstByte: time it took to receive the first byte of the first response (that was not a redirect) -* timeToLastByte: time it took to receive the last byte of the first response (that was not a redirect) -* bodySize: size of the content of all responses -* contentLength: size of the content of all responses (based on ``Content-Length`` header) -* httpTrafficCompleted: time it took to receive the last byte of the last HTTP response - -### AJAX requests - -* ajaxRequests: number of AJAX requests - -### Assets types - -> Due to [PhantomJS issue #10156](https://github.com/ariya/phantomjs/issues/10156) **body size related metrics are not reliable** - -* htmlCount: number of HTML responses -* htmlSize: size of HTML responses -* cssCount: number of CSS responses -* cssSize: size of CSS responses -* jsCount: number of JS responses -* jsSize: size of JS responses -* jsonCount: number of JSON responses -* jsonSize: size of JSON responses -* imageCount: number of image responses -* imageSize: size of image responses -* webfontCount: number of web font responses -* webfontSize: size of web font responses -* videoCount: number of video responses -* videoSize: size of video responses -* base64Count: number of base64 encoded "responses" (no HTTP request was actually made) -* base64Size: size of base64 encoded "responses" -* otherCount: number of other responses -* otherSize: size of other responses - -### Cache Hits - -> Metrics are calculated based on ``Age`` and ``X-Cache`` headers added by Varnish / Squid servers - -* cacheHits: number of cache hits -* cacheMisses: number of cache misses -* cachePasses: number of cache passes - -### Headers - -* headersCount: number of requests and responses headers -* headersSentCount: number of headers sent in requests -* headersRecvCount: number of headers received in responses -* headersSize: size of all headers -* headersSentSize: size of sent headers -* headersRecvSize: size of received headers -* headersBiggerThanContent: number of responses with headers part bigger than the response body - -### Domains - -* domains: number of domains used to fetch the page -* maxRequestsPerDomain: maximum number of requests fetched from a single domain -* medianRequestsPerDomain: median of requests fetched from each domain - -### Cookies - -* cookiesSent: length of cookies sent in HTTP requests -* cookiesRecv: length of cookies received in HTTP responses -* domainsWithCookies: number of domains with cookies set -* documentCookiesLength: length of `document.cookie` -* documentCookiesCount: number of cookies in `document.cookie` - -### DOM complexity - -> Metrics listed below are generated after the full page load - -* globalVariables: number of JS globals variables -* globalVariablesFalsy: number of JS global variables that cast to false -* bodyHTMLSize: the size of body tag content (``document.body.innerHTML.length``) -* commentsSize: the size of HTML comments on the page -* whiteSpacesSize: the size of text nodes with whitespaces only -* DOMelementsCount: total number of HTML element nodes -* DOMelementMaxDepth: maximum level on nesting of HTML element node -* DOMidDuplicated: number of duplicated IDs found in DOM -* iframesCount: number of iframe nodes -* nodesWithInlineCSS: number of nodes with inline CSS styling (with `style` attribute) -* imagesScaledDown: number of nodes that have images scaled down in HTML -* imagesWithoutDimensions: number of ```` nodes without both ``width`` and ``height`` attribute - -### DOM hidden content - -* hiddenContentSize: the size of content of hidden elements on the page (with CSS ``display: none``) -* hiddenImages: number of hidden images that can be lazy-loaded - -### DOM queries - -* DOMqueries: the sum of all four metrics below -* DOMqueriesWithoutResults: number of DOM queries that returned nothing -* DOMqueriesById: number of `document.getElementById` calls -* DOMqueriesByClassName: number of `document.getElementsByClassName` calls -* DOMqueriesByTagName: number of `document.getElementsByTagName` calls -* DOMqueriesByQuerySelectorAll: number of `document.querySelectorAll` calls -* DOMinserts: number of DOM nodes inserts -* DOMqueriesDuplicated: number of DOM queries called more than once -* DOMqueriesAvoidable: number of repeated uses of a duplicated query - -### DOM mutations - -> These metrics are only available when running phantomas using **Gecko engine** (``--engine=gecko``) - -* DOMmutationsInserts: number of node inserts -* DOMmutationsRemoves: number of node removes -* DOMmutationsAttributes: number of DOM nodes attributes changes - -### Event listeners - -* eventsBound: number of ``EventTarget.addEventListener`` calls -* eventsDispatched: number of ``EventTarget.dispatchEvent`` calls -* eventsScrollBound: number of `scroll` event bounds to `window` or `document` - -### Window performance - -> Times below are relative to ``responseEnd`` entry in NavigationTiming (represented by ``timeToLastByte`` metric). See [NavigationTiming spec](http://w3c-test.org/webperf/specs/NavigationTiming/) for more information. - -* domInteractive: time it took to parse the HTML and construct the DOM -* domContentLoaded: time it took to construct both DOM and CSSOM, no stylesheets are blocking JavaScript execution (i.e. onDOMReady) -* domContentLoadedEnd: time it took to finish handling of onDOMReady event **experimental** -* domComplete: time it took to load all page resources, the loading spinner has stopped spinning -* timeBackend: time to the first byte compared to the total loading time (in %) -* timeFrontend: time to window on load compared to the total loading time (in %) - -### Requests statistics - -> **Time** is total duration, from the start of the request to the receipt of the final byte in the response. **Latency** is the time to load the first byte in the response. -> https://developers.google.com/chrome-developer-tools/docs/network -> -> Includes ``HTTP 200`` responses only - -* smallestResponse: the size of the smallest response -* biggestResponse: the size of the biggest response -* fastestResponse: the time to the last byte of the fastest response -* slowestResponse: the time to the last byte of the slowest response -* smallestLatency: the time to the first byte of the fastest response -* biggestLatency: the time to the first byte of the slowest response -* medianResponse: median value of time to the last byte for all responses -* medianLatency: median value of time to the first byte for all responses - -### Requests to - -* requestsToFirstPaint: number of HTTP requests it took to make the first paint -* domainsToFirstPaint: number of domains used to make the first paint -* requestsToDomContentLoaded: number of HTTP requests it took to make the page reach `DomContentLoaded` state -* domainsToDomContentLoaded: number of domains used to make the page reach DomContentLoaded state -* requestsToDomComplete: number of HTTP requests it took to make the page reach `DomComplete` state -* domainsToDomComplete: number of domains used to make the page reach DomComplete state - -### keepAlive - -> Monitors the use of ``Connection: close`` and ``Keep-Alive`` - -* closedConnections: number of requests not keeping the connection alive and slowing down the next request - -### localStorage - -* localStorageEntries: number of entries in local storage - -### jQuery - -> Requires jQuery 1.8.0+ - -* jQueryVersion: version of jQuery framework (if loaded) -* jQueryVersionsLoaded: number of loaded jQuery "instances" (even in the same version) -* jQueryOnDOMReadyFunctions: number of functions bound to `onDOMReady` event -* jQueryWindowOnLoadFunctions: number of functions bound to `windowOnLoad` event -* jQuerySizzleCalls: number of calls to [Sizzle](http://sizzlejs.com/) (including those that will be resolved using ``querySelectorAll``) -* jQueryEventTriggers: number of jQuery event triggers -* jQueryDOMReads: number of DOM read operations -* jQueryDOMWrites: number of DOM write operations -* jQueryDOMWriteReadSwitches: number of read operations that follow a series of write operations (will cause repaint and can cause reflow) - -### Static assets - -* assetsNotGzipped: static assets that were not gzipped -* assetsWithQueryString: static assets requested with query string (e.g. ?foo) in URL -* assetsWithCookies: number of static assets requested from domains with cookie set -* smallImages: images smaller than 2 KiB that can be base64 encoded -* smallCssFiles: number of CSS assets smaller than 2 KiB that can be inlined or merged -* smallJsFiles: number of JS assets smaller than 2 KiB that can be inlined or merged -* multipleRequests: number of static assets that are requested more than once - -### Caching - -* cachingNotSpecified: responses with no caching header sent (either `Cache-Control` or `Expires`) -* cachingTooShort: responses with too short (less than a week) caching time -* cachingDisabled: responses with caching disabled (`max-age=0`) -* oldCachingHeaders: responses with old, HTTP 1.0 caching headers (``Expires`` and ``Pragma``) -* cachingUseImmutable: responses with a long TTL that can benefit from [`Cache-Control: immutable`](https://hacks.mozilla.org/2017/01/using-immutable-caching-to-speed-up-the-web/) - -### Time to first asset - -* timeToFirstCss: time it took to receive the last byte of the first CSS -* timeToFirstJs: time it took to receive the last byte of the first JS -* timeToFirstImage: time it took to receive the last byte of the first image - -### Redirects - -* redirects: number of HTTP redirects (either 301, 302 or 303) -* redirectsTime: time it took to send and receive redirects - -### JavaScript bottlenecks - -* documentWriteCalls: number of calls to either ``document.write`` or ``document.writeln`` -* evalCalls: number of calls to ``eval`` (either direct or via ``setTimeout`` / ``setInterval``) - -### JavaScript errors - -> Error message and backtrace will be emitted as offenders - -* jsErrors: number of JavaScript errors - -### JavaScript console and alert - -* windowAlerts: number of calls to ``alert`` -* windowConfirms: number of calls to ``confirm`` -* windowPrompts: number of calls to ``prompt`` -* consoleMessages: number of calls to ``console.*`` functions - -### Main request - -> Analyzes bits of data pertaining to the main request only - -* statusCodesTrail: comma-separated list of HTTP status codes that main request followed through (could contain a single element if the main request is a terminal one) - -### Document height - -* [documentHeight](http://www.stevesouders.com/blog/2014/06/08/http-archive-new-stuff/): the page height in pixels - -### Lazy-loadable images - -* lazyLoadableImagesBelowTheFold: number of images displayed below the fold that can be lazy-loaded - -### Optional metrics - -> The following metrics are emitted only when certain options are passed to phantomas - -* blockedRequests: number of requests blocked due to domain filtering (emitted only when in `--no-externals` / `--block-domain` mode) - -### CSS metrics - -> This is an experimental feature. Use `--analyze-css` option to enable it. - -Take a look at [analyze-css README](https://github.com/macbre/analyze-css) for the full list of metrics. - -* cssParsingErrors: number of CSS files (or embeded CSS) that failed to be parse by analyze-css - -### Reporters - -phantomas provides a number of reporters that can format the run results and send them to various tools. Use ``--reporter`` (or ``-R`` shortcut) option to use one. - -#### Formatters - -Results can be emitted as TAP, CSV and JSON. ``plain`` format is most useful for human beings :) - -#### Parameters - -Formatters can be provided with colon separated list of options: - -``` -$ phantomas http://foo.net -R csv:no-header:timestamp -``` - -This will omit CSV headers row and add current timestamp as the first column, so you can append the results line to a growing file. - -##### CSV -* ``no-header`` - omit CSV header -* ``timestamp`` - add the current timestamp as the first column -* ``url`` - add the URL as the first column - -##### JSON -* ``pretty`` - emits pretty printed JSON - -##### Plain -* ``no-color`` - disable ANSI colors - -##### StatsD -* ``::`` - shorthand for ``--statsd-host``, ``--statsd-port`` and ``--statsd-prefix`` (you don't need to provide all three options) - -##### TAP -* ``no-skip`` - don't print out metrics that were skipped - -##### StatsD -* ``::`` - shorthand for ``--statsd-host``, ``--statsd-port`` and ``--statsd-prefix`` (you don't need to provide all three options) - -#### StatsD integration - -Metrics from phantomas run can be sent directly to [StatsD](http://codeascraft.com/2011/02/15/measure-anything-measure-everything/) and then graphed using [graphite](http://graphite.wikidot.com/), [graphene](http://jondot.github.io/graphene/) or any other tool of your choice. For instance: - -``` -$ phantomas http://app.net/start -R statsd --statsd-host stats.app.net --statsd-port 8125 --statsd-prefix 'myApp.mainPage.' -``` - -or - -``` -$ phantomas http://app.net/start -R statsd:stats.app.net:8125:myApp.mainPage. -``` - -will sent metrics to StatsD running on ``stats.app.net:8125`` and prefix them with 'myApp.mainPage'. - -### 3rd-party reporters - -> These need to be installed on demand as 3rd party npm packages - -* [AWS CloudWatch](https://github.com/EFF/phantomas-reporter-cloudwatch) -* [elasticsearch](https://github.com/macbre/phantomas-reporter-elasticsearch) - -## Engines - -phantomas can be run using [PhantomJS 2.1.x](https://github.com/macbre/phantomas/issues/488) (WebKit-powered headless browser) or [SlimerJS](https://slimerjs.org/) (Gecko-based non headless browser, run using [`xfvb`](http://www.x.org/releases/X11R7.6/doc/man/man1/Xvfb.1.xhtml)). - -**PhantomJS 2.1.x is a default engine**. - -You can choose the engine by using either: - -* cli option: ``--engine=[webkit|gecko]`` or ``--webkit`` / ``--gecko`` -* `PHANTOMAS_ENGINE` environmental variable: e.g. `PHANTOMAS_ENGINE=gecko` - -> Please note that **support for SlimerJS is experimental at this point**. - -### PhantomJS - -All required binaries have already been installed by npm. No extra work needed here :) - -### SlimerJS - -In order to use SlimerJS install the following Debian/Ubuntu packages: - -``` -sudo aptitude install xvfb libasound2 libgtk2.0-0 -``` - -You will also need to install the module: - -```bash -npm install --global slimerjs@^0.906.1 -``` +> Please refer to `/docs/metrics.md` file for **a full, up-to-date list of all available modules and metrics** that phantomas emits. ## For developers @@ -525,16 +70,6 @@ npm install --global slimerjs@^0.906.1 * Description of [events fired by phantomas core](https://github.com/macbre/phantomas/wiki/Events) * Description of [helper functions available to the browser in window.__phantomas](https://github.com/macbre/phantomas/wiki/Phantomas-scope) -## Custom modules - -You can load your own, custom phantomas modules using `--include-dirs` option: - -``` -phantomas --include-dirs /my/path/to/custom/modules/ --url http://example.com -``` - -> `/my/path/to/custom/modules/` directory should contain custom modules, **each in its own directory**, e.g. `/my/path/to/custom/modules/fooBar/fooBar.js`. - ## Let's make Web a bit faster! * [Best Practices for Speeding Up Your Web Site](http://developer.yahoo.com/performance/rules.html) (by Yahoo!) From 33ce9ce3705e045cbf391b246e17669aa1c05f4b Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 3 Feb 2019 18:31:59 +0100 Subject: [PATCH 176/248] README: add npm module usage example --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index f807d4417..d30183dee 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,27 @@ npm install phantomas > This will install [a recent version of Chromium](https://github.com/GoogleChrome/puppeteer#installation) supported by `puppeteer` module. +## Usage example + +```js +const phantomas = require('phantomas'), + promise = phantomas('http://example.com/'); + +promise. + then(results => { + console.log('Metrics', results.getMetrics()); + console.log('Offenders', results.getAllOffenders()); + }). + catch(res => { + console.error(res); + }); + +// events handling +promise.on('recv', response => { + console.log('Response: %s %s [%s]', response.method, response.url, response.contentType); +}); +``` + ### Development version To get the latest development version of phantomas (and install all required dependencies): From 77eeba6275c82357e1c1a3b996e7d8453d464068 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 14:57:33 +0100 Subject: [PATCH 177/248] phantomas: expose getMetric function --- lib/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/index.js b/lib/index.js index 6fdfe8dee..3c98be708 100644 --- a/lib/index.js +++ b/lib/index.js @@ -73,6 +73,8 @@ function phantomas(url, opts) { setMetric: results.setMetric, addToAvgMetric: results.addToAvgMetric, + getMetric: results.getMetric, + // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args evaluate: page.evaluate.bind(page), From 4ba60551177d41ee220d998112753b81b3793ab8 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 14:57:55 +0100 Subject: [PATCH 178/248] windowPerformance in headless Chrome (resolves #732) --- .../windowPerformance/windowPerformance.js | 133 +++++------------- 1 file changed, 34 insertions(+), 99 deletions(-) diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index 93173e48b..8b38f0d44 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -9,6 +9,7 @@ module.exports = function(phantomas) { // @see https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp // times below are calculated relative to performance.timing.responseEnd (#117) + // all values are in miliseconds! phantomas.setMetric('domInteractive'); // @desc time it took to parse the HTML and construct the DOM phantomas.setMetric('domContentLoaded'); // @desc time it took to construct both DOM and CSSOM, no stylesheets that are blocking JavaScript execution (i.e. onDOMReady) phantomas.setMetric('domContentLoadedEnd'); // @desc time it took to finish handling of onDOMReady event @unreliable @@ -18,115 +19,49 @@ module.exports = function(phantomas) { phantomas.setMetric('timeBackend'); // @desc time to the first byte compared to the total loading time [%] phantomas.setMetric('timeFrontend'); // @desc time to window.load compared to the total loading time [%] - /** - - // measure dom... metrics from the moment HTML response was fully received - var responseEndTime = Date.now(); - - phantomas.on('responseEnd', function() { - responseEndTime = Date.now(); - phantomas.log('Performance timing: responseEnd = %d', responseEndTime); - }); - - phantomas.once('init', function() { - phantomas.evaluate(function(responseEndTime) { - (function(phantomas) { - phantomas.spyEnabled(false, 'installing window.performance metrics'); - - // extend window.performance - // "init" event is sometimes fired twice, pass a value set by "responseEnd" event handler (fixes #192) - if (typeof window.performance === 'undefined') { - window.performance = { - timing: { - responseEnd: responseEndTime - } - }; - - phantomas.log('Performance timing: emulating window.performance'); - } else { - phantomas.log('Performance timing: using native window.performance'); - } - - // onDOMReady - document.addEventListener("DOMContentLoaded", function() { - var time = Date.now() - responseEndTime; - - phantomas.setMetric('domContentLoaded', time, true); - phantomas.log('Performance timing: document reached "DOMContentLoaded" state after %d ms', time); - - setTimeout(function() { - // use NavigationTiming if possible - var time = window.performance.timing.domContentLoadedEventEnd ? - (window.performance.timing.domContentLoadedEventEnd - window.performance.timing.responseEnd) : - (Date.now() - responseEndTime); - - phantomas.setMetric('domContentLoadedEnd', time, true); - phantomas.log('Performance timing: document reached "DOMContentLoadedEnd" state after %d ms', time); - }, 0); - }); - - // emulate Navigation Timing - document.addEventListener('readystatechange', function() { - var readyState = document.readyState, - responseEndTime = window.performance.timing.responseEnd, - time = Date.now() - responseEndTime, - metricName; - - // @see http://www.w3.org/TR/html5/dom.html#documentreadystate - switch (readyState) { - // the browser has finished parsing all of the HTML and DOM construction is complete - case 'interactive': - metricName = 'domInteractive'; - break; - - // the processing is complete and all of the resources on the page have finished downloading - case 'complete': - metricName = 'domComplete'; - phantomas.log('Performance timing: %j', window.performance.timing); - break; - - default: - phantomas.log('Performance timing: unhandled "%s" state!', readyState); - return; - } - - phantomas.setMetric(metricName, time, true); - - phantomas.log('Performance timing: document reached "%s" state after %d ms', readyState, time); - }); - - phantomas.spyEnabled(true); - })(window.__phantomas); - }, responseEndTime); - }); - - /** - * Emit metrics with backend vs frontend time - * - * Performance Golden Rule: - * "80-90% of the end-user response time is spent on the frontend. Start there." - * - * @see http://www.stevesouders.com/blog/2012/02/10/the-performance-golden-rule/ - */ - /** - phantomas.on('report', function() { + phantomas.on('beforeClose', async function() { + const performance = await phantomas.evaluate(() => { + return window.performance.toJSON(); + }), + timing = performance.timing; + + phantomas.log('window.performance timing: %j', timing); + + // https://developer.mozilla.org/en-US/docs/Web/API/PerformanceTiming + const base = timing.responseEnd; + + /** + domInteractive: 60, + domContentLoaded: 61, + domContentLoadedEnd: 63, + domComplete: 63, + */ + phantomas.setMetric('domInteractive', timing.domInteractive - base); + phantomas.setMetric('domContentLoaded', timing.domContentLoadedEventStart - base); + phantomas.setMetric('domContentLoadedEnd', timing.domContentLoadedEventEnd - base); + phantomas.setMetric('domComplete', timing.domComplete - base); + + /** + * Emit metrics with backend vs frontend time + * + * Performance Golden Rule: + * "80-90% of the end-user response time is spent on the frontend. Start there." + * + * @see http://www.stevesouders.com/blog/2012/02/10/the-performance-golden-rule/ + */ // The “backend” time is the time it takes the server to get the first byte back to the client. // The “frontend” time is measured from the last byte of the response (responseEnd) until all resources are fetched (domComplete) - var backendTime = parseInt(phantomas.getMetric('timeToFirstByte'), 10), + const backendTime = parseInt(phantomas.getMetric('timeToFirstByte'), 10), frontendTime = parseInt(phantomas.getMetric('domComplete'), 10), - totalTime = backendTime + frontendTime, - backendTimePercentage; + totalTime = backendTime + frontendTime; if (totalTime === 0) { return; } - backendTimePercentage = Math.round(backendTime / totalTime * 100); + const backendTimePercentage = Math.round(backendTime / totalTime * 100); phantomas.setMetric('timeBackend', backendTimePercentage); phantomas.setMetric('timeFrontend', 100 - backendTimePercentage); - - phantomas.log('Performance timing: backend vs frontend time - %d% / %d%', backendTimePercentage, 100 - backendTimePercentage); }); - **/ }; From b35d47000f3cc19d697cc61e1653077e81bc60bd Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 15:03:07 +0100 Subject: [PATCH 179/248] timeToFirstByte: fix setting timeToFirstByte and timeToLastByte --- .../timeToFirstByte/timeToFirstByte.js | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/core/modules/timeToFirstByte/timeToFirstByte.js b/core/modules/timeToFirstByte/timeToFirstByte.js index 4001681d2..4b195f181 100644 --- a/core/modules/timeToFirstByte/timeToFirstByte.js +++ b/core/modules/timeToFirstByte/timeToFirstByte.js @@ -4,8 +4,7 @@ 'use strict'; module.exports = function(phantomas) { - var measured = false, - reqId = 1; // request ID to consider when calculating TTFB / TTLB + var measured = false; phantomas.setMetric('timeToFirstByte'); // @desc time it took to receive the first byte of the first response (that was not a redirect) phantomas.setMetric('timeToLastByte'); // @desc time it took to receive the last byte of the first response (that was not a redirect) @@ -17,24 +16,18 @@ module.exports = function(phantomas) { } if (entry.isRedirect) { - // wait for the next request - reqId = entry.id + 1; - - phantomas.log('Time to first byte: response #%d <%s> is a redirect (waiting for response #%d)', entry.id, entry.url, reqId); + phantomas.log('Time to first byte: <%s> is a redirect', entry.url); return; } - // check the first response which is not a redirect (issue #74) - if (entry.id === reqId) { - phantomas.setMetric('timeToFirstByte', entry.timeToFirstByte, true); - phantomas.setMetric('timeToLastByte', entry.timeToLastByte, true); + phantomas.setMetric('timeToFirstByte', entry.timeToFirstByte, true); + phantomas.setMetric('timeToLastByte', entry.timeToLastByte, true); - measured = true; + measured = true; - phantomas.log('Time to first byte: set to %d ms for #%d request to <%s> (HTTP %d)', entry.timeToFirstByte, entry.id, entry.url, entry.status); - phantomas.log('Time to last byte: set to %d ms', entry.timeToLastByte); + phantomas.log('Time to first byte: set to %d ms for request to <%s> (HTTP %d)', entry.timeToFirstByte, entry.url, entry.status); + phantomas.log('Time to last byte: set to %d ms', entry.timeToLastByte); - phantomas.emit('responseEnd', entry, res); // @desc The first response (that was not a redirect) fully received - } + phantomas.emit('responseEnd', entry, res); // @desc The first response (that was not a redirect) fully received }); }; From f6650235505fcdb971f8ebbf275cbd5ba7609646 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 15:32:54 +0100 Subject: [PATCH 180/248] New metric proposal "synchronousXHR" Ported #618 to a new phantomas-v2 branch. Resolves #620 --- modules/ajaxRequests/ajaxRequests.js | 1 + modules/ajaxRequests/scope.js | 8 +++++++- test/integration-spec.yaml | 12 ++++++++++++ test/webroot/ajax.html | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 test/webroot/ajax.html diff --git a/modules/ajaxRequests/ajaxRequests.js b/modules/ajaxRequests/ajaxRequests.js index e30301322..9e54422a1 100644 --- a/modules/ajaxRequests/ajaxRequests.js +++ b/modules/ajaxRequests/ajaxRequests.js @@ -6,4 +6,5 @@ module.exports = function(phantomas) { phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests @offenders + phantomas.setMetric('synchronousXHR'); // @desc number of synchronous XMLHttpRequest @offenders }; diff --git a/modules/ajaxRequests/scope.js b/modules/ajaxRequests/scope.js index 2f34c77dd..ab1777611 100644 --- a/modules/ajaxRequests/scope.js +++ b/modules/ajaxRequests/scope.js @@ -1,8 +1,14 @@ (function(phantomas) { - phantomas.spy(window.XMLHttpRequest.prototype, 'open', (result, method, url) => { + phantomas.spy(window.XMLHttpRequest.prototype, 'open', (_, method, url, async) => { phantomas.incrMetric('ajaxRequests'); phantomas.addOffender('ajaxRequests', {url, method}); phantomas.log('Ajax request: ' + url); + + if (async === false) { + phantomas.incrMetric('synchronousXHR'); + phantomas.addOffender('synchronousXHR', {url, method}); + phantomas.log('synchronous XMLHttpRequest call to <%s>', url); + } }, true); })(window.__phantomas); diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index c9f4c87c3..d4a9fc22a 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -437,6 +437,18 @@ options: "wait-for-network-idle": true +# synchronous AJAX requests (#618) +- url: "/ajax.html" + metrics: + ajaxRequests: 2 + synchronousXHR: 1 + offenders: + ajaxRequests: + - { url: 'jquery.html', method: 'GET' } + - { url: 'inline-css.html', method: 'GET' } + synchronousXHR: + - { url: 'inline-css.html', method: 'GET' } + # multiple cookies (#597) - url: "/cookies.html" options: diff --git a/test/webroot/ajax.html b/test/webroot/ajax.html new file mode 100644 index 000000000..f7467c14b --- /dev/null +++ b/test/webroot/ajax.html @@ -0,0 +1,14 @@ + From 1c0d4f7c2de293a2ffbe4c43dc527b8b29c62793 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 15:34:40 +0100 Subject: [PATCH 181/248] synchronousXHR: update auto-generate docs and metadata file --- docs/metrics.md | 13 ++++++++++++- lib/metadata/metadata.json | 12 ++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index a7d717719..20c964c84 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,7 +1,7 @@ Modules and metrics =================== -This file describes all [`phantomas` modules](https://github.com/macbre/phantomas/tree/devel/modules) (34 of them) and 172 metrics that they emit. +This file describes all [`phantomas` modules](https://github.com/macbre/phantomas/tree/devel/modules) (34 of them) and 173 metrics that they emit. When applicable, [offender](https://github.com/macbre/phantomas/issues/140) example is provided. @@ -22,6 +22,17 @@ number of AJAX requests (number, with offenders) } ``` +##### `synchronousXHR` + +number of synchronous XMLHttpRequest (number, with offenders) + +```json +{ + "url": "inline-css.html", + "method": "GET" +} +``` + ## [alerts](https://github.com/macbre/phantomas/tree/devel/modules/alerts/alerts.js) diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index 56e37ebd8..c713df1b2 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -170,7 +170,8 @@ "desc": "Analyzes AJAX requests", "events": [], "metrics": [ - "ajaxRequests" + "ajaxRequests", + "synchronousXHR" ] }, "alerts": { @@ -619,6 +620,13 @@ "module": "ajaxRequests", "testsCovered": true }, + "synchronousXHR": { + "desc": "number of synchronous XMLHttpRequest", + "offenders": true, + "unit": "number", + "module": "ajaxRequests", + "testsCovered": true + }, "windowAlerts": { "desc": "number of calls to window.alert", "offenders": true, @@ -1739,7 +1747,7 @@ "testsCovered": false } }, - "metricsCount": 172, + "metricsCount": 173, "modulesCount": 34, "extensionsCount": 11, "version": "2.0.0-beta" From 5d7684070a01b7c26628f6a271862b6cb7fd76ba Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 21:50:36 +0100 Subject: [PATCH 182/248] DNS and Connect timings (resolves #477) --- modules/windowPerformance/windowPerformance.js | 12 ++++++++++++ test/integration-spec.yaml | 3 +++ 2 files changed, 15 insertions(+) diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index 8b38f0d44..848d16be9 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -15,6 +15,12 @@ module.exports = function(phantomas) { phantomas.setMetric('domContentLoadedEnd'); // @desc time it took to finish handling of onDOMReady event @unreliable phantomas.setMetric('domComplete'); // @desc time it took to load all page resources, the loading spinner has stopped spinning + // get values from Resource Timing API (issue #477) + phantomas.setMetric('performanceTimingConnect'); // @desc time it took to connect to the server before making the first HTTP request + phantomas.setMetric('performanceTimingDNS'); // @desc time it took to resolve the domain before making the first HTTP request + phantomas.setMetric('performanceTimingPageLoad'); // @desc time it took to fully load the page + phantomas.setMetric('performanceTimingTTFB'); // @desc time it took to receive the first byte of the first HTTP response + // backend vs frontend time phantomas.setMetric('timeBackend'); // @desc time to the first byte compared to the total loading time [%] phantomas.setMetric('timeFrontend'); // @desc time to window.load compared to the total loading time [%] @@ -41,6 +47,12 @@ module.exports = function(phantomas) { phantomas.setMetric('domContentLoadedEnd', timing.domContentLoadedEventEnd - base); phantomas.setMetric('domComplete', timing.domComplete - base); + // see #477 + phantomas.setMetric('performanceTimingConnect', timing.connectEnd - timing.connectStart); + phantomas.setMetric('performanceTimingDNS', timing.domainLookupEnd - timing.domainLookupStart); + phantomas.setMetric('performanceTimingPageLoad', timing.loadEventStart - timing.navigationStart); + phantomas.setMetric('performanceTimingTTFB', timing.responseStart - timing.navigationStart); + /** * Emit metrics with backend vs frontend time * diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index d4a9fc22a..099a7b5a9 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -333,6 +333,9 @@ domainsToDomContentLoaded: 2 requestsToDomComplete: 3 domainsToDomComplete: 2 + # test are made using a HTTP server running locally, it should be pretty fast to connect to it + performanceTimingConnect: 0 + performanceTimingDNS: 0 offenders: domainsToDomContentLoaded: - { domain: '127.0.0.1', requests: 2 } From 5b1cffe7013fbc31b0839c64a5632289ad62bbbc Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 21:52:08 +0100 Subject: [PATCH 183/248] performanceTiming:: update docs and metadata --- docs/metrics.md | 18 +++++++++++++++++- lib/metadata/metadata.json | 30 +++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 20c964c84..e1920dd22 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,7 +1,7 @@ Modules and metrics =================== -This file describes all [`phantomas` modules](https://github.com/macbre/phantomas/tree/devel/modules) (34 of them) and 173 metrics that they emit. +This file describes all [`phantomas` modules](https://github.com/macbre/phantomas/tree/devel/modules) (34 of them) and 177 metrics that they emit. When applicable, [offender](https://github.com/macbre/phantomas/issues/140) example is provided. @@ -1253,6 +1253,22 @@ time it took to finish handling of onDOMReady event (ms) time it took to parse the HTML and construct the DOM (ms) +##### `performanceTimingConnect` + +time it took to connect to the server before making the first HTTP request (ms) + +##### `performanceTimingDNS` + +time it took to resolve the domain before making the first HTTP request (ms) + +##### `performanceTimingPageLoad` + +time it took to fully load the page (ms) + +##### `performanceTimingTTFB` + +time it took to receive the first byte of the first HTTP response (ms) + ##### `timeBackend` time to the first byte compared to the total loading time (%) diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index c713df1b2..d320cd1dc 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -541,6 +541,10 @@ "domContentLoaded", "domContentLoadedEnd", "domComplete", + "performanceTimingConnect", + "performanceTimingDNS", + "performanceTimingPageLoad", + "performanceTimingTTFB", "timeBackend", "timeFrontend" ] @@ -1734,6 +1738,30 @@ "module": "windowPerformance", "testsCovered": false }, + "performanceTimingConnect": { + "desc": "time it took to connect to the server before making the first HTTP request", + "unit": "ms", + "module": "windowPerformance", + "testsCovered": true + }, + "performanceTimingDNS": { + "desc": "time it took to resolve the domain before making the first HTTP request", + "unit": "ms", + "module": "windowPerformance", + "testsCovered": true + }, + "performanceTimingPageLoad": { + "desc": "time it took to fully load the page", + "unit": "ms", + "module": "windowPerformance", + "testsCovered": false + }, + "performanceTimingTTFB": { + "desc": "time it took to receive the first byte of the first HTTP response", + "unit": "ms", + "module": "windowPerformance", + "testsCovered": false + }, "timeBackend": { "desc": "time to the first byte compared to the total loading time", "unit": "%", @@ -1747,7 +1775,7 @@ "testsCovered": false } }, - "metricsCount": 173, + "metricsCount": 177, "modulesCount": 34, "extensionsCount": 11, "version": "2.0.0-beta" From 11439ecd41f10bf0a97ba491a054aa44cb20297a Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 23 Feb 2019 23:12:40 +0100 Subject: [PATCH 184/248] Update dependencies HeadlessChrome/72.0.3617.0 -> HeadlessChrome/73.0.3679.0 --- package-lock.json | 242 +++++++++++++++++++++++++--------------------- package.json | 4 +- 2 files changed, 134 insertions(+), 112 deletions(-) diff --git a/package-lock.json b/package-lock.json index c6a5d75aa..6eb5082db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,9 +43,9 @@ "dev": true }, "acorn": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.5.tgz", - "integrity": "sha512-i33Zgp3XWtmZBMNvCr4azvOFeWVw1Rk6p3hfi3LUDvIFraOMywb1kAtrbi+med14m4Xfpqm3zRZMT+c0FNE7kg==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.0.tgz", + "integrity": "sha512-MW/FjM+IvU9CgBzjO3UIPCE2pyEwUsoFl+VGdczOPEdxfGFjuKny/gN54mOuX7Qxmb9Rg9MCn2oKiSUeW+pjrw==", "dev": true }, "acorn-jsx": { @@ -63,9 +63,9 @@ } }, "ajv": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz", - "integrity": "sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg==", + "version": "6.9.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz", + "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==", "dev": true, "requires": { "fast-deep-equal": "^2.0.1", @@ -94,9 +94,9 @@ } }, "ansi-escapes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", - "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", "dev": true }, "ansi-regex": { @@ -187,12 +187,6 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, - "circular-json": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", - "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==", - "dev": true - }, "cli": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cli/-/cli-1.0.1.tgz", @@ -251,7 +245,7 @@ }, "concat-stream": { "version": "1.6.2", - "resolved": "http://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "requires": { "buffer-from": "^1.0.0", @@ -336,9 +330,9 @@ "dev": true }, "doctrine": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", - "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", "dev": true, "requires": { "esutils": "^2.0.2" @@ -384,6 +378,12 @@ "sigmund": "^1.0.1" } }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, "es6-promise": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", @@ -404,56 +404,47 @@ "dev": true }, "eslint": { - "version": "5.12.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.12.1.tgz", - "integrity": "sha512-54NV+JkTpTu0d8+UYSA8mMKAG4XAsaOrozA9rCW7tgneg1mevcL7wIotPC+fZ0SkWwdhNqoXoxnQCTBp7UvTsg==", + "version": "5.14.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.14.1.tgz", + "integrity": "sha512-CyUMbmsjxedx8B0mr79mNOqetvkbij/zrXnFeK2zc3pGRn3/tibjiNAv/3UxFEyfMDjh+ZqTrJrEGBFiGfD5Og==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "ajv": "^6.5.3", + "ajv": "^6.9.1", "chalk": "^2.1.0", "cross-spawn": "^6.0.5", "debug": "^4.0.1", - "doctrine": "^2.1.0", + "doctrine": "^3.0.0", "eslint-scope": "^4.0.0", "eslint-utils": "^1.3.1", "eslint-visitor-keys": "^1.0.0", - "espree": "^5.0.0", + "espree": "^5.0.1", "esquery": "^1.0.1", "esutils": "^2.0.2", - "file-entry-cache": "^2.0.0", + "file-entry-cache": "^5.0.1", "functional-red-black-tree": "^1.0.1", "glob": "^7.1.2", "globals": "^11.7.0", "ignore": "^4.0.6", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", - "inquirer": "^6.1.0", + "inquirer": "^6.2.2", "js-yaml": "^3.12.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.3.0", - "lodash": "^4.17.5", + "lodash": "^4.17.11", "minimatch": "^3.0.4", "mkdirp": "^0.5.1", "natural-compare": "^1.4.0", "optionator": "^0.8.2", "path-is-inside": "^1.0.2", - "pluralize": "^7.0.0", "progress": "^2.0.0", "regexpp": "^2.0.1", "semver": "^5.5.1", "strip-ansi": "^4.0.0", "strip-json-comments": "^2.0.1", - "table": "^5.0.2", + "table": "^5.2.3", "text-table": "^0.2.0" - }, - "dependencies": { - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", - "dev": true - } } }, "eslint-scope": { @@ -479,12 +470,12 @@ "dev": true }, "espree": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.0.tgz", - "integrity": "sha512-1MpUfwsdS9MMoN7ZXqAr9e9UKdVHDcvrJpyx7mm1WuQlx/ygErEQBzgi5Nh5qBHIoYweprhtMkTCb9GhcAIcsA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", "dev": true, "requires": { - "acorn": "^6.0.2", + "acorn": "^6.0.7", "acorn-jsx": "^5.0.0", "eslint-visitor-keys": "^1.0.0" } @@ -620,27 +611,42 @@ } }, "file-entry-cache": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz", - "integrity": "sha1-w5KZDD5oR4PYOLjISkXYoEhFg2E=", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", "dev": true, "requires": { - "flat-cache": "^1.2.1", - "object-assign": "^4.0.1" + "flat-cache": "^2.0.1" } }, "flat-cache": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.4.tgz", - "integrity": "sha512-VwyB3Lkgacfik2vhqR4uv2rvebqmDvFu4jlN/C1RzWoJEo8I7z4Q404oiqYCkq41mni8EzQnm95emU9seckwtg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", "dev": true, "requires": { - "circular-json": "^0.3.1", - "graceful-fs": "^4.1.2", - "rimraf": "~2.6.2", - "write": "^0.2.1" + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "dependencies": { + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } } }, + "flatted": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", + "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==", + "dev": true + }, "follow-redirects": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.0.tgz", @@ -692,15 +698,9 @@ } }, "globals": { - "version": "11.10.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.10.0.tgz", - "integrity": "sha512-0GZF1RiPKU97IHUO5TORo9w1PwrH/NBPl+fS7oMLdaTRiYmYbwK4NWoZWrAdd0/abG9R2BU+OiwyQpTpE6pdfQ==", - "dev": true - }, - "graceful-fs": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", - "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", + "version": "11.11.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.11.0.tgz", + "integrity": "sha512-WHq43gS+6ufNOEqlrDBxVEbb8ntfXrfAUU2ZOpCxrBdGKW3gyv8mCxAfIBD0DroPKGrJ2eSsXsLtY9MPntsyTw==", "dev": true }, "has-flag": { @@ -837,21 +837,21 @@ "dev": true }, "inquirer": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.2.1.tgz", - "integrity": "sha512-088kl3DRT2dLU5riVMKKr1DlImd6X7smDhpXUCkJDCKvTEJeRiXh0G132HG9u5a+6Ylw9plFRY7RuTnwohYSpg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.2.2.tgz", + "integrity": "sha512-Z2rREiXA6cHRR9KBOarR3WuLlFzlIfAEIiB45ll5SSadMg7WqOh1MKEjjndfuH5ewXdixWCxqnVfGOQzPeiztA==", "dev": true, "requires": { - "ansi-escapes": "^3.0.0", - "chalk": "^2.0.0", + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", "cli-cursor": "^2.1.0", "cli-width": "^2.0.0", - "external-editor": "^3.0.0", + "external-editor": "^3.0.3", "figures": "^2.0.0", - "lodash": "^4.17.10", + "lodash": "^4.17.11", "mute-stream": "0.0.7", "run-async": "^2.2.0", - "rxjs": "^6.1.0", + "rxjs": "^6.4.0", "string-width": "^2.1.0", "strip-ansi": "^5.0.0", "through": "^2.3.6" @@ -1040,12 +1040,6 @@ "osenv": "^0.1.4" } }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true - }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -1158,12 +1152,6 @@ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, - "pluralize": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-7.0.0.tgz", - "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==", - "dev": true - }, "portfinder": { "version": "0.4.0", "resolved": "http://registry.npmjs.org/portfinder/-/portfinder-0.4.0.tgz", @@ -1214,9 +1202,9 @@ "dev": true }, "puppeteer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.11.0.tgz", - "integrity": "sha512-iG4iMOHixc2EpzqRV+pv7o3GgmU2dNYEMkvKwSaQO/vMZURakwSOn/EYJ6OIRFYOque1qorzIBvrytPIQB3YzQ==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.12.2.tgz", + "integrity": "sha512-xWSyCeD6EazGlfnQweMpM+Hs6X6PhUYhNTHKFj/axNZDq4OmrVERf70isBf7HsnFgB3zOC1+23/8+wCAZYg+Pg==", "requires": { "debug": "^4.1.0", "extract-zip": "^1.6.6", @@ -1236,7 +1224,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -1282,11 +1270,11 @@ } }, "rimraf": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", - "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", "requires": { - "glob": "^7.0.5" + "glob": "^7.1.3" } }, "run-async": { @@ -1299,9 +1287,9 @@ } }, "rxjs": { - "version": "6.3.3", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.3.3.tgz", - "integrity": "sha512-JTWmoY9tWCs7zvIk/CvRjhjGaOd+OVBM987mxFo+OW66cGpdKjZcpmc74ES1sB//7Kl/PAe8+wEakuhG4pcgOw==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.4.0.tgz", + "integrity": "sha512-Z9Yfa11F6B9Sg/BK9MnqnQ+aQYicPLtilXBp2yUtDt2JRCE0h26d33EnfO3ZxoNxG0T92OUucP3Ct7cpfkdFfw==", "dev": true, "requires": { "tslib": "^1.9.0" @@ -1352,9 +1340,9 @@ "dev": true }, "slice-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.0.0.tgz", - "integrity": "sha512-4j2WTWjp3GsZ+AOagyzVbzp4vWGtZ0hEZ/gDY/uTvm6MTxUfTUIsnMIFb1bn8o0RuXiqUw15H1bue8f22Vw2oQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", "dev": true, "requires": { "ansi-styles": "^3.2.0", @@ -1412,7 +1400,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" @@ -1427,6 +1415,12 @@ "ansi-regex": "^3.0.0" } }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -1437,15 +1431,43 @@ } }, "table": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/table/-/table-5.2.1.tgz", - "integrity": "sha512-qmhNs2GEHNqY5fd2Mo+8N1r2sw/rvTAAvBZTaTx+Y7PHLypqyrxr1MdIu0pLw6Xvl/Gi4ONu/sdceP8vvUjkyA==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/table/-/table-5.2.3.tgz", + "integrity": "sha512-N2RsDAMvDLvYwFcwbPyF3VmVSSkuF+G1e+8inhBLtHpvwXGw4QRPEZhihQNeEN0i1up6/f6ObCJXNdlRG3YVyQ==", "dev": true, "requires": { - "ajv": "^6.6.1", + "ajv": "^6.9.1", "lodash": "^4.17.11", - "slice-ansi": "2.0.0", - "string-width": "^2.1.1" + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.0.0.tgz", + "integrity": "sha512-iB5Dda8t/UqpPI/IjsejXu5jOGDrzn41wJyljwPH65VCIbk6+1BzFIMJGFwTNrYXT1CrD+B4l19U7awiQ8rk7w==", + "dev": true + }, + "string-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.0.0.tgz", + "integrity": "sha512-rr8CUxBbvOZDUvc5lNIJ+OC1nPVpz+Siw9VBtUjB9b6jZehZLFt0JMCZzShFHIsI8cbhm0EsNIfWJMFV3cu3Ew==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.0.0" + } + }, + "strip-ansi": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.0.0.tgz", + "integrity": "sha512-Uu7gQyZI7J7gn5qLn1Np3G9vcYGTVqB+lFTytnDJv83dd8T22aGH451P3jueT2/QemInJDfxHB5Tde5OzgG1Ow==", + "dev": true, + "requires": { + "ansi-regex": "^4.0.0" + } + } } }, "text-table": { @@ -1554,18 +1576,18 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz", - "integrity": "sha1-X8A4KOJkzqP+kUVUdvejxWbLB1c=", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", "dev": true, "requires": { "mkdirp": "^0.5.1" } }, "ws": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.2.tgz", - "integrity": "sha512-rfUqzvz0WxmSXtJpPMX2EeASXabOrSMk1ruMOV3JBTBjo4ac2lDjGGsbQSyxj8Odhw5fBib8ZKEjDNvgouNKYw==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz", + "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==", "requires": { "async-limiter": "~1.0.0" } diff --git a/package.json b/package.json index 65e9ac58b..5380a681c 100644 --- a/package.json +++ b/package.json @@ -26,10 +26,10 @@ "dependencies": { "analyze-css": "^0.12.7", "debug": "^4.1.1", - "puppeteer": "^1.11.0" + "puppeteer": "^1.12.2" }, "devDependencies": { - "eslint": "^5.12.1", + "eslint": "^5.14.1", "glob": "^7.1.2", "http-serve": "^1.0.1", "js-beautify": "^1.6.14", From 80120d526b7e3a4e140d6aa40361451f08e29cbe Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 19:44:09 +0100 Subject: [PATCH 185/248] Install puppeteer@1.13.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit phantomas/2.0.0-beta (HeadlessChrome/74.0.3723.0) ✗ Broken » 418 honored ∙ 21 broken (47.299s) --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6eb5082db..699797967 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1202,9 +1202,9 @@ "dev": true }, "puppeteer": { - "version": "1.12.2", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.12.2.tgz", - "integrity": "sha512-xWSyCeD6EazGlfnQweMpM+Hs6X6PhUYhNTHKFj/axNZDq4OmrVERf70isBf7HsnFgB3zOC1+23/8+wCAZYg+Pg==", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.13.0.tgz", + "integrity": "sha512-LUXgvhjfB/P6IOUDAKxOcbCz9ISwBLL9UpKghYrcBDwrOGx1m60y0iN2M64mdAUbT4+7oZM5DTxOW7equa2fxQ==", "requires": { "debug": "^4.1.0", "extract-zip": "^1.6.6", diff --git a/package.json b/package.json index 5380a681c..9e4b9a6fa 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "analyze-css": "^0.12.7", "debug": "^4.1.1", - "puppeteer": "^1.12.2" + "puppeteer": "^1.13.0" }, "devDependencies": { "eslint": "^5.14.1", From 3be88bb49805915323751474663954c776a5ca69 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:05:50 +0100 Subject: [PATCH 186/248] browser: take encodedDataLength value from "Network.loadingFinished" event --- lib/browser.js | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index d045774dc..412cfece1 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -113,9 +113,7 @@ Browser.prototype.init = async () => { responses[data.requestId] = { _chunks: 0, - _dataLength: 0, - // Actual bytes received (might be less than dataLength for compressed encodings). - _encodedDataLength: 0, + _dataLength: 0 }; }); @@ -147,8 +145,12 @@ Browser.prototype.init = async () => { } } + networkDebug(eventName, data); + + // Actual bytes received (might be less than dataLength for compressed encodings). + response.encodedDataLength = data.encodedDataLength; + response.dataLength = meta._dataLength; - response.encodedDataLength = meta._encodedDataLength; response.chunks = meta._chunks; response._timestamp = data.timestamp; @@ -163,13 +165,12 @@ Browser.prototype.init = async () => { return resp.body; }).bind(this); - //networkDebug('Network.loadingFinished', data); - networkDebug('Network.%s < %s %s %s %s (%f kB fetched, %f kB uncompressed)', + networkDebug('Network.%s < %s %s %s %s (%s kB fetched, %s kB uncompressed)', eventName, response.protocol, response.status, response.statusText, response.url, - 1.0 * (response.encodedDataLength || response.headers['content-length'] || 0) / 1024, - 1.0 * response.dataLength / 1024); - + (1.0 * (response.encodedDataLength || response.headers['content-length'] || 0) / 1024).toFixed(1), + (1.0 * response.dataLength / 1024).toFixed(1) + ); this.events.emit('response', response); // @desc Emitted when page received a HTTP response }; @@ -188,8 +189,6 @@ Browser.prototype.init = async () => { responses[data.requestId]._chunks++; responses[data.requestId]._dataLength += data.dataLength; - // Actual bytes received (might be less than dataLength for compressed encodings) - responses[data.requestId]._encodedDataLength += data.encodedDataLength; }); return this.page; From 1892d6dca43137f3f0734cddf696a1bfb4e2218d Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:10:11 +0100 Subject: [PATCH 187/248] performanceTimingConnect: test is flaky --- lib/metadata/metadata.json | 2 +- test/integration-spec.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index d320cd1dc..f8a8325bd 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -1742,7 +1742,7 @@ "desc": "time it took to connect to the server before making the first HTTP request", "unit": "ms", "module": "windowPerformance", - "testsCovered": true + "testsCovered": false }, "performanceTimingDNS": { "desc": "time it took to resolve the domain before making the first HTTP request", diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 099a7b5a9..bc05556c3 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -334,7 +334,6 @@ requestsToDomComplete: 3 domainsToDomComplete: 2 # test are made using a HTTP server running locally, it should be pretty fast to connect to it - performanceTimingConnect: 0 performanceTimingDNS: 0 offenders: domainsToDomContentLoaded: From d13a2963fe3f080c18f9601f3a82eee7695d2a0f Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:20:33 +0100 Subject: [PATCH 188/248] responseSize: do not add headersSize to transferedSize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✗ Broken » 431 honored ∙ 7 broken (33.141s) --- core/modules/requestsMonitor/requestsMonitor.js | 2 +- lib/browser.js | 2 +- test/integration-spec.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index a0c87fd94..50653fb10 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -199,7 +199,7 @@ module.exports = function(phantomas) { // that's the response size as reported by Chrome's dev tools (headers + compressed body) // note: base64-encoded resources do not have "resp.headersText" set entry.headersSize = (resp.headersText || '').length; - entry.responseSize = entry.headersSize + entry.transferedSize; + entry.responseSize = entry.transferedSize; entry = parseEntryUrl(entry); diff --git a/lib/browser.js b/lib/browser.js index 412cfece1..a7e74d689 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -145,7 +145,7 @@ Browser.prototype.init = async () => { } } - networkDebug(eventName, data); + networkDebug('%s: %j', eventName, data); // Actual bytes received (might be less than dataLength for compressed encodings). response.encodedDataLength = data.encodedDataLength; diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index bc05556c3..11ceda30f 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -33,7 +33,7 @@ offenders: gzipRequests: - - {url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js', bodySize: 84245, transferedSize: 29415 } + - {url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js', bodySize: 84245, transferedSize: 29730 } htmlCount: - { url: 'http://127.0.0.1:8888/dom-operations.html', size: 2385 } cssCount: @@ -185,7 +185,7 @@ eventsScrollBound: 2 offenders: gzipRequests: - - { url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js', transferedSize: 29415, bodySize: 84245 } + - { url: 'http://127.0.0.1:8888/static/jquery-2.1.1.min.js', transferedSize: 29730, bodySize: 84245 } eventsDispatched: - { eventType: 'click', path: 'body > div#foo > span.bar' } eventsBound: From f942013ef9d329491ccfb4bb90c14f25cd3b3e20 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:33:36 +0100 Subject: [PATCH 189/248] Improved handling of HTTP 404 responses --- lib/browser.js | 7 +++---- test/integration-spec.yaml | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index a7e74d689..da16ad1d4 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -124,8 +124,7 @@ Browser.prototype.init = async () => { var response = data.response; response._requestId = data.requestId; - //networkDebug('responseReceived', response); - //networkDebug('Network.responseReceived < %s %s %s %s', response.protocol, response.status, response.statusText, response.url); + // networkDebug('Network.responseReceived', response); // next event tells us that the response was fully fetched responses[data.requestId].response = response; @@ -148,7 +147,7 @@ Browser.prototype.init = async () => { networkDebug('%s: %j', eventName, data); // Actual bytes received (might be less than dataLength for compressed encodings). - response.encodedDataLength = data.encodedDataLength; + response.encodedDataLength = data.encodedDataLength || meta.response.encodedDataLength; /* "or" fallback for 404 response */ response.dataLength = meta._dataLength; response.chunks = meta._chunks; @@ -185,7 +184,7 @@ Browser.prototype.init = async () => { // https://chromedevtools.github.io/devtools-protocol/tot/Network#event-dataReceived // Fired when data chunk was received over the network this.cdp.on('Network.dataReceived', data => { - // networkDebug('Network.dataReceived', data); + networkDebug('Network.dataReceived: %j', data); responses[data.requestId]._chunks++; responses[data.requestId]._dataLength += data.dataLength; diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index 11ceda30f..b411e82b2 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -508,8 +508,8 @@ - url: "/headers.html" metrics: requests: 2 - bodySize: 1065 - contentLength: 1065 # these are equal as no gzip compression took place + bodySize: 1065 # uncompressed content of all responses (without headers) + contentLength: 1643 # transfered bytes smallestResponse: 330 # headers + body size biggestResponse: 1313 offenders: @@ -606,7 +606,7 @@ options: phone: true metrics: - userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3452.0 Mobile Safari/537.36' + userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3679.0 Mobile Safari/537.36' viewport: '360x640' - url: "/devices.html" @@ -614,7 +614,7 @@ options: "phone-landscape": true metrics: - userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3452.0 Mobile Safari/537.36' + userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3679.0 Mobile Safari/537.36' viewport: '640x360' - url: "/devices.html" From 8fddc2175b924bc45077db7ac203342396117ede Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:47:20 +0100 Subject: [PATCH 190/248] Improve header size calculation --- core/modules/requestsMonitor/requestsMonitor.js | 7 ++++++- modules/headers/headers.js | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/modules/requestsMonitor/requestsMonitor.js b/core/modules/requestsMonitor/requestsMonitor.js index 50653fb10..167d85a34 100644 --- a/core/modules/requestsMonitor/requestsMonitor.js +++ b/core/modules/requestsMonitor/requestsMonitor.js @@ -198,9 +198,10 @@ module.exports = function(phantomas) { // that's the response size as reported by Chrome's dev tools (headers + compressed body) // note: base64-encoded resources do not have "resp.headersText" set - entry.headersSize = (resp.headersText || '').length; entry.responseSize = entry.transferedSize; + phantomas.log('headers: %j', resp); + entry = parseEntryUrl(entry); /** @@ -234,11 +235,15 @@ module.exports = function(phantomas) { phantomas.addOffender('postRequests', entry.url); } + entry.headersSize = 0; + // response content type // https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceType Object.keys(entry.headers).forEach(headerName => { const headerValue = entry.headers[headerName]; + entry.headersSize += headerName.length + headerValue.length + 2 /* ": " */ + 2; /* line break - CR+LF */ + switch(headerName) { // detect content type case 'content-type': diff --git a/modules/headers/headers.js b/modules/headers/headers.js index 9a5681e60..95793ac55 100644 --- a/modules/headers/headers.js +++ b/modules/headers/headers.js @@ -43,6 +43,8 @@ module.exports = function(phantomas) { phantomas.on('recv', function(entry) { var headers = processHeaders(entry.headers); + phantomas.log('%s: %d / %d', entry.url, entry.headersSize, entry.transferedSize); + // phantomas.log('Headers: <%s> %d bytes', entry.url, headers.size); phantomas.incrMetric('headersCount', headers.count); From a62c16e2139bacb763127375f97ac06f9d2d8781 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:47:54 +0100 Subject: [PATCH 191/248] test/integration-spec.yaml: comment out headersBiggerThanContent section for now --- lib/metadata/metadata.json | 2 +- test/integration-spec.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/metadata/metadata.json b/lib/metadata/metadata.json index f8a8325bd..0d67ab469 100644 --- a/lib/metadata/metadata.json +++ b/lib/metadata/metadata.json @@ -1423,7 +1423,7 @@ "offenders": true, "unit": "number", "module": "headers", - "testsCovered": true + "testsCovered": false }, "documentWriteCalls": { "desc": "number of calls to either document.write or document.writeln", diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index b411e82b2..fff8b1786 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -489,11 +489,11 @@ headersCount: 20 headersSentCount: 4 headersRecvCount: 16 - headersRecvSize: 578 - headersBiggerThanContent: 1 - offenders: - headersBiggerThanContent: - - { url: 'http://127.0.0.1:8888/static/blank.gif', contentSize: 43, headersSize: 287 } + headersRecvSize: 540 +# headersBiggerThanContent: 1 +# offenders: +# headersBiggerThanContent: +# - { url: 'http://127.0.0.1:8888/static/blank.gif', contentSize: 43, headersSize: 287 } # caching - url: "/caching.html" From cb96ec66968236e1083865e343ddd4f4086d9908 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 5 Mar 2019 20:55:34 +0100 Subject: [PATCH 192/248] eslint: remove "global" annotations --- core/modules/navigationTiming/navigationTiming.js | 1 - modules/ajaxRequests/ajaxRequests.js | 1 - modules/analyzeCss/analyzeCss.js | 1 - modules/documentHeight/documentHeight.js | 1 - modules/domComplexity/domComplexity.js | 1 - modules/domHiddenContent/domHiddenContent.js | 1 - modules/domMutations/domMutations.js | 1 - modules/domQueries/domQueries.js | 1 - modules/events/events.js | 1 - modules/globalVariables/globalVariables.js | 1 - modules/jQuery/jQuery.js | 1 - modules/lazyLoadableImages/lazyLoadableImages.js | 1 - modules/localStorage/localStorage.js | 1 - modules/windowPerformance/windowPerformance.js | 2 +- 14 files changed, 1 insertion(+), 14 deletions(-) diff --git a/core/modules/navigationTiming/navigationTiming.js b/core/modules/navigationTiming/navigationTiming.js index 58aa6c583..7495fff9d 100644 --- a/core/modules/navigationTiming/navigationTiming.js +++ b/core/modules/navigationTiming/navigationTiming.js @@ -7,7 +7,6 @@ * * Code taken from windowPerformance module */ -/* global document: true, window: true */ 'use strict'; // no-op, see scope.js diff --git a/modules/ajaxRequests/ajaxRequests.js b/modules/ajaxRequests/ajaxRequests.js index 9e54422a1..e04f18357 100644 --- a/modules/ajaxRequests/ajaxRequests.js +++ b/modules/ajaxRequests/ajaxRequests.js @@ -1,7 +1,6 @@ /** * Analyzes AJAX requests */ -/* global window: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/modules/analyzeCss/analyzeCss.js b/modules/analyzeCss/analyzeCss.js index e5ac893da..9b8763a4e 100644 --- a/modules/analyzeCss/analyzeCss.js +++ b/modules/analyzeCss/analyzeCss.js @@ -39,7 +39,6 @@ * setMetric('cssNotMinified') @desc [number] set to 1 if the provided CSS is not minified @optional @offenders * setMetric('cssSelectorLengthAvg') @desc [number] average length of selector (e.g. for ``.foo .bar, #test div > span { color: red }`` will be set as 2.5) @optional @offenders */ -/* global document: true, window: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/modules/documentHeight/documentHeight.js b/modules/documentHeight/documentHeight.js index bae793a78..65faeb14e 100644 --- a/modules/documentHeight/documentHeight.js +++ b/modules/documentHeight/documentHeight.js @@ -1,7 +1,6 @@ /** * Measure document height */ -/* global document: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/modules/domComplexity/domComplexity.js b/modules/domComplexity/domComplexity.js index dd10e220c..eeec849d7 100644 --- a/modules/domComplexity/domComplexity.js +++ b/modules/domComplexity/domComplexity.js @@ -1,7 +1,6 @@ /** * Analyzes DOM complexity */ -/* global document: true, Node: true, window: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/modules/domHiddenContent/domHiddenContent.js b/modules/domHiddenContent/domHiddenContent.js index 62104b5a5..127767806 100644 --- a/modules/domHiddenContent/domHiddenContent.js +++ b/modules/domHiddenContent/domHiddenContent.js @@ -1,7 +1,6 @@ /** * Analyzes DOM hidden content */ -/* global document: true, Node: true, window: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/modules/domMutations/domMutations.js b/modules/domMutations/domMutations.js index a8a3a904f..d821c877f 100644 --- a/modules/domMutations/domMutations.js +++ b/modules/domMutations/domMutations.js @@ -1,7 +1,6 @@ /** * Analyzes DOM changes via MutationObserver API */ -/* global window: true, document: true, MutationObserver: true */ 'use strict'; module.exports = phantomas => { diff --git a/modules/domQueries/domQueries.js b/modules/domQueries/domQueries.js index 5838b4200..d9f520cd1 100644 --- a/modules/domQueries/domQueries.js +++ b/modules/domQueries/domQueries.js @@ -1,7 +1,6 @@ /** * Analyzes DOM queries done via native DOM methods */ -/* global Element: true, Document: true, Node: true, window: true */ 'use strict'; module.exports = phantomas => { diff --git a/modules/events/events.js b/modules/events/events.js index 7d2f3c37c..132c03720 100644 --- a/modules/events/events.js +++ b/modules/events/events.js @@ -1,7 +1,6 @@ /** * Analyzes events bound to DOM elements */ -/* global Document: true, Element: true, window: true */ 'use strict'; module.exports = phantomas => { diff --git a/modules/globalVariables/globalVariables.js b/modules/globalVariables/globalVariables.js index daed6bd08..42596edd0 100644 --- a/modules/globalVariables/globalVariables.js +++ b/modules/globalVariables/globalVariables.js @@ -1,7 +1,6 @@ /** * Counts global JavaScript variables */ -/* global document: true, window: true */ 'use strict'; module.exports = phantomas => { diff --git a/modules/jQuery/jQuery.js b/modules/jQuery/jQuery.js index 3050aeddb..b757b3b74 100644 --- a/modules/jQuery/jQuery.js +++ b/modules/jQuery/jQuery.js @@ -1,7 +1,6 @@ /** * Analyzes jQuery activity */ -/* global document: true, window: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/modules/lazyLoadableImages/lazyLoadableImages.js b/modules/lazyLoadableImages/lazyLoadableImages.js index 181d8d307..d743ea21b 100644 --- a/modules/lazyLoadableImages/lazyLoadableImages.js +++ b/modules/lazyLoadableImages/lazyLoadableImages.js @@ -1,7 +1,6 @@ /** * Analyzes images and detects which one can be lazy-loaded (are below the fold) */ -/* global document: true, window: true */ 'use strict'; module.exports = phantomas => { diff --git a/modules/localStorage/localStorage.js b/modules/localStorage/localStorage.js index a6ed5cedd..2edc90fd9 100644 --- a/modules/localStorage/localStorage.js +++ b/modules/localStorage/localStorage.js @@ -1,7 +1,6 @@ /** * localStorage metrics */ -/* global window: true */ 'use strict'; module.exports = phantomas => { diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index 848d16be9..7b3ff9459 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -1,7 +1,7 @@ /** * Measure when the page reaches certain states */ -/* global document: true, window: true */ +/* global window: true */ 'use strict'; module.exports = function(phantomas) { From 50c07cf17db3fc944a0bb46d53ea62c592ab3b80 Mon Sep 17 00:00:00 2001 From: macbre Date: Wed, 1 May 2019 16:53:58 +0200 Subject: [PATCH 193/248] puppeteer: use v1.15.0 (HeadlessChrome/75.0.3765.0) --- package-lock.json | 18 +++++++++--------- package.json | 2 +- test/integration-spec.yaml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 699797967..0d8197f1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -965,9 +965,9 @@ } }, "mime": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz", - "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==" + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.2.tgz", + "integrity": "sha512-zJBfZDkwRu+j3Pdd2aHsR5GfH2jIWhmL1ZzBoc+X+3JEti2hbArWcyJ+1laC1D2/U/W1a/+Cegj0/OnEU2ybjg==" }, "mimic-fn": { "version": "1.2.0", @@ -1202,9 +1202,9 @@ "dev": true }, "puppeteer": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.13.0.tgz", - "integrity": "sha512-LUXgvhjfB/P6IOUDAKxOcbCz9ISwBLL9UpKghYrcBDwrOGx1m60y0iN2M64mdAUbT4+7oZM5DTxOW7equa2fxQ==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.15.0.tgz", + "integrity": "sha512-D2y5kwA9SsYkNUmcBzu9WZ4V1SGHiQTmgvDZSx6sRYFsgV25IebL4V6FaHjF6MbwLK9C6f3G3pmck9qmwM8H3w==", "requires": { "debug": "^4.1.0", "extract-zip": "^1.6.6", @@ -1585,9 +1585,9 @@ } }, "ws": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz", - "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", "requires": { "async-limiter": "~1.0.0" } diff --git a/package.json b/package.json index 9e4b9a6fa..60adb5c2b 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "analyze-css": "^0.12.7", "debug": "^4.1.1", - "puppeteer": "^1.13.0" + "puppeteer": "^1.15.0" }, "devDependencies": { "eslint": "^5.14.1", diff --git a/test/integration-spec.yaml b/test/integration-spec.yaml index fff8b1786..a4db61d82 100644 --- a/test/integration-spec.yaml +++ b/test/integration-spec.yaml @@ -606,7 +606,7 @@ options: phone: true metrics: - userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3679.0 Mobile Safari/537.36' + userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3765.0 Mobile Safari/537.36' viewport: '360x640' - url: "/devices.html" @@ -614,7 +614,7 @@ options: "phone-landscape": true metrics: - userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3679.0 Mobile Safari/537.36' + userAgent: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3765.0 Mobile Safari/537.36' viewport: '640x360' - url: "/devices.html" From ed53b0145dc1a35664ab5f4659eefdcaf2afd46c Mon Sep 17 00:00:00 2001 From: macbre Date: Wed, 1 May 2019 16:54:17 +0200 Subject: [PATCH 194/248] Travis | do not use system-wide Chrome binary --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9b336f48..448c450ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,9 @@ before_install: # Enable user namespace cloning - "sysctl kernel.unprivileged_userns_clone=1" # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. - - "export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true" + #- "export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true" # Point Phantomas to Chrome stable binary provided by Travis - - "export PHANTOMAS_CHROMIUM_EXECUTABLE=`which google-chrome-stable`" + #- "export PHANTOMAS_CHROMIUM_EXECUTABLE=`which google-chrome-stable`" before_script: - sh test/server-start.sh & From 1e29d90b4e944750c061436c66f7ba4004358106 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 3 May 2019 19:15:34 +0200 Subject: [PATCH 195/248] Dockerfile - initial commit REPOSITORY TAG IMAGE ID CREATED SIZE phantomas latest 6c6452ee6c75 6 minutes ago 1.06GB --- Dockerfile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..681e860da --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# based on https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-on-alpine +FROM node:10-alpine + +# Installs latest Chromium (72) package. +RUN apk update && apk upgrade && \ + echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ + echo @edge http://nl.alpinelinux.org/alpine/edge/main >> /etc/apk/repositories && \ + apk add --no-cache \ + chromium@edge \ + nss@edge \ + freetype@edge \ + harfbuzz@edge \ + ttf-freefont@edge + +WORKDIR /opt/phantomas + +# Puppeteer v1.15.0 works with Chromium 75 // keep in sync with version in package.json +ENV PUPPETEER_VERSION 1.15.0 + +RUN npm i --no-save puppeteer@$PUPPETEER_VERSION + +# Add user so we don't need --no-sandbox. +RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ + && mkdir -p /opt/phantomas \ + && chown -R phantomas:phantomas /opt/phantomas + +# Run everything after as non-privileged user. +USER phantomas + From a38291a28e0f6fdb7fd92c7ea550dc605fecd012 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 3 May 2019 19:23:06 +0200 Subject: [PATCH 196/248] alpine | use a system-wide chromium binary /opt/phantomas $ /usr/bin/chromium-browser --version Chromium 73.0.3683.103 --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 681e860da..674f040a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,9 @@ RUN apk update && apk upgrade && \ WORKDIR /opt/phantomas +# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + # Puppeteer v1.15.0 works with Chromium 75 // keep in sync with version in package.json ENV PUPPETEER_VERSION 1.15.0 From 99cb082c07e62cdde4530a2794374e52ad10d517 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 3 May 2019 19:31:00 +0200 Subject: [PATCH 197/248] Upgrade js-yaml to resolve security issues --- package-lock.json | 22 +++++++++++----------- package.json | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0d8197f1a..94b04346b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -245,7 +245,7 @@ }, "concat-stream": { "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "resolved": "http://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "requires": { "buffer-from": "^1.0.0", @@ -339,9 +339,9 @@ } }, "ecstatic": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-2.2.1.tgz", - "integrity": "sha512-ztE4WqheoWLh3wv+HQwy7dACnvNY620coWpa+XqY6R2cVWgaAT2lUISU1Uf7JpdLLJCURktJOaA9av2AOzsyYQ==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-2.2.2.tgz", + "integrity": "sha512-F1g29y3I+abOS+M0AiK2O9R96AJ49Bc3kH696HtqnN+CL3YhpUnSzHNoUBQL03qDsN9Lr1XeKIxTqEH3BtiBgg==", "dev": true, "requires": { "he": "^1.1.1", @@ -358,7 +358,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true } @@ -917,9 +917,9 @@ "dev": true }, "js-yaml": { - "version": "3.12.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.1.tgz", - "integrity": "sha512-um46hB9wNOKlwkHgiuyEVAybXBjwFUV0Z/RaHJblRd9DXltue9FTYvzCr9ErQrK9Adz5MU4gHWVaNUfdmrC8qA==", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", "dev": true, "requires": { "argparse": "^1.0.7", @@ -1224,7 +1224,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -1384,7 +1384,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, @@ -1400,7 +1400,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" diff --git a/package.json b/package.json index 60adb5c2b..7d9b7a053 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "glob": "^7.1.2", "http-serve": "^1.0.1", "js-beautify": "^1.6.14", - "js-yaml": "^3.12.1", + "js-yaml": "^3.13.1", "mockery": "^2.0.0", "vows": "^0.8.2" }, From d3928d09dde049c3f99f43cec535a74e975a319a Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 3 May 2019 21:59:42 +0200 Subject: [PATCH 198/248] puppeteer.launch - catch and log exceptions --- lib/browser.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/browser.js b/lib/browser.js index da16ad1d4..5fc6bbb03 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -34,11 +34,17 @@ Browser.prototype.init = async () => { options.executablePath = env['PHANTOMAS_CHROMIUM_EXECUTABLE']; } - debug('Launching Puppeter: %j', options); + debug('Launching Puppeteer: %j', options); - // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#puppeteerlaunchoptions - this.browser = await puppeteer.launch(options); - this.page = await this.browser.newPage(); + try { + // https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#puppeteerlaunchoptions + this.browser = await puppeteer.launch(options); + this.page = await this.browser.newPage(); + } + catch(ex) { + debug('Puppeteer failed to launch: %s', ex); + throw ex; + } // A Chrome Devtools Protocol session attached to the target this.cdp = this.page._client; From c55e13898b4d05e4907841e2847862e8021a60b0 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 3 May 2019 21:59:02 +0200 Subject: [PATCH 199/248] Tell phantomas were Chromium binary is --- .dockerignore | 3 +++ Dockerfile | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..67b4468a0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +Dockerfile +node_modules/ +docs/ diff --git a/Dockerfile b/Dockerfile index 674f040a8..bf7e31eef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,8 @@ WORKDIR /opt/phantomas # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# Puppeteer v1.15.0 works with Chromium 75 // keep in sync with version in package.json -ENV PUPPETEER_VERSION 1.15.0 - -RUN npm i --no-save puppeteer@$PUPPETEER_VERSION +# Tell phantomas were Chromium binary is +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser # Add user so we don't need --no-sandbox. RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ @@ -30,3 +28,12 @@ RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ # Run everything after as non-privileged user. USER phantomas +# Copy the content of the repository into a container +COPY . /opt/phantomas + +# and install dependencies +RUN npm i + +# test it (if needed) +#RUN ./test/server-start.sh & +#RUN sleep 2 && npm t From 84b9cd6249c930dd29d3c62ce8c44e818b543206 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 3 May 2019 22:24:34 +0200 Subject: [PATCH 200/248] Docker - run using --no-sandbox (we are already in a isolated namespace) See https://github.com/jessfraz/dockerfiles/issues/65#issuecomment-402368346 --- Dockerfile | 4 +++- lib/browser.js | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf7e31eef..89809297b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,12 +13,14 @@ RUN apk update && apk upgrade && \ ttf-freefont@edge WORKDIR /opt/phantomas +ENV HOME /opt/phantomas # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# Tell phantomas were Chromium binary is +# Tell phantomas where Chromium binary is and that we're in docker ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV DOCKERIZED yes # Add user so we don't need --no-sandbox. RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ diff --git a/lib/browser.js b/lib/browser.js index 5fc6bbb03..637d8c735 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -34,6 +34,13 @@ Browser.prototype.init = async () => { options.executablePath = env['PHANTOMAS_CHROMIUM_EXECUTABLE']; } + // allow Chromium to be run inside a container + // @see https://github.com/jessfraz/dockerfiles/issues/65 + if (env['DOCKERIZED']) { + debug('Docker environment detected, enabling -no-sandbox switch'); + options.args.push('--no-sandbox'); + } + debug('Launching Puppeteer: %j', options); try { From f30db9bff5f069a5488c85e71117754a3e50b659 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 3 May 2019 22:27:16 +0200 Subject: [PATCH 201/248] Dockerfile - improved caching of dependencies --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 89809297b..affcb47b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,12 +30,14 @@ RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ # Run everything after as non-privileged user. USER phantomas -# Copy the content of the repository into a container -COPY . /opt/phantomas - -# and install dependencies +# Install dependencies +COPY package.json /opt/phantomas +COPY package-lock.json /opt/phantomas RUN npm i +# Copy the content of the rest of the repository into a container +COPY . /opt/phantomas + # test it (if needed) #RUN ./test/server-start.sh & #RUN sleep 2 && npm t From 6083d90999152714a15d393cf8d322de2053fbef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2019 16:23:44 +0000 Subject: [PATCH 202/248] build(deps): bump eslint-utils from 1.3.1 to 1.4.3 Bumps [eslint-utils](https://github.com/mysticatea/eslint-utils) from 1.3.1 to 1.4.3. - [Release notes](https://github.com/mysticatea/eslint-utils/releases) - [Commits](https://github.com/mysticatea/eslint-utils/compare/v1.3.1...v1.4.3) Signed-off-by: dependabot[bot] --- package-lock.json | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 94b04346b..eff3e3dc5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -458,10 +458,21 @@ } }, "eslint-utils": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.3.1.tgz", - "integrity": "sha512-Z7YjnIldX+2XMcjr7ZkgEsOj/bREONV60qYeB/bjMAqqqZ4zxKyWX+BOUkdmRmA9riiIPVvo5x86m5elviOk0Q==", - "dev": true + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz", + "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==", + "dev": true + } + } }, "eslint-visitor-keys": { "version": "1.0.0", @@ -985,7 +996,7 @@ }, "minimist": { "version": "0.0.8", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "mkdirp": { From b2fd5fbb9f47b02f6d71a77f653a7852cca673c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2019 21:40:11 +0000 Subject: [PATCH 203/248] build(deps): bump lodash from 4.17.11 to 4.17.15 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.11 to 4.17.15. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.11...4.17.15) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index eff3e3dc5..eef5b4117 100644 --- a/package-lock.json +++ b/package-lock.json @@ -960,9 +960,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "lru-cache": { From 9a06bccebc180b91e7681f6c586f3dd4d2a47aa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Mar 2020 01:14:10 +0000 Subject: [PATCH 204/248] build(deps): bump acorn from 6.1.0 to 6.4.1 Bumps [acorn](https://github.com/acornjs/acorn) from 6.1.0 to 6.4.1. - [Release notes](https://github.com/acornjs/acorn/releases) - [Commits](https://github.com/acornjs/acorn/compare/6.1.0...6.4.1) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index eef5b4117..1953364df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,9 +43,9 @@ "dev": true }, "acorn": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.0.tgz", - "integrity": "sha512-MW/FjM+IvU9CgBzjO3UIPCE2pyEwUsoFl+VGdczOPEdxfGFjuKny/gN54mOuX7Qxmb9Rg9MCn2oKiSUeW+pjrw==", + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", "dev": true }, "acorn-jsx": { From 09ba67ef08048218f7c8c3f303293fb61651bd85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:31:54 +0000 Subject: [PATCH 205/248] build(deps): bump https-proxy-agent from 2.2.1 to 2.2.4 Bumps [https-proxy-agent](https://github.com/TooTallNate/node-https-proxy-agent) from 2.2.1 to 2.2.4. - [Release notes](https://github.com/TooTallNate/node-https-proxy-agent/releases) - [Commits](https://github.com/TooTallNate/node-https-proxy-agent/compare/2.2.1...2.2.4) Signed-off-by: dependabot[bot] --- package-lock.json | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index eef5b4117..0394523b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -778,14 +778,22 @@ } }, "https-proxy-agent": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", - "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", + "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", "requires": { - "agent-base": "^4.1.0", + "agent-base": "^4.3.0", "debug": "^3.1.0" }, "dependencies": { + "agent-base": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", + "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", + "requires": { + "es6-promisify": "^5.0.0" + } + }, "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", From 9be7ee70021728090e2a7c015112eded2af01eac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2020 04:31:55 +0000 Subject: [PATCH 206/248] build(deps): bump lodash from 4.17.15 to 4.17.19 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.19. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.19) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a8232af63..afea4ad77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -968,9 +968,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", "dev": true }, "lru-cache": { From b6a41bd73bdea25136882fb1515be3bcf0fb8285 Mon Sep 17 00:00:00 2001 From: macbre Date: Thu, 23 Jul 2020 18:47:27 +0200 Subject: [PATCH 207/248] Use Puppeteer v3.0.0 --- package-lock.json | 306 +++++++++++++++++++++++++++------------------- package.json | 2 +- 2 files changed, 183 insertions(+), 125 deletions(-) diff --git a/package-lock.json b/package-lock.json index 94b04346b..3d39668ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,8 +27,7 @@ "@types/node": { "version": "10.12.18", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", - "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==", - "dev": true + "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==" }, "@types/semver": { "version": "5.5.0", @@ -36,6 +35,15 @@ "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==", "dev": true }, + "@types/yauzl": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", + "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", + "optional": true, + "requires": { + "@types/node": "*" + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -135,11 +143,6 @@ "integrity": "sha1-rDYTsdqb7RtHUQu0ZRuJMeRxRsc=", "dev": true }, - "async-limiter": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" - }, "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", @@ -150,6 +153,28 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "bl": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", + "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + } + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -159,10 +184,19 @@ "concat-map": "0.0.1" } }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, "callsites": { "version": "3.0.0", @@ -187,6 +221,11 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "cli": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cli/-/cli-1.0.1.tgz", @@ -243,17 +282,6 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, - "concat-stream": { - "version": "1.6.2", - "resolved": "http://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, "config-chain": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz", @@ -264,11 +292,6 @@ "proto-list": "~1.2.1" } }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "corser": { "version": "2.0.1", "resolved": "http://registry.npmjs.org/corser/-/corser-2.0.1.tgz", @@ -384,6 +407,14 @@ "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", "dev": true }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "es6-promise": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", @@ -539,29 +570,14 @@ } }, "extract-zip": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", - "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", - "requires": { - "concat-stream": "1.6.2", - "debug": "2.6.9", - "mkdirp": "0.5.1", - "yauzl": "2.4.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "requires": { + "@types/yauzl": "^2.9.1", + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" } }, "eyes": { @@ -594,9 +610,9 @@ "integrity": "sha512-HtS5uSqMiwfxFFyukKP/F0f3o8/8oqHtbInsaq2s0+V2J2MEHGyukWajWqzKS57sWLTOgJ7bKMRhA4fG5cTQ3Q==" }, "fd-slicer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", - "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", "requires": { "pend": "~1.2.0" } @@ -673,6 +689,11 @@ } } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -684,6 +705,14 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "get-stream": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", + "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", + "requires": { + "pump": "^3.0.0" + } + }, "glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", @@ -767,21 +796,18 @@ } }, "https-proxy-agent": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", - "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", "requires": { - "agent-base": "^4.1.0", - "debug": "^3.1.0" + "agent-base": "5", + "debug": "4" }, "dependencies": { - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "requires": { - "ms": "^2.1.1" - } + "agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" } } }, @@ -794,6 +820,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -886,11 +917,6 @@ "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", "dev": true }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -965,9 +991,9 @@ } }, "mime": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.2.tgz", - "integrity": "sha512-zJBfZDkwRu+j3Pdd2aHsR5GfH2jIWhmL1ZzBoc+X+3JEti2hbArWcyJ+1laC1D2/U/W1a/+Cegj0/OnEU2ybjg==" + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, "mimic-fn": { "version": "1.2.0", @@ -992,10 +1018,16 @@ "version": "0.5.1", "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, "requires": { "minimist": "0.0.8" } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "mockery": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mockery/-/mockery-2.1.0.tgz", @@ -1168,11 +1200,6 @@ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -1185,9 +1212,9 @@ "dev": true }, "proxy-from-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", - "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "pseudomap": { "version": "1.0.2", @@ -1195,6 +1222,15 @@ "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", "dev": true }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -1202,18 +1238,20 @@ "dev": true }, "puppeteer": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.15.0.tgz", - "integrity": "sha512-D2y5kwA9SsYkNUmcBzu9WZ4V1SGHiQTmgvDZSx6sRYFsgV25IebL4V6FaHjF6MbwLK9C6f3G3pmck9qmwM8H3w==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-3.3.0.tgz", + "integrity": "sha512-23zNqRltZ1PPoK28uRefWJ/zKb5Jhnzbbwbpcna2o5+QMn17F0khq5s1bdH3vPlyj+J36pubccR8wiNA/VE0Vw==", "requires": { "debug": "^4.1.0", - "extract-zip": "^1.6.6", - "https-proxy-agent": "^2.2.1", + "extract-zip": "^2.0.0", + "https-proxy-agent": "^4.0.0", "mime": "^2.0.3", "progress": "^2.0.1", "proxy-from-env": "^1.0.0", - "rimraf": "^2.6.1", - "ws": "^6.1.0" + "rimraf": "^3.0.2", + "tar-fs": "^2.0.0", + "unbzip2-stream": "^1.3.3", + "ws": "^7.2.3" } }, "qs": { @@ -1223,17 +1261,13 @@ "dev": true }, "readable-stream": { - "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" } }, "regexpp": { @@ -1270,9 +1304,9 @@ } }, "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "requires": { "glob": "^7.1.3" } @@ -1296,9 +1330,9 @@ } }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer": { "version": "2.1.2", @@ -1399,11 +1433,11 @@ } }, "string_decoder": { - "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "requires": { - "safe-buffer": "~5.1.0" + "safe-buffer": "~5.2.0" } }, "strip-ansi": { @@ -1470,6 +1504,29 @@ } } }, + "tar-fs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", + "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, + "tar-stream": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.3.tgz", + "integrity": "sha512-Z9yri56Dih8IaK8gncVPx4Wqt86NDmQTSh49XLZgjWpGZL9GK9HKParS2scqHCC4w6X9Gh2jwaU45V47XTKwVA==", + "requires": { + "bl": "^4.0.1", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -1479,8 +1536,7 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "tmp": { "version": "0.0.33", @@ -1506,10 +1562,14 @@ "prelude-ls": "~1.1.2" } }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + "unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "requires": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } }, "union": { "version": "0.4.6", @@ -1585,12 +1645,9 @@ } }, "ws": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", - "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", - "requires": { - "async-limiter": "~1.0.0" - } + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", + "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==" }, "yallist": { "version": "2.1.2", @@ -1599,11 +1656,12 @@ "dev": true }, "yauzl": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", - "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", "requires": { - "fd-slicer": "~1.0.1" + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" } } } diff --git a/package.json b/package.json index 7d9b7a053..5cab77a3f 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "analyze-css": "^0.12.7", "debug": "^4.1.1", - "puppeteer": "^1.15.0" + "puppeteer": "^3.0.0" }, "devDependencies": { "eslint": "^5.14.1", From bb0ad12c4f7e08c8e1d8a7f2af42227467b414a6 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 Jul 2020 21:40:50 +0200 Subject: [PATCH 208/248] Use node:14-alpine --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index affcb47b9..254614d7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ # based on https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-on-alpine -FROM node:10-alpine +FROM node:14-alpine -# Installs latest Chromium (72) package. +# Installs latest Chromium package +# https://pkgs.alpinelinux.org/package/edge/community/x86_64/chromium RUN apk update && apk upgrade && \ - echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ +# echo @ege http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ echo @edge http://nl.alpinelinux.org/alpine/edge/main >> /etc/apk/repositories && \ apk add --no-cache \ chromium@edge \ From 9aaf496cd7bab7080db2cf926544a5cc1a51d282 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 25 Jul 2020 00:26:34 +0200 Subject: [PATCH 209/248] Set up GitHub Actions and Dependabot --- .github/dependabot.yml | 10 ++++++++++ .github/workflows/tests.yml | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..3dc395c87 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# Basic set up +# https://help.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#package-ecosystem + +version: 2 +updates: + + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..7b321c60d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,35 @@ +# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions + +name: Node.js CI + +on: + push: + branches: [ phantomas-v2 ] + pull_request: + branches: [ phantomas-v2 ] + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [14.x] + + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: | + sh test/server-start.sh & + npm test + npm run-script lint From 1d41f01f663c9aff93f931a88d1e9b6fb6817398 Mon Sep 17 00:00:00 2001 From: macbre Date: Sat, 25 Jul 2020 00:37:56 +0200 Subject: [PATCH 210/248] chromium-browser with flags --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 254614d7c..2a895f368 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,3 +42,6 @@ COPY . /opt/phantomas # test it (if needed) #RUN ./test/server-start.sh & #RUN sleep 2 && npm t + +# Autorun chrome headless +#ENTRYPOINT ["chromium-browser", "--headless", "--use-gl=swiftshader", "--disable-software-rasterizer", "--disable-dev-shm-usage"] From 878eb097ca92c4915722348208cdd631365642b2 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 01:05:48 +0200 Subject: [PATCH 211/248] Run chromium-browser --no-sandbox --version when building Docker image --- Dockerfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a895f368..952d5ab66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,14 @@ FROM node:14-alpine # Installs latest Chromium package # https://pkgs.alpinelinux.org/package/edge/community/x86_64/chromium RUN apk update && apk upgrade && \ -# echo @ege http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ + echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ echo @edge http://nl.alpinelinux.org/alpine/edge/main >> /etc/apk/repositories && \ apk add --no-cache \ chromium@edge \ - nss@edge \ freetype@edge \ harfbuzz@edge \ + libstdc++@edge \ + nss@edge \ ttf-freefont@edge WORKDIR /opt/phantomas @@ -30,6 +31,7 @@ RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ # Run everything after as non-privileged user. USER phantomas +RUN chromium-browser --no-sandbox --version # Install dependencies COPY package.json /opt/phantomas @@ -39,9 +41,6 @@ RUN npm i # Copy the content of the rest of the repository into a container COPY . /opt/phantomas -# test it (if needed) -#RUN ./test/server-start.sh & -#RUN sleep 2 && npm t - # Autorun chrome headless -#ENTRYPOINT ["chromium-browser", "--headless", "--use-gl=swiftshader", "--disable-software-rasterizer", "--disable-dev-shm-usage"] +ENTRYPOINT ["sh"] +#CMD ["chromium-browser", "--no-sandbox", "--headless", "--use-gl=swiftshader", "--disable-software-rasterizer", "--disable-dev-shm-usage"] From 6541ced3e0d8aa8681c8a0a423c752ebd70c60ce Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 01:06:46 +0200 Subject: [PATCH 212/248] GitHub Actions: build an image --- .github/workflows/dockerimage.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/dockerimage.yml diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml new file mode 100644 index 000000000..f2e695e69 --- /dev/null +++ b/.github/workflows/dockerimage.yml @@ -0,0 +1,16 @@ +name: Check if a Docker image can be built + +on: [push] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Build the Docker image + run: | + docker build . --tag ${{ github.repository }}:$(date +%s) + docker images From 3bd584ab7e9af51cd29cffa1340a362c233b0cda Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 01:11:48 +0200 Subject: [PATCH 213/248] Get rid of Travis integration --- .travis.yml | 28 ---------------------------- README.md | 2 +- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 448c450ad..000000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -# https://docs.travis-ci.com/user/languages/javascript-with-nodejs/ -language: node_js -dist: xenial -node_js: 10 - -addons: - chrome: stable - -# allow headful tests -# https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/troubleshooting.md#running-puppeteer-on-travis-ci -before_install: - # Enable user namespace cloning - - "sysctl kernel.unprivileged_userns_clone=1" - # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. - #- "export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true" - # Point Phantomas to Chrome stable binary provided by Travis - #- "export PHANTOMAS_CHROMIUM_EXECUTABLE=`which google-chrome-stable`" - -before_script: - - sh test/server-start.sh & - - SERVER_PID=$! - - sleep 1 -after_script: - - kill $SERVER_PID -script: "npm test && npm run-script lint" -cache: - directories: - - node_modules diff --git a/README.md b/README.md index d30183dee..6d9d691b4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -phantomas [![npm](https://img.shields.io/npm/dt/phantomas.svg)]() [![Build Status](https://api.travis-ci.org/macbre/phantomas.png?branch=devel)](http://travis-ci.org/macbre/phantomas) [![Known Vulnerabilities](https://snyk.io/test/github/macbre/phantomas/badge.svg)](https://snyk.io/test/github/macbre/phantomas) +phantomas [![npm](https://img.shields.io/npm/dt/phantomas.svg)]() ========= [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md)-based modular web performance metrics collector. And why phantomas? Well, [because](http://en.wikipedia.org/wiki/Fantômas) :) From 2f0f4baeb1de0e8e7b95c51bbdb85328fad025ea Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 14:22:37 +0200 Subject: [PATCH 214/248] .dockerignore: skip more files --- .dockerignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 67b4468a0..2db517099 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,7 @@ Dockerfile -node_modules/ docs/ +examples/ +node_modules/ +*.png +*.html +*.md From 6f11eaadb5705794b3ff5cc8f4d88bbc39e0b2db Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 14:25:59 +0200 Subject: [PATCH 215/248] Install tini --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 952d5ab66..0fd8e4b09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ RUN apk update && apk upgrade && \ harfbuzz@edge \ libstdc++@edge \ nss@edge \ + tini@edge \ ttf-freefont@edge WORKDIR /opt/phantomas @@ -41,6 +42,4 @@ RUN npm i # Copy the content of the rest of the repository into a container COPY . /opt/phantomas -# Autorun chrome headless -ENTRYPOINT ["sh"] -#CMD ["chromium-browser", "--no-sandbox", "--headless", "--use-gl=swiftshader", "--disable-software-rasterizer", "--disable-dev-shm-usage"] +ENTRYPOINT ["tini", "--"] From e541c228ff6e94f031a49809937b17ff85a55ccb Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 15:01:23 +0200 Subject: [PATCH 216/248] Dockerfile: paths simplified --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0fd8e4b09..602ec8502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,8 @@ RUN apk update && apk upgrade && \ tini@edge \ ttf-freefont@edge -WORKDIR /opt/phantomas ENV HOME /opt/phantomas +WORKDIR $HOME # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true @@ -27,19 +27,18 @@ ENV DOCKERIZED yes # Add user so we don't need --no-sandbox. RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ - && mkdir -p /opt/phantomas \ - && chown -R phantomas:phantomas /opt/phantomas + && chown -R phantomas:phantomas $HOME # Run everything after as non-privileged user. USER phantomas RUN chromium-browser --no-sandbox --version # Install dependencies -COPY package.json /opt/phantomas -COPY package-lock.json /opt/phantomas +COPY package.json . +COPY package-lock.json . RUN npm i # Copy the content of the rest of the repository into a container -COPY . /opt/phantomas +COPY . . ENTRYPOINT ["tini", "--"] From 354b03dc531c727b7ab889f20a80cd12672b5e07 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 15:10:23 +0200 Subject: [PATCH 217/248] Update dependencies --- package-lock.json | 460 +++++++++++++++++++++++----------------------- package.json | 8 +- 2 files changed, 238 insertions(+), 230 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3d39668ef..ad2ae3fd7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,35 +5,41 @@ "requires": true, "dependencies": { "@babel/code-frame": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", - "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", "dev": true, "requires": { - "@babel/highlight": "^7.0.0" + "@babel/highlight": "^7.10.4" } }, + "@babel/helper-validator-identifier": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", + "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==", + "dev": true + }, "@babel/highlight": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz", - "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==", + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", "dev": true, "requires": { + "@babel/helper-validator-identifier": "^7.10.4", "chalk": "^2.0.0", - "esutils": "^2.0.2", "js-tokens": "^4.0.0" } }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/node": { "version": "10.12.18", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", - "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==" - }, - "@types/semver": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", - "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==", - "dev": true + "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==", + "optional": true }, "@types/yauzl": { "version": "2.9.1", @@ -51,54 +57,74 @@ "dev": true }, "acorn": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.0.tgz", - "integrity": "sha512-MW/FjM+IvU9CgBzjO3UIPCE2pyEwUsoFl+VGdczOPEdxfGFjuKny/gN54mOuX7Qxmb9Rg9MCn2oKiSUeW+pjrw==", + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", "dev": true }, "acorn-jsx": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.1.tgz", - "integrity": "sha512-HJ7CfNHrfJLlNTzIEUTj43LNWGkqpRLxm3YjAlcD0ACydk9XynzYsCBHxut+iqt+1aBXkx9UP/w/ZqMr13XIzg==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", "dev": true }, "agent-base": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", - "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", "requires": { - "es6-promisify": "^5.0.0" + "debug": "4" } }, "ajv": { - "version": "6.9.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz", - "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==", + "version": "6.12.3", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz", + "integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", "dev": true, "requires": { - "fast-deep-equal": "^2.0.1", + "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "analyze-css": { - "version": "0.12.7", - "resolved": "https://registry.npmjs.org/analyze-css/-/analyze-css-0.12.7.tgz", - "integrity": "sha512-UAuqBMSxqoXyWXvEsrjLBD1U1sylyl8S2EBTWgRq7Od3odN6Wu51RvEci731ln5szOZ6yJijWtE2H6eQQk54Dg==", + "version": "0.12.10", + "resolved": "https://registry.npmjs.org/analyze-css/-/analyze-css-0.12.10.tgz", + "integrity": "sha512-7HQrlqsWnfvcBBO++LrDvHnaov6luVyh2phuNYnZXxdy2S/QBs2Q6SlvtUW9Ehp3/MkGHRo7TGXGIHEd9NWnHQ==", "requires": { "cli": "^1.0.1", + "commander": "^5.1.0", "css": "^2.2.4", "css-shorthand-properties": "^1.1.1", "debug": "^4.1.1", "fast-stats": "0.0.5", - "glob": "^7.1.3", - "http-proxy-agent": "^2.1.0", - "node-fetch": "^2.3.0", + "glob": "^7.1.6", + "http-proxy-agent": "^4.0.1", + "node-fetch": "^2.6.0", "onecolor": "^3.1.0", - "optimist": "0.6.x", "slick": "~1.12.1", "specificity": "^0.4.1" + }, + "dependencies": { + "commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "ansi-escapes": { @@ -199,9 +225,9 @@ "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, "callsites": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.0.0.tgz", - "integrity": "sha512-tWnkwu9YEq2uzlBDI4RcLn8jrFvF9AOi8PxDNU3hZZjJcjkcRAq3vCI+vZcg1SuxISDYe86k9VZFwAxDiJGoAw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, "chalk": { @@ -245,9 +271,9 @@ } }, "cli-width": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", - "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", "dev": true }, "color-convert": { @@ -272,9 +298,9 @@ "dev": true }, "commander": { - "version": "2.19.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", - "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, "concat-map": { @@ -347,9 +373,9 @@ "dev": true }, "diff": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.8.tgz", - "integrity": "sha1-NDJ2MI7Jkbe8giZ+1VvBQR+XFmY=", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, "doctrine": { @@ -388,15 +414,13 @@ } }, "editorconfig": { - "version": "0.15.2", - "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.2.tgz", - "integrity": "sha512-GWjSI19PVJAM9IZRGOS+YKI8LN+/sjkSjNyvxL5ucqP9/IqtYNXBaQ/6c/hkPNYQHyOHra2KoXZI/JVpuqwmcQ==", + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", + "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", "dev": true, "requires": { - "@types/node": "^10.11.7", - "@types/semver": "^5.5.0", "commander": "^2.19.0", - "lru-cache": "^4.1.3", + "lru-cache": "^4.1.5", "semver": "^5.6.0", "sigmund": "^1.0.1" } @@ -415,19 +439,6 @@ "once": "^1.4.0" } }, - "es6-promise": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", - "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==" - }, - "es6-promisify": { - "version": "5.0.0", - "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", - "requires": { - "es6-promise": "^4.0.3" - } - }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -435,9 +446,9 @@ "dev": true }, "eslint": { - "version": "5.14.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.14.1.tgz", - "integrity": "sha512-CyUMbmsjxedx8B0mr79mNOqetvkbij/zrXnFeK2zc3pGRn3/tibjiNAv/3UxFEyfMDjh+ZqTrJrEGBFiGfD5Og==", + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", @@ -446,7 +457,7 @@ "cross-spawn": "^6.0.5", "debug": "^4.0.1", "doctrine": "^3.0.0", - "eslint-scope": "^4.0.0", + "eslint-scope": "^4.0.3", "eslint-utils": "^1.3.1", "eslint-visitor-keys": "^1.0.0", "espree": "^5.0.1", @@ -460,7 +471,7 @@ "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "inquirer": "^6.2.2", - "js-yaml": "^3.12.0", + "js-yaml": "^3.13.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.3.0", "lodash": "^4.17.11", @@ -479,9 +490,9 @@ } }, "eslint-scope": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.0.tgz", - "integrity": "sha512-1G6UTDi7Jc1ELFwnR58HV4fK9OQK4S6N985f166xqXxpjU6plxFISJa2Ba9KCQuFa8RCnj/lSFJbHo7UFDBnUA==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", "dev": true, "requires": { "esrecurse": "^4.1.0", @@ -489,15 +500,18 @@ } }, "eslint-utils": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.3.1.tgz", - "integrity": "sha512-Z7YjnIldX+2XMcjr7ZkgEsOj/bREONV60qYeB/bjMAqqqZ4zxKyWX+BOUkdmRmA9riiIPVvo5x86m5elviOk0Q==", - "dev": true + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } }, "eslint-visitor-keys": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", - "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", "dev": true }, "espree": { @@ -518,12 +532,20 @@ "dev": true }, "esquery": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz", - "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", "dev": true, "requires": { - "estraverse": "^4.0.0" + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz", + "integrity": "sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==", + "dev": true + } } }, "esrecurse": { @@ -536,15 +558,15 @@ } }, "estraverse": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", - "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", "dev": true }, "esutils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, "eventemitter3": { @@ -559,9 +581,9 @@ "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" }, "external-editor": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.0.3.tgz", - "integrity": "sha512-bn71H9+qWoOQKyZDo25mOMVpSmXROAsTJVVVYzrrtol3d4y+AsKjf4Iwl2Q+IuT0kFSQ1qo166UuIwqYq7mGnA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", "dev": true, "requires": { "chardet": "^0.7.0", @@ -587,15 +609,15 @@ "dev": true }, "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, "fast-levenshtein": { @@ -658,9 +680,9 @@ } }, "flatted": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", - "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, "follow-redirects": { @@ -727,9 +749,9 @@ } }, "globals": { - "version": "11.11.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.11.0.tgz", - "integrity": "sha512-WHq43gS+6ufNOEqlrDBxVEbb8ntfXrfAUU2ZOpCxrBdGKW3gyv8mCxAfIBD0DroPKGrJ2eSsXsLtY9MPntsyTw==", + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true }, "has-flag": { @@ -756,27 +778,13 @@ } }, "http-proxy-agent": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", - "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", "requires": { - "agent-base": "4", - "debug": "3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" } }, "http-serve": { @@ -832,9 +840,9 @@ "dev": true }, "import-fresh": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.0.0.tgz", - "integrity": "sha512-pOnA9tfM3Uwics+SaBLCNyZZZbK+4PTu0OPZtLlMIrv17EdBoC15S9Kn8ckJ9TZTyKb3ywNE5y1yeDxxGA7nTQ==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", "dev": true, "requires": { "parent-module": "^1.0.0", @@ -868,9 +876,9 @@ "dev": true }, "inquirer": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.2.2.tgz", - "integrity": "sha512-Z2rREiXA6cHRR9KBOarR3WuLlFzlIfAEIiB45ll5SSadMg7WqOh1MKEjjndfuH5ewXdixWCxqnVfGOQzPeiztA==", + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", "dev": true, "requires": { "ansi-escapes": "^3.2.0", @@ -879,28 +887,28 @@ "cli-width": "^2.0.0", "external-editor": "^3.0.3", "figures": "^2.0.0", - "lodash": "^4.17.11", + "lodash": "^4.17.12", "mute-stream": "0.0.7", "run-async": "^2.2.0", "rxjs": "^6.4.0", "string-width": "^2.1.0", - "strip-ansi": "^5.0.0", + "strip-ansi": "^5.1.0", "through": "^2.3.6" }, "dependencies": { "ansi-regex": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.0.0.tgz", - "integrity": "sha512-iB5Dda8t/UqpPI/IjsejXu5jOGDrzn41wJyljwPH65VCIbk6+1BzFIMJGFwTNrYXT1CrD+B4l19U7awiQ8rk7w==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true }, "strip-ansi": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.0.0.tgz", - "integrity": "sha512-Uu7gQyZI7J7gn5qLn1Np3G9vcYGTVqB+lFTytnDJv83dd8T22aGH451P3jueT2/QemInJDfxHB5Tde5OzgG1Ow==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", "dev": true, "requires": { - "ansi-regex": "^4.0.0" + "ansi-regex": "^4.1.0" } } } @@ -911,12 +919,6 @@ "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, - "is-promise": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", - "dev": true - }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -924,16 +926,24 @@ "dev": true }, "js-beautify": { - "version": "1.8.9", - "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.8.9.tgz", - "integrity": "sha512-MwPmLywK9RSX0SPsUJjN7i+RQY9w/yC17Lbrq9ViEefpLRgqAR2BgrMN2AbifkUuhDV8tRauLhLda/9+bE0YQA==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.11.0.tgz", + "integrity": "sha512-a26B+Cx7USQGSWnz9YxgJNMmML/QG2nqIaL7VVYPCXbqiKz8PN0waSNvroMtvAK6tY7g/wPdNWGEP+JTNIBr6A==", "dev": true, "requires": { "config-chain": "^1.1.12", - "editorconfig": "^0.15.2", + "editorconfig": "^0.15.3", "glob": "^7.1.3", - "mkdirp": "~0.5.0", - "nopt": "~4.0.1" + "mkdirp": "~1.0.3", + "nopt": "^4.0.3" + }, + "dependencies": { + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + } } }, "js-tokens": { @@ -975,9 +985,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", "dev": true }, "lru-cache": { @@ -1012,7 +1022,8 @@ "minimist": { "version": "0.0.8", "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true }, "mkdirp": { "version": "0.5.1", @@ -1058,14 +1069,14 @@ "dev": true }, "node-fetch": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz", - "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==" + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" }, "nopt": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", - "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", "dev": true, "requires": { "abbrev": "1", @@ -1104,42 +1115,35 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dev": true, "requires": { "minimist": "~0.0.1", "wordwrap": "~0.0.2" } }, "optionator": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", - "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", "dev": true, "requires": { "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.4", + "fast-levenshtein": "~2.0.6", "levn": "~0.3.0", "prelude-ls": "~1.1.2", "type-check": "~0.3.2", - "wordwrap": "~1.0.0" - }, - "dependencies": { - "wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", - "dev": true - } + "word-wrap": "~1.2.3" } }, "os-homedir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-tmpdir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -1154,9 +1158,9 @@ } }, "parent-module": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.0.tgz", - "integrity": "sha512-8Mf5juOMmiE4FcmzYc4IaiS9L3+9paz2KOiXzkRviCP6aDmN49Hz6EMWz0lGNp9pX80GvvAuLADtyGfW/Em3TA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "dev": true, "requires": { "callsites": "^3.0.0" @@ -1312,18 +1316,15 @@ } }, "run-async": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", - "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", - "dev": true, - "requires": { - "is-promise": "^2.1.0" - } + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true }, "rxjs": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.4.0.tgz", - "integrity": "sha512-Z9Yfa11F6B9Sg/BK9MnqnQ+aQYicPLtilXBp2yUtDt2JRCE0h26d33EnfO3ZxoNxG0T92OUucP3Ct7cpfkdFfw==", + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.0.tgz", + "integrity": "sha512-3HMA8z/Oz61DUHe+SdOiQyzIf4tOx5oQHmMir7IZEu6TMqCLHT4LRcmNaUS0NwOz8VLvmmBduMsoaUvMaIiqzg==", "dev": true, "requires": { "tslib": "^1.9.0" @@ -1341,9 +1342,9 @@ "dev": true }, "semver": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", - "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "dev": true }, "shebang-command": { @@ -1368,9 +1369,9 @@ "dev": true }, "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, "slice-ansi": { @@ -1395,11 +1396,11 @@ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, "source-map-resolve": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", - "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", "requires": { - "atob": "^2.1.1", + "atob": "^2.1.2", "decode-uri-component": "^0.2.0", "resolve-url": "^0.2.1", "source-map-url": "^0.4.0", @@ -1465,41 +1466,41 @@ } }, "table": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/table/-/table-5.2.3.tgz", - "integrity": "sha512-N2RsDAMvDLvYwFcwbPyF3VmVSSkuF+G1e+8inhBLtHpvwXGw4QRPEZhihQNeEN0i1up6/f6ObCJXNdlRG3YVyQ==", + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", "dev": true, "requires": { - "ajv": "^6.9.1", - "lodash": "^4.17.11", + "ajv": "^6.10.2", + "lodash": "^4.17.14", "slice-ansi": "^2.1.0", "string-width": "^3.0.0" }, "dependencies": { "ansi-regex": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.0.0.tgz", - "integrity": "sha512-iB5Dda8t/UqpPI/IjsejXu5jOGDrzn41wJyljwPH65VCIbk6+1BzFIMJGFwTNrYXT1CrD+B4l19U7awiQ8rk7w==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true }, "string-width": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.0.0.tgz", - "integrity": "sha512-rr8CUxBbvOZDUvc5lNIJ+OC1nPVpz+Siw9VBtUjB9b6jZehZLFt0JMCZzShFHIsI8cbhm0EsNIfWJMFV3cu3Ew==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, "requires": { "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.0.0" + "strip-ansi": "^5.1.0" } }, "strip-ansi": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.0.0.tgz", - "integrity": "sha512-Uu7gQyZI7J7gn5qLn1Np3G9vcYGTVqB+lFTytnDJv83dd8T22aGH451P3jueT2/QemInJDfxHB5Tde5OzgG1Ow==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", "dev": true, "requires": { - "ansi-regex": "^4.0.0" + "ansi-regex": "^4.1.0" } } } @@ -1548,9 +1549,9 @@ } }, "tslib": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz", - "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", + "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", "dev": true }, "type-check": { @@ -1606,12 +1607,12 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "vows": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/vows/-/vows-0.8.2.tgz", - "integrity": "sha1-aR95qybM3oC6cm3en+yOc9a88us=", + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/vows/-/vows-0.8.3.tgz", + "integrity": "sha512-PVIxa/ovXhrw5gA3mz6M+ZF3PHlqX4tutR2p/y9NWPAaFVKcWBE8b2ktfr0opQM/qFmcOVWKjSCJVjnYOvjXhw==", "dev": true, "requires": { - "diff": "~1.0.8", + "diff": "^4.0.1", "eyes": "~0.1.6", "glob": "^7.1.2" } @@ -1625,10 +1626,17 @@ "isexe": "^2.0.0" } }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true }, "wrappy": { "version": "1.0.2", diff --git a/package.json b/package.json index 5cab77a3f..9abddeebb 100644 --- a/package.json +++ b/package.json @@ -24,18 +24,18 @@ "node": ">=8.0" }, "dependencies": { - "analyze-css": "^0.12.7", + "analyze-css": "^0.12.10", "debug": "^4.1.1", "puppeteer": "^3.0.0" }, "devDependencies": { - "eslint": "^5.14.1", + "eslint": "^5.16.0", "glob": "^7.1.2", "http-serve": "^1.0.1", - "js-beautify": "^1.6.14", + "js-beautify": "^1.11.0", "js-yaml": "^3.13.1", "mockery": "^2.0.0", - "vows": "^0.8.2" + "vows": "^0.8.3" }, "scripts": { "test": "vows --spec", From 05e7a1649a67ef14ff45262884bacbbaade53eb5 Mon Sep 17 00:00:00 2001 From: macbre Date: Sun, 26 Jul 2020 15:15:39 +0200 Subject: [PATCH 218/248] Run tests inside the Docker container --- .github/workflows/dockerimage.yml | 6 +++++- run_tests.sh | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100755 run_tests.sh diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index f2e695e69..9e9d9f93f 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -12,5 +12,9 @@ jobs: - uses: actions/checkout@v1 - name: Build the Docker image run: | - docker build . --tag ${{ github.repository }}:$(date +%s) + docker build . --tag ${{ github.repository }} docker images + + - name: Run tests inside the Docker container + run: | + docker run -it ${{ github.repository }} sh run_tests.sh diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 000000000..af012e21c --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +sh test/server-start.sh & +SERVER_PID=$! +sleep 1 + +npm test && npm run-script lint + +kill $SERVER_PID From 1ef5ec05641792b41862ff7d65f263a9cdbf324f Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Sun, 26 Jul 2020 17:15:16 +0200 Subject: [PATCH 219/248] docker run - the input device is not a TTY --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 9e9d9f93f..171194e1b 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -17,4 +17,4 @@ jobs: - name: Run tests inside the Docker container run: | - docker run -it ${{ github.repository }} sh run_tests.sh + docker run ${{ github.repository }} sh run_tests.sh From 95a22de876042ce97e44e331381a3214f8fd8e4c Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 27 Jul 2020 16:51:28 +0200 Subject: [PATCH 220/248] Use the latest puppeteer (v5.2.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * HeadlessChrome/85.0.4182.0 * removal of require('puppeteer/DeviceDescriptors') (#6043) == Tests == ✗ Errored » 432 honored ∙ 2 broken ∙ 2 errored (41.285s) httpbin's /basic-auth (with --auth-user and --auth-pass) ✗ should be generated » No error should be thrown: got Error: net::ERR_NETWORK_CHANGED at https://httpbin.org/basic-auth/foo/bar // /home/macbre/src/git/phantomas/node_modules/vows/lib/assert/macros.js:31 ✗ should have "performanceTimingDNS" metric properly set » expected 0, got NaN (strictEqual) // /home/macbre/src/git/phantomas/node_modules/vows/lib/assert/macros.js:14 --- extensions/devices/devices.js | 3 +- package-lock.json | 70 ++++++++++++++++++++++++++++++++--- package.json | 2 +- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/extensions/devices/devices.js b/extensions/devices/devices.js index 6d95333e8..92cd87d1e 100644 --- a/extensions/devices/devices.js +++ b/extensions/devices/devices.js @@ -4,7 +4,8 @@ 'use strict'; module.exports = function(phantomas) { - const devices = require('puppeteer/DeviceDescriptors'), + const puppeteer = require('puppeteer'), + devices = puppeteer.devices, // @see https://github.com/GoogleChrome/puppeteer/blob/master/DeviceDescriptors.js availableDevices = { 'phone': 'Galaxy S5', // 360x640 diff --git a/package-lock.json b/package-lock.json index ad2ae3fd7..bbbbb768a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,9 +36,9 @@ "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, "@types/node": { - "version": "10.12.18", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", - "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==", + "version": "14.0.26", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.26.tgz", + "integrity": "sha512-W+fpe5s91FBGE0pEa0lnqGLL4USgpLgs4nokw16SrBBco/gQxuua7KnArSEOd5iaMqbbSHV10vUDkJYJJqpXKA==", "optional": true }, "@types/yauzl": { @@ -372,6 +372,11 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, + "devtools-protocol": { + "version": "0.0.781568", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.781568.tgz", + "integrity": "sha512-9Uqnzy6m6zEStluH9iyJ3iHyaQziFnMnLeC8vK0eN6smiJmIx7+yB64d67C2lH/LZra+5cGscJAJsNXO+MdPMg==" + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -657,6 +662,15 @@ "flat-cache": "^2.0.1" } }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, "flat-cache": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", @@ -984,6 +998,14 @@ "type-check": "~0.3.2" } }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, "lodash": { "version": "4.17.19", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", @@ -1157,6 +1179,27 @@ "os-tmpdir": "^1.0.0" } }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -1166,6 +1209,11 @@ "callsites": "^3.0.0" } }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, "path-is-absolute": { "version": "1.0.1", "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -1188,6 +1236,14 @@ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "requires": { + "find-up": "^4.0.0" + } + }, "portfinder": { "version": "0.4.0", "resolved": "http://registry.npmjs.org/portfinder/-/portfinder-0.4.0.tgz", @@ -1242,14 +1298,16 @@ "dev": true }, "puppeteer": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-3.3.0.tgz", - "integrity": "sha512-23zNqRltZ1PPoK28uRefWJ/zKb5Jhnzbbwbpcna2o5+QMn17F0khq5s1bdH3vPlyj+J36pubccR8wiNA/VE0Vw==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.2.1.tgz", + "integrity": "sha512-PZoZG7u+T6N1GFWBQmGVG162Ak5MAy8nYSVpeeQrwJK2oYUlDWpHEJPcd/zopyuEMTv7DiztS1blgny1txR2qw==", "requires": { "debug": "^4.1.0", + "devtools-protocol": "0.0.781568", "extract-zip": "^2.0.0", "https-proxy-agent": "^4.0.0", "mime": "^2.0.3", + "pkg-dir": "^4.2.0", "progress": "^2.0.1", "proxy-from-env": "^1.0.0", "rimraf": "^3.0.2", diff --git a/package.json b/package.json index 9abddeebb..cad26e5d6 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "analyze-css": "^0.12.10", "debug": "^4.1.1", - "puppeteer": "^3.0.0" + "puppeteer": "^5.2.1" }, "devDependencies": { "eslint": "^5.16.0", From b468a099f9b18948aa5c1096e130bd65d74b8d1b Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 27 Jul 2020 17:04:48 +0200 Subject: [PATCH 221/248] Install Chrome as provided by puppeteer npm module --- Dockerfile | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 602ec8502..957fce2a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,26 +3,29 @@ FROM node:14-alpine # Installs latest Chromium package # https://pkgs.alpinelinux.org/package/edge/community/x86_64/chromium -RUN apk update && apk upgrade && \ - echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ - echo @edge http://nl.alpinelinux.org/alpine/edge/main >> /etc/apk/repositories && \ - apk add --no-cache \ - chromium@edge \ - freetype@edge \ - harfbuzz@edge \ - libstdc++@edge \ - nss@edge \ - tini@edge \ - ttf-freefont@edge + +RUN apk update && apk upgrade && apk add --no-cache tini + +#RUN apk update && apk upgrade && \ +# echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ +# echo @edge http://nl.alpinelinux.org/alpine/edge/main >> /etc/apk/repositories && \ +# apk add --no-cache \ +# chromium@edge \ +# freetype@edge \ +# harfbuzz@edge \ +# libstdc++@edge \ +# nss@edge \ +# tini@edge \ +# ttf-freefont@edge ENV HOME /opt/phantomas WORKDIR $HOME # Tell Puppeteer to skip installing Chrome. We'll be using the installed package. -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +#ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true # Tell phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +#ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser ENV DOCKERIZED yes # Add user so we don't need --no-sandbox. @@ -31,13 +34,14 @@ RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ # Run everything after as non-privileged user. USER phantomas -RUN chromium-browser --no-sandbox --version # Install dependencies COPY package.json . COPY package-lock.json . RUN npm i +RUN echo "Chrome installed in: "`find -name chrome` + # Copy the content of the rest of the repository into a container COPY . . From d67e2126d7f55017ae5003f47a1f7765cc2bcb99 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 27 Jul 2020 17:24:56 +0200 Subject: [PATCH 222/248] Install Chrome binary via npm install (and its dependencies via apt) --- Dockerfile | 68 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 957fce2a0..12cb48efd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,41 @@ -# based on https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-on-alpine -FROM node:14-alpine - -# Installs latest Chromium package -# https://pkgs.alpinelinux.org/package/edge/community/x86_64/chromium - -RUN apk update && apk upgrade && apk add --no-cache tini - -#RUN apk update && apk upgrade && \ -# echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \ -# echo @edge http://nl.alpinelinux.org/alpine/edge/main >> /etc/apk/repositories && \ -# apk add --no-cache \ -# chromium@edge \ -# freetype@edge \ -# harfbuzz@edge \ -# libstdc++@edge \ -# nss@edge \ -# tini@edge \ -# ttf-freefont@edge +# We need glibc distro in order to run Chrome binaries provided by puppeteer npm module +FROM node:14-slim + +# install Chrome binaries depedencies +RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + fonts-liberation \ + libappindicator3-1 \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libgcc1 \ + libgdk-pixbuf2.0-0 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libuuid1 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 ENV HOME /opt/phantomas WORKDIR $HOME @@ -28,21 +47,20 @@ WORKDIR $HOME #ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser ENV DOCKERIZED yes -# Add user so we don't need --no-sandbox. -RUN addgroup -S phantomas && adduser -S -g phantomas phantomas \ - && chown -R phantomas:phantomas $HOME +RUN chown -R nobody:nogroup $HOME # Run everything after as non-privileged user. -USER phantomas +USER nobody # Install dependencies COPY package.json . COPY package-lock.json . RUN npm i -RUN echo "Chrome installed in: "`find -name chrome` +RUN ldd `find -name chrome` +RUN `find -name chrome` --no-sandbox --version # Copy the content of the rest of the repository into a container COPY . . -ENTRYPOINT ["tini", "--"] +ENTRYPOINT ["sh"] From d5c1d7d671f860ac9c782b386e7461879c09a707 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 27 Jul 2020 17:30:47 +0200 Subject: [PATCH 223/248] dockerimage.yml: tests invocation changed --- .github/workflows/dockerimage.yml | 2 +- Dockerfile | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 171194e1b..4a2bdb93c 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -17,4 +17,4 @@ jobs: - name: Run tests inside the Docker container run: | - docker run ${{ github.repository }} sh run_tests.sh + docker run ${{ github.repository }} ./run_tests.sh diff --git a/Dockerfile b/Dockerfile index 12cb48efd..0ba4dfa07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,5 +62,3 @@ RUN `find -name chrome` --no-sandbox --version # Copy the content of the rest of the repository into a container COPY . . - -ENTRYPOINT ["sh"] From 1fc182a6df6f20aa7da9b632f284792f9005cfc3 Mon Sep 17 00:00:00 2001 From: macbre Date: Mon, 27 Jul 2020 18:59:42 +0200 Subject: [PATCH 224/248] run_tests.sh: exit with error when npm test fails --- run_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index af012e21c..5277508ef 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -3,6 +3,4 @@ sh test/server-start.sh & SERVER_PID=$! sleep 1 -npm test && npm run-script lint - -kill $SERVER_PID +npm test && npm run-script lint && kill $SERVER_PID From 86ba09c842ac9b5bb3f58f2ce3e75bdef4d297e3 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 12:30:38 +0200 Subject: [PATCH 225/248] Bring back .travis.yml --- .travis.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..8b1790eec --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +# https://docs.travis-ci.com/user/languages/javascript-with-nodejs/ +language: node_js +dist: xenial +node_js: 14 + +# allow headful tests +# https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/troubleshooting.md#running-puppeteer-on-travis-ci +before_install: + # Enable user namespace cloning + - "sysctl kernel.unprivileged_userns_clone=1" + +before_script: + - sh test/server-start.sh & + - SERVER_PID=$! + - sleep 1 +after_script: + - kill $SERVER_PID +script: "npm test && npm run-script lint" +cache: + directories: + - node_modules From 6206da120ae0553699b898875488e7648f57c1ab Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 12:34:21 +0200 Subject: [PATCH 226/248] eslint upgraded --- .eslintrc.yml | 2 +- .../windowPerformance/windowPerformance.js | 1 - package-lock.json | 555 ++++++++---------- package.json | 2 +- 4 files changed, 245 insertions(+), 315 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index 6e641785b..2871bb69a 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -228,7 +228,7 @@ rules: quote-props: 'off' quotes: 'off' radix: 'off' - require-atomic-updates: error + require-atomic-updates: 'off' require-await: 'off' require-jsdoc: 'off' require-unicode-regexp: 'off' diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index 7b3ff9459..aa06ef5e7 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -1,7 +1,6 @@ /** * Measure when the page reaches certain states */ -/* global window: true */ 'use strict'; module.exports = function(phantomas) { diff --git a/package-lock.json b/package-lock.json index bbbbb768a..d45670758 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,19 @@ "@babel/helper-validator-identifier": "^7.10.4", "chalk": "^2.0.0", "js-tokens": "^4.0.0" + }, + "dependencies": { + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + } } }, "@tootallnate/once": { @@ -35,6 +48,12 @@ "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, + "@types/color-name": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true + }, "@types/node": { "version": "14.0.26", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.26.tgz", @@ -57,9 +76,9 @@ "dev": true }, "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz", + "integrity": "sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA==", "dev": true }, "acorn-jsx": { @@ -127,16 +146,16 @@ } } }, - "ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", "dev": true }, "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "ansi-styles": { @@ -231,22 +250,57 @@ "dev": true }, "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", "dev": true, "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "requires": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } } }, - "chardet": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", - "dev": true - }, "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -261,21 +315,6 @@ "glob": "^7.1.1" } }, - "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", - "dev": true, - "requires": { - "restore-cursor": "^2.0.0" - } - }, - "cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", - "dev": true - }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -325,16 +364,14 @@ "dev": true }, "cross-spawn": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", - "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, "requires": { - "nice-try": "^1.0.4", - "path-key": "^2.0.1", - "semver": "^5.5.0", - "shebang-command": "^1.2.0", - "which": "^1.2.9" + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" } }, "css": { @@ -444,6 +481,15 @@ "once": "^1.4.0" } }, + "enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, + "requires": { + "ansi-colors": "^4.1.1" + } + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -451,53 +497,61 @@ "dev": true }, "eslint": { - "version": "5.16.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", - "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.5.0.tgz", + "integrity": "sha512-vlUP10xse9sWt9SGRtcr1LAC67BENcQMFeV+w5EvLEoFe3xJ8cF1Skd0msziRx/VMC+72B4DxreCE+OR12OA6Q==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "ajv": "^6.9.1", - "chalk": "^2.1.0", - "cross-spawn": "^6.0.5", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", "debug": "^4.0.1", "doctrine": "^3.0.0", - "eslint-scope": "^4.0.3", - "eslint-utils": "^1.3.1", - "eslint-visitor-keys": "^1.0.0", - "espree": "^5.0.1", - "esquery": "^1.0.1", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.0", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^1.3.0", + "espree": "^7.2.0", + "esquery": "^1.2.0", "esutils": "^2.0.2", "file-entry-cache": "^5.0.1", "functional-red-black-tree": "^1.0.1", - "glob": "^7.1.2", - "globals": "^11.7.0", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", "ignore": "^4.0.6", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", - "inquirer": "^6.2.2", - "js-yaml": "^3.13.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.3.0", - "lodash": "^4.17.11", + "levn": "^0.4.1", + "lodash": "^4.17.19", "minimatch": "^3.0.4", - "mkdirp": "^0.5.1", "natural-compare": "^1.4.0", - "optionator": "^0.8.2", - "path-is-inside": "^1.0.2", + "optionator": "^0.9.1", "progress": "^2.0.0", - "regexpp": "^2.0.1", - "semver": "^5.5.1", - "strip-ansi": "^4.0.0", - "strip-json-comments": "^2.0.1", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", "table": "^5.2.3", - "text-table": "^0.2.0" + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + } } }, "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.0.tgz", + "integrity": "sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w==", "dev": true, "requires": { "esrecurse": "^4.1.0", @@ -505,9 +559,9 @@ } }, "eslint-utils": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", - "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", "dev": true, "requires": { "eslint-visitor-keys": "^1.1.0" @@ -520,14 +574,14 @@ "dev": true }, "espree": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", - "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.2.0.tgz", + "integrity": "sha512-H+cQ3+3JYRMEIOl87e7QdHX70ocly5iW4+dttuR8iYSPr/hXKFb+7dBsZ7+u1adC4VrnPlTkv0+OwuPnDop19g==", "dev": true, "requires": { - "acorn": "^6.0.7", - "acorn-jsx": "^5.0.0", - "eslint-visitor-keys": "^1.0.0" + "acorn": "^7.3.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" } }, "esprima": { @@ -585,17 +639,6 @@ "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" }, - "external-editor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", - "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", - "dev": true, - "requires": { - "chardet": "^0.7.0", - "iconv-lite": "^0.4.24", - "tmp": "^0.0.33" - } - }, "extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", @@ -644,15 +687,6 @@ "pend": "~1.2.0" } }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", - "dev": true, - "requires": { - "escape-string-regexp": "^1.0.5" - } - }, "file-entry-cache": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", @@ -762,11 +796,23 @@ "path-is-absolute": "^1.0.0" } }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, "globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } }, "has-flag": { "version": "3.0.0", @@ -833,15 +879,6 @@ } } }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, "ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", @@ -889,43 +926,11 @@ "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true }, - "inquirer": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", - "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", - "dev": true, - "requires": { - "ansi-escapes": "^3.2.0", - "chalk": "^2.4.2", - "cli-cursor": "^2.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^2.0.0", - "lodash": "^4.17.12", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rxjs": "^6.4.0", - "string-width": "^2.1.0", - "strip-ansi": "^5.1.0", - "through": "^2.3.6" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } - } + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true }, "is-fullwidth-code-point": { "version": "2.0.0", @@ -933,6 +938,15 @@ "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -989,13 +1003,13 @@ "dev": true }, "levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, "requires": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" } }, "locate-path": { @@ -1027,12 +1041,6 @@ "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "dev": true - }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1072,24 +1080,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, - "mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", - "dev": true - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, - "nice-try": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", - "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", - "dev": true - }, "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", @@ -1118,15 +1114,6 @@ "resolved": "https://registry.npmjs.org/onecolor/-/onecolor-3.1.0.tgz", "integrity": "sha512-YZSypViXzu3ul5LMu/m6XjJ9ol8qAy9S2VjHl5E6UlhUH1KGKWabyEJifn0Jjpw23bYDzC2ucKMPGiH5kfwSGQ==" }, - "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", - "dev": true, - "requires": { - "mimic-fn": "^1.0.0" - } - }, "opener": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/opener/-/opener-1.4.3.tgz", @@ -1144,17 +1131,17 @@ } }, "optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", "dev": true, "requires": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" } }, "os-homedir": { @@ -1219,16 +1206,10 @@ "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", - "dev": true - }, "path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true }, "pend": { @@ -1255,9 +1236,9 @@ } }, "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true }, "progress": { @@ -1333,9 +1314,9 @@ } }, "regexpp": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", - "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", "dev": true }, "requires-port": { @@ -1355,16 +1336,6 @@ "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" }, - "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", - "dev": true, - "requires": { - "onetime": "^2.0.0", - "signal-exit": "^3.0.2" - } - }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -1373,32 +1344,11 @@ "glob": "^7.1.3" } }, - "run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "dev": true - }, - "rxjs": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.0.tgz", - "integrity": "sha512-3HMA8z/Oz61DUHe+SdOiQyzIf4tOx5oQHmMir7IZEu6TMqCLHT4LRcmNaUS0NwOz8VLvmmBduMsoaUvMaIiqzg==", - "dev": true, - "requires": { - "tslib": "^1.9.0" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true - }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -1406,18 +1356,18 @@ "dev": true }, "shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, "requires": { - "shebang-regex": "^1.0.0" + "shebang-regex": "^3.0.0" } }, "shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, "sigmund": { @@ -1426,12 +1376,6 @@ "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=", "dev": true }, - "signal-exit": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", - "dev": true - }, "slice-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", @@ -1482,13 +1426,31 @@ "dev": true }, "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, "requires": { + "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" + "strip-ansi": "^5.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } } }, "string_decoder": { @@ -1500,18 +1462,18 @@ } }, "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "requires": { - "ansi-regex": "^3.0.0" + "ansi-regex": "^5.0.0" } }, "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, "supports-color": { @@ -1533,34 +1495,6 @@ "lodash": "^4.17.14", "slice-ansi": "^2.1.0", "string-width": "^3.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } } }, "tar-fs": { @@ -1597,30 +1531,21 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, - "tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, "requires": { - "os-tmpdir": "~1.0.2" + "prelude-ls": "^1.2.1" } }, - "tslib": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", - "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", "dev": true }, - "type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "dev": true, - "requires": { - "prelude-ls": "~1.1.2" - } - }, "unbzip2-stream": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", @@ -1664,6 +1589,12 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "v8-compile-cache": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", + "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==", + "dev": true + }, "vows": { "version": "0.8.3", "resolved": "https://registry.npmjs.org/vows/-/vows-0.8.3.tgz", @@ -1676,9 +1607,9 @@ } }, "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "requires": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index cad26e5d6..c45d60fb6 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "puppeteer": "^5.2.1" }, "devDependencies": { - "eslint": "^5.16.0", + "eslint": "^7.5.0", "glob": "^7.1.2", "http-serve": "^1.0.1", "js-beautify": "^1.11.0", From 029af5002d0e4af182fbd13beb5918dd32684448 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 12:35:14 +0200 Subject: [PATCH 227/248] Upgrade glob and js-yaml --- package-lock.json | 12 ++++++------ package.json | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index d45670758..6f5d1ade3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -784,9 +784,9 @@ } }, "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -981,9 +981,9 @@ "dev": true }, "js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", "dev": true, "requires": { "argparse": "^1.0.7", diff --git a/package.json b/package.json index c45d60fb6..ade3e0cee 100644 --- a/package.json +++ b/package.json @@ -30,10 +30,10 @@ }, "devDependencies": { "eslint": "^7.5.0", - "glob": "^7.1.2", + "glob": "^7.1.6", "http-serve": "^1.0.1", "js-beautify": "^1.11.0", - "js-yaml": "^3.13.1", + "js-yaml": "^3.14.0", "mockery": "^2.0.0", "vows": "^0.8.3" }, From b0015191498cfe2f77b387b49a66960473dc101b Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 17:57:44 +0200 Subject: [PATCH 228/248] Install optimist-config-file to make phantomas "binary" work --- package-lock.json | 46 +++++++++++++++++++++++++++++++++++----------- package.json | 1 + 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6f5d1ade3..58bd7f034 100644 --- a/package-lock.json +++ b/package-lock.json @@ -167,11 +167,15 @@ "color-convert": "^1.9.0" } }, + "ansistyles": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/ansistyles/-/ansistyles-0.1.3.tgz", + "integrity": "sha1-XeYEFb2gcbs3EnhUyGT0GyMlRTk=" + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, "requires": { "sprintf-js": "~1.0.2" } @@ -587,8 +591,7 @@ "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" }, "esquery": { "version": "1.3.1", @@ -984,7 +987,6 @@ "version": "3.14.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", - "dev": true, "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -1052,8 +1054,7 @@ "minimist": { "version": "0.0.8", "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "mkdirp": { "version": "0.5.1", @@ -1124,12 +1125,37 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "dev": true, "requires": { "minimist": "~0.0.1", "wordwrap": "~0.0.2" } }, + "optimist-config-file": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/optimist-config-file/-/optimist-config-file-1.0.1.tgz", + "integrity": "sha1-tUBUiKztcKjnhJ/V+Jx4aCckvpM=", + "requires": { + "ansistyles": "^0.1.3", + "debug": "^2.6.8", + "js-yaml": "^3.8.4", + "optimist": "^0.6.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -1422,8 +1448,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", - "dev": true + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "string-width": { "version": "3.1.0", @@ -1624,8 +1649,7 @@ "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" }, "wrappy": { "version": "1.0.2", diff --git a/package.json b/package.json index ade3e0cee..685e3b4b1 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dependencies": { "analyze-css": "^0.12.10", "debug": "^4.1.1", + "optimist-config-file": "^1.0.1", "puppeteer": "^5.2.1" }, "devDependencies": { From 88706405d284877ab0c147dec9d9ecfc6cca870b Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 18:12:12 +0200 Subject: [PATCH 229/248] windowPerformance.js: fix the way window.performance.timing are passed from the browser context to phantomas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes: /timing.html ✗ should have "performanceTimingDNS" metric properly set » expected 0, got NaN (strictEqual) // /home/travis/build/macbre/phantomas/node_modules/vows/lib/assert/macros.js:14 --- modules/windowPerformance/windowPerformance.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/windowPerformance/windowPerformance.js b/modules/windowPerformance/windowPerformance.js index aa06ef5e7..e3cffc492 100644 --- a/modules/windowPerformance/windowPerformance.js +++ b/modules/windowPerformance/windowPerformance.js @@ -25,10 +25,9 @@ module.exports = function(phantomas) { phantomas.setMetric('timeFrontend'); // @desc time to window.load compared to the total loading time [%] phantomas.on('beforeClose', async function() { - const performance = await phantomas.evaluate(() => { - return window.performance.toJSON(); - }), - timing = performance.timing; + const timing = await phantomas.evaluate(() => { + return window.performance.timing.toJSON(); + }); phantomas.log('window.performance timing: %j', timing); From 5b5a1b4c89521d3cd26491c774292f058f354821 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 18:13:18 +0200 Subject: [PATCH 230/248] bin/phantomas.js: temp change to make it work using more recent Node.js versions --- bin/phantomas.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/phantomas.js b/bin/phantomas.js index ee1f4b001..233f8b0b0 100755 --- a/bin/phantomas.js +++ b/bin/phantomas.js @@ -159,7 +159,7 @@ function task(callback) { }); // pipe --verbose messages to stderr - child.stderr.pipe(process.stderr); + // child.stderr.pipe(process.stderr); } // @see https://github.com/caolan/async#seriestasks-callback From b0899ad637536c65a7ebd566a20d365e6606bb27 Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 18:18:57 +0200 Subject: [PATCH 231/248] run_tests is run inside Docker container where examples/ does not exist - eslint fails --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 5277508ef..56d65768f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -3,4 +3,4 @@ sh test/server-start.sh & SERVER_PID=$! sleep 1 -npm test && npm run-script lint && kill $SERVER_PID +npm test && kill $SERVER_PID From c847816a2d26b5237f870d3df03016cfbd12157d Mon Sep 17 00:00:00 2001 From: macbre Date: Tue, 28 Jul 2020 18:25:40 +0200 Subject: [PATCH 232/248] npm audit fix --- package-lock.json | 64 +++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 58bd7f034..578761217 100644 --- a/package-lock.json +++ b/package-lock.json @@ -452,9 +452,9 @@ "dev": true }, "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "dev": true } } @@ -632,9 +632,9 @@ "dev": true }, "eventemitter3": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", - "integrity": "sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.4.tgz", + "integrity": "sha512-rlaVLnVxtxvoyLsQQFBx53YmXHDxRIzzTLbdfxqi4yocpSjAxXwkU0cScM5JgSKMqEhrZpnvQ2D9gjylR0AimQ==", "dev": true }, "exit": { @@ -737,30 +737,10 @@ "dev": true }, "follow-redirects": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.0.tgz", - "integrity": "sha512-4Oh4eI3S9OueVV41AgJ1oLjpaJUhbJ7JDGOMhe0AFqoSejl5Q2nn3eGglAzRUKVKZE8jG5MNn66TjCJMAnpsWA==", - "dev": true, - "requires": { - "debug": "=3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - } - } + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.12.1.tgz", + "integrity": "sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg==", + "dev": true }, "fs-constants": { "version": "1.0.0", @@ -830,12 +810,12 @@ "dev": true }, "http-proxy": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.17.0.tgz", - "integrity": "sha512-Taqn+3nNvYRfJ3bGvKfBSRwy1v6eePlm3oc/aWVxZp57DQr5Eq3xhKJi7Z4hZpS8PC3H4qI+Yly5EmFacGuA/g==", + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", "dev": true, "requires": { - "eventemitter3": "^3.0.0", + "eventemitter3": "^4.0.0", "follow-redirects": "^1.0.0", "requires-port": "^1.0.0" } @@ -1057,12 +1037,20 @@ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "mkdirp": { - "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "0.0.8" + "minimist": "^1.2.5" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + } } }, "mkdirp-classic": { From f969faacab266a95a3ac703eef582dde84437403 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2020 05:19:36 +0000 Subject: [PATCH 233/248] build(deps): bump async from 2.6.3 to 3.2.0 Bumps [async](https://github.com/caolan/async) from 2.6.3 to 3.2.0. - [Release notes](https://github.com/caolan/async/releases) - [Changelog](https://github.com/caolan/async/blob/master/CHANGELOG.md) - [Commits](https://github.com/caolan/async/compare/v2.6.3...v3.2.0) Signed-off-by: dependabot[bot] --- package-lock.json | 12 +++++------- package.json | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 85c63dd45..34e44e8c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -177,12 +177,9 @@ "dev": true }, "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "requires": { - "lodash": "^4.17.14" - } + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", + "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" }, "atob": { "version": "2.1.2", @@ -1000,7 +997,8 @@ "lodash": { "version": "4.17.19", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "dev": true }, "lru-cache": { "version": "4.1.5", diff --git a/package.json b/package.json index bee7adeb5..5e03574b5 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "ansicolors": "~0.3.2", "ansistyles": "~0.1.0", "ascii-table": "0.0.9", - "async": "^2.6.1", + "async": "^3.2.0", "csv-string": "^3.1.3", "debug": "^4.1.1", "js-yaml": "^3.13.0", From 8d013bb7c27deaba91f6b19d30935776421842d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2020 05:19:52 +0000 Subject: [PATCH 234/248] build(deps): bump csv-string from 3.2.0 to 4.0.1 Bumps [csv-string](https://github.com/Inist-CNRS/node-csv-string) from 3.2.0 to 4.0.1. - [Release notes](https://github.com/Inist-CNRS/node-csv-string/releases) - [Commits](https://github.com/Inist-CNRS/node-csv-string/compare/v3.2.0...v4.0.1) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 85c63dd45..b494ec766 100644 --- a/package-lock.json +++ b/package-lock.json @@ -379,9 +379,9 @@ "integrity": "sha512-Md+Juc7M3uOdbAFwOYlTrccIZ7oCFuzrhKYQjdeUEW/sE1hv17Jp/Bws+ReOPpGVBTYCBoYo+G17V5Qo8QQ75A==" }, "csv-string": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/csv-string/-/csv-string-3.2.0.tgz", - "integrity": "sha512-JN3iAuFJ+r7+CwF6UtP3U8ryorRkQp8NT+9VufeiRV+Xyv+Q8HPPBHGm4LAq7YihTQYmUnIeYy5CPQ8Y2GhMkg==" + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/csv-string/-/csv-string-4.0.1.tgz", + "integrity": "sha512-nCdK+EWDbqLvZ2MmVQhHTmidMEsHbK3ncgTJb4oguNRpkmH5OOr+KkDRB4nqsVrJ7oK0AdO1QEsBp0+z7KBtGQ==" }, "debug": { "version": "4.1.1", diff --git a/package.json b/package.json index bee7adeb5..4c6f05d5f 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "ansistyles": "~0.1.0", "ascii-table": "0.0.9", "async": "^2.6.1", - "csv-string": "^3.1.3", + "csv-string": "^4.0.1", "debug": "^4.1.1", "js-yaml": "^3.13.0", "node-statsd": "0.1.1", From e9d2b2cbc19bd23a6bfab068fe2ef86566c8cd31 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Thu, 30 Jul 2020 17:43:14 +0200 Subject: [PATCH 235/248] Add inch-ci.org badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d9d691b4..b92b7ec5e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -phantomas [![npm](https://img.shields.io/npm/dt/phantomas.svg)]() +phantomas [![npm](https://img.shields.io/npm/dt/phantomas.svg)]() [![Inline docs](http://inch-ci.org/github/macbre/phantomas.svg?branch=phantomas-v2)](http://inch-ci.org/github/macbre/phantomas) ========= [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md)-based modular web performance metrics collector. And why phantomas? Well, [because](http://en.wikipedia.org/wiki/Fantômas) :) From 60814df06ba791f874f785ae705e9b48ff5962d4 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Thu, 30 Jul 2020 17:44:26 +0200 Subject: [PATCH 236/248] package.json - update project description --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f58b12703..d224cb72a 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "phantomas", "version": "2.0.0-beta", "author": "macbre (http://macbre.net)", - "description": "PhantomJS-based web performance metrics collector", + "description": "Headless Chromium-based web performance metrics collector and monitoring tool", "main": "./lib/index.js", "repository": { "type": "git", From 356bb0a21f0dbc61cf673b62e524bdfc2539d9ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 05:03:23 +0000 Subject: [PATCH 237/248] build(deps-dev): bump eslint from 7.5.0 to 7.6.0 Bumps [eslint](https://github.com/eslint/eslint) from 7.5.0 to 7.6.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Changelog](https://github.com/eslint/eslint/blob/master/CHANGELOG.md) - [Commits](https://github.com/eslint/eslint/compare/v7.5.0...v7.6.0) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 93e80f6c4..7ce858af8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -501,9 +501,9 @@ "dev": true }, "eslint": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.5.0.tgz", - "integrity": "sha512-vlUP10xse9sWt9SGRtcr1LAC67BENcQMFeV+w5EvLEoFe3xJ8cF1Skd0msziRx/VMC+72B4DxreCE+OR12OA6Q==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.6.0.tgz", + "integrity": "sha512-QlAManNtqr7sozWm5TF4wIH9gmUm2hE3vNRUvyoYAa4y1l5/jxD/PQStEjBMQtCqZmSep8UxrcecI60hOpe61w==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", diff --git a/package.json b/package.json index d224cb72a..d6dd1964e 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "puppeteer": "^5.2.1" }, "devDependencies": { - "eslint": "^7.5.0", + "eslint": "^7.6.0", "glob": "^7.1.6", "http-serve": "^1.0.1", "js-beautify": "^1.11.0", From 03fb4f3c23cc66c8957cd0263738eae76fa91f40 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 7 Aug 2020 00:25:47 +0200 Subject: [PATCH 238/248] Run tests using Node.js 12.x too --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7b321c60d..3c543e966 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [12.x, 14.x] steps: - uses: actions/checkout@v2 From a85ecfba18e6abbc82b7e78a467ce8d762923021 Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 7 Aug 2020 00:38:48 +0200 Subject: [PATCH 239/248] Delete .travis.yml GitHub Actions run fine now --- .travis.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8b1790eec..000000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -# https://docs.travis-ci.com/user/languages/javascript-with-nodejs/ -language: node_js -dist: xenial -node_js: 14 - -# allow headful tests -# https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/troubleshooting.md#running-puppeteer-on-travis-ci -before_install: - # Enable user namespace cloning - - "sysctl kernel.unprivileged_userns_clone=1" - -before_script: - - sh test/server-start.sh & - - SERVER_PID=$! - - sleep 1 -after_script: - - kill $SERVER_PID -script: "npm test && npm run-script lint" -cache: - directories: - - node_modules From 828955585727ec5c21b613ed8fe7dbe7e1b141c7 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 00:54:45 +0200 Subject: [PATCH 240/248] Use http-server for tests (fixes qs security issue) --- package-lock.json | 208 ++++++++++++++++++++++--------------------- package.json | 5 +- test/server-start.sh | 2 +- 3 files changed, 108 insertions(+), 107 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7ce858af8..48c4cc77c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -196,6 +196,11 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, + "basic-auth": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.1.0.tgz", + "integrity": "sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ=" + }, "bl": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", @@ -316,12 +321,6 @@ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "dev": true }, - "colors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", - "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", - "dev": true - }, "commander": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", @@ -345,8 +344,7 @@ "corser": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", - "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=", - "dev": true + "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=" }, "cross-spawn": { "version": "7.0.3", @@ -419,32 +417,6 @@ "esutils": "^2.0.2" } }, - "ecstatic": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-2.2.2.tgz", - "integrity": "sha512-F1g29y3I+abOS+M0AiK2O9R96AJ49Bc3kH696HtqnN+CL3YhpUnSzHNoUBQL03qDsN9Lr1XeKIxTqEH3BtiBgg==", - "dev": true, - "requires": { - "he": "^1.1.1", - "mime": "^1.2.11", - "minimist": "^1.1.0", - "url-join": "^2.0.2" - }, - "dependencies": { - "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true - }, - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true - } - } - }, "editorconfig": { "version": "0.15.3", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", @@ -626,8 +598,7 @@ "eventemitter3": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.4.tgz", - "integrity": "sha512-rlaVLnVxtxvoyLsQQFBx53YmXHDxRIzzTLbdfxqi4yocpSjAxXwkU0cScM5JgSKMqEhrZpnvQ2D9gjylR0AimQ==", - "dev": true + "integrity": "sha512-rlaVLnVxtxvoyLsQQFBx53YmXHDxRIzzTLbdfxqi4yocpSjAxXwkU0cScM5JgSKMqEhrZpnvQ2D9gjylR0AimQ==" }, "exit": { "version": "0.1.2", @@ -731,8 +702,7 @@ "follow-redirects": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.12.1.tgz", - "integrity": "sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg==", - "dev": true + "integrity": "sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg==" }, "fs-constants": { "version": "1.0.0", @@ -798,14 +768,12 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" }, "http-proxy": { "version": "1.18.1", "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", - "dev": true, "requires": { "eventemitter3": "^4.0.0", "follow-redirects": "^1.0.0", @@ -822,20 +790,93 @@ "debug": "4" } }, - "http-serve": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/http-serve/-/http-serve-1.0.1.tgz", - "integrity": "sha1-u4JL+P7SCS6RlYN7C/eztvo8ovg=", - "dev": true, - "requires": { - "colors": "1.0.3", - "corser": "~2.0.0", - "ecstatic": "^2.0.0", - "http-proxy": "^1.8.1", - "opener": "~1.4.0", - "optimist": "0.6.x", - "portfinder": "0.4.x", - "union": "~0.4.3" + "http-server": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.3.tgz", + "integrity": "sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==", + "requires": { + "basic-auth": "^1.0.3", + "colors": "^1.4.0", + "corser": "^2.0.1", + "ecstatic": "^3.3.2", + "http-proxy": "^1.18.0", + "minimist": "^1.2.5", + "opener": "^1.5.1", + "portfinder": "^1.0.25", + "secure-compare": "3.0.1", + "union": "~0.5.0" + }, + "dependencies": { + "async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "requires": { + "lodash": "^4.17.14" + } + }, + "colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ecstatic": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-3.3.2.tgz", + "integrity": "sha512-fLf9l1hnwrHI2xn9mEDT7KIi22UDqA2jaCwyCbSUJh9a1V+LEUSL/JO/6TIz/QyuBURWUHrFL5Kg2TtO1bkkog==", + "requires": { + "he": "^1.1.1", + "mime": "^1.6.0", + "minimist": "^1.1.0", + "url-join": "^2.0.5" + } + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "opener": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.1.tgz", + "integrity": "sha512-goYSy5c2UXE4Ra1xixabeVh1guIX/ZV/YokJksb6q2lubWu6UbvPQ20p542/sFIll1nl8JnCyK9oBaOcCWXwvA==" + }, + "portfinder": { + "version": "1.0.28", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.28.tgz", + "integrity": "sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==", + "requires": { + "async": "^2.6.2", + "debug": "^3.1.1", + "mkdirp": "^0.5.5" + } + }, + "qs": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==" + }, + "union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "requires": { + "qs": "^6.4.0" + } + } } }, "https-proxy-agent": { @@ -997,8 +1038,7 @@ "lodash": { "version": "4.17.19", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", - "dev": true + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" }, "lru-cache": { "version": "4.1.5", @@ -1032,7 +1072,6 @@ "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", - "dev": true, "requires": { "minimist": "^1.2.5" }, @@ -1040,8 +1079,7 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" } } }, @@ -1100,12 +1138,6 @@ "resolved": "https://registry.npmjs.org/onecolor/-/onecolor-3.1.0.tgz", "integrity": "sha512-YZSypViXzu3ul5LMu/m6XjJ9ol8qAy9S2VjHl5E6UlhUH1KGKWabyEJifn0Jjpw23bYDzC2ucKMPGiH5kfwSGQ==" }, - "opener": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/opener/-/opener-1.4.3.tgz", - "integrity": "sha1-XG2ixdflgx6P+jlklQ+NZnSskLg=", - "dev": true - }, "optimist": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", @@ -1236,24 +1268,6 @@ "find-up": "^4.0.0" } }, - "portfinder": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-0.4.0.tgz", - "integrity": "sha1-o/+t/6/k+5jgYBqF7aJ8J86Eyh4=", - "dev": true, - "requires": { - "async": "0.9.0", - "mkdirp": "0.5.x" - }, - "dependencies": { - "async": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.0.tgz", - "integrity": "sha1-rDYTsdqb7RtHUQu0ZRuJMeRxRsc=", - "dev": true - } - } - }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -1316,12 +1330,6 @@ "ws": "^7.2.3" } }, - "qs": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz", - "integrity": "sha1-6eha2+ddoLvkyOBHaghikPhjtAQ=", - "dev": true - }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -1341,8 +1349,7 @@ "requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", - "dev": true + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" }, "resolve-from": { "version": "4.0.0", @@ -1368,6 +1375,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha1-8aAymzCLIh+uN7mXTz1XjQypmeM=" + }, "semver": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", @@ -1573,15 +1585,6 @@ "through": "^2.3.8" } }, - "union": { - "version": "0.4.6", - "resolved": "https://registry.npmjs.org/union/-/union-0.4.6.tgz", - "integrity": "sha1-GY+9rrolTniLDvy2MLwR8kopWeA=", - "dev": true, - "requires": { - "qs": "~2.3.3" - } - }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -1599,8 +1602,7 @@ "url-join": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/url-join/-/url-join-2.0.5.tgz", - "integrity": "sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=", - "dev": true + "integrity": "sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=" }, "util-deprecate": { "version": "1.0.2", diff --git a/package.json b/package.json index d6dd1964e..7fefc7537 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "async": "^3.2.0", "csv-string": "^4.0.1", "debug": "^4.1.1", - "js-yaml": "^3.13.0", + "js-yaml": "^3.14.0", "node-statsd": "0.1.1", "optimist": "^0.6.1", "optimist-config-file": "^1.0.1", @@ -40,9 +40,8 @@ "devDependencies": { "eslint": "^7.6.0", "glob": "^7.1.6", - "http-serve": "^1.0.1", + "http-server": "^0.12.3", "js-beautify": "^1.11.0", - "js-yaml": "^3.14.0", "mockery": "^2.0.0", "vows": "^0.8.3" }, diff --git a/test/server-start.sh b/test/server-start.sh index 56cb82c1f..05694b558 100755 --- a/test/server-start.sh +++ b/test/server-start.sh @@ -1,5 +1,5 @@ PORT=8888 DIR=`dirname $0` -$DIR/../node_modules/.bin/http-serve $DIR/webroot -p $PORT -c 84600 --gzip +$DIR/../node_modules/.bin/http-server $DIR/webroot -p $PORT -c 84600 --gzip From e9ad7393f9c862e3655883fcfcd4dc0a761039a7 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 01:15:19 +0200 Subject: [PATCH 241/248] Browser.close - check if there is a browser to call close() on --- lib/browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/browser.js b/lib/browser.js index 637d8c735..913bc9825 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -246,7 +246,7 @@ Browser.prototype.visit = (url, waitUntil, timeout) => { // we're done Browser.prototype.close = async () => { - await this.browser.close(); + if (this.browser) await this.browser.close(); this.events.emit('close'); // @desc Chromium has been closed debug('Browser closed'); }; From aa91a0cdd27efaccfa97e70182615345e3ad5e52 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 01:28:10 +0200 Subject: [PATCH 242/248] 2.0.0-alpha1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 48c4cc77c..45cd04023 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "phantomas", - "version": "2.0.0-beta", + "version": "2.0.0-alpha1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 7fefc7537..d27d1abf6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "phantomas", - "version": "2.0.0-beta", + "version": "2.0.0-alpha1", "author": "macbre (http://macbre.net)", "description": "Headless Chromium-based web performance metrics collector and monitoring tool", "main": "./lib/index.js", From 1212311b802d79f72ab9289e14109df72c866d03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2020 05:02:54 +0000 Subject: [PATCH 243/248] build(deps): bump analyze-css from 0.12.10 to 0.12.11 Bumps [analyze-css](https://github.com/macbre/analyze-css) from 0.12.10 to 0.12.11. - [Release notes](https://github.com/macbre/analyze-css/releases) - [Commits](https://github.com/macbre/analyze-css/compare/v0.12.10...v0.12.11) Signed-off-by: dependabot[bot] --- package-lock.json | 106 ++++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/package-lock.json b/package-lock.json index 45cd04023..1ea3f55e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -108,13 +108,13 @@ } }, "analyze-css": { - "version": "0.12.10", - "resolved": "https://registry.npmjs.org/analyze-css/-/analyze-css-0.12.10.tgz", - "integrity": "sha512-7HQrlqsWnfvcBBO++LrDvHnaov6luVyh2phuNYnZXxdy2S/QBs2Q6SlvtUW9Ehp3/MkGHRo7TGXGIHEd9NWnHQ==", + "version": "0.12.11", + "resolved": "https://registry.npmjs.org/analyze-css/-/analyze-css-0.12.11.tgz", + "integrity": "sha512-tDrFLNilLq9TwqVvAEIy6FcP/B96Y7aQza2Z1YdjGXLC1Z0udgqtsaX7jpMwv3c5FbXmffvQqml3RnxpV7ROzA==", "requires": { "cli": "^1.0.1", - "commander": "^5.1.0", - "css": "^2.2.4", + "commander": "^6.0.0", + "css": "^3.0.0", "css-shorthand-properties": "^1.1.1", "debug": "^4.1.1", "fast-stats": "0.0.5", @@ -199,7 +199,8 @@ "basic-auth": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.1.0.tgz", - "integrity": "sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ=" + "integrity": "sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ=", + "dev": true }, "bl": { "version": "4.0.2", @@ -322,9 +323,9 @@ "dev": true }, "commander": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", - "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.0.0.tgz", + "integrity": "sha512-s7EA+hDtTYNhuXkTlhqew4txMZVdszBmKWSPEMxGr8ru8JXR7bLUFIAtPhcSuFdJQ0ILMxnJi8GkQL0yvDy/YA==" }, "concat-map": { "version": "0.0.1", @@ -344,7 +345,8 @@ "corser": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", - "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=" + "integrity": "sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=", + "dev": true }, "cross-spawn": { "version": "7.0.3", @@ -358,14 +360,13 @@ } }, "css": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz", - "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/css/-/css-3.0.0.tgz", + "integrity": "sha512-DG9pFfwOrzc+hawpmqX/dHYHJG+Bsdb0klhyi1sDneOgGOXy9wQIC8hzyVp1e4NRYDBdxcylvywPkkXCHAzTyQ==", "requires": { - "inherits": "^2.0.3", + "inherits": "^2.0.4", "source-map": "^0.6.1", - "source-map-resolve": "^0.5.2", - "urix": "^0.1.0" + "source-map-resolve": "^0.6.0" } }, "css-shorthand-properties": { @@ -598,7 +599,8 @@ "eventemitter3": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.4.tgz", - "integrity": "sha512-rlaVLnVxtxvoyLsQQFBx53YmXHDxRIzzTLbdfxqi4yocpSjAxXwkU0cScM5JgSKMqEhrZpnvQ2D9gjylR0AimQ==" + "integrity": "sha512-rlaVLnVxtxvoyLsQQFBx53YmXHDxRIzzTLbdfxqi4yocpSjAxXwkU0cScM5JgSKMqEhrZpnvQ2D9gjylR0AimQ==", + "dev": true }, "exit": { "version": "0.1.2", @@ -702,7 +704,8 @@ "follow-redirects": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.12.1.tgz", - "integrity": "sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg==" + "integrity": "sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg==", + "dev": true }, "fs-constants": { "version": "1.0.0", @@ -768,12 +771,14 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true }, "http-proxy": { "version": "1.18.1", "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, "requires": { "eventemitter3": "^4.0.0", "follow-redirects": "^1.0.0", @@ -794,6 +799,7 @@ "version": "0.12.3", "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.3.tgz", "integrity": "sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==", + "dev": true, "requires": { "basic-auth": "^1.0.3", "colors": "^1.4.0", @@ -811,6 +817,7 @@ "version": "2.6.3", "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "dev": true, "requires": { "lodash": "^4.17.14" } @@ -818,12 +825,14 @@ "colors": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", + "dev": true }, "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, "requires": { "ms": "^2.1.1" } @@ -832,6 +841,7 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/ecstatic/-/ecstatic-3.3.2.tgz", "integrity": "sha512-fLf9l1hnwrHI2xn9mEDT7KIi22UDqA2jaCwyCbSUJh9a1V+LEUSL/JO/6TIz/QyuBURWUHrFL5Kg2TtO1bkkog==", + "dev": true, "requires": { "he": "^1.1.1", "mime": "^1.6.0", @@ -842,22 +852,26 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true }, "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true }, "opener": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.1.tgz", - "integrity": "sha512-goYSy5c2UXE4Ra1xixabeVh1guIX/ZV/YokJksb6q2lubWu6UbvPQ20p542/sFIll1nl8JnCyK9oBaOcCWXwvA==" + "integrity": "sha512-goYSy5c2UXE4Ra1xixabeVh1guIX/ZV/YokJksb6q2lubWu6UbvPQ20p542/sFIll1nl8JnCyK9oBaOcCWXwvA==", + "dev": true }, "portfinder": { "version": "1.0.28", "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.28.tgz", "integrity": "sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==", + "dev": true, "requires": { "async": "^2.6.2", "debug": "^3.1.1", @@ -867,12 +881,14 @@ "qs": { "version": "6.9.4", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", - "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==" + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==", + "dev": true }, "union": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dev": true, "requires": { "qs": "^6.4.0" } @@ -1038,7 +1054,8 @@ "lodash": { "version": "4.17.19", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "dev": true }, "lru-cache": { "version": "4.1.5", @@ -1072,6 +1089,7 @@ "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, "requires": { "minimist": "^1.2.5" }, @@ -1079,7 +1097,8 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true } } }, @@ -1349,7 +1368,8 @@ "requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", + "dev": true }, "resolve-from": { "version": "4.0.0", @@ -1357,11 +1377,6 @@ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, - "resolve-url": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" - }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -1378,7 +1393,8 @@ "secure-compare": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", - "integrity": "sha1-8aAymzCLIh+uN7mXTz1XjQypmeM=" + "integrity": "sha1-8aAymzCLIh+uN7mXTz1XjQypmeM=", + "dev": true }, "semver": { "version": "7.3.2", @@ -1429,22 +1445,14 @@ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, "source-map-resolve": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", - "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.6.0.tgz", + "integrity": "sha512-KXBr9d/fO/bWo97NXsPIAW1bFSBOuCnjbNTBMO7N59hsv5i9yzRDfcYwwt0l04+VqnKC+EwzvJZIP/qkuMgR/w==", "requires": { "atob": "^2.1.2", - "decode-uri-component": "^0.2.0", - "resolve-url": "^0.2.1", - "source-map-url": "^0.4.0", - "urix": "^0.1.0" + "decode-uri-component": "^0.2.0" } }, - "source-map-url": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", - "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=" - }, "specificity": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/specificity/-/specificity-0.4.1.tgz", @@ -1594,15 +1602,11 @@ "punycode": "^2.1.0" } }, - "urix": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" - }, "url-join": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/url-join/-/url-join-2.0.5.tgz", - "integrity": "sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=" + "integrity": "sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=", + "dev": true }, "util-deprecate": { "version": "1.0.2", From 82162903806d09c514f2d04bde004884df4dce64 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 18:28:37 +0200 Subject: [PATCH 244/248] .npmignore: make npm package a bit smaller Resolves #762 Before: $ npm publish --dry-run 2>&1 | egrep 'size|files' npm notice package size: 392.1 kB npm notice unpacked size: 781.0 kB npm notice total files: 164 After: npm notice package size: 67.1 kB npm notice unpacked size: 258.3 kB npm notice total files: 86 --- .npmignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.npmignore b/.npmignore index d838da986..0abe3450e 100644 --- a/.npmignore +++ b/.npmignore @@ -1 +1,9 @@ +.github/ +docs/ examples/ +test/ + +.* +*.html +*.png +*.sh From 936ccca2b8ee5ad81318ef56f462ee08717f5f08 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 18:33:18 +0200 Subject: [PATCH 245/248] Bump minimum Node.js version to 12.x --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d27d1abf6..b29ace448 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "license": "BSD-2-Clause", "engines": { - "node": ">=8.0" + "node": ">=12.0" }, "dependencies": { "analyze-css": "^0.12.10", From 6a736f9dd30df5c3db7a3a6c879532961c8bbe61 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 18:37:14 +0200 Subject: [PATCH 246/248] .eslintrc.yml: make settings more explicit --- .eslintrc.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index 2871bb69a..e462f09d2 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -1,10 +1,8 @@ env: - es6: true + es2017: true node: true browser: true extends: 'eslint:recommended' -parserOptions: - ecmaVersion: 2017 rules: accessor-pairs: error array-bracket-newline: error From a29410571525e17107de46caeed8495f89bca3ed Mon Sep 17 00:00:00 2001 From: Maciej Brencz Date: Fri, 7 Aug 2020 19:35:39 +0200 Subject: [PATCH 247/248] Fixing "No usable sandbox" --- Troubleshooting.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Troubleshooting.md b/Troubleshooting.md index 1fe1ca331..d6a4306f8 100644 --- a/Troubleshooting.md +++ b/Troubleshooting.md @@ -1,6 +1,24 @@ Troubleshooting =============== +## Fixing "No usable sandbox" + +So you want to run `phantomas` and get the following error: + +``` +Error: Failed to launch the browser process! +[0807/192619.437769:FATAL:zygote_host_impl_linux.cc(117)] No usable sandbox! Update your kernel or see https://chromium.googlesource.com/chromium/src/+/master/docs/linux/suid_sandbox_development.md for more information on developing with the SUID sandbox. If you want to live dangerously and need an immediate workaround, you can try using --no-sandbox. +``` + +Run the following as `root`: + +``` +echo kernel.unprivileged_userns_clone=1 > /etc/sysctl.d/00-local-userns.conf +service procps restart +``` + +> See https://github.com/iridium-browser/tracker/issues/208#issuecomment-450572959 for more details. + ## Debugging Before reporting an issue please run phantomas in verbose and debug mode: From 42c3637e68eeb5446dec83b3cb7bcbfcf799b8b2 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 7 Aug 2020 19:56:38 +0200 Subject: [PATCH 248/248] Remove mentions of PhantomJS --- bin/phantomas.js | 10 +++++----- test/modules/mock.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/phantomas.js b/bin/phantomas.js index 233f8b0b0..34487bcda 100755 --- a/bin/phantomas.js +++ b/bin/phantomas.js @@ -1,7 +1,7 @@ #!/usr/bin/env node /** - * PhantomJS-based web performance metrics collector + * Headless Chromium-based web performance metrics collector and monitoring tool * * Run "node phantomas.js" to get help * @@ -20,7 +20,7 @@ var phantomas = require('..'), // parse options program - .usage('PhantomJS-based web performance metrics collector\n\nphantomas [options]') + .usage('Headless Chromium-based web performance metrics collector and monitoring tool\n\nphantomas [options]') // mandatory .header('General options') .describe('url', 'Set URL to work with').string('url') @@ -28,7 +28,7 @@ program .describe('version', 'Show version number and quit').boolean('version').alias('version', 'V') .describe('help', 'This help text').boolean('help').alias('help', 'h') .describe('verbose', 'print debug messages to the console').boolean('verbose').alias('verbose', 'v') - .describe('debug', 'run PhantomJS in debug mode').default('debug') + .describe('debug', 'run phantomas in debug mode').default('debug') .describe('modules', 'run selected modules only [moduleOne],[moduleTwo],...') .describe('include-dirs', 'load modules from specified directories [dirOne],[dirTwo],...') .describe('skip-modules', 'skip selected modules [moduleOne],[moduleTwo],...') @@ -39,7 +39,7 @@ program .describe('engine', '[experimental] select engine used to run the phantomas [webkit|gecko]').string('engine') .describe('phone', 'force viewport and user agent of a mobile phone') .describe('tablet', 'force viewport and user agent of a tablet') - .describe('viewport', 'phantomJS viewport dimensions [width]x[height [default: 1280x1024]') + .describe('viewport', 'viewport dimensions [width]x[height [default: 1280x1024]') .describe('user-agent', 'provide a custom user agent') .header('HTTP options') .describe('auth-user', 'sets the user name used for HTTP authentication') @@ -206,7 +206,7 @@ async.series( // this function is called when phantomas is done with all runs function doneFn() { - // pass error code from PhantomJS process + // pass error code from Chromium process debug('Exiting with code #%d', err || 0); process.exit(err); } diff --git a/test/modules/mock.js b/test/modules/mock.js index 44ba38ad8..c3c49b3c2 100644 --- a/test/modules/mock.js +++ b/test/modules/mock.js @@ -71,7 +71,7 @@ phantomas.prototype = { return this.offenders[name]; }, - // mock core PhantomJS events + // mock core phantomas events sendRequest: function(req) { req = req || {}; req._requestId = req._requestId || 1;