From cba6b85051c3aa404c9b77d28f4ee9a5b2d38266 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 17 Sep 2020 12:58:39 -0700 Subject: [PATCH] Dist rpc merge (#1158) * Create distributed_rpc_profiling.rst * Update recipes_index.rst * Add files via upload * Update recipes_index.rst --- _static/img/thumbnails/cropped/profile.png | Bin 0 -> 42945 bytes recipes_source/distributed_rpc_profiling.rst | 314 +++++++++++++++++++ recipes_source/recipes_index.rst | 7 + 3 files changed, 321 insertions(+) create mode 100644 _static/img/thumbnails/cropped/profile.png create mode 100644 recipes_source/distributed_rpc_profiling.rst diff --git a/_static/img/thumbnails/cropped/profile.png b/_static/img/thumbnails/cropped/profile.png new file mode 100644 index 0000000000000000000000000000000000000000..372db8bbe8771eb7c788f3c70d895e5629896816 GIT binary patch literal 42945 zcmeFZ^OGA#ogT@xVuYnC=S6X6qn#`!3#x;7iqB|#VKyZp+JzL#l5)S^qlja z_ul{D{_rH(nc0)k@1C`0*(dSZn#$N1jj|FWF_zWz5wEQ0^PJr*JSpV|mUMTq}5|F4~xVj%>Upu4LWd%?kR zGyJ>YL1oguu&#`q^o_iY-fD>1xP9OT+PYcWaf3d%|7!&&0TP2{KiGK#X+a-cT)o6V zlJx&oLkyPxSIk3C`(IVOoh9jw-fGjzyLsBt3UUi@1L&nNXlZFBJZJ0>Oxq31DcPIa+9|b!v z8&4;9Zznfb+JF55t=)XQCF$w^4fMaS|DLC}ll}i4$<^zBmIYfN&%a-Ic)0;Q|JygL zsl>lhF)dFgJJ`to`j_IB_^+D(pR)fMM}p_y;Qz15{P#-#s}#0WDGUjo|AkEoL!*zy z7!FPbPE|ox9|V8gh2F%d?7v5mW9*Z)cI&FzrS^N-uPF={Uk($2F_p9jMYF|x6N%a_yxYaUR$#pA-@kD&^ag?MhN@}$W4o#D*(*Vlx& zb|2HFQjq!2=RXMi2Z8?}@E-*JgTQ|f z_zwdALE!(_2-xS!@XaAlUnt5T?J|ag&Ndq^f}`*j2fK$&iLogLZq0;#xIq=%U_UDq zbs;ea5;(g#Wptp$^|;XW1gc|f^N~++fvLGK&xIFTIFjBa8^y2=EE~Mp;Xt4V45OXz zmNFvH(<10G;Ua4g>W#-*m!Xt`a$KxKn2d-MLvN#Qb&x#Fi44OBZ7x7&BrUxw=q#D} z!!|!O#v|=gwi1pq{t1*;x=35tcDmwDHh8$6lS<*W*@sF+fC!9wZ&I4Y7fgLKq1uvD2+^l13Bg0keY*i0`szxwPv3P-;$X_#rtE=a$7TELUWWhZZYkcDL34p=kz#tQfhm4R#h*lBk+8V-e%m+1q zZ~(LVSj+e8|IWtJe@lGZc4f`%H{W>}bD3M3dJ*q`Q83r~hs8`-=-P^4x1Wp4@`tRo z;c7dE-#l7ks9yEOb3%IXPp$|Lh99>SdSTb)U&m(g|7yJ5FgW)>rb)>p%j+Pq61Or1 zrXU9*jO|%fR6s2`16n4HAaj1R>##XBrt~DZxip?p`Y6HfY|s_zFuPxmW4+tapou<< z{jO2wh#g4H#JdJ>khO4pxqvxV>uMGr75l^u10TpI>df_hM|HhxOu|&Gd9KDK_EY)5 z@apF+IM~+0p!SdW3v=5<1kuE=iOO+jT_odf24m*3Gtp(}OoSDrF@}HI)h37#_ZtSg zwXiz2(CDLCDw()=-;tAnY+f!sJj=dBY0b80m)`x2-*RF(u7J?={J?y~OSGsPQkWz( zTTKfd%I^LM1DTSp$29(5=9oohrhlkl-Ty}My#~kmFgC;#DBN2cR)3bUp1bzNfmhJyqUMrDoBlRwY%# za*6$SaYDJin_OL{>ruf}N;89#*Sr_wQ2#=4RTo8e6L1eF^YrnF3vubp*Laylf{oj} z60n>j+CnATepx$}FKXh}Aw8*?stTcgFobgyFQ5wLj8KvAaueTtX|K)pnm&s&Jlh1i z_c26)jIBI?^-C*HqH+a@Je}72d_9pTXo@ha)a~g(wL4~euzwk!$86{}$v+pZzxN<4 z@|>OB;2oB%6}FsrP77v4uV%+4D~k9J{D%6^t7JFwvo04#WjnFm-j84<3UFgX5;6WYC*;Z56WvRY~VN`EIl_ z&2mt{S5Hu7vMDO#6EFfyHE8a6!IxESwvQiBu`5q;MK`_}*{`iKn0%B;WT$m~q5>m=J-p~|aIJeb zs{(m+Mtr=EBJf*{=_z+bmSn;DL0`h&m^OnViF^XfQegscaJ%;IfV9`Z>yp+ea2AWY z=NEP|p6M}Q3a};J_d+&wKyPi0mPW5`QKO=uj<;`(@Y@sf*m{7y9j)mT95P-sQUBZe zOKqApJ?$=JU-3jQZGSBTY4Q;8H+XNQZ-lE^nT*aPt+v~qS=Bus>7)ttp+W%_ou>E6k=a1!Fqv%m35m#H_iM#^LGXK=H7Tq| zXQ+eK#e;`F%SKflF)1r-AL3`%v<%gD(XzLqX{5qYSP2arn-a~&(4oUtTD{gKM)uiR1zb$96N;N2 zu;`X$?)_C?9}j?VY^mrmctDiRn)yO>oDW$69!s1=*$*|_mxr5R%fE> z*y(X0=@20k-BwUe{|;xI%pZwoYuK8VO9*GHi0fe;eCk1ecPt`c=)#*g;yn81?^V?O z!1*bORp^^X)b5`aHWeWB8Wh$54upIS+BIUawnX+kw74*u&EYfWz)p`wqJoh?I{q>*Ufz%x~{;tGWKUnAHMTkT~>%N8v*W-&9NBhq=XKNX0Uy`0$l zg-&=ZP9OSb8-2W|cf?a#pE^3;25m+nJa1!|bs_0Vx$o9wLnB`pAg>h7lGR^W(oF{f z@C=>l5a_4b{tdmOR!K<@$b^#*)wnVQcpz;ATDisCpMAj4GC(02-G5S7?Zc(*GTT@8Fmafw#z-UauilK)ol?Y<)T)F}uhR{npvUZj<+|jdafadB5vP z9U2<_%=o$g;F_^|K@KLBl%wBKTx(>>4U@p*(S{G5*<4w65*~$8_(}^yxITV~u>+6t zKHp?luy^_ApP5Gce{*qVzd7JdF@O&HE~qQxZ4m7eeVHziW%Vkkd0LX^eqE?F`W11R zDr>dbPw$NTGc*DF1@rh3D8H*)M>*VsSV-w$MsQ5dOD!bOEYs#H(uL|K|6@dpucv z^njZEdAU2ldf@7g)~U~6vQ4SOCnkowQY0RGEW8lXP(4RU>;zRgj*rv2NcLfHZm0hw zu{o`<0@iOO`*zT{+ivtnJJMXuOZ{Kds4T7JMfre^fB^KUdG@P1c6lQm{rXTM!Q3CC zF&`E>v#0x(zD%oJy-(8TeZZAFs(3ulrar0|)Z^0DVr{yUPyn+9w7l>@QLP@WG^)ui z_)UvRU^II`SG4l?%6&mcffnD&X86pU^7%W`Styl~$9xJ(IO$gkr#ITe&mnjXejOM4a*$Gu7)RK_)LrbQuUpb{t9jylDMGntnJ3g;WRh-aH zjKxd~bI7;m_@r8c^EX~`eF=vKUV=ZHCbY4^=%6V(49PGdoLw9b4GCH~V?*?4^31vR zN_|XEU~t)}cdFn+{c_V-yWZ40Wg|#ql0db1VMKJv>!?qQ%a;EMPLX&APmx$`V5Jn~ zOX+=II^|!-B5iI&?#@zm8JQS>W!g=mR7Xs3#O9SMoFI^)P!tVgG$v?z#_=+=Jt6w? zo6p3YuQ;U0#9bKKs`w0uv7VTy<-#>S2brI^jpb^h2i$49Kj9n3$ny{|;zkocI_P*& zN$tE3THCz6dBA$$5BPSs;kNL$Tsrod+lA#-=6!WzNRkDN@N(&C!{-U5PB;;4(6j}5 zKWXEj|G+$>iKuD5uO_zdKXsgc7-TffB+h0OxS)d8V@4A_qaB;V)Ga-x9&{D9Nwy~K zioe18h1aF#wV<2UDGv{)X)0wZTXU~dQK(z*3pkWk$JU?DKJ-u#rrMDqwC4%SYdwN{ zSi_ZYdhDx1q{p?0#b_+5OlC0a42Fhyx-RLnJ{ZMJp6ny&MMA8lInb+gJ_amO!hbxiq%#1vehp(h74N`Y}mg;tTbagbuLV}bw3`YmqH6y?8Dk{KQF zgCu~sqh&S>Wj{9Q(b z#`{xQSDEJR2#BoQW+=l%4>u=LIVG2Yqg-4#a#wm9=8^3WaG8WgpO?dl(tUE*IRA~s zhM$V+4BlwULkp8(BYiXl?iW7RK`f)_pb>ZxWld3&gE<@(+#P`l>Wgwl9?QtSRt~Jn zeLuAi$|jJk&25zLioaD!hC+7=1D9#MJ=zeo3|$FU8U$HaL;H{1`tBmbH(ip$u)C4Y z|9L>H`d~YRIZ=%Q+(U|hh0d4Q7dHL@g40_YA=*M}QDs9Ea{BRTq4gMiY2HBkfk+ZV zNDS9fJoqwB*{K~3lG*Uh(**U0ctoWjHKVyiNSF^E1NK<|X=`2M-JK~U6!B3?gHB#X z8$bLju6CR|`>&=cGZp|1oVCjNZ^t63#O~sf={h^rHWZKOHsa~bu3h{rVBfSY^`nU( zsD`l?Hi#HHWRllK#f^BKd*A2aJ|W*U`Q>!w4NLZZ^PkM?>Ab;|#bWMmqA_{MSuY_U z5)u<$HYfb=Ll$+7l_@)frymfvpKMTiXkiImK*BvFU=DA*z}&OqJ)06*`|(f`@DN$R zk?$pjsOr9#2FZ{&&vgq*7s=3krU4~Tei{N*2tLwI9ohK-Xt z+5OVF9;)Mn65)hm#o#!H zsEhISOPy`;JIwAI*%e*0K{g)7rlC^F2Y*`=1nIc>phcZ`j}d+I+8{F?g6$7zVM zNPbY=u#s}7l0|3!(+Q2y_HROo0VZ4s;WFD#hB&f-=VZYvHtNa_z}Ra2`=7D=4#IAp zqx3F6hgUbcqvjb8ak{h#4Fu4>Jw+36uLu7k3Opa_46d(Q$+=YL&OXCWoYa`K+?xF~ z-6=WU9u&iXOc9+qJAjrgjQO-V)9~ipaz%)oPXc+zgyW}S1!iFKE~11_DWwvwHSr-03)bAlKU_lJ7 z`06-^Cp$(B-&SgaWWJ|K*Iy@1aQC@BQMYZ=k&k0^^6}Ldqwf<$)o9lsyZZ-6O^O@U zNTkGnC}paEuW%h?nH~4W;mOAfl@4_szo@4Zuca6(-O`J*r`65r%edFQ#^-z`&b3aBv;|1-P89XN`T#X0zk^u|TmJX^_o`$vUv z0zKx~vXF#`*kq0%PgPquIZ$^s1n&lg6g5c27Cz!5+GkA}zk<2={j)aBuyu3{)*GHS z{2Ly~H6RK}Jx5&Ut*Ei%K{$t5gJ-c^1RVc?LVX52|HIe$0_f4-I;o?^k&k^y1T5}$ z3}=c&8Cj3$iRuGbTwKFP_E159CL6A#!EvQQ11^`LU0pX;QAZ*z#0yG~o4l^c#y%DdzPKKvNB`pL4owXKE;(s?I>PS2#-D0EObCR_9# znWyzE&j+8uHF&8qI2$LgL<#q+zYUj|brT5v)_iT{+cc<>ATwk_YiEvfoSmexS zgUPJxKeB(Bp7B6*Yz_Nu0occw0KPpRr%s_eJ(z)jIvk{z7wAySPaLj;@I@cV^3vs*dx+-+9%g?EBd z-_R)7-loDGiM0V;g#)!kA38}o0Blb1H%y7;6DKBJRBe*zHk@gT(dCQ?Ub%2?nrifb zEcA16+@t3yv2tp4kpz79ri#4=5GD)K)%)^Z;~=)Sx}QtWfyhU!hrh z`oF!dMg9hmB?bqO-XPj`FcVxvxKp@J9jR=8$+nE`!gl6Q4BI9|bgF-?R2Sy*Q+qO(*5uEZTg0zrz3T(zF2)Uk2=TSdtaiG+DeY-E*F|gLB4AZr^g$F z|2#k1@beRMoUfHXW+HXvJ;3{KjxH$vUI!-lUQ}2e6I>ZO-;r_u*%!W|QkybOoL5;j zks_Ao-6UBE?r_o8{H*QgL{n%paugpxX9+KRDX4hie|$!iS(5r5#>Gl+PH z-Ud0Fl!TlBz*Tw>dxEYs!eOj;^x=!>Z^oMhTG96d4lD&f5`9=Qzb)BpY4hi9-P4*C zaK?^^g%)#!1oR|SOT|@xOWOG2US1i%$jQ`Te^qj@!7mG&8FS@PHot0^@Pmlu)Z@ik zLY2Sq=zu{cahb5Z881@b-g-Un!3~NBvs2zHz1MxRd@VHDT{p%(|17T&ajY=>!tEQB zRrQv2O7JI;g0rm_S zvC3W=qlqj@Sm7PyKiLf!Y)GdfWmdmvE%8YJwBwe_iF!6lZ=-3tUa*sPI}^4`ILC9! zXy=NvYJ~fpQ<6(-z{(weI9z*S3Ik9wHU{! zOPW*fQveF8X8}9b&jFS{Q@Hq#8j8IMEnHgSsoOOoyY^533U>yj1owfb$OM$cg&iS5 z4uilw*Q&3LXgJoa{;V~ABz9s*i?-hM}Oox@2K#S<1P=I zhr-;XN^cukf=0O&Gk;a|L}`hO)rOH$B7;yhok(<*dQZW_kW%d6J)vHqKgM(FM)8VI z=k6P9!{6~3HM_|vzgEM;OQ8OoYz1Q3f2L>?W8|4Zv_7*qIOIoAeMO5ZC&6?}T;tI# zf9`ntgst}>hJtCITHi4QYO4IiE9m4-T33Af%81aI9)`4V+*vDDRgbjKSEOV0EOv|t ziFoz^v}1YQf2qCNFjy#5*ut+{Od84AL7oDcn6UAY zHuqW(O{>8xIMc=G5SdsCawFyH6k50)y#=POl)cJ{g?tMpk0iR)X6VCguT;=-#CZq+ zE+a_D*Rxy?g#fX=RroYh>EGTxn9VzYB!+0g-J!R8&!#6yA%kwN1^%pO8lzSwAz7J@6 zZdbY4KYY^FiLZ+>svI*|qNB?o!H_llL7Y{fRtZ?t~LZ z+va_QNHtE|(sY(3jLBbRM=0wi>YFvZFIErW-S-fRuU5qV_bgi^lR;V7`$SG zIsk)wx2Q=SN{7Q&O`u7T#n$3w4y)jE$E2b+nc!gy73cGqPp&Hw2L?J+qG=VM@J;=k zOIR@RX1xU;cz;}|y{7J`&r?TXZuVAPZ#})>e>}AKy!Ot@eRIjL(R9jihdA5|f1^$n zwul-eXpfFr7uHRS??*}8ck)8b=g+`7IZ>=e;IUtxA9yPu>G}! z9=3Er^}O1!XnC3-hUPI}2owlALX1qY+4%$r5oT-1P_QzDSxj<2W%+0p66R?!gE=(K z{PB*{AyQFQO4A#8dtl*^ym|3SuW0Oy@+29`(WO(wHMftohVFecjDv|!qY)M$qYcBn zb6OP3{P!s*vMe|0AQ8cWLJ-#0NUYc^<(RltVYmm?5wWC+0A=M%K^j6p(+i5}Z}5(E zKc63(^#?MA1p-v_4=drS8R&t2p7eaPMI$CHzwGL|1_Jr03XYZ{wLfgS(`ne&^cvzD z=Lc~aG?$~PSP4dJiTja;gPiy3tic=uY^&@2Aci_xPCcb=(kz(D`8sVP07smD@qu-X z<9h}ig>ggDrF4?L=EX`a;SpSaBn||JRJ{4p3ccTu6SGT~K=sP=6EEU3CX3lijd>gk zL!DkCmkG;BchBE^00s-pI?sV2`tp{-6*LtL-HThPZ;|ew$6_hpMeiTKxhI*aJ7R4@ ziC#xah}q4w92qj6zS+y+9?a&d=ISH;U@>u7-0ObsLBc7%citB#t4~d zqpf3>&vntcSJe%pvR9jbG0{2qy+FP z3wk7|+p1Xj^}H~CrbTwDWCbP>(@*cfPP6tWgl~5?Xw!PVD4z~erdu1_G&VCc2|ySh zuXew4ci4Wv#hp&zH6`a0|NSNi-dav`7j+Nm^=7;dr|dgd-=iT9HN@D+5V>_HL>5bIN1tD|WAlled zUdOqN%RXdjYjA5cX9U5EvvF+7YxH_g^h$aRm)!1c5#~J=WIdes!X&3<=4Cbk&07>6 zy_&l_jQxca;R#tI{Mv2C>X|%LwBO5M8(Xq1D0-~c;)tSd3BVXK?&SBECfn2-&nJ#) z)lCD~MHtGP=)5%?`m5oc&~j&8S8tRWbzkHjmr$=Q^{04h48@6CniSL0>YKYP9_v8{ zKmd@Lqt@C2E1ox)@P;68aoC7C*&Xd60bz{2E_n8thtQa?Ia|Kk`$R0u#o}i#0#lH% z|Fba@b>C5*)zi@5KS{`$Fi*xjjJfOLhcC&%UC=n6TSTv1(k#~m(Cw~^ny|GQJac#W z6GX(`d5lv?y4|Uru*i&O2lR+#*46h~x2_Ws%4oNro@$2~#zb#DA2a_6COd>rG4SNz zb;q6>yIho_yf(uWy8I z5u&$bnw)>#cShM!H^I?yT$cCQ6vJ(6Mh%BrYf$q{;&*q?(L4am4SvUjw;zCT5%cSs zrGy}R4hb57)3ea~f+c+1AW6J~w7V|L{@dp(4o4KXVh5#0R^BWz0{w};BQFcv@P_C92)zi zc0G(i*#z@z;v1O9w&5|QN=WiKJ6$e2I7b6l>x8!X-7)3Fi@CdTt}W2vy|mY(0hCl>sIdDdL>;WfvX*GRo=u8`CArIJ`}CIADcTzI@0HUIOT3C8ZNIA z&3Bga2V`m=kdIRHiRl4dFg{S8)snnmg5aXRL@x;!SZ6n(d7?b?^;t|}#qUzh)Gn#i zb*^AgqZ4K3qq&c)gjgMl;sP;rxKAyhutWu;iurroEguavCx*}@_vW9H8&Y!0WNWw1 zf#s-|cf`~|->9j-WHs?hkRoq^+VYPsjYwl!`T^>$)b9$N7r9>(7sDe+^6wkR01fAS zXlL+=U;gwj2A7 zS!qd((5SBnIRFX%$AfiuJ0*psdfl09f;QPVqKi-dBIh3lGhRV^h4Mss+47J zYOO#x3RQ1)H~UdR*KzWaM02R-B9Suik*SkUVnkqO^=ZK&B8^D>A{k_(r>arj z=`h)Gx#;mmMkcN!9z}bt(*kN%VKmn~%I6K^L7am^B=6i)48T!x1FvtuE1z%r{VDfKE< zUu3r2Nz=Ji)oBi=^xW9V{M2liHhcL9`zCHps(YpfYIpX?HxruC<2PdbX(;D_X!Zxq>dcZK2~1xBvApAC#VZEc!+^>U9DDH5XYBO>gXbH2Z3v@(Z`>-Leu_~_3Lfv{03GZ!@fXYs z?9|`%2Z>2;+HQ`j@fdB*t_VH!Er9(uWY4a;W1c@-9(;|uHHnjXhnw1S2p;<)Cxjd4 zleq@IU3rK$#@>wn^j{! zgKKc8-}^4>oM$+MEc4D~rYueyG%~4CMvFixH3!6&h(R9^ES@A^%ZmM(O6;{-zi&w` z?D`WOV?Y(utzwFc6}#1D-aoq)GEyr+(@0djPQ)T`9EB2{R45o=7NeA$gf1z%pRPYW zrbIUqr33&L^2WqkM~+OjN(uv|8tKvy6^WIIGDA+q-UWg<`aCK?zIFoN7(^EF6(**( z-Io1;gu>EcClOSU^=V~bd$a>z=#f>z67O#X6DA(h!mn%NWg;;bc`D|#&YC*_Ge!So zdi5_BW^usgqto;t+4u7XadMm2BaNtZyaAVHp4dnzG<*HdQZo$3=JgHt&+pOP8r<}7 zxvKrpj|h|Yl!+zY@gK*w5J(w{XpE6apvGK2Zv5^H>8g_Q`2v@M--^|i_PJnGI{G)_ z_8=`;rim(&SOLs2H*85y1i+O!(k@)R5I%n~Tn$70ts2Mi<+yo5wSg3|0otnZMRrH? zM8gFE>OMUUC5j}X3Fs@;2FtUUd*JF+1XC8%EQ5cHGgZ-{iR8nJM{f5HjLK=?DE5vB zAuZS+pYT-%yS9#>sbWRHS2)3q`IdA+BB`p!PnC3u}l zXn+YMRn24A5K^oz>Lo3v{Wz-lA;qg3dMyV!7pa}nBc@K2D@X?he+BN@Q}|^Tk`#=_ zqKaY<01C23+WItHHgHA0dk~1qRg+W{si(YY-GVzJa-orMuP_Q?ykb+I?7u}Q;9cOr zSjkA?K18NyD21jp44(1w*Gaft1>4Xx9nYJ7Ty`j6cYph+Jw#{L-=R}!8C$PK+>N{W z=b!gTEW1bH#}JiS5YdM>cpFJXhocYsL389q{>IS^DYK>v_j_0|C=N=<#Cet>Rr_34 z+kTW@Dq9{L_gXg_bS=k*))QoTO+zucfdk!3aC7)lja-V-CEgSCiTsf_MTvB4jxQ9f?)~L)-42OtMR>;b3r#U!C-rsf)ZE?yHBS}0R=>_)8%cq; z-E&biZF4`Zc_5-VY=|kJ%`JIu7|T=EyXt%|SicUk70doUf z2lC-fX^F}9T#`r8D9^jO`VHcESlC4cRUv28#eW;qo^`P5T*EgGEBdvB^r!dm_^5Q& zA3?{g{zu5P;lvcm>gaN(?MVcE7fA5oyy&Ft7Tg`>z&?nAV7HmLdfyz${d(9~FJPd- z5N=P%Ki=4Dr4mKJIW!PchO}MzUEnJ)!@LHxtMc_`JguqOzF^E1y20}HJMKPXl>B%% zK~|@sETG(ZW|WMo^Kwipo0h9UJfk(Tzl*t%HKP-dv=uIwAUCM%d7+!-V)RBzy>l_e zf(#_jY(@h5Z4?S)`D3h%x;$pm^1iFD2k)AWiC8`yNOj)Wv}4>gzE}0+>W`cT6X3n0 ziw)WX^3?7!r0^-(TAymhsr};mtTi6HViJez#s~WOZ7@z8kg>Mb4Lj zPMtlt!yZ$38}%Y_F+Ls8$^dBp4ob%I>ASX`vb}WPl~4UNbN`4&4NkDlzuPQUC58!o zgoI||nD}|4te&hW?yiKDLb+#1qy;^JQAS*!2j>(m1uw5?e|_RsV)gMr^XJ96>9)yD zR4F3u-5=V5#XLQC(3IW}@nZ$f;n$MPFBm}b$MF_iBHu9S-vWZM&)mZICwaS6KK>3h z{#a|dtN2Dr#Cc1!p^h4)-+hxvbJE(x66=>yk{*BUM1$oa$H5cL%IACcG3r;!^J4?v zaX$?wTgAkPMf0d!+**3HjYf>+c#FZ9yX0RgRPz(dke_X@qtv{bszYcG@dDGGAl6M|d_{Nk%V z#4a8-YWWw81+brjqVt=#Z?S4x9_bU4j+9E(8F(i>c^IW_OlA%Fs0Rr}8+uTbRk@fv!)6DYBHDmXh#>EJ@61cGxM8xTAR1SU zvAOTM9k)3SmC`Q%L$Dn1E~)pn?cw^vn) zEq3@d`=uw|=WfWDxwKrQ{QgsWdH0CxT9h%^J;lz^ro8hVCsj06vgJxU4Yojg9OM_K zIs>l4e&vVIU{vQ2)S=1ySF}KUU$~)|&(x?d_zn(-<}f$46#63zDMk9&G=!7yY;m?A zS6t%bZpw+*LAf1T?56UQ%VD}=vH;(X|6nQ+o~}l{o={!mhsY%ImiR~OMO7vYbg*a? z;U&woJxveOf$~*i{%-yw4}|6|d$*Sx%-b$|t_hXAam>aOajZ2>xBYApwHI7FFy;SZ zWCC(W75KF^3+OBzzhwX{)a?PUI_ov-T|M6LVB8N|6-GP@x`?|VDjbnVmGHNGkw`q# z|0<)xZ4946i(+Jy%a#_cCy<6ZlgAl8`OI63*{%HftQI2Wd{jR;4D3`A%!Xmc2>(J4 zbN6d9^_X(anOfbJrpy8!s_t|mg?AUFnEViAy(=xgoM z_2@4A8z3n%0fl@3--DI5%K3=@c3^GAMdtxZDQh58BDT$Yx_rfR5n6-Sn_*H>6bH0^ z#Ve8V@`1*R)XD_5C#o)eLW7_wz4A)%ipKR*4$gpSxVCNmHLlm!FqS%GNg2yc48}E< zyl4$xTxFcKehoTOf3D!TJbJg)D0*$jhB{IdOzqlQa(@TbORKN}@BOLPuxx0AFjPWP zV8J&G=o)`af2Se8E4G(}Uj=dv)WeNQ14h#t6crP^PskmfYXN&HHG4mER0xHz7CXbF z-W3*8%c)J%`H47e`{KhwW~KX3tOzyDv~W-;OoSX~#(H8R&e{rwE^YgJPYT1j-rUHV z26pBa^D3|~VU-15kqPjQBqz~zVK)kr!?s}h!qZdW zJfoA19W6Nb7UDck;kT+J$|%&fA5}{uThz)zhf??k!5PQVSN*cimE^U1+@2Fsj!_8k@O=MntAeunP^$G z7EMpY2}!pycmTv4Z~}lSs#LzR`k%$Zjf6ufoEM5y#%Wz4l19W!JA0c-2`NiF!_Ku2 znsYZX*z{vzlSAL#TyA4e-c}hZh0ay5J~)tu`ZVg;^be!(2;qVP7X}Q|u~8j_aW2RO zVxk|Xnuk>d#xH4)3nX2Es+U_()PNj9cnf)q>Y16y=p*@1W*&CXiFNLXV8Tkl2eQjn z5(-YpIc-XaWWfX!=R~leXKU?AsmSFzXP2bk{7~6QDpBjEC)>zxGBX=9TH=UUk$7&{ z$}POUF}Tn)YFbEfA;lNS5$CsXaq%;TYWZU&OaJXSHQEV|we!V|E;R6&{V^oE1*Smk z;cmS9$AOdJj+jC}SRy>Wbw;WZRRpVB*5h^G6^q(S2#oaIX`T#ni&Tjqg&@iQ34Rde zqz-VgmZ0KI+=@;RWivjp)Qr}aHv>l^j2~J>@Z47Q9eK{Cg>@SUm zXLvU5q`vC;5OyIh=T--!dBSigt+#%$ezvF>lNVxAby{j;91oQEQEHe}yJ{^dG`Y4-fGdph$ zXOL~dc{Y!kr+JI8mwXg?e~RlEyEfPfZ1wGoNC4tQ&kh6*d6Q@k6NES7XCIUNttLA_ z#B;=h#~^5?^y{M@NNS473*ouctTW9Lk*Z4alBxIjJQ_8`y~lr)Y+#hiZc8}UN;mUlqekc~fg+!D<}}`RM{O_D^5%^HH*WKxegy-KSuRZI>X-GSUCzryTnu$$mE}P4 z9N(TF5spD^{PQ44fP>SJDHwxF^65FMv2m>c!UeY2I#G55Cc}*kN#$n;h0ON&I@?P(d+U-K zZxy8MslXQ1-Tja7prj@AkS{$~48+Wzolqst|;2N%U|*XFh_ zw+8YCTw(ZT_`rfN2ahk^XMVV0v^~{aRV{h(sZKk8d7uu!%5&!YxR&$4c9G}`Y zq@rTDeoN;lVJO5xdGqU_O+t`pye&$)IUcCqkkfrsa6iSTA#;ljHSH`=ZWOvH30@}S ztbW6Vu-3)?hJG3y$$Dgay$C?k5BM%5wn}f8wPF9$@*QciTgw_zY=S=iU|h9g1% zE3^rSm-okfz$=?Ms`II-LSj$MUs_5LDw({Yi@CvjID{l%FsWAxY>lxmU3D$p9j&UQL=zT>yi z+lNG#6Z|?d`XjK~WCA}&ZB%m|mD)D%TQohW>GGKidc4H zyw71g{blRPAQg(17UuNx8S&#>f}t<*=V=iyPc=r(aa@X-XMXKz$)sUUrO*lC+J#46 zBwAb;n$HWYv}#VrEtg9WwzGtpQUb%b0kTen6d?h1)*wiR*~Cm_<=<^5qL4{5Rur75 z^NmhU&8P=fsrXIVfc)II5+*J<<2A97muH~X#Ni8g8$6mrD*?dQwFa(=g{Chpvd&y`E3w0Nca~E( zPN=nniSk%~=tYK+f(SRtZ$Rf`%Wv5Y~OC*1%_H^kPu%00bRe7WS;=l{2`Y!v{Du(DSr?t~QJlLOD zh)aznYL7w-YiF~IPD9Ehf`kJ+ZrNvVJ8+=!XYjpk`R$x0EIMU5?I^4QFK(^M5BSXd zI4JdTB}wXcQ{Stn)Q8xd*Yv@J4?0}}{1*wv>+51K=$*y37}O&HAa}Zgqy7HKl#RIx zd2Ssu4cfUzSN!PdYv`7eL`3XnFH$<_Ays&cw;-zVJqqtg&Mq$t(ccky&^g(k`%mtF z3^N1^6gcrhj|bX6L7Nyk-B_49GFchV@HB>bY0O7|f-qkKs%rEcZo^~~a_Dip75IIE zsQg?B;<^QlahBThgEfv9;+3~rZQ)NZ_uXc6);Kbg%TDTV(tF4E6*2q*3I>|W1e282 z#A@(dIL(whhX)fjWi|2DdEVv@oLgO?UGz26OOQ&k`_Xns;~G|rmi_rTPmpIuNMRaL z!93hQkaqrAAqq7o!OJr8B(k1)XnW&>Ui{&}-lQ=>Ds`6;@xrFbj z7S-6DADI$s6D|H&TD1ZHT7SoFX~>m;Iq^pM=csEKdMtlxykZ{nbuhhoOjbB3-)Svg zdHyZ5A4hU2R-{2XXuFjnd|{%4Jz1?z{&OTRP+afI1Z4lCc+gA}9G8emwdk$^az{%s zchUl-(kEF=bX051<5ElDZ1&!*j$FJ)KccAk2n9~5RtriT@!b$@-?A(qi)#?@aK0tX zBr%clm%Ydo(_{e#e-=A_9N8)|JBYC}Tt(qopfd94wlqFtAba^Ghg6;dPnD{0D9n{Cd?f3%pipzH!s zI&-m>W*$jaPWX=F<{)vATk^xyk~5igz{S2dVNmx6Lq>`PCx$sJ&cPSqLS66#bpd`) zBAhhWpG&24+)-bHigBUNnjU*vgKpqN+?+CI>2i1C&Dfe=y*D|S@bP9f+=eUSGe3`f zgX=|QPs5gmylVR)u458g)Dmzt7L^wfaXM1wUeTfxGfY&N=na^+FEyrThpi}5IDg3B7k=~SE%AQ;72c4K3#wvR*Tnm^-I_vKWTK}frBfszh z_nrvZAkb_R_}vNowJyQ@Za~esu*kY_+W%Mv!ooiC*76+12psVo4p8#{PTK%;U2Y!V z385d!{Xa~7gLfp{6Xhh6Ol*5%bnHx=Osokzwryi#O>En?ZQHhOZ-2kD=j{Fi-Tht{ zZdKj74^N(xD61V|uwQ<(y%EH*tbkZuL6$xXJ=WC{`MU`SBEsHn1;np|Bd-KbQ{S_) zu8jvuYjo=N4Mjd3C&l)65|MIw3u(MPb?v#f$IQXaMqXixX)<4N>zcM-a0M)a%AHRG zqak~P6=Rwj`Wy3XV*1@WD6H>mAt!NFdvTP)GpP^SOn7C`s@e!1DS=Q>=E?AA4+VTJ z#}G6er*XU<*IbUaS-~_1V!tj0GLia{pFhl$!*>1oQN0IFZ~t6V=u|^6!=>7h-Neqc_+D;%6Wl)*yB&D;$qUq;YEJCH?INDO>(eSbD~9#3Au~ zO+U1A65zZJ3!74Z$-X%oCnG1)rya%8JkaF31+Q6d_QMlh@f7*UVZMV8Vjd|++Z zpsl82psJ{LlnLzIhqrPHcuCy{5_x1WenjLli@fR{BZx5y&x=KY`VCJ!- zy_%>72KuduAYC;e`!=G1w#vBm39@~^UAfws|M;xkzjipQir04X{Qxg0>jI$qu#u7t z5udMIqDglqR0nOBKi<1Y3i3vXWvBfn*2_-dOn9YC&ugp1W}vd zhdOsXH4X?UcPcRRAXGD`cxLo?w%n4>ZVE$ocklFs?X6XGZ?s3x#vP$7!5}!!gt&a6 zo2XeS6r1F&gR&K&O~=1wwI1-+_$+J09Es_clfKi-cPj?mU4@b@_F(H9W;XSG!Zf1$ z86pqunuxf6y!KB(QP`d#?nKT7Bafq<&W)&d35fJNYcxdQc%927wS66x(cX)XC1fqR*R1g8g?|&l zMzeWr?}-B&z{GQ6cb;j^jKJg1LpCC04N~|bTK<9&uG1>rwWEHVYj=DqxYpC!tOKaBwBwAofo=`l;aD+P3hPZj!rt($EF6aw=8Wlc&gD3oT2}h z#8<@x5-Pkb6YfBM1v#j*gS7kM89-`_#!$Q8%H~~}G4kF^l=cMw2?_i@nxWM?Sis<4 zxNBO)dNMG~5JtLr@`e8>v0SDreQ*B9^2XtRmx4gfZy-GSJkG7BQynFzD_?Y;B=hzX|ImYoxJ&I(v^EnknyMzVsC5)uWD-bURM{JPBO!3 zJ){#Pi)!#;epu_IqoNcoGj6i#VZ|Tbn-+&52`Akzg+|>9;ePP^tb312^NVL zsTYZHx9TcySQeQMrF(9uv3YYpto2~@der$TCuD54;tjC`=M>AU+t*vuP)F10g^u0< zTt89$YWeqG?xKJWkf=Dt8!YJVW*lH|#2`iZMC%rPp&>d(|KtgSj zwLq}l`*ou;QkJ8lesMGc%G-D~!#vM8R0{#=0*%Oj`*Lfw-3EW((|MY7&0M@`SB_(# zcu}SAdCCJj&>zFAzaTuYEXiMQl|XdV>+tkV=%kJt$)rrAe;fOHx;-W&H?P^COiS68 z!5vaEEtOn7CpUFXjDEz;Xn*8nN>S5O*sbRdIOn^M!5g%5j|Heu;uL{Or+~3g3TxP$ zxO?GzVogVk~W~3S}i+6f1u@qc7i9kP}@BQY6O8FU*3O%I7$SZ^l!zm z02}LeZ@_ZKA7ntM35TH#b6I2xV<`{(#sg+UWnwY3wbDUFdDO0-`((~6G<~&zQ)LM0 ztBgOu_BC&SxRn7r*#Iw!uGG}i`6GO(jAgW30i-)qZVGB&S%ae^*PwW05R@b3ekUzo zCT82|GjuaBbU;oEb51H=c&QWI<;9<(-VX>KSvzc5m3wO0ee;mIpWU&=2zY)jA0Twi z#O$qe!ohWSZxlrn3%An-EevPd5!U>$t!6*f+TIg*7BFF7+!!~%|5tRN<$X;SrMRv3 zm%9D0=a&`s07hKJ^?#g_vK~jSa}3XR_*b_CuGEP`N`nmJkIYij9q)8Zh>r_N2DK~u z`>0Yc11O=OKJ?8NUZ1M?czKiKjJ1#h(K4dv=}tL#VuS7c1v*(B+wxdl?#nY&$J{AK zUf3dhCOx(kZ~gIXG_-ab&vrTY8=52^2Otiy@@MrPQu|UjRW;RbH@RLc(4G`1?tqv5 z7k8+@ioxMREBbEP&sbOukzy|zChIM~VaImomZ9MZowfN5Lr{FuZdCUiU{%_JLEYVXTa-hrfny?aC(meU6345GHmijo=piYvbjpbhhRKjydK;| z!6~E}v*zo9U+fT(NLrY^CK<|SjP}-~^@T60impt8Uqn$qlaHm?^>#HQ`{4){)3`#k z93GntsNBEbYISxWxEAW)JZIJ~a|_f2j2h_AEES{55yB{zrv7aDS&R^=Tf9lmk*jT` zsTb$4y6*69H~kOVurMQf*9BR-Jis3=Syn)f316M=vQLNTubm$g3}|t}oB>qsJ>Du` zW(vfAujB`NUYW093g`OPvBA}bCJ;VC&*ILy;Xi0Va_TOGTB#ky zNG=Q8sow1l&0qFu*W`&B-ex?_%sMK|G^r*0_^yV~Fd4^tjCRB*u#f9$|NT6i(XHy6V99^Hy45jYi-(OgJkq9tfaEb;-&Q=l zTY@~~%a|KGxrMzLo-p38$uwRQZ=EQ_DbwjN6jpO|nFtsJ?Ow!IWaITBdLA59!&Ewi zZ2mMew+T=mMwg$$nAaG;%l|t+2IA)F^|lo;&oZdEx)mtjRH5vZPgAQ#WaJMcarH(E z_-7Sav4H#s{f6{<@lSv}C({h~ziOvmC||`tGZv$uQqA&4{e^|VYvAOjEn)mSp6Axj z>S^k-?Q}itFta?EyzW>mi}=*d244{O1}~dtj0r+@Xc~99{ZG>}!e>5p5LsZ-AnxB4 zWI}w0(VZx&kn?w3TppfTQr`}7Mvx=_NugFLpD->Y(o2 zeb=c3+`q;>7=i#V_VkpIxNGmya95+2_N~_!_~&LD$*5QvtiOHrQvIxpQG=~$XNsEa z-h(Vw>o%n(n|=TO4Q|s7AHT)mt1diPNUFPQH4+o^Ry?Q?%KGn#_w13^ zIIHBVXC}KTB{%?qixTTV30Hbxo0LkFW3j8VXO?#hJZRX`73_ zV*cG^O8KgpMgHKlS&F#~7QK)8*fLOTulj39Vk<8(FPoNU_(uqbFgc1T2G}CyqeI`Z z3|SD=o>ydD5{unwxG}HlVt|IVcdgtwqee9tLyZk_5lk`$sX-r%^9D%}%uG@ZAw@q= z9lKVb!#P}-jteLSXoKGj&TZyVJ+(;wlq@>?whfENJAD09^AVGfEMb|*uo89Fttk^k z-=}19zdj-LR*>QA9Oio8;UhpazUlq%##NhIzP@7;-VfQaKZv*W`76>rXs%AR2ScOT zM^-7bXJ~S{;@Ch0NgblXW}J zdk^BFdE@t69Cg)X%JPqyj-IdS_)@SJg+;*#ump92Yt=kcfj&@~Anyuj^bZ zv_%!%6ruDBfT~%DCSGeg#XMfTrVK{{^oh=gK4e z4u~c2{?qBF9FQ=}A<=nxC(xn8yME`rL$QRAkk%nlMhYIuI(Q}xw{P)LFplAT=q_v< zg}O}y8cBcAd*DY}{_5z4+IP-WWbJ`{p%9(OTWPeg^M-0b3plkq>BoN?otP+~stiTk zh$RSQ5a%`nj&U8BkVO{5mcNm>zom7DU1V8-gjGx?9$rz1cwyQuxeZRn{4uWbKhmgm2#(zq;JKLw3 z!NJJpY5A)ZV@Ggtm1=g=bhAuS?}3-Miiu}f_K0YIVD%<3pBY`TkJBaDz#Xf&Euls(E=o>`r zKK}}AUhJ`v>+Oo19Qqc>>u5%v;%S%tIW;NGBtw)}@5kAe$kuJ{R)<&Y$(bTDa4XOZ zZ2_~~;g{mu1U9s0 z!%K(KAsdDF6^Ax3&YF~0Eq6gw)>9ub8S`x{WhR}Dd zT{PHO3|J&Ti6aVFopCM;F99`4SX=(#`+}V5nn}k?u3|$iv7eiat(Ds}wnL~PUEoXE z0ZAHPW#AuG%VI{}AZedVBLa_y)*4+Vo@Y&LUe&U<=uf_ML`W5YLc7jpFsW{-$7gD`-LR+TJ+m4R1lujVx=2rN(Ag|Hsx z*65wLuYi&p$YERtgP=MfZc)$Th&|Y|E5wJ)6Crk{I&c?&2u|E65Xv8F z#k56T23nwiQ{eV242NQx2s%d{M8j7KwC!ytUCx*CY2r4H)kltho3)&eG0PY0MPk*<2nMc0b=}WW$8CBsSnrLHpNAWZI7vZ;K&Cf zew@pNn@Hh#LT(XGE~%XWCdx=MUp7kmyZQ{ncEOA&(n^F&7akv5uvfnAPYnKy#@%Cq zy2Nj1s3v=aJ^Q&+YsZ4eg!f&yPUmPl?Hc4B%g@XmV1qm|_2fwv^T&b;CyD(4u__ZH z2+T|xU-Y#u1XmwI;LnEA$>;U^Uqvweun<|1-B8xg|l8<4l#FP0&c*j zTD`zR73Lr&(FyMf0YOmw1h+PUPhn~UjC`dfPN^E={KTlr|DvPQ&SklH->`pc#Sh zdmv*=v0Fm$x7CD5?695 z(?`zZAZmu)ax;y0Sgna}!Ej7aE;k4JZFEBdjs@bxksV!(lc#$Gq|L@YX6e4aT z?!3^kMkfb|VoCriMiy(*&$!qmi{L8!+ib4W>ku!|Rol}*W*i4`y3->F&I;3V7o~9)LoJr0ALUcf9_Rn(cNthSVksglF*BH6 z<=4?~U-9}0d^{JFnfOM#kvxqQPO#wx8;c{CaZUY63-4o246gW}$2&Q%-i<>rG9ioZ zKc^;8jLq~A)KNC&w{YLx)H~CoKyL!@c$)z{xJ3c%v*8(Ca8&9}Iu`i6$pdP;ab-Z# z^9b)ip2A-L4d5G9$D6HOZxnyNDvWCr7Z!!FUEDwQP{6&=5I;|1=q%lk`> z^qfwomx!r%L+^%?r;s9efvarQ zXSBh3b5@}dgsf31t9pC0$_&lM?auErO%R(-VbuIQeoG!QoETcm--N;s|64d6lJkf{ z4T-?rrBqcLxHegD2L)t<8kB%{)u#%1O?M76@OwXfh-AFM{oJ;`W5Mx4(2n5Eiho6G z(SYZ2OUJm{T8gJDvQM=RE!-%s4T;ttC5#$> znEo})@aDnS0tnxO*Szjt$_Ndo>yctp0?EI)1e<^0kgOZ7LC{k_J@~SmAwWSahv+kX z*6(Q`r~2s3QmvO2S^UI3f)HM_)kSXugV8m5)2VOj!K;t7_B0BuKoKIw52^7Yd z$Q^BXBzGi*0p)T6!>ux{xY4t9yx>T#OLt4>iiHD3HbkQP9l^6MhfDZ-GY(_{rD*+? zqGvQlSq4&Hj1kinbYFJ_3mPntwp!*lydrYq-^tA)&w(K?#vlkl4GtRHbPRc_c-cn0 zUT=R{(p+I(;p45nUCVc!?crM^1DEvAfAA81CNpeGS+s_KBS$BC8SlLY?s7Nhbky3x|UQ#z61J7tqSxb$srtv zOkqTgP@%{BW{t137mfB>vNakmU^V~&qw0$66p{yJ;9AWwAyj-xglBKI=&=x;;6eA9eD!+q zm`t`r&9e+`(%9ZKB);l@e5N0^3HTHTX|5#xmnvWexw0fI*QqMqg|mOih4{aD)svp2 z2a|nM{Mnrf;vd5NBlKhrfO3FCvM-#e3czK+WA_d;Uuu+TEd$Br`Z+b|>mLLCYgmPnK}TR;Y1XgF zd9&+e$b@zAM7?$CD?RO+NJ#efGe_X)i?zmzNUz0J>&4#u6! zk>;7XJgn0Ii=su*1UWI(>GFe^(kqaXuCRbyQ%Ko4|ho`a0zbRb)%k8<-(cBfMZaD7x zq*xGpVpy?cn|Sv;WRkyg6hmA|2>yUA&Xs7t4$4jWs`j74P^h%j$tpP?kEx>G3Z=tc z;8MQ4q56l-JF4z?>%RVl#h#JR)AOVm``UZj7Eu)EM~t)5SqWXIg@e=Q-a(CW2CA>H zrV3fw)&wMm_6h~~S&d#i>%8T#gUB7(ppxZf`;iY(n`t{Kx+0;U%u&)SE>-`iSP~mZ zzh}DWs5|ws!Im@hI7=Tenydux2*T2UdPjB}OaV~xM6m@9p7*-xPvQQ`jE&u$q{XiW z{g^p}XM<h*`}^R*HM8m;Hv z*M3{1ge9kHCnw4&vS>*LZcs$CzAnV)zc2=x@JJXKtJskjUiy@+@S~X>!tu!(baiD_ zOU+-0`>pvuo#Y|^rr*cphMf&NA}eYsThI?RP&E>4!=N~FDHX`Er4ui5Watw<{VWP( z=TIm}rC1uc)}4EC(fQGiDE;nz?V(WeGK9Xvz3VFTOrZA>1r8Zkee0pOh2$xB{?_vW zSPpV?YMs8_(of_}=iK`LPQVLNmpbLBL4m}?liB=xVJw3&3r5Y3kG~V5vpUeDNWbMV zkI@#@vdcBDG4f+a)66JKioV$k8L zyucpTSz8JY;21`Fzuyn*P_p@Hp#zT*}=$x8E5jC2RsiHl;- zEfu!g2w^wY3dVquMVEZ#+w4VHM}Hm_SG@MA)XKWmMZ>^2R{Y%*l$eKVl|!EdU`z4z zme%?~gJ~gMGO45GcyVVhTfOJK?|w~E1 zx%>-V%G)^w;*hVYrlLGP`DUI9Zk3L=0}dOYi-a>odPMf#+Ob7V&SKX%k`LZ8?FLr8 z=)F!Tk}7-R*pEkct(T>#ks1HF!hEcU?ST}R-TE;!o?7&gUV#x5?3>Dtmcgw(4dMdr z-%KVG|K9lP)lCJGR|R3|=nbG_wo5WawcTt6Ly6svXYvk*+N(LQT6Sfk1JU`Q=MJw` z7CIXzTCkR{wiFPS^24T>h6sM(Tzgn)KBJ<(nZ}^pQC)u&qqp-N>~gnhVI*$4#f-66 z;D6^}DAd%C#L z>|T!e7`EPmn*S)jylI}1RJ>Uf^h`~L5UzzxF|vzV9RSTPHtXmTwmi!_Ei{m7+oS*; zL78XWypJno}%5Cqfq$LVKgSUsIz^p{P zUpIIk*uLHMr~*Byi|GV1t(xD6u&_E1^UAX|1xv^JI(ah~*7MBOI|M5^H|Ug9R?J?x zUq9^v@i6rHzFlCUecdF$Ua1t^Ow}jlh0^3SDlsSv97LEu5GBi95Ot|01W*3DkW$mM zYvZ0)=@WAx;&I&yeM=Wa85tjcir|Kuw$5#aedZVph*;q9Xs^*@(B^ zzF<-#@T>GX2^NW(Cn>!u?n){1GtB@eRE42QoB#fPhlU_m*2~ZB~uSeV&b%_bBs&(eSp!1OO5MSlRuR*{gxh zS1?_s*^+&M@QS-|(Xsla5eUHGtF*wFX|)-_Ewo7Du#w$pJ7_|^)Oa0Gh)=&cjjeDd z63?jG%o?pK)^7ZYs337xT`C&(w{DP9`U>ytQrp1|)KRe&`TW}}bU*|2enEU4F$=Fc za8~y?ZRR93XOnw*KdK#!y(h@0N*L~u_ zc8n@twZ+_t@Jn!Gd@2bpZr*Ea85C8rmD$e6HL0xazq!u&>rMQl_vp&Yi)V}%g|OFh z^&{P;${XyU%o6Y}rr;;f*HwaWmm2<#5=v4Jm#|uy<1UB96sE^JvsZC~*}A3FQO!V$ zru<9Yyxr{|-y2}ykOuOK&Kt`e5W%@zcfxBj7QEE^mfb4{IFHL+ScLAkj>X7-&y1&) zquY*A9fGs{u!ISjJCoFi5_lqBsK{yuC`Xw&9$JotB^+u+3#p49&^hQ_I~;tJ!EPxh z0~wD3-C|bGl(v4S{-#ph?BcClO}L7_yqK}c+^)aejAWT>F)RL+HL9^drOK^w!fThb zU5TD9ZsGs=bO)(3gb_wi)s??ZUwH8f(#weOQ zNx0HFYK-XldC9*8 zeETI53t2?STP*D;U8pwCrg}L`HJO%wMsK6ha{Fd+FbZlXD?L~GW!dqcc^_t7bqpV6 z_PUl_4^$fHD?l=0Pe z_w(%Zf30q!StOnjJA^&BUz&_c>>%S>o2+4tNMY~H7%K4&M$sCa4VmC|G7oCi-G26X zri|g#`K0t-;YRrZbomKB@yF|lf}S%Q6VdDvd!|dHI#Vu#F{FoBWLEL=tZ{=Thx75?(5%jk0fLZq~Q^@0S_V8N5jf zbVMOn4F{trXGxqJD!89fer-aUu0s!F9tWfLN)?7tn)sOhA>GmJmFNWm?yy0&+RlL; z#)b3n=dxW1Wo;Pi2qr^Lz}-nWz)oI%42A`NSrg6hqC$nV_Sv746Ys~&UgHi1;=>jvb z^^cZvqdtUhrv#~Kq?jbQuDl4>WlCKx9p%*p)1AQ%sb{w z{Te4(mRgstBDcuK;oCar#xY0OyqIke9At4Ouzm>5|x56uwV#va$<&ZBN2w1 zPD9DI>>3tM#Y=a4t$2~`r3uSw^H|W^JfkUB|LG{ogGHcbSNAJc44rR*DpB4YxTfr@ z`U1kkbf}2{3?gUKE)R>_S*jdOUNn|Q^_JKC-YKeP)CLhU4fY2++Wr@#@KlEbe(El7 zF20Ma@lkLr44qyoTY+~97O-aL|Hs>5- zT_E3ZyeK4wgw#ebTf_pv=?^W8yy^6n%HLXQU2UO$31#O=OZd2H834thrLaXJnkhc? zs8elfHIp?4iMfJ}Th)z$vr!-1QceEY)7nZxtJW!tvxEd97aNUJ5YBw^NIOfrI_-5K zZ$TD6ETel{psIa~Gm37ix{7WBsXm}>*-xH|AdK8tcGmiLpP+0D{^GTMx3B}JCwJk% zh|w6LhON;{NU7vGZ*E=s97UBg$1!ppN6bvfdR^CTeAqZWaCR=PtR^oiXCEOtkDiv6 zBDZpWbckbVP7;7Q89v=X(HD75?3DAfKQMh88OMYJsCQMFJQ>*CpS+~p=eAT5JhtKO zhg|3&o^dT!{2);wy@wyHYu5O6#q3pLGVMaK*l9ZC22TR}@=@*fa;TbVt_)8+FRc+K z*LZ8}WP;4Z=`3SZSY8F!|DBO`KAa7z(M0?&)CoM;?Nb~yk6Z6wk1DQ1aEIxHI5-pj z3G;;29rauZ9M}pgtb&IOK1PU}34Aygw)LfT2J|*bZ%HPa3NFr?wnB?(%;4Kb87Jn& z2|~%>wy}hvwBH}Rzf22^(&>OWKHkg(Tbh`kt+Tpt>8Y?{a8W_`tDXsm;i-w0y@Tre zPo!N5NK_5h392JJ5@*vj-l1EN9KUFv`Wq`mIyRtd{0>|mRO5Hgbve8#EkcEahzcY~ zk&lT$iR=CWMgd`fV4@`h&rW!j;ejo};0CL4kz# zZso%wA4MF!Y-ojTDYVqv#4vMFWqqF6R6FRolG)GLMhy&2vqL^2FMJET(sYwZn~cbK zzU(PH^m=%wi{GY{pV$be_j1km(lXI#bu|*P80D{S1k$)(7v;JOSB5C(W2f2I!F~sL zNX)Q9Xoo$FL;VQB?h_(cELc9GKW$e*#xILU;B!tqtwF+va$wMA9VSYASl4HsvRaj! zxkfQGl84LIPG^V>Y0^1Vd6us2@=zLEWgF4}~Fmdvc-036Iv!&%= z^!dv9h$>gCZw8p8fqdKb{6jsXX>Ft-d$pmudBF5Sr`D62A5Tq8DN!M<#Cv7PvhPe4 zCgQCv>qtd(sEa++4PQR5W==kpwJ(j$e8RO(=D$%4ldT)^3##zyGqptiAg=8w7udrA>YEu<$ec`FWj{BEOPjTSE=2TTI_s;sSl{S)Hd+Dkvo z>?ugTe~C|r1tmX1wJK|Q-u%hBdY8?xkcH!|c@@P)xqz(%gLhAT9pbTI3_{>*m};4v z&N`bq{}@-=N1a66Ys@**K!uqC4(y_@f~ztr(y%z5?kknc zqjwyrHmW3R$Dm(yT56CLTu{sp7nC>

z}$o{YF3K;~D-7apA0t;y(7?C{8c>Yh6| z+Lv>;A}B(M&WVwkI?|zf0A|^`=eb~Z`F48P*dm0ZO6#Zxz-dhFsBMvGHyE%7sQPz$ zIShGUrMti~*j9s z%$W0nGy2b=UW40!nixp^4CeG{MCs^t-uA3D47<2s_h5Z6sLK74fdDy{4W+8ue_#~) zaI5^F>DD5zdmwo@5)I=epSE0whX=wl25}gQXeiv!HsRKS06cDGxfVt|Ly!pE%9B)L zpnk=fdnhH&1MXkyj*T>8aqq0kxej`jkF(x3WzAZeKtBwv0^yK>zaT>?sR6HyTbf8b z5ucaZnyF~Vk0b}CKQxVO{o?hGF8}s!M|Na7-==)vR>R|4neZxsDh9j~uMujUHxKN4 zRp=sOSV&o7lc>pMqZ%1aYa`qG$v+W~IG@L#D~}#QmVv{nvOy`T(Av&fNnWm12jn zmnVl$M%|I>&EmIq;*5f&g7^c=k1(ZjM|9d3%7-Rj%MaK?lz>PEJUfR5rZ9(*GXwunU-Td!w7Qh`K;O;pOXS2GxR#n@Q!mX9=PWPM;^UMank*K#bl z+zl(EqSsDQKQcEbAF`N+HRe~W+g6Vdwk_bwQP1I7k}Bg_ey0Vd!+o-0=+xmpq=?I? zl_@W%$PgTa(oizV75u56T=?0^R$F>`fj;qOS2Fu#Arpy^t38arRp`;qFma0Mao9Jm zS)LTQ1UQ86>%}wr9p$$$>)X0NI%V(wmjGfiz5AzjK1&n^F!?t)ec|ohy(!sG27h9) zC)FnaON;F)g*okDy599CKt82W!>NZsf9B4O`rhVvVP1gCDss3 z6;^Dh!x+0o2=uD;P8L_@Z5LDFJxO3nwe6m&w_XO?8Hi~uR-7^18EL2^QWf3k9KWTZ zztej3E_jDxPy=?nW-{<2g{L|EV-<%WagGW`aQ5SlNl>jk;ERD3RTb5M%o$CtfU(*r>C>{?8-dF`{) z2Nd7Iu!GAs4#2y&7h~gx>!+4kbA9LU9iKtvHIWh#M-cjJ)2CQ*U$`!B^f>ne*}J|3 zPwyqWPxL-dRhP`x;%FXS<{E}K_K1ZgAt5ohZJFh|Ei&FqlMW5wit7|%Juk3g%3N)v zR`H7nkY1)6;a>TKK5WF%~ zC5B`zUk%?HP95ls^(QIPuEEkGrl$FJR zD4w_uB-!lWf%R@W1b2Bkx-HDDRT7f;$=!w>(Nr2eweylj^1sP7FB(vcg$6K5*@RYY zI%;JFHZp9;&u9V}Xv*a|vvmv39Je%G($?&aUcLuD0mh3~RJXYrHa*qMGm0uDIRt{p zY4G%aJb9s1c?bLqf(Xn85f+~ugSTxUXxSfJd7uyB^_)j5Swl- zul4md-`E)pKyFZ3tLprHVb?Hww9vp(_PxPVscl{BSaJa4xTF)VQ&eTta->CeM-~z& ztu^tyaSM@N(d8xLsQBYMz%!1fGWu3%0g%urYj|T!ZUT(UH@hU%Ot#=ZZm@Bc8C-3$ z6}kh4`JDAOlsqJknd_9-nS2mFd%61ePQWq%t1(ZcpICfoU>G0VIDJZB4W zRAibJeQYGQU9B~0za;!Sk&-=_Z51NQAL`lA+D!CF9YbV5oj_QT;qXyFzK;-~XMA%Oq_5A1X|5lgn*WfWU)%sQs(;+TKn^rRH}{81 zNGNwuDNO15I(_w4sWn!fZG;74S1CM4`iX^XWN*_Wh)pU_=?ezWiNJno_9R?%Rh8i( zIAzh5Yl#A3)?Vjz%v%c2Ea4M;8M++sK8oW_`N|ovS^ho>L+9Q;# z`30agnaNJ4n-~O$SFvuoKM>#_B-^{A36*ebP#F6ggrvcdg_KWI%Bdf%sylPS=zw31 zh8ayX6YXzoB@T1nsd-W8mQ#v<<8RfvC+)t_51i^j0yM2#Y*en1Di}u&3 zNO(1X;r!LxzJjIt#m*arif(MlN)2tF<(OJ#$sh}!F zeo}%CqQ{Qt0V*?`kHNt(|HA-EUu zU%v&cf13Hh0`zXYG)H3g&z&k8CJ6*xTFn@zm3R_yTY+74OeQ6ImGtemvvrDYQ;$l# z&+(F4P6&&WYnSZtEmCW}J|Ux^<%?K;bpS=@TSiI4Z@Pz>*2~y=_mh_g1)|H_>iGoe z54YJSS+5RFO`EsP^u%0F|GX|%)U2#~v!6Rdyq-T6JiG_EN?fNb6knz%l{RaYUn+yd zKfJ}^;B;s^U}k=Xa)=BWp2P|%zp=FJ*p+KjSMo75=ui_?-2s;nqV$~M9uLC}N~u() zRK*Z)-Yi>48!dk_eghxU4gk|klcJXacdAh?`hYSY=;W?QPm#FGkn~nfr4lUifW6Bh zx%Dy-Z>EWPIMZi8yG{KlS-o|nX%Z(|_ny8I=DpdjQ={0gQqZiS$55D7R|hXrrAtEl zT7*-N%2nnw{acwVE4!p{f~zUcVM@!x$+ndzlQZ88(3xteP^$n;N*%Dvm^Hb%loGD& zc~;?JH83~0&N8USs3&E_sVAAHqH`4UNO@R$roYqDc}I8?Hjtdxm*o86wfdglKjyk1 zxkvZlYl3quSmj6C_+E2uhfnl*s7kH? z_)W(!XV-I@96ygHrsAxbSWr>hyML-AVvSwLY7z~|fG*qjBh5XVPoqo~ZSGTdoQaSibVn3+_1_MpA*@>|hm*)*YG z`t8qi$29k>dc}u*4s3y_XqVob#)0@j$6m<_vHoY7mUBz&MX4681v;%!>Yk~&eJNt;h$u&RYJJItJv{4bQ-877BQR`;3Stxi~lP31q@W2>97B0%CuKZS8%OYmTQB~%C?B-F}p zzN<$A4GHlf(%ip1F!~PK&M8hCa zA+n979wtk6Lbf4{8Cw$BCCk{$64@zcMz$$gV(bai*oG{bkTu&RG-R(QjIF$P&vAUe z@Bi=~-9OzwoyU27uFv^7&-=R0`?`iR2l6?8Cu=`%XRa@K(QZrgGAW3Lilvy;&AjX; zwfMTVRiFv#uj^g9@7{(^9t&VpG^9nN?jtOt=uz2jaen#}HqL=t+> ziQO$x)46p3a%e6A~GPFPRqp>Dg-dRa5A5RI?QH98uc_ ztMIr#SEA!A&6{o_CfN(cR7clObtx9uKl_(%zVoPb@a(uzQ6ebsjJ`IX_56z+geaGZ zBH9fzLSq-?=U=eAGb)dYdfRAO+^(XTRpVER`4$r7cPKpVx&u%{Jud-lyRO{Rwc+PMtRg4jLC7{0stxS#cmGia% ze*ERUPM<7BE(!!MDD4BIlF&Nc3uD>+^nN03%RpR1Y5Gno+6xhKE!!{zGH`r-n5iVk)Gx*1r+=76>py``MDr5AYFCK5-u%MIw(ene^T-O={{;prd8oEN?4kQfzrd?2 zYaUYfN^yT>HtF~jHoB6b>P#4s_`RzF*w+nCuJpCJA; z;fF+YEbQ~X;!54?tz`U_h&K;Ojlq5T_i8|S>r}HuVXS{@U9<02HQA`&vcO|Nj^WP&lCaoQ!-;|G=b()*^UbIX*+A#dz# z9YwVG$8S#!mO%P)xNCoZamW@%taZYI5gQonm6wo$e*VfPnMe~Ii2_&nf~QA=YPY%J zMB(ppO_TRNHsKFaCSYqY<(z4@!m4u|KF`mR%)UBrUWn`;&YDv1zX+FIT^?UhFWqM2 z^*O1-z~T+ICf-$uaflAntR#)T*xVU5Yc^T??df)iEG+f+OiYzybk88Mz3FlIk-JlV zK;9)b`0aX3D{fW4J}uSU$u4%_PG9I_Othx(%f1C=wF=9I@sT= zP0wyDBt`h^LG~-PGeYbVsqg;uTufHxBBO&_`_52$3EwmWTS8<)Wlk*9i(_^@(@gp7SHe9~g9aWf`9KqFQAju^%Y({%SV) zxVFs`Jjw@Sy(!wx{(m&97Yh}Z%;c!wTKVWGXcih}r)V`5bPw|2HjDR&?}gq`wF3Ms z%>_C!YwFc{)H&7~f*V1-Q3~_9bgS~N50zKKG0?h{7N9*7rS#S?C9oLo(keKY>X!|r z*~#FantWkIRbf3Xdgi}fXb1?t1qn0G|1r>Ao^+OdV$79>lzjJcaQ)bsb+kstp~erm z9H2jdGFmke|8GX4O|+k`=-E5(5tV^%-Qfk1b( zD42S?V8XSgrV86?c-=Y(gZ-FV=SnN08gF>Z8`HuFu*FhVm-n%-sv;%ZdgN5UT_qN- z1xZPNXb#W_e{y*C4wO)usLCUTquPdALBf=G9r#7T0#b~;l`#vn1r6qa81(D(s9RKH zWe@S(r4T%vRvlH|+H+zgGf70mLKHDY^Ib1MwHh>ovQOOcB=xDN;tH&vQ4q(ZO@zP@vYB9~>Es>@~Z zR+Ct+eomKhQXMz|XpH}x`a6fpi>?~smvMRnqG+(d zDF95eKdmAKqEgh=nCoZVY)QtvZnqu3J7DE08N$p2j_%;NzBsAWIUiK4n^$m6~3kPx^siH8JNm+I|i=W*SJ=xT@5^(%4Mc|oHE`^9{fDv zkYLGP)|oZ8od#s_Ig5Z7R^rZ(D75-7>!W4oPw8cDVRVEXM-@3CO`%)bDFd&=z+{rE zP1Qi2SAW_stm(&d-fSTA@ug$8xSk2%XNRK)d)A145IpXAmDcfm_%)HWkv3S^Z%O!rPHY^A(gP*M-kA*A!RNy~)QUi`= zV2+Irg4)+Q+_lczm8iO}hTjc|49e1_h7$N>jk6ef?&bjUlyD<2Vu$B@HYbI5({@ic z|9cS`jee*tH;lAUPOpz)U>5QJhml~~=yV>lvr0a%xm7~c*hrx%vS&y)r&CUNZ*#|V zeE;K91}sGFSu~#vMOZ3y4E|)$%VspkxtIT8Z32MQ+(Mj5V&+;S;3z~CdGu;u0N-a@n@@5hIpUYs zobO{fnMD|4a-$1oH0r*^%}3C%IT&cla_d`ra{OaC+W$wI=y^KELF%{pxjY=7Hv)3< zx4)rpckZy*Dk}HC>1t)RR(iCBk^Nln6fC%gN#Kz<47O9xJh^7`+*y$wAoR=UgBlTR zlgV!FNj1`uF!RYQT`wE@gEXmk$$UnWz*K24cZS=Pj=W6wo+CKmqAcg`S$2fG{nKZ} z_VsZ<@slr0CWYFKE>W^ZK^0i+kaUIchAB-RX5>z_?oNL570<(V(;V2z}zbT*?8Ogm<P4KI^2#CeaOU`r9D%VV}O*j1E{K0mlNcw7^e$NNCXF&<-AeLZ~D-x5H=0*E@on&X@cvKi=tBvFi4jVVXVcLxJ(n4B4@}<(q$@VM_HoB zOw|B(-WVdUWV9F9Dj-5zr^^gS90&!C8wx(}Wd?5*B+~>jWvJ1KwN_Iqw(;(UmICFobk%i z=WbAD0#{|EMk;_N=xeqm6Y<_koknSPP&>b!KfH70sfA6SK&j=HN6jdfAI2Z91<7Uw z1dC@w?Q2@5yWlXkZMw>Eq&w^XLQxnWT~!&-Vzde|RhDkK+7`iUsSy`w=@9t93%CQ! zt7`P$JLqV-)kdtKn~4Y*l&TtYRDBSoA=B+ekFowt(y$Ryn!^Yo-ER=pG+%rR310VQ z-{cAEf=Y-@T!`tIiW0s8UE4cVg%=C^w8t!LDkiY4#%-aK2iP=qOOUYNDc zZ`E~t9C)*__|ZK64agjzzbWX^&G8$U{7EPjaoSUcAAZq`kS)ohQB15>v%5;b!oBG~ z*{@nf^q`NbnHm4n=u&2f=XdvCX*Tnsb?i$pzGr&mea_Do1>#9Q}5fQM!m6#*>18T6|d&g+Fl=>^6!HUkeD}!8rxRNPk zt(loQl`@K6*Rh<}$vpV$4KN36hp%MKfAO9gi3@iYPSr4JnLtZq`%uRSe z5r{q@e9tk;6edyrx!3|?)apePS6UEl*8NVt{fon268sH@ rze?~|5&x!xziILRPEDY9>H9I`09f3GMVFe#06aIYo9kEWI>-JG=7v}j literal 0 HcmV?d00001 diff --git a/recipes_source/distributed_rpc_profiling.rst b/recipes_source/distributed_rpc_profiling.rst new file mode 100644 index 00000000000..da9c003d1c7 --- /dev/null +++ b/recipes_source/distributed_rpc_profiling.rst @@ -0,0 +1,314 @@ +Profiling PyTorch RPC-Based Workloads +====================================== + +In this recipe, you will learn: + +- An overview of the `Distributed RPC Framework`_ +- An overview of the `PyTorch Profiler`_ +- How to use the profiler to profile RPC-based workloads + +Requirements +------------ + +- PyTorch 1.6 + +The instructions for installing PyTorch are +available at `pytorch.org`_. + +What is the Distributed RPC Framework? +--------------------------------------- + +The **Distributed RPC Framework** provides mechanisms for multi-machine model +training through a set of primitives to allow for remote communication, and a +higher-level API to automatically differentiate models split across several machines. +For this recipe, it would be helpful to be familiar with the `Distributed RPC Framework`_ +as well as the `RPC Tutorials`_. + +What is the PyTorch Profiler? +--------------------------------------- +The profiler is a context manager based API that allows for on-demand profiling of +operators in a model's workload. The profiler can be used to analyze various aspects +of a model including execution time, operators invoked, and memory consumption. For a +detailed tutorial on using the profiler to profile a single-node model, please see the +`Profiler Recipe`_. + + + +How to use the Profiler for RPC-based workloads +----------------------------------------------- + +The profiler supports profiling of calls made of RPC and allows the user to have a +detailed view into the operations that take place on different nodes. To demonstrate an +example of this, let's first set up the RPC framework. The below code snippet will initialize +two RPC workers on the same host, named ``worker0`` and ``worker1`` respectively. The workers will +be spawned as subprocesses, and we set some environment variables required for proper +initialization. + +:: + + import torch + import torch.distributed.rpc as rpc + import torch.autograd.profiler as profiler + import torch.multiprocessing as mp + import os + import logging + import sys + + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + logger = logging.getLogger() + + def random_tensor(): + return torch.rand((3, 3), requires_grad=True) + + + def worker(rank, world_size): + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "29500" + worker_name = f"worker{rank}" + + # Initialize RPC framework. + rpc.init_rpc( + name=worker_name, + rank=rank, + world_size=world_size + ) + logger.debug(f"{worker_name} successfully initialized RPC.") + + pass # to be continued below + + logger.debug(f"Rank {rank} waiting for workers and shutting down RPC") + rpc.shutdown() + logger.debug(f"Rank {rank} shutdown RPC") + + + if __name__ == '__main__': + # Run 2 RPC workers. + world_size = 2 + mp.spawn(worker, args=(world_size,), nprocs=world_size) + +Running the above program should present you with the following output: + +:: + + DEBUG:root:worker1 successfully initialized RPC. + DEBUG:root:worker0 successfully initialized RPC. + DEBUG:root:Rank 0 waiting for workers and shutting down RPC + DEBUG:root:Rank 1 waiting for workers and shutting down RPC + DEBUG:root:Rank 1 shutdown RPC + DEBUG:root:Rank 0 shutdown RPC + +Now that we have a skeleton setup of our RPC framework, we can move on to +sending RPCs back and forth and using the profiler to obtain a view of what's +happening under the hood. Let's add to the above ``worker`` function: + +:: + + def worker(rank, world_size): + # Above code omitted... + if rank == 0: + dst_worker_rank = (rank + 1) % world_size + dst_worker_name = f"worker{dst_worker_rank}" + t1, t2 = random_tensor(), random_tensor() + # Send and wait RPC completion under profiling scope. + with profiler.profile() as prof: + fut1 = rpc.rpc_async(dst_worker_name, torch.add, args=(t1, t2)) + fut2 = rpc.rpc_async(dst_worker_name, torch.mul, args=(t1, t2)) + # RPCs must be awaited within profiling scope. + fut1.wait() + fut2.wait() + + print(prof.key_averages().table()) + +The aformentioned code creates 2 RPCs, specifying ``torch.add`` and ``torch.mul``, respectively, +to be run with two random input tensors on worker 1. Since we use the ``rpc_async`` API, +we are returned a ``torch.futures.Future`` object, which must be awaited for the result +of the computation. Note that this wait must take place within the scope created by +the profiling context manager in order for the RPC to be accurately profiled. Running +the code with this new worker function should result in the following output: + +:: + + # Some columns are omitted for brevity, exact output subject to randomness + ---------------------------------------------------------------- --------------- --------------- --------------- --------------- --------------- --------------- --------------- + Name Self CPU total % Self CPU total CPU total % CPU total CPU time avg Number of Calls Node ID + ---------------------------------------------------------------- --------------- --------------- --------------- --------------- --------------- --------------- --------------- + rpc_async#aten::add(worker0 -> worker1) 0.00% 0.000us 0 20.462ms 20.462ms 1 0 + rpc_async#aten::mul(worker0 -> worker1) 0.00% 0.000us 0 5.712ms 5.712ms 1 0 + rpc_async#aten::mul(worker0 -> worker1)#remote_op: mul 1.84% 206.864us 2.69% 302.162us 151.081us 2 1 + rpc_async#aten::add(worker0 -> worker1)#remote_op: add 1.41% 158.501us 1.57% 176.924us 176.924us 1 1 + rpc_async#aten::mul(worker0 -> worker1)#remote_op: output_nr 0.04% 4.980us 0.04% 4.980us 2.490us 2 1 + rpc_async#aten::mul(worker0 -> worker1)#remote_op: is_leaf 0.07% 7.806us 0.07% 7.806us 1.952us 4 1 + rpc_async#aten::add(worker0 -> worker1)#remote_op: empty 0.16% 18.423us 0.16% 18.423us 18.423us 1 1 + rpc_async#aten::mul(worker0 -> worker1)#remote_op: empty 0.14% 15.712us 0.14% 15.712us 15.712us 1 1 + ---------------------------------------------------------------- --------------- --------------- --------------- --------------- --------------- --------------- --------------- + Self CPU time total: 11.237ms + +Here we can see that the profiler has profiled our ``rpc_async`` calls made to ``worker1`` +from ``worker0``. In particular, the first 2 entries in the table show details (such as +the operator name, originating worker, and destination worker) about each RPC call made +and the ``CPU total`` column indicates the end-to-end latency of the RPC call. + +We also have visibility into the actual operators invoked remotely on worker 1 due RPC. +We can see operations that took place on ``worker1`` by checking the ``Node ID`` column. For +example, we can interpret the row with name ``rpc_async#aten::mul(worker0 -> worker1)#remote_op: mul`` +as a ``mul`` operation taking place on the remote node, as a result of the RPC sent to ``worker1`` +from ``worker0``, specifying ``worker1`` to run the builtin ``mul`` operator on the input tensors. +Note that names of remote operations are prefixed with the name of the RPC event that resulted +in them. For example, remote operations corresponding to the ``rpc.rpc_async(dst_worker_name, torch.add, args=(t1, t2))`` +call are prefixed with ``rpc_async#aten::mul(worker0 -> worker1)``. + +We can also use the profiler to gain insight into user-defined functions that are executed over RPC. +For example, let's add the following to the above ``worker`` function: + +:: + + # Define somewhere outside of worker() func. + def udf_with_ops(): + import time + time.sleep(1) + t1, t2 = random_tensor(), random_tensor() + torch.add(t1, t2) + torch.mul(t1, t2) + + def worker(rank, world_size): + # Above code omitted + with profiler.profile() as p: + fut = rpc.rpc_async(dst_worker_name, udf_with_ops) + fut.wait() + print(p.key_averages().table()) + +The above code creates a user-defined function that sleeps for 1 second, and then executes various +operators. Similar to what we've done above, we send an RPC to the remote worker, specifying it to +run our user-defined function. Running this code should result in the following output: + +:: + + # Exact output subject to randomness + -------------------------------------------------------------------- --------------- --------------- --------------- --------------- --------------- --------------- --------------- + Name Self CPU total % Self CPU total CPU total % CPU total CPU time avg Number of Calls Node ID + -------------------------------------------------------------------- --------------- --------------- --------------- --------------- --------------- --------------- --------------- + rpc_async#udf_with_ops(worker0 -> worker1) 0.00% 0.000us 0 1.008s 1.008s 1 0 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: rand 12.58% 80.037us 47.09% 299.589us 149.795us 2 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: empty 15.40% 98.013us 15.40% 98.013us 24.503us 4 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: uniform_ 22.85% 145.358us 23.87% 151.870us 75.935us 2 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: is_complex 1.02% 6.512us 1.02% 6.512us 3.256us 2 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: add 25.80% 164.179us 28.43% 180.867us 180.867us 1 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: mul 20.48% 130.293us 31.43% 199.949us 99.975us 2 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: output_nr 0.71% 4.506us 0.71% 4.506us 2.253us 2 1 + rpc_async#udf_with_ops(worker0 -> worker1)#remote_op: is_leaf 1.16% 7.367us 1.16% 7.367us 1.842us 4 1 + -------------------------------------------------------------------- --------------- --------------- --------------- --------------- --------------- --------------- --------------- + +Here we can see that the user-defined function has successfully been profiled with its name +``(rpc_async#udf_with_ops(worker0 -> worker1))``, and has the CPU total time we would roughly expect +(slightly greater than 1s given the ``sleep``). Similar to the above profiling output, we can see the +remote operators that have been executed on worker 1 as part of executing this RPC request. + +Lastly, we can visualize remote execution using the tracing functionality provided by the profiler. +Let's add the following code to the above ``worker`` function: + +:: + + def worker(rank, world_size): + # Above code omitted + # Will generate trace for above profiling output + trace_file = "/tmp/trace.json" + prof.export_chrome_trace(trace_file) + logger.debug(f"Wrote trace to {trace_file}") + +Now, we can load the trace file in Chrome (``chrome://tracing``). We should see output similar to +the following: + +.. image:: ../_static/img/rpc_trace_img.png + :scale: 25 % + +As we can see, we have traced our RPC requests and can also visualize traces of the remote operations, +in this case, given in the trace row for ``node_id: 1``. + +Putting it all together, we have the following code for this recipe: + +:: + + import torch + import torch.distributed.rpc as rpc + import torch.autograd.profiler as profiler + import torch.multiprocessing as mp + import os + import logging + import sys + + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + logger = logging.getLogger() + + def random_tensor(): + return torch.rand((3, 3), requires_grad=True) + + def udf_with_ops(): + import time + time.sleep(1) + t1, t2 = random_tensor(), random_tensor() + torch.add(t1, t2) + torch.mul(t1, t2) + + def worker(rank, world_size): + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "29500" + worker_name = f"worker{rank}" + + # Initialize RPC framework. + rpc.init_rpc( + name=worker_name, + rank=rank, + world_size=world_size + ) + logger.debug(f"{worker_name} successfully initialized RPC.") + + if rank == 0: + dst_worker_rank = (rank + 1) % world_size + dst_worker_name = f"worker{dst_worker_rank}" + t1, t2 = random_tensor(), random_tensor() + # Send and wait RPC completion under profiling scope. + with profiler.profile() as prof: + fut1 = rpc.rpc_async(dst_worker_name, torch.add, args=(t1, t2)) + fut2 = rpc.rpc_async(dst_worker_name, torch.mul, args=(t1, t2)) + # RPCs must be awaited within profiling scope. + fut1.wait() + fut2.wait() + print(prof.key_averages().table()) + + with profiler.profile() as p: + fut = rpc.rpc_async(dst_worker_name, udf_with_ops) + fut.wait() + + print(p.key_averages().table()) + + trace_file = "/tmp/trace.json" + prof.export_chrome_trace(trace_file) + logger.debug(f"Wrote trace to {trace_file}") + + + logger.debug(f"Rank {rank} waiting for workers and shutting down RPC") + rpc.shutdown() + logger.debug(f"Rank {rank} shutdown RPC") + + + + if __name__ == '__main__': + # Run 2 RPC workers. + world_size = 2 + mp.spawn(worker, args=(world_size,), nprocs=world_size) + + +Learn More +------------------- + +- `pytorch.org`_ for installation instructions, and more documentation + and tutorials. +- `Distributed RPC Framework`_ for RPC framework and API reference. +- `Full profiler documentation`_ for profiler documentation. + +.. _pytorch.org: https://pytorch.org/ +.. _Full profiler documentation: https://pytorch.org/docs/stable/autograd.html#profiler +.. _Pytorch Profiler: https://pytorch.org/docs/stable/autograd.html#profiler +.. _Distributed RPC Framework: https://pytorch.org/docs/stable/rpc.html +.. _RPC Tutorials: https://pytorch.org/tutorials/intermediate/rpc_tutorial.html +.. _Profiler Recipe: https://pytorch.org/tutorials/recipes/recipes/profiler.html diff --git a/recipes_source/recipes_index.rst b/recipes_source/recipes_index.rst index f8986363092..3a05d729b67 100644 --- a/recipes_source/recipes_index.rst +++ b/recipes_source/recipes_index.rst @@ -166,6 +166,13 @@ Recipes are bite-sized, actionable examples of how to use specific PyTorch featu :image: ../_static/img/thumbnails/cropped/android.png :link: ../recipes/android_native_app_with_custom_op.html :tags: Mobile + +.. customcarditem:: + :header: Profiling PyTorch RPC-Based Workloads + :card_description: How to use the PyTorch profiler to profile RPC-based workloads. + :image: ../_static/img/thumbnails/cropped/profile.png + :link: ../recipes/distributed_rpc_profiling.html + :tags: Production .. Automatic Mixed Precision