From b3dc119f340faf871099c4c6518b929f6ab67586 Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Tue, 9 Jul 2024 17:02:45 +0000 Subject: [PATCH 1/6] add support for confidential pods --- charts/latest/azurefile-csi-driver-v0.0.0.tgz | Bin 13537 -> 13611 bytes .../templates/rbac-csi-azurefile-node.yaml | 53 +++ deploy/csi-azurefile-controller.yaml | 6 + deploy/csi-azurefile-node.yaml | 6 + deploy/example-cc/cc-deployment.yaml | 53 +++ deploy/example-cc/cc-nginx-pod-azurefile.yaml | 24 + deploy/example-cc/cc-statefulset.yaml | 44 ++ .../shared-pvc-across-runtimes.yaml | 72 +++ deploy/rbac-csi-azurefile-controller.yaml | 23 + deploy/rbac-csi-azurefile-node.yaml | 45 ++ go.mod | 8 +- go.sum | 21 +- pkg/azurefile/azure.go | 16 + pkg/azurefile/azure_test.go | 69 +++ pkg/azurefile/azurefile.go | 11 +- pkg/azurefile/azurefile_options.go | 2 + pkg/azurefile/directvolume_interface.go | 54 +++ pkg/azurefile/directvolume_mock.go | 120 +++++ pkg/azurefile/nodeserver.go | 95 +++- pkg/azurefile/nodeserver_test.go | 96 ++++ pkg/azurefile/resolver_interface.go | 32 ++ pkg/azurefile/resolver_mock.go | 62 +++ pkg/azurefile/utils.go | 25 + pkg/azurefile/utils_test.go | 65 +++ .../Microsoft/go-winio/.golangci.yml | 27 +- .../github.com/Microsoft/go-winio/hvsock.go | 6 +- .../Microsoft/go-winio/internal/fs/doc.go | 2 + .../Microsoft/go-winio/internal/fs/fs.go | 202 ++++++++ .../go-winio/internal/fs/security.go | 12 + .../go-winio/internal/fs/zsyscall_windows.go | 64 +++ .../go-winio/internal/socket/socket.go | 4 +- .../go-winio/internal/stringbuffer/wstring.go | 132 ++++++ vendor/github.com/Microsoft/go-winio/pipe.go | 22 +- .../Microsoft/go-winio/zsyscall_windows.go | 19 - vendor/github.com/golang/mock/AUTHORS | 12 + vendor/github.com/golang/mock/CONTRIBUTORS | 37 ++ vendor/github.com/golang/mock/LICENSE | 202 ++++++++ vendor/github.com/golang/mock/gomock/call.go | 445 ++++++++++++++++++ .../github.com/golang/mock/gomock/callset.go | 113 +++++ .../golang/mock/gomock/controller.go | 336 +++++++++++++ .../github.com/golang/mock/gomock/matchers.go | 341 ++++++++++++++ .../kata-containers/src/runtime/LICENSE | 201 ++++++++ .../src/runtime/pkg/direct-volume/utils.go | 126 +++++ .../moby/sys/mountinfo/mountinfo_linux.go | 50 +- vendor/modules.txt | 14 +- 45 files changed, 3300 insertions(+), 69 deletions(-) create mode 100644 deploy/example-cc/cc-deployment.yaml create mode 100644 deploy/example-cc/cc-nginx-pod-azurefile.yaml create mode 100644 deploy/example-cc/cc-statefulset.yaml create mode 100644 deploy/example-cc/shared-pvc-across-runtimes.yaml create mode 100644 pkg/azurefile/directvolume_interface.go create mode 100644 pkg/azurefile/directvolume_mock.go create mode 100644 pkg/azurefile/resolver_interface.go create mode 100644 pkg/azurefile/resolver_mock.go create mode 100644 vendor/github.com/Microsoft/go-winio/internal/fs/doc.go create mode 100644 vendor/github.com/Microsoft/go-winio/internal/fs/fs.go create mode 100644 vendor/github.com/Microsoft/go-winio/internal/fs/security.go create mode 100644 vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go create mode 100644 vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go create mode 100644 vendor/github.com/golang/mock/AUTHORS create mode 100644 vendor/github.com/golang/mock/CONTRIBUTORS create mode 100644 vendor/github.com/golang/mock/LICENSE create mode 100644 vendor/github.com/golang/mock/gomock/call.go create mode 100644 vendor/github.com/golang/mock/gomock/callset.go create mode 100644 vendor/github.com/golang/mock/gomock/controller.go create mode 100644 vendor/github.com/golang/mock/gomock/matchers.go create mode 100644 vendor/github.com/kata-containers/kata-containers/src/runtime/LICENSE create mode 100644 vendor/github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume/utils.go diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz index 50e43d5ea84d2232598dbb3836d6231fe8a1e467..bf65de26d8136f751dc741ac4012e47b7eb3dbd9 100644 GIT binary patch delta 13282 zcmV<8Gaby~X{&0GO@GwIaxYIfHZ+pjtwPP;!ZRFxz#DgreCiW1xHh)$QTcyzk$#d7H|#`>!Y#HyJO;WCfGoBeKy`e*{oGX z5#kUrK_uhTKH9qeUTpb-Y!yUF7n69HsuA^f6o=s%3yDANqm#iYlV=Xa&yW49~<$pN{^7}m1aq~g0s_pLf&g{BE zG64!goN;#L>O)6$$tRKeBv;{1cYAMkg_H$wq-XOv42h5&&(TWtvDF10Pd+cQFtg1Y z&z_-k7yt+A1Ck6e3W#wuW*m?JpMwtUV#{ZAK!z~~FJgh`Co9vIqUUL1Uc|`+OGw%?-P+K&J5hYn*Q*%*cR3PRlu90Z6^$g0$B zBtS|vIeiv~f$Fn>2z(VnphPNbBnlV^M1Pq?0YpFp#3=gt;w1Te1F0Fpi7(C=mwmMJ zpZnW;ZVfpC9LiBvO!3ZR`kwxbg;e9M1v>fB3c^{4DP*s(l-M7g!#`ufA;^9Y+3+1q zAnc=8>aYAH#gsCMmA#w%60x8#jUyHuR0?qrEapB4!@O%jQ>`2mDU!bs@}szqwtv$< z$1v84`Bks?GqJyN_-72F`cZFO`D`cos=Tz_XT=wG0?N57-&wM9hX}QcO9s+C zwOQ!^-%j;TN6GGBKq!$@E0apSUO))GWZeEFSqR(;t|sfDYnb*?N2_AaLg*Hk*|DB% z9m@iX)Ypny=lGTgDUug37=SPPsDE?HF8mP$aR{AO^sVNI)V#g(TD&Ey+0YBvO#L~= zw;w2;U=k|jXvX|uw7!0OLPJ8~7Z%22_z9*5z7InDjmXhS090yGNpxaGFTZN1mv>A0 zbR$`VRdJwtjmTS#xngrHT)Z`<3iPkg%!LF2M!x!5q~ZmCXD_l zGa~9LDz4|VB(k)=xmLL98dtN*Ro1!Inr?RqO}8-sEMpFtK)nTNWcos4Wx89DH!sbb zpOty{ma{lYUk`--I~k92z`@@*k?@}SpytF67pez0HjFu*8FWgqcYQ&+ddAv4v9a$z z2;c`ppwepKIQjb`9t_BBAAfb;UtD-22*;gdAN`hH9R2hzoiLpadWKSkfsFG-{rrM5 zHLt|WO>}hp*AH(u(aGr#?>Euk4$efd@CeZTlN>%Iv1#o8SA3}t_Y@1Hq5I{G=P9{tkj#5hcn<~ zB7~9X-s{(|(_TsT$x7OdvP*jD4y?mgrpl^plUiTE-Bmf$97?{}TdE?5G2>j*9d?kz zQaVU}+DN!jdxoQ!6Ms6iW&okXlObj4&&Rjm$J#Y+S_)g$K`cPc$TQBAfsi`) zMGIZw-4N>17wL?Ce%Svw!Te)Lu0GlypSx!Dqo0Vc782IIWq&5(>?(Jgs%$+G7u<;K z)lS6B9xs{VOe|{Vucxt{#*ANw3@-;lp z_@9Jvq{B8_r|&P1FS_zpE>p*>`2X(yb}9b1|9Wq49sj$J=Thl~Tu}ew1ep~|&HbVE z_y!P#YzhIA%)|=pwC`?gTr%X3z`w2`60txaA+e0bCVz@Tzyc7DX@4VG{$+?fk0$<& zH=jSF?m6gk-PG%WzI;)v7-F6$tX_W8!BJFNTNI@V5%lFt2YKF&Qrl~#P<&?0|4$K1XlduVYf7{>P zeO+AtceZz5@2}VY`*=!|Egfg;9@I0y^Sq5bP8df7ZY5Cd{ODePZ&cX!#xgxf1<{VZSC*B>LkD)!ssO#gOGR}^%0H3FpUXKbdV|e zI@fi=z@BZ#Y={UnkZ5QkQFX-(S=$??CvAYseqh~|LjL4<`HmzI%P{6RtZ-+dgd@gH zEIIMfP3#LohjEB``Lhj#gwG=AqjWD8Y@o!T(o{TzgMb)E)0r}jfFCLvf0-Y!fFD1h z0r8b)Q9`GEO13Js-Q<9SX&w_O8RDS&2@tldb*+EwAA->{EJuCRHQURs{`!)A5Q*E( z4AXh}XQ^YPSB!;#sguV6!id66#v26)L=h;qX)sZCi4%FFIQ8J^_UT!G&VD&W1J1_i zW<>mv#Zx8ZZ?Hfw7*Ftfe=n0dTy2fO?;4t2wJ#$=0=1FQ!w{44sd2q$?Z424&we>9 zG+#(=uvqTar>x=73<_xXY(t~ddR5a7O|^;fG?GGY(n9L4hx@WbELTg zCNAnV9Z)LOvNO&T7EN2ngpzSQMuPmV+iVy}UCc&aN7H~SBTNl47e(#1Q*3#$h;3iQ0h{#^t4oTroK+v7q;y zZvG`hKt;^6z<~Z9e{(<)BW5CCi2(Z}#N-Hg7yZ@p^7Ds_OQn7cuMntxDB}jei-#q( zLxgRAy`){Sm&J4gsgo|72HCBKO1IG?)RSZkMfE4~HPA#1V;TStr+GzU7fWQ2 zPxZ~pFWQAh99CbL7Ni&%FHbJ9Rk{@`pg{FtSVPR32nGcsfBX@QaX$-s(Sx&-Uv@7F zKciyzm1K$tgmu{Dc{*+N77K_H6Kks|=_z({E|`>dH&Y5eTRW^0+eD0~f-jjW!ojQ6 z$OuK8MZhH~hOBaCVj+-KXxnque7mcO7>_whT_l!({9^)zEptVX&bM)+C zDfF{4QwSZf?7YH|$izJM*?1gNBBut4NUmbZxY!C{0^yb*Ll1L*L?rk!=CGw~Tg^Jv zCUSQiJWJ!&&kMX(rlEPX0ogDD%7!#TWj-~ns4mIpf5#V>X~6BDfW0W}Eo6y6V9_#K{}KKiv&JM%lg&G@ob6e?fVY|v(C-sg@jqLwsL-RL`tIy!UK{?bK8 z*QhLqE0-DxVIUDDp_;_Rx{}Nu&McrNvRK^ZW>Odn_k=${LoxxHpc-`bBRl~#jzdYJ zFr#c3E>_@P9T?5Nih$=Z5nbbADe1jbe_@$;Ibl?fis}=6GFXn3F;Clym{t>dCxq=F z=4#MZdx5AP)hhss#+YAMY$(Ha(#}n@Y0Sn*b3DK#)Rc!nK|6Ysl2nFb7wMBYe-4_~ z#U2ZT>O@W+UZ>;s0bG23|nO7ih)L^uApg8v)LiMYN^|_`&9HJLjQ?;8R0k&oj zmh-u3O0}51O2IAj5?#SS305pje-#yni~Y3LfNXZZ%+l1&U$;=$&W2`Pn)k2~;Wp8n zuCMG$|}awe`W}HzGx!N za)v+ArFL`Pvl<6u*|CUDC<>+&kBN^&*{57>3-Ymm&72lbSG55N^~w0O&Xk$TnZf*% zBBnb@6`xe&I;np~#+=$suU_gp#WkwqpgBB0J48MX!!G(kFO*KtUPCLE^ef+p_6sFc~0Br-kiUnzvyDVWd_{ z$xs|A<1~Z`ggLc>}M#QfHA|jJGjFl=*(l zMKTP-SErtk4TowftHp|>UMFhD3=@h3P(dV_K;HUvEJ0|t zM@BlAph}0y=H>kSt9ZgV>oX6ufbTsOELGUUSy4U-kjnP4q|7_KgKKr>v6XZ=#y zFcsP?p1CDRv-?T1Pq6^ks?v7tY+flF42&iVpoUGulv8uHe^@$aE|jhnL2ep@?uOLZ zEW)IT>Rj>|I*xVLNfeC%TPTQXAfX`2A&aI}wTi~ccSVYNs0w!DWGP8~N~XM@odM`@ z%e`5UQn614Di82zoCz*_av3vHirX$wy*xYqjlBHhR~nD6f6ZTiy!aK4e``tP<;(y2Q-zgV zF9+RJJ3hueGw;)b;knUqEt6G$nWwvGqxLIfcjxHaePaHX_^Of$#k$qZS>!b|(v1@Q zCLxjEJQQ*bfxIUODi<-Q- zDWr@De-tRq;%D0Z89F#S$z#9Fa3_Q~_&_FV*}K7<>bROsPoZ&Vigd)Nu*+daJhyaP zhM7HSDhkofglcMnDwAKM)C3x1Jq41s<@qdDM;QH^q0>j5nj;@u37IO^GvoUI(FuC% zDU?1o+uw%+oIW9#>1u$bYba?wlypVl(KTGOe+^vp9sr_`3ljR&0HDi4J(mP^E(zaU z8mRg8LNZqXzFZz=xip~i4xy6|f>&G?owy1t;_}GDrE!MKq6$}tA-n^`;IjC@<i0(NU{HMzo0p_Ve*B|Z+MvAV=-TH!yK-{rW!VaZo2#^|I;iq$q`84V;$a0 zb|iJ)l4Q#vN1VmOBCOWs1`zsCPM402T8W_7eV29@;?^@M95M}iW)5STQL?wiCkksSPIP6Zc#O2m&uRY3A z7>6LCRX2YUhjS2yJZa1UQAV>H7=lNDyeO@4>jYm8JGnRu8^)qoQ0>B7Y8jWSv8Zzv z(lsTr^_mA=82d6E0WGYZb5Q&Fe~uopj#fC(DH~{Mp=594bXzCd)Hn?UibhjGd>qyt z{AIM>?WuXYhDdf(rdm~As-l*a*_D?RcJjEv4M5{@Ie=u?F8ydavXwxlS7z1;VtTDi z1rTmqV@v2B8(|uRT5VKP&N-y64#cs}j`vvMPT9g*9EiNZRp*SXbsZETe>U0Dt%vd& zrk)wcF^Th&80s(n7#UlLA7}OG!LsKla6?58K`bFNl zT)`QZwx?Dejm`V5+PZ7x_1z$^jQ}kJzt$91g~48AFS5!V$jbK}k8`)N#(5`wde=LO z`_SR@>>5_JPq>qv!7BCwe^0{tf7JH=`Ut~UV$*;1R($(d%R0L9@X-~c!|x%yl5-=~ z23Sm)Ss@n7a(#>?``3)nO3Krc!IiHrJwoY>ywApwT2%|d?u>1UbTGbwt@%#Et1cW` zRWf^@?OIxEERR$<m#6li-d@I1S5ee`|>Tk_f?_lFlJ$-0q9f02ccXlSg%5X9e6qUBnmd1z?PX{pTD6_d3+RKk9l6cwn$s231%-EoC&(xVQ|^YamSAVnIQfJLRX5w}L)FarN+*_4)%yx0LSevq^(UsH2 zkaEymiF3@w>x$fE^L9fE2bzQAidRcdsK2qvXWYbde*(v;b>o9m4R_*^#imX) z;Sw(m@=eYj=R2SftC&Zsl0~vUjYHMniRyfwsLt<+njz~h|0ini18v-RINChFBC)5a zY;G+F$uz|S&asz9;xhR|Z$X}g@sLnuRK9p=kl_Pos%SkO1A7|1eta7Gb>Rdd$%(Fl zo8SfG+0OK(e}mzgX^a@Jr2SL6|02wAl=Grr_y+??sHyFn4H^dh|F?@lX9X3GoHe`*RFonh9PpII`d1@jbE^IOU;pz13Bk;QfB)-$mDc(9zy9a1A@i^Q{jdLR zosheT2C>S~F!x*-j&o;QiX-fzn2vc#7%)XRj7Hezu$V`!FP8CTRz{b)Y_g6<)#jH) zr)h3Xf1`oZWq{D(W?j{e8tlo`#75VrfRp&QHqHVv7=X^GAwZ|gGbXuNS+I+wi2c!K z{&4G+TA?)&A9SJw=~Yh!FFr2k;FyvPxsc9bP|{yccbLmiSinJ6&ewAhf#02#?lb=w zK^~hm-1cIMd2$kZZjmqcB)Dr=MgqdYH&Kmzf1<-HFj*v==lJF`bvF$YI#E_T|6{$? zr=>fv32Acdj&Kw~d+WqOc~N_%&e@hc1^GKTJ1N85>`*#)i!)bl-BRrZon80ylYyy> zXSpm1n4w60cZg6S0S66c5=B5I;gF85(Ot%al}kr}HQq`K5&C~G-k)y0Wi=$tcwwCe ze?T9(gE3IKsr84+Xrwa-UZ^+SF{WeyLUwH(z{RgSztx!oN_gQGkE0MalR#W*f+cf^ z2-6)YMXm508!FoaC}K&{@EhG+wLB_25nB&lNUoueI@$rAvH4j^{}p0w^9kWFzLiN&1J0r!EN_?wo@|Cqw*Rwj&o1V=RWe6za1j5iu_Wn0`s(K+AIGfHD zbJ-}ZX^Ox1NALTFGbK<%n&h||5T*M}!mno20vWex5+m_Pm=4o?gM(Pc92QH_e^B)l zCy&EpTvi+p=5uY?INB~x;q!c?t+8=Uxj)$i_=LdC)(zv=gbqC=nx0LSw58Xdt!Mf_ z3%5ltU~L;zSnckzRjMAbwNRsy&_erl?D<6kQ!0I1vW!w4TQix6Kw}(0mOAH-MOk&I zP7CKxJ&W1#Fb&kdO{!9Vx=;_sf27s){cv)0KLQ?;h30DQmbWg6UkyqOxsYP0F72OJ zHOn2Lod>!SxZe2t#_tC`yg11_@?Gp!T>I-gLg&hzmO4gi+~374YDFIj*epzG_&#%k zpbwPXI@dPJ1o%uRSyU8(`YbR>MKMK$%Kg1AdQbKDn+alUtPH3B4&2w6fBxmq;?Jl5 zZM&{G{|wKxP^%!y9N+%W*FCSd?e(@D6nnkC{{QI1*)f;Ied8-t2HTK>5OV~*))$7Z zPsGf9B4&jrVZQbgFzb^q_j>YWed6UYpLkhkX1SxxELLQe%FSYEf2=G!%bcf@zM`{8 z>x?MoS*MNoy3Z7S&8LVS$mCq7&uMwGXX$f1D>${Yk~2DUoXBak zksqa5-NBisdrW4j12M}Sefc^Mxh#2{lM$*sSxHXb~ zjig^==GU0{HD>lH)4R9l zM2D~M-+oN@Z?6Qyy)0sTCHU<1J=$OGJ=$L>LV9UD^s?yZf5(A!ex%obfBx(tpaJr} zeA)PJ{WL%L_n5b<%oRzm<>*fFIBZ)-uh;AC@9pXT_Ikb2 ze|L8Gc3yqAy|@2*@74DHtKHr2dON$@yS?vF?{1rJo>)lCzw0esSGjRt$z!J45kw)I zj)BV3)h?0Pf0}F;?rJem*Uz6(_ZNfN%L#>cu7CNmk@rUGqUR6-7SPT7j=EHcuOPH@ z9!Jq^vIwckO{gEn0iaGQnO#GvgSzMsL8TCHp%bhv*Mf zrl8cx3}U%EYz(i3OKnjw~4y3{O5#CJz=3Kb4kGWStD8tSX#v(EqTNE zb;xjVqKFbX?Hcsu$x(s4;v_BFCAREVETE4%Qf&YV^jnBj#DaHu5w+6O&E)6%IRrFf z!9iI@e>LMJIh|b#DuFgfOvqD^s_k_>oGX&mtn3Kl!-21N;HPE9Qhc4GmQkQAfQy3k zYxAMtx@Mw}LPF!)#@@+OYTey6@{*sZF?~;e#zF$ml~z8a zA)LiwcmX~Knd83vrcg`W*Tu>!TE_g4#kljfzFE+fp_c*o_=aV7gvFUsu(znQ)hSlr zfAs}9%f3)+5)(u+KFv=J+nzq^EVWoRK91-uIk$Cf}*Hq#v z=0d*N?zsvL0Lz$zH&lC+H$O7U_}MS#;D?xuO&GAk0MJQ!1nikE)5B%hnHuUoQu`nE ze_g@+>#_`~`Df1opjqM(rDcnfF|iq?V=5~WGFHEu18Rzg5{HI+#TQ5~AbBT9fBq`S z&t6$~6wF9lu*7c8gxcOIGF+?F)#Kv`Uy+bFPfl~Cqh2W&?T&a_ii(3Xuf7Z3yndP(`?qj+=_sq@Qo|zaPp|Jo@2UFi@*YESn z2RZveRNod?@^kRn1h|17AKyk;Un=|qCSlA$FX_)8c1cghc9G@$p7n)g^_0ZjrJ(+9O|Upl+>e<&dgnre+&%#bC&N$ z&A}oshT=%d$V*NrzX_R-L-kqT?F}k6bp$w+qcg^3x^b>_v8_dy^diRPn*}MCyV4Mz zyxEyc6%?y^?AoBf?ZYaB)L+|17i7-emFj^Sveg-x6rP?S2Sv&0rDy^A=L%x7UAT^^n#YKg*PD`6{En-(b- z-=@UxGx@IQi$X=kUL6>t`f|iUj93`-(W}CHwV=NRS=Q>2S&|Ztma-d6;HZyMYoT1% zf?Zf=K_nFO%scD>e?o~QI6Q(7PcIB?xsP^>5F~3lrLVV(uOr}u1@4dQcP;<_hx6l4 zhv&yf$ETMk2k$=paPschwTJnEGeB$U(;{argrX;V>an z?LGBe!?dr3!10bn=4fgzsZ_a*`Gv;4J&P&1?Qd<>+ppCLe`-~hKDctm+3gf{I@OtJ z82It>^6cW%+4=kb@9*yVtEwd^zCYJq@P~^Hqq@i>}MP zTdp9wCR;xCB#(vM((xd_dZT+oZB9X5V4ssp)I;oXjIcT)t4=A@M$-}DHuG!o3#Y`* z_`)GtYDul0QjcZ}tck)O>0|?pPv3KcrryYp*PIgTf0ovn99BiQUNUDAn<1t-DST*9 z-aS5Z;XkeJp9lN--Aw#vZ@ahqx`hAS+1cG)<3I1?@jR~p`K)Bo+%O2y!;2G}ptlhW zTHJ$|AfEGLH%r(wMZy=vu+i*pNxUd1cN}1zU~2Mxf^x%BVt;g=UM^j;!7Y?BEr?if zLI>rF5F)`%I06NUwnP&~Y?^F>np68i==?#gW6RvZ0i2 zp@aokvTLB{Hb-Pq4UPi>)K{xVMEYA>2deiDjxJT%<6DzK;#<;$-{v!y{?{PF8$tti zBRsXe}5m3wdF7kGPtuD;4V;n6+-p*Siq0U z)-q#%n1TwGuCd1nFpL?!fCWwyB$Wi>bcKfrTvS(ZmLMU!{6T_*(n?PenDoCF1!TTj zst6r^(616xd5mvAP&~mTR3l#KIW^_G#rH|~RY3l_Y!%)=^LRqL(OTvh!+a+U+VV61 ze_4duabE${KS%10aQ^e*^_4U*jY3Og`5U1I5;X z3wQ2dKPi_{-R_|lIDWmnmq7fF0JgrWa|Fpt+g0zQ+`b~`LNLo(#ZARydLTXs;HJwu zriyRk7|(3N=>B5l)k@1Q z#MwURI?uDsaZ`!4N9+c;7(p0z;no3g_bWf0!i{$#VT|P0bitRe3V%Mm*4fBZPX z{Nu-vTz&kgR8|P`)^4vqenj36jQa&f?##j2bb3T<4#S+sVMvq_lWi&p z0rQh~DljUWKfO3QKl$bO{L}gI+fNto4}bb}c5wNli-wVSCyBFnAKspvY7$2$=abMX zEPws@(OJV!AO3p$>Gb{4@u$;+pXZ2V>+~x%A?z&muC`3|Y{^~Gg<5HS*A%N;37W%D ziC|8}uA_kKDBwB@_$C6+o`s(~dQmQAHHN2KDguYLVahySY1vd_Cydqii>*TKn$3r;?o*rJ|v<-D$ijm~+Fx{=zpXKCTK ze1UyUh`?~O)fky|e6TNv zB$1)Uyd@oHCzq#Y#V)F)=P;|1&kM3Vi%7ok5)0fq(EH7$R$0ep*Rk0r8k;qY5*iYk zCilCBsb{W|HurlbDec|NpXhH*ML)sxK!5l*B1cKqe0RmmM|kH<88d6c=6~F`x5>bg z{eI_ZS4uAnSxxe9)~QS$QEa{0hFu5ApLme`@g!b(I(ENJd!grfBLVkr&E9!%JM{*_ zi%0}sWpHz3K7QExkP`V4tLkr;AUy6r4^_!J4{ftPK&9CDlEBIq{X?Jt8(Y+wSRO^G&modxT|Do znz2YW@mX0h%RTXg0de2hGdKQk?(9!9@qfMj?fnw|Z+rjs8vl1MPbnm361iFA-zt!N zHZ$BxaeP&pD?srXqRV6W44pNCuV$Pc4!`H-X7T$(%faOPyd>1LWd(4-l+b-#4Vv?z z1ynfv6LyG_cli_$i+^Aa&9_V55o&n48$6)Z6?cUO%vgVnSiqXzSScR#i6af$WIkTw z3?J#8JLLi`uF#n)c6E36+BAE`%ru@C3AhI9HFNe*V7tqOS~3=u5B#@Tq|qi zd0=a6WZ5^0EX(HM8dvtM;mR!dODi64$XOiKBg=ltbfeBWevg1P!(Rk^OoUMT!m9il z-I%#U2OWpU@Fx_X= zTAurLh^hvi@Z>L>dsr-03;a)VRhE~#>S`_!(2V=vUVrWH?UwU@^?K|3-|popIo#80 z(URm}UIqe!C}h(y&;?LHDcR&7hnXm;!Ycv>wW?+XSfQqRmPZgZQC}&ph(i<-AB)y~ zn0&Ax9;oH;ECUhJ0P*z7Ka^=Kbi?R#iLL;RF~5cYVWDlX6azllMu+D|22>trASq3g zQ1t9oZ-1{q+so4?VTj*C^EEFSz8o4Wq*yJS~QU##zF$WH$YL&URe>UsHuuP zjLVLs3!C+oLzd>7r;YK#`YTwAOU^2Si1XLgpZDvz!;SaNQ|rLwjw4zT0uF$CS{D#T z>k_t?ij{(JE|ad)+R4#unb;BiB z?SH+}{%^0hyFUMMFHdQ?QRCh9waJ;=`W>2u`<#VP*!$_yIF=*E$?tk0&D|T`xFOVE zS_h#Aw-PwTp%-yBA&N9`k27t2FLQr5k3%!(JcJ{nBh;dge(iLA+bC(9UprBH>9+Na z+CmfHSGvFuWM>m~LL&5kZ`5{2{lj#Y`hQoH{1iY45;TDYYW71+#;vP2XG2_d6>oBP z)oe@n%1~?B;FfKI2~fE}7uwdOow-o6d_k1my3z(MMKD{qX7X-@&wkKY<0>msI(^oN zIqLHn;{l=hneVDgFZy+~Qz=Ng(axTNG?XXVZyTQHZB(_>UkMEe9o~hKvPYy-{eN_2 zt6EM)iTLUt;7hBeOnWX0`!h|pYI_&=aDHX)*2bBpRIH6DSM~fdN;?sfhR)xp8NoYU zU$B(eAALoOOTBhqxsuYjkG8s%&;eUq9kx_XX0*ohtRyR0KPt6v35jLAN+!NF#hFgL zwbrarMUPcwCKiIcf34{@BA2rsoquMq@r}8r*-CA!<;?7r+jd#M%~oWYYN^+&%ko#F zW!KuYqD}L2a;;G76;*4UdcP`_gCM_ueMacKYIO;}#!@vbu1D@`5qe;Kb>Hf0h4TIc z^i-)g9!5v0VW}Omv~!Y^RQIf?8tk!5rhlvHl@-YHe4X>HEWitiOdIL`{C_zKO+Iui zJ( z&s-_wtnUKa}E$nXzBWGUl9W%{d^NLzVMqe^?}q!#o#uw7aaM za9q3TGeh1ZLGd22CN5@iEP>)S9Gh}8tIm^iB&d1tTs7-w_0KZ;zilLVrvBfrod0~? z>#gt^e=inaBTse~Uxi ge*AuxinX3wKkH}xd<)P24FCZD|AIU=bpYxC01#+S!~g&Q delta 13208 zcmV;JGiS`JYT;>+O@EXu%e_3|*w9F7w@$ldg(UZ!Ij#P@P*sw|s0h>mC`xR%Bj!PN zUv8e{3tZ}sDi$xgSZ00SPFpNgA^{{4iOe4o`1hE@fP~=rf_MQZ6X071Bg|!YipSx1 zt9W|7UT<%ASO2%y>y`i8+u7OsZhLp{&CcuH*W10F?|R#>cYj{(e201~seQ&13yJx6 zy`}3aH|{HWa72CqE(oK2G}+$3QI!4}VhKWS1R#7)BB_7dSK@+xAR(XwMkOW`I69Y% z;~}79DuEBM59r0g`SHv2_tP*Q61qY082ZR1a2u19g-m2l4;kYj^fwT?!UE19Vtq9Bd3Q{F&IB9CuFu9BD4Vsa zC_)?}CWvHw+DBX0--|6@kgb9!>0%NOQ#GRgj^Z#pWg+pWeRMoHVe*uN0IFX4f}C2B}Nu8^_-4)tt4gCP-;<2hQXKDN5R_-NDHI6~gFYHb8HxT@ zj@2$k@tR`O5}E&(QVv!oyKivkq))oBX~cyBqup z`+w#BSB)qdF&i7X{z_lh^_wCWvN$-<;+n~nCFtpwMx_PyrA35&=p!{5JTVm#w1N&p zc5^h2Cgqw#6cQhcKH5=TCpnfdoL)>L=%X{H=iJ9Az*3bRF(FSt-Y|ZhN}U`L zfdojYCa2HhFi?FK5P`2k2$V==jYI(hfqy8ID1ZoPfEYzTpC2cmZy+^8IPt|PgofPCz+#S$HWSqRm1z@Aw}{61_SVAAAfaD*ttJ~AP%9^ioVqxk(#%6UW>OxH5+;Xo2fs? z`1T{k6HG#-9L<$p($L16`N!-2!bSbu~IhY6!U z%8ZD*ii+#`EQu_wZ>|-ty2jP4a+P(iwWfQugr?gV0G2U_OrYL@G%|fIu`=DQ$eWku z&Cklbd&gNErLPA<|DBA-8Q|b=oJjaUeNc1ahjZ0~8ym(P&kQ=H*t@GaGd;o9uEfOwttU0AI{Ic5rpGTvX6ep&JTZjpH7%g2R%cn!a&CPqJDls znVMJPGco9OuDhYy?RZ~JE_$0zSL(b3u2hcjiesr}I@XSY+!g!&=-dm|(? zzAa2f+aG~AaTu&h&!J~1z%XW1fYhC!N>h>a+#w8b97-qS_T45r2!CU>bo@ZLDE+1S z`bT4DZlV)L!LDT?$UfSdV7?WSt1bJBYn=@HQ2!URw19l@^svs7iX5=Yn%0Nh+ z`=W)e@NNio>5FtmKR@VSPB6a=$*T}5(Ro+i%4O=975{&=w_S?=?Y()myN>_e$8(|dLN2I(evHftrRM(7 zdVB+jLNf1?Yvpf|ND48f8P2JG$G@@Ub&SZD8uFLSapj(f ztuJ3THdJ)6quU76KriUj2Gg_+pwVKw+~?083SdAepiU}TT~)Y)yzISp9dr%KOi$46 zF9va^af0D8BsGyFQ5d#|Ne`^Le4s??b37!JjPm`|+e}CP3 z_3BM={omPswYRrk|L@}|O}2ELt$R?<0MGL_@;G4}5xA8=wezEU{k>6P+Z)$}27QZ+ zI>8pwR*gYNf-bx=*LN~`@IdV;fQ!Ria47EGh}UVn4Yu&F8hIXR|@%)nxA2pz^D=H<^e5E4F%ppVkMSg?T-gGy8J5Do%j98IUnGy;C0Xn$mWzyf~! zmL)TtC+0>5i$cGbR&2np0iLJvYr#wW)0p0)p67e4*v zpwN6Fxxr$&Tc5IqLo+C#-Lnmi*8kR}61wUs3h4o(id)34{sF#J*0BkN6FMw>p3RZw z4w$&8*K|OsRLjmdOIS2*9TQ5%@fZp6yKb{#AayYtc^ypyu8c4>+14xcvJp&M!(sQyPg73z4|_vn^4vP+jj&s-&u&o`2ht<&%1Li-IFrI*rOD(+;;q zl|rI+?h>_kWnFY?+)z}R)N+KS(r09lx~B)aq!L5y%NU2@G$m>WS{Rp?Dssi-sKkQa zbGrGL3;`7}&jJJbdwS|#bO#bxvU?{xu#CKY#L;@8YGT9h^hhZLbq5+Q=OyMIiXe$ZT@G= zMfsEqXX9>#C_)!n=m>;b(8+ilON|f11DR?}M!Q)%Dm3bwqTkk;->$-?R@Q!-Yk0}H zb`nMC)J3V*!nOu$wiBtLn(J<>NV5*_nx0iV8h@oggoww+^t}QL6sRGp;R9)?FF_EL z7P>@_=SI(PP($~!D7D|#2nw!g zPhl}{zbnJEx~FqBF0&7+16Mahc57Kf*NmGUHb%p#9Xnhy1(d-3l&Q8{QE;p0HShy$rJ{pn<&;-?>s~_PBpm7{Z z5``IM!*HKT7S$+bb2Qe) zWGvB5(tB74<`cc77EmBYcjo)I-RS4&SsWOadCCc+dQ?=O=##;6q>OpmR>ZWL&^sY) z2QgQJw%Q9s^{8F}P&CH;x?)2awv%>lnoVOiMw;URCZVQ01Pa>Gqm-mF6uU^D#D8(n zv^KUy)9GV5_IR#L6;{cNLQSeDRSPBHp)sBu#%V=kneB3ed1?ye?0RFZ$DFIRLK&BQ zoH|UfV}EF#14fm7pmqbXu%4D_y3f1{SYGnV0Aa21>AEVSlQqI9%+fwFYFf`(>7(ac3jR?1i z=5(zc4;(Zztwtc&L`0^lTV%T1UB?LPDdQljCvi6F;nZ}gaa}YTI8%^abn@Zi$fdR~ z7{U`q0u(VJh}td)@MCVUup$PriToy!c5*tOkh{g&L3q`Ij^-BjIaF3Lo_{t&!1F~D zX_hnmi7vI9^Pbf>7|V`DbWBk&rFcwyB+5SJYFm(x1#ITDc)F?$NT^T7r*)>xRL%_M zpA<3ONvimy8rMnvGcxAXZhG}n*D0=19S6d4|<_=diEMxu`HjeoaXgC z5i;RaG_soP77QQ(i8zWFSATOtYEz-AQjD|7s1*m{RL`U%r+v_@8ib7LHMOZQijP_G zjE~nXaSg%`oHIT{>UEh!+(RssCJ?|U5ZG6NKC6(EB-@rvM}x@#VLB~r$I`qFBM2k4 zQc8y6NExRgOd!k&rg%ZR&~=f`1NpP^FBPc(d=h|~o!TWWM=;HKVSgo|F`$a`R?8A5 zS`v*tlpBp=W{xW?Kqr&X+F%;#(bFB{+B3~^lo`FcZE0$-hkzGwLybX*^gwm_DVhzoXuxdc@@Og1k+W>{~q$akCDsW4l2%;dVk^^hSSW@wnKV9o?<0mN{1p#z%XT0HBQ z%7&@XX7S7|L7Lr9l6{H=xK@?6YiILH*Qgf1{p<`t zhgeVYT2i&sRYs9hRn?a2ie7J$aEe!D9&VZ z<}7dVN#l%l~@$NAkyCUcP+s;@6*kes^(t^c#8k$FDRVUw@mw{&?{#9R1dk$jg`i^`{Ce zw_XmqsdjvfduHCJ2g7rt<60)G{xVN@(MIi8#_rD1xBJBWFY#3+6^eDMnX|}iXrvn@ z_)S70zj-L+8UlGw5XhB6A6Ex?TqV@;DM1{+T4>`HNNn(p&y1HTZ)QMJAU%DV92PZs zb5lqe5q~I9n#Iqw`!lqEdYs38nc+?daqxjm)UtPjIn{ABo1Q}B&J^j0QDK+EjCgM8 zwhS|S(o__pn+es_1XU)#MyUxj#(D}QZOijntd21HH$$h7IyFZ=wh}T`tY^mc|DzN1 z)>9~bY_`7-2RMB~Fw@llOV?1+dMN3Nz@uxpXnz~H=sf^L9~UI_sR2Njg?cUt>Rb}O zxinDo>xE>l0DQST%yMZ!5JPweh{0v?fy<)- zza{M7iqU>|h3|X9h`#0Vdn-WetpuC5G!pM?6}QKPsaq9-Zbf*x<V zJ#qZjD$rUhlf@;GSj$V|(kQDHVXBrzP%Vp>TBD;@g^o(D^vc1baz8cWqcSf9_I-#X z1SjSEcO4VWOj6qEKYd0GrpjE`YHEhymwzNBo%(T|uW)w0Lf<$L?>uW^hJL9_U8qaU z$X{r=HFD+1V(6!}q<=wkYQy9Uo8Ryty~kp{K{#b-)hKN8YuneIUx}6{d@GIEhLKXFsXOpK;ioWQohI*Is*+ zr7#XbLaT26Bo5~w40+O+1EP#(H!uW`0C`bbw#JswJvPEL2({X%q?~g|T^)#HogMG7!kw~(wKx!YgR9ONS?f9|LVs+srCSf> zHB3D-j$<5|ZE;EPlI6HmU5=2x)gW2Yv((?{Vgv&6YKleppE0NaF8|%Mu*=+cqQjX zstvH1GP6P~mgV{wOZKlBp_P=UC4(zpU3!Gl8F`s~?a>`pEjVRFeUf_9l$WWEK50l7pzY`)3_NA~3PMJQVwvOy^#QBJd2RzOLZ!}{eayt|0H{Z>H@UYe?U?Q2YOywDPopcR zjUnZrxf17?i`Nyo%jWHd77jEA$rZWO9F*HT*?wR=fe-7HNoo^HjLvt^P5lj3&@PT8 z8)Iq7xDdt%=f@2sC2%B206xy+j*C`t$5v=2X;6P-lh3$`=YIr_Q|rbDryB0WBa2O) zXu>648swXtJbjd$rA+ro=IOip-mD7^hclTLJlJ@_oUFt2)Efqf~ zCAHA!1CkP-NYKe&JX?Hf&MM4a^4zkdo+OV+*3a%KP6|Wwe_z?#O74{_rLz<2NHsr1^@on|0=EX?|=QzUqj|!|NCG6 z+d3h45e;IMqhap3FdXO3wiHL$MKK-ok}zP3a2SoS&0#T*T3;;V%dCtpb=hPcjjGKr zi%!zqn14nCr^^7L!_B&?9W~gKsfmrQPXQc3SN9%&cHDx8*(9?!JwqSobE7}p|F61temgsJOaNvE8S=Q zF@iicYq;&j6!YXH^xPs}>`8Ffu8ahPgKwf5`F})*S75S8IM4CTXXxN`tA^+LIMsN%p{6{O2Q!>U8B2<2P>D30BgLJ79#Zjo_{#mddF%=n(@Lq z4S#?>atC9ea#QOMlhH_L4m?+Hx?@bq0EFz?I)IB`cYdog2bA!_EgnZ9Y$k!Y)C5cB z5D}(3Qi@vPIW|3{rAFO^CsYDLbGpf*`gxm+bv-@Kp6eGH^DX zE9SCMTGJGN9}YkC4QEQAhBV1>H6Tj&nS@`>rUf!?(IiIVk1!pk`348Ej5#cpqJN?4 zDNY`T$GEIG9?a+3vT?Lsp2Fw(NLyp$nsR@#3GfMlo2?thuL&J`N;Ey2DrrlvKU>fA ze->_wUclNmsIc1IWvf&@Vr!vBC834(>)7-21g2E_wqzNlI<{sq5rM`yfGl;+9gDK+ zP@NXepL!Ov<6#=8f16aL{&b-pjDJb1>HG2c@O}h5CJW8g+AVKg62BUh7;+)SP+i(T zuWFV%LOTz1C2+m*_l@5VdT@T6cjUX+t+@8rcZANBJ1upL)VRNkS=5R?60ljA((rxe z20lnL;ePO_*d0`*y7l8R!A2$g%gUG#zK?>7^~*jO1({~frmF@OEbpT(a~ z{@Zq4asC;eX`xm@lsUfrpKp3zZ`~Nab8ugZkon_C1U3i+$ zfo8XJ&?({PM_2AWY5y)cvf&~XC-HJ<~WhlXvaO_12{_`w^{yh z%_Bcbv$}&bPxqM2QU_v|Ir{Q-9&%aoILl+cxpvWW468o1u=E*)r*b0U9?l;uf7)P` zXA71+NwBJO1M5=)<$qHGOPvQ;3G#nAjQ_G|{-tsID?#WlgT-G4b-xV0evPD`yK!qI z{TfNX#>}rV^J~ofwC1n+IzIWQiSx-B_U{g^9~Udz#);&Irvj$W_V+uPmM|Lyg9 zrT^}{+TD5m-S+O@o1NFYueWKNA-;mp z&UqX~v&kZ)CO4se7zcnlsbqEyr4H(%KM*A}Q2Wna^^Gd8mu?4L^N}f~zm)9ncpRcX zP?>^KCo_oU?yxbu7B01ESwRAY;}C_!$D*!BlJ}-20e>O*GTH8qvGhmpb3I$zX~}o* z9i6wAba=~=5t3sGLko#BONRTKq~ygkvfn1^#`2#NHuZ#srpzS)-)D_zEnsOCgS6xg z!LcGrPWQ#D9~>qQV|Q@=S9>?PdAgF@8=NE zhz0v)8GqG`m*jMIEvN+A95EqJK&rOa^>D68R+C{TT}hJXc;-=5ZE6@usS+u`ZZeJ~>fo zU)h2A`!vK9T2z@tAR<-1vF3;q#)+I}C>Lw&dVjvT-^ueik-7VV1o*N3fKdszG8-QO zzS>oua;9c(-at2c<1D6^?YogtsRa)X9`tqe?#vl~X zlAqPpS=0^)VLlY)uvN#Kyxr+ke$-|-4g$5ad;6ys`-g{TN9X4+9R)+c0dP<2EW+sQ z64|x8Z|iI_2N5+BSuEdf@AP`Y@g8m^$A6wdjd{u1{kFK?rG=I5@o+@Ee@5c%>t0id ztC$PpFJ`fhJfv8f}#p&XquF4K*3rHgGXx}+B|F5fOlx!je8 z@Z{~zT&kd0&12UF1#TZ!A*BA=Ho72l?ygi1)R3*t$gJRk^MBcz+F1?Wq;4{>;d_`s z=-$|;v-M9e4$hB19i1GWet$SVxp27}#vnQIMXFtTUH$)RX16VEr(Mk+-)D3{hBb_5 z1>6Kom{Lhpo`5|aYn}Nn*GkA@jk6w*p(nKG%qtVgaiNPJ=+ef`b}~2R%(`kxzQGY= zvc!0?0Pf)-Q0Zv9Df3YP-&ZBvgfa@AhzA35>mU>-B+hjSHz6E`?0?3CaU`dO9;j)s zE`Fp+A7o80QPx11vWTaCOC?fu5r(4FIh`f8NaRN9bNJI|m$etP+PmkPr?M zGS%Kw&oxZ@S_mBPSY(c-=8{U4+n8Ty+}pF5lH2~)R=xdNoqwQKW$A+}r<~nRQKwU# znTCNMFD_2cKb@X^`2YUyuD_}Zb975YYXqo~!!``g&)+=k73J;Shfn8+KYe=t;oYbA zN534sZ(FMtopKreB%o&qgUHrSxqN$ULc{PIyNI!RCYO97& z%)#@^cS}`usDF-DcLO3MXQ(rPfN@+aFhZ1bp_cg)BHZ$!n#z|04nELu+Av>*2)*dK z+`HupqHD6{V^8u}$SoZY@~bzxH`L}7)CKlAsYE@*9>)l)BeLq0LTxl15^giU7Qb*x z+>9?AqNSG9>M8YTw!oSw{Gm=Z!1(k7H)!gO{CLeNv43u9oylQUbn7K^Cb1b}nv=qZ z2Ibx3GZ+5T>i&7KpWn^IfA+R}uiljKpF2A{uh#g_`*=LhD?mOgSu{5cLiFJL*e2+0 z1cMg$;3bIXyx7eWHcgT61u<+iyIT@33d$V^m?xN;e4n7)u$0&zou!vc*KBYLrA!MV z797(7`+q>S8Ivx4X0>=v2H;QqP+uvXCiy;9VJgFt*4Do2z5T-rRrcuCWRUolG~u`T%%%S|i13Ec zfZYgBt+x?pJ_s>mMQGOl+q=7a+a>+~`pwSnT7UoF$75|dOoI&WYzDXs6kml<{XG`& zqq4Qk*dL^zLZxf$aRLluM$ch^(*#K+fjC*=VFDM`6`UnV$S!}7AfdFzb;#a_s=|@&~CJrImR&G34^x$ z%zu9tp?2I?0QJw2x+9$be0Y5&4ULSh2)k{*YkLyFR$kB55(feDxbV99odc84b;Lli zweP~6JJ?UkWmLC&s0EH+FYhG~|095{uj(8@^3rzIhbXtN$hi>AvQ}|Z@tE$5j{>;q zvd+l`+W&a*$M&+OZgs_VLFgRb*D+Z;EQSbM~7fQu1?VHa*40C&Ig(+S*o#}dX!|F2#YQ^y+ePO%)JgZ|~^ z1oO+wkX&6}DwP$2yu6Hb9%ylSS-ZWyyhPp)jQa&f?##j2bb3T<4#S+sVMvq_(35T| z2LZE_b}BF`+dsWHJUjm7=}>ql<=-cqfU|_aEOKpJ)Kj}OGOM{h2RZACMSPAgv3(3b%YyC(RUzi!gy7#k!aq3>{$nL z-FeX#z*FO(s)d|B5x7;-YmPFjgk9Z<&9g>qtq8Go7Y0?{K|k5PD~oGY*3|UdR9L<7pB9dqJ6O9Rz&2jsmWufa@sWItsXs0zQAC zDBx;CaE~CS*c8kt9a_(t)bc#SLI1MuM1u8j2c%VSWPKKX!L#slM=#0;;vGe5&Yl~8 zUHxga`-oH=^tG?w>F{6GS@wB%YLDp5`8qiHaKXt(6I;}iuAJAku+cfoQ8!ZC_AD*j zmM^fc2@x1>wi+X|jt>^-0VkC)3*djPn^b9~t2qU`*L({0RnIL1cdOwZQ&9Cuggj@b zElFgkF>gu7*~#UpS+R?1={d}*x4)lI=sa4jo*>!C8iN{m-o{rsb(_ZL#-blc`TeEi_+)llL z@FEg{R~g(Knaj)Wr7Cop`;3HOz2>Dat5t~c)k1Zs?@FIMLvjQHwDJ-Nb4V|R(=n(3 z4H(z=HeyNizCvF|3)AP^Tnc|$BF1PqO|B%=;hGUnu9;ImU4g`CVlPp*Hw_+~A0y3P z>wZ;KmgwB-=Z!r)e{H6l%inz^_!RT++6*)^@Y#mIxVIms<7luF9RO*V2DI(cpY+;;xdV zX~rVi#Ajv2Ece6{2E=`1&)oRGxwAjb#Q*j7w)aZ-zwN!fHU95jo>EB6ByzLJzf~al zY-YHX;`pjGSAgO(M3=|#89HkOU(Gl@9DdKu&EofomV?Rnc}b{g%L?FvDWUtg8Z_ra z3#f4TC+rX<@A4@i7QufUns1l9Bh>J8H+Vp+EA9#nn6dsCv4Azbu~Izf6Gs}h$$Y%V z89vfGcgh7?T%j{p?CS3BwQ2T>nQ1&P5^xRHYv$~sz4gG~9mzmv}7#UE5j+xK`H2 z^T5{D$g*z~S(eSiHLmPi!ohQA2-mB02p{(@%4mb&k)VY<(# zwLJIh5LFF2;mKb%_pn&17Wkj!sw^*e)zw@epc(hSz21M@eO12y?aiC@{crd3lpOBq zwP;E5FE0ZDK@_s-80Z2hpps-Ljx+0Gmw;~ zNho^uy0?E@pzY;plQ2YZOC-Vr2^?Y0Vj7?~-PdZf!e}5=EktT!K30qj2n`DL3Wnr} z2(z_9V#MMwQ0xx%TFj9gVXFQ|5g8MDrg+ascMuY-0AlAcA3`7XvR?|2F|&WHK*E}a zu8I^}4ye^yOjeM2t*NdRz^i7Ee-s(os;n)Fow9#MP%WCs4`U&L-y5JPXRoXXRn$~P z9>!%y(uK|X${|bh&C|wsVf_`X#U*DILB#p%>d*W2+~LOi=BagHa>o&^2muGcJ*^7} zqjd?}OT|h-IG0J+Y3<}_woL4o4F~UngIful;?RpYn-E1BxW}0`zL&W_oX4S=a~{GG(GhCVN56JDzipJX&99v(y>#39 zMs1-9@GD(l2(q(@Iw2AIzc*^TqyAw!OZ|T8dtZ|hUDV;uR z#2oecjPZcb{LFXNr5F7=+Nl&I-Dqb|K^n@F?6(ci^ERs5>92$agbwdQN!cUPseXUD zvQ;goqC|Z45AdbcQl>rUh5eZ(TeZE5dpN(ccWdKJQ!3WRl&gAv8Ks>FNkivv)QsSr zt}j?h?2o>p#id@muUtuK+(%p8O6Y(st`1u&Co@{(c~+8@tRI!yw}iwpUL_OXn&M0+ z-dbzcsG`TJG7}3y-oMs#8-)1YaOtsYO)n)ms z(Xwl8TG6KYIk{G-^@^&sPQ721%0ZCdzdj>$UbVV}Ut_5n7S|*9wFo`1zPfL9wL*D+ z0(z>{8xNzS)UebJS=u?tNveBRR1Nl6Cey#w^vVikdA`p1Rug9gizx$QPBF}P zxB@+(C-~k)zSOJh8PFtb_qM^AVgK8H!ls`VM1s%q|McGMlp={?osU9OIoQVSBq|?S> /mnt/azurefile/kata-cc.txt; sleep 1; done + volumeMounts: + - name: cc-azurefile + mountPath: "/mnt/azurefile" + readOnly: false + volumes: + - name: cc-azurefile + persistentVolumeClaim: + claimName: pvc-azurefile + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate diff --git a/deploy/example-cc/cc-nginx-pod-azurefile.yaml b/deploy/example-cc/cc-nginx-pod-azurefile.yaml new file mode 100644 index 0000000000..1a03193afe --- /dev/null +++ b/deploy/example-cc/cc-nginx-pod-azurefile.yaml @@ -0,0 +1,24 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: cc-nginx-azurefile +spec: + runtimeClassName: kata-cc + nodeSelector: + "kubernetes.io/os": linux + containers: + - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 + name: nginx-azurefile + command: + - "/bin/bash" + - "-c" + - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata-cc.txt; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: "/mnt/azurefile" + readOnly: false + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: pvc-azurefile diff --git a/deploy/example-cc/cc-statefulset.yaml b/deploy/example-cc/cc-statefulset.yaml new file mode 100644 index 0000000000..61d2c35b40 --- /dev/null +++ b/deploy/example-cc/cc-statefulset.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: cc-statefulset-azurefile + labels: + app: nginx +spec: + podManagementPolicy: Parallel # default is OrderedReady + serviceName: cc-statefulset-azurefile + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-cc + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: cc-statefulset-azurefile + image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 + command: + - "/bin/bash" + - "-c" + - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata-cc.txt; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: /mnt/azurefile + readOnly: false + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: nginx + volumeClaimTemplates: + - metadata: + name: persistent-storage + spec: + storageClassName: azurefile-csi + accessModes: ["ReadWriteMany"] + resources: + requests: + storage: 100Gi diff --git a/deploy/example-cc/shared-pvc-across-runtimes.yaml b/deploy/example-cc/shared-pvc-across-runtimes.yaml new file mode 100644 index 0000000000..efee0e14f4 --- /dev/null +++ b/deploy/example-cc/shared-pvc-across-runtimes.yaml @@ -0,0 +1,72 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: nginx-azurefile +spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 + name: nginx-azurefile + command: + - "/bin/bash" + - "-c" + - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/runc.txt; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: "/mnt/azurefile" + readOnly: false + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: pvc-azurefile +--- +kind: Pod +apiVersion: v1 +metadata: + name: kata-nginx-azurefile +spec: + runtimeClassName: kata + nodeSelector: + "kubernetes.io/os": linux + containers: + - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 + name: nginx-azurefile + command: + - "/bin/bash" + - "-c" + - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata.txt; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: "/mnt/azurefile" + readOnly: false + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: pvc-azurefile +--- +kind: Pod +apiVersion: v1 +metadata: + name: kata-cc-nginx-azurefile +spec: + runtimeClassName: kata-cc + nodeSelector: + "kubernetes.io/os": linux + containers: + - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 + name: nginx-azurefile + command: + - "/bin/bash" + - "-c" + - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata-cc.txt; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: "/mnt/azurefile" + readOnly: false + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: pvc-azurefile +--- diff --git a/deploy/rbac-csi-azurefile-controller.yaml b/deploy/rbac-csi-azurefile-controller.yaml index 59a0f8c2af..dbbc819f2f 100644 --- a/deploy/rbac-csi-azurefile-controller.yaml +++ b/deploy/rbac-csi-azurefile-controller.yaml @@ -192,3 +192,26 @@ roleRef: kind: ClusterRole name: csi-azurefile-controller-secret-role apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-controller-pod-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-controller-pod-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azurefile-controller-pod-role + apiGroup: rbac.authorization.k8s.io +--- diff --git a/deploy/rbac-csi-azurefile-node.yaml b/deploy/rbac-csi-azurefile-node.yaml index 3752f36aa4..432e91e7a5 100644 --- a/deploy/rbac-csi-azurefile-node.yaml +++ b/deploy/rbac-csi-azurefile-node.yaml @@ -28,3 +28,48 @@ roleRef: kind: ClusterRole name: csi-azurefile-node-secret-role apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-node-pod-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-node-pod-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azurefile-node-pod-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-azurefile-node-rc-role +rules: + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-azurefile-node-rc-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azurefile-node-rc-role + apiGroup: rbac.authorization.k8s.io +--- diff --git a/go.mod b/go.mod index 20bc77ca13..6651456928 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/container-storage-interface/spec v1.9.0 + github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 @@ -65,7 +66,7 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect @@ -74,7 +75,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect @@ -104,12 +105,13 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kata-containers/kata-containers/src/runtime v0.0.0-20240702121346-ef3f6515cf8a github.com/klauspost/compress v1.17.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect github.com/moby/spdystream v0.4.0 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index e6464d869b..eec2226a62 100644 --- a/go.sum +++ b/go.sum @@ -672,8 +672,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -735,8 +735,8 @@ github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXt github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 h1:OoRAFlvDGCUqDLampLQjk0yeeSGdF9zzst/3G9IkBbc= +github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09/go.mod h1:m2r/smMKsKwgMSAoFKHaa68ImdCSNuKE1MxvQ64xuCQ= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -836,7 +836,7 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -862,6 +862,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -1009,6 +1010,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kata-containers/kata-containers/src/runtime v0.0.0-20240702121346-ef3f6515cf8a h1:R+RUmZNLAxUevzW/NQH8pQOkwYjsL0xK4iRTB/KokTc= +github.com/kata-containers/kata-containers/src/runtime v0.0.0-20240702121346-ef3f6515cf8a/go.mod h1:E4YBwwlTgi4TmfBfGDbMRhVaD6iHRaMLM7v8g1reHT8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1060,8 +1063,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1167,8 +1170,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= diff --git a/pkg/azurefile/azure.go b/pkg/azurefile/azure.go index 953bd11104..f48b877264 100644 --- a/pkg/azurefile/azure.go +++ b/pkg/azurefile/azure.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -49,6 +50,21 @@ var ( storageService = "Microsoft.Storage" ) +func getRuntimeClassForPod(ctx context.Context, kubeClient clientset.Interface, podName string, podNameSpace string) (string, error) { + if kubeClient == nil { + return "", fmt.Errorf("kubeClient is nil") + } + // Get runtime class for pod + pod, err := kubeClient.CoreV1().Pods(podNameSpace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return "", err + } + if pod.Spec.RuntimeClassName != nil { + return *pod.Spec.RuntimeClassName, nil + } + return "", nil +} + // getCloudProvider get Azure Cloud Provider func getCloudProvider(ctx context.Context, kubeconfig, nodeID, secretName, secretNamespace, userAgent string, allowEmptyCloudConfig, enableWindowsHostProcess bool, kubeAPIQPS float64, kubeAPIBurst int) (*azure.Cloud, error) { var ( diff --git a/pkg/azurefile/azure_test.go b/pkg/azurefile/azure_test.go index 7127d6af55..cfa218dcaa 100644 --- a/pkg/azurefile/azure_test.go +++ b/pkg/azurefile/azure_test.go @@ -33,6 +33,9 @@ import ( "sigs.k8s.io/azurefile-csi-driver/test/utils/testutil" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/subnetclient/mocksubnetclient" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fake "k8s.io/client-go/kubernetes/fake" azureprovider "sigs.k8s.io/cloud-provider-azure/pkg/provider" ) @@ -42,6 +45,72 @@ func skipIfTestingOnWindows(t *testing.T) { } } +func TestGetRuntimeClassForPod(t *testing.T) { + ctx := context.TODO() + + // Test the case where kubeClient is nil + _, err := getRuntimeClassForPod(ctx, nil, "test-pod", "default") + if err == nil || err.Error() != "kubeClient is nil" { + t.Fatalf("expected error 'kubeClient is nil', got %v", err) + } + + // Create a fake clientset + clientset := fake.NewSimpleClientset() + + // Test the case where the pod exists and has a RuntimeClassName + runtimeClassName := "my-runtime-class" + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + RuntimeClassName: &runtimeClassName, + }, + } + _, err = clientset.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + runtimeClass, err := getRuntimeClassForPod(ctx, clientset, "test-pod", "default") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if runtimeClass != runtimeClassName { + t.Fatalf("expected runtime class name to be '%s', got '%s'", runtimeClassName, runtimeClass) + } + + // Test the case where the pod exists but does not have a RuntimeClassName + podWithoutRuntimeClass := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-no-runtime", + Namespace: "default", + }, + Spec: corev1.PodSpec{}, + } + _, err = clientset.CoreV1().Pods("default").Create(ctx, podWithoutRuntimeClass, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + runtimeClass, err = getRuntimeClassForPod(ctx, clientset, "test-pod-no-runtime", "default") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if runtimeClass != "" { + t.Fatalf("expected runtime class name to be '', got '%s'", runtimeClass) + } + + // Test the case where the pod does not exist + _, err = getRuntimeClassForPod(ctx, clientset, "nonexistent-pod", "default") + if err == nil { + t.Fatalf("expected an error, got nil") + } +} + // TestGetCloudProvider tests the func GetCloudProvider(). // To run this unit test successfully, need to ensure /etc/kubernetes/azure.json nonexistent. func TestGetCloudProvider(t *testing.T) { diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index 2c0bf5f521..b9b3dd7d29 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -120,6 +120,7 @@ const ( storageEndpointSuffixField = "storageendpointsuffix" fsGroupChangePolicyField = "fsgroupchangepolicy" ephemeralField = "csi.storage.k8s.io/ephemeral" + podNameField = "csi.storage.k8s.io/pod.name" podNamespaceField = "csi.storage.k8s.io/pod.namespace" serviceAccountTokenField = "csi.storage.k8s.io/serviceAccount.tokens" clientIDField = "clientID" @@ -220,6 +221,7 @@ type Driver struct { allowInlineVolumeKeyAccessWithIdentity bool enableVHDDiskFeature bool enableGetVolumeStats bool + enableKataCCMount bool enableVolumeMountGroup bool appendMountErrorHelpLink bool mountPermissions uint64 @@ -270,8 +272,10 @@ type Driver struct { // azcopy for provide exec mock for ut azcopy *fileutil.Azcopy - kubeconfig string - endpoint string + kubeconfig string + endpoint string + resolver Resolver + directVolume DirectVolume } // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version & @@ -288,6 +292,7 @@ func NewDriver(options *DriverOptions) *Driver { driver.allowEmptyCloudConfig = options.AllowEmptyCloudConfig driver.allowInlineVolumeKeyAccessWithIdentity = options.AllowInlineVolumeKeyAccessWithIdentity driver.enableVHDDiskFeature = options.EnableVHDDiskFeature + driver.enableKataCCMount = options.EnableKataCCMount driver.enableVolumeMountGroup = options.EnableVolumeMountGroup driver.enableGetVolumeStats = options.EnableGetVolumeStats driver.appendMountErrorHelpLink = options.AppendMountErrorHelpLink @@ -310,6 +315,8 @@ func NewDriver(options *DriverOptions) *Driver { driver.azcopy = &fileutil.Azcopy{} driver.kubeconfig = options.KubeConfig driver.endpoint = options.Endpoint + driver.resolver = new(NetResolver) + driver.directVolume = new(directVolume) var err error getter := func(_ string) (interface{}, error) { return nil, nil } diff --git a/pkg/azurefile/azurefile_options.go b/pkg/azurefile/azurefile_options.go index a0ea5808f5..c331a90aff 100644 --- a/pkg/azurefile/azurefile_options.go +++ b/pkg/azurefile/azurefile_options.go @@ -31,6 +31,7 @@ type DriverOptions struct { EnableVHDDiskFeature bool EnableVolumeMountGroup bool EnableGetVolumeStats bool + EnableKataCCMount bool AppendMountErrorHelpLink bool MountPermissions uint64 FSGroupChangePolicy string @@ -67,6 +68,7 @@ func (o *DriverOptions) AddFlags() *flag.FlagSet { fs.BoolVar(&o.EnableVHDDiskFeature, "enable-vhd", true, "enable VHD disk feature (experimental)") fs.BoolVar(&o.EnableVolumeMountGroup, "enable-volume-mount-group", true, "indicates whether enabling VOLUME_MOUNT_GROUP") fs.BoolVar(&o.EnableGetVolumeStats, "enable-get-volume-stats", true, "allow GET_VOLUME_STATS on agent node") + fs.BoolVar(&o.EnableKataCCMount, "enable-kata-cc-mount", true, "enable Kata Confidential Containers mount") fs.BoolVar(&o.AppendMountErrorHelpLink, "append-mount-error-help-link", true, "Whether to include a link for help with mount errors when a mount error occurs.") fs.Uint64Var(&o.MountPermissions, "mount-permissions", 0777, "mounted folder permissions") fs.StringVar(&o.FSGroupChangePolicy, "fsgroup-change-policy", "", "indicates how the volume's ownership will be changed by the driver, OnRootMismatch is the default value") diff --git a/pkg/azurefile/directvolume_interface.go b/pkg/azurefile/directvolume_interface.go new file mode 100644 index 0000000000..276b4cbbf5 --- /dev/null +++ b/pkg/azurefile/directvolume_interface.go @@ -0,0 +1,54 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azurefile + +import ( + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" +) + +type DirectVolume interface { + Add(volumePath string, mountInfo string) error + Remove(volumePath string) error + VolumeMountInfo(volumePath string) (*volume.MountInfo, error) + RecordSandboxID(sandboxID string, volumePath string) error + GetSandboxIDForVolume(volumePath string) (string, error) +} + +// Ensure the existing functions implement the interface +var _ DirectVolume = &directVolume{} + +type directVolume struct{} + +func (dv *directVolume) Add(volumePath string, mountInfo string) error { + return volume.Add(volumePath, mountInfo) +} + +func (dv *directVolume) Remove(volumePath string) error { + return volume.Remove(volumePath) +} + +func (dv *directVolume) VolumeMountInfo(volumePath string) (*volume.MountInfo, error) { + return volume.VolumeMountInfo(volumePath) +} + +func (dv *directVolume) RecordSandboxID(sandboxID string, volumePath string) error { + return volume.RecordSandboxId(sandboxID, volumePath) +} + +func (dv *directVolume) GetSandboxIDForVolume(volumePath string) (string, error) { + return volume.GetSandboxIdForVolume(volumePath) +} diff --git a/pkg/azurefile/directvolume_mock.go b/pkg/azurefile/directvolume_mock.go new file mode 100644 index 0000000000..b835ebf306 --- /dev/null +++ b/pkg/azurefile/directvolume_mock.go @@ -0,0 +1,120 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azurefile + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" +) + +// MockDirectVolume is a mock of DirectVolume interface. +type MockDirectVolume struct { + ctrl *gomock.Controller + recorder *MockDirectVolumeMockRecorder +} + +// MockDirectVolumeMockRecorder is the mock recorder for MockDirectVolume. +type MockDirectVolumeMockRecorder struct { + mock *MockDirectVolume +} + +// NewMockDirectVolume creates a new mock instance. +func NewMockDirectVolume(ctrl *gomock.Controller) *MockDirectVolume { + mock := &MockDirectVolume{ctrl: ctrl} + mock.recorder = &MockDirectVolumeMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDirectVolume) EXPECT() *MockDirectVolumeMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockDirectVolume) Add(volumePath, mountInfo string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", volumePath, mountInfo) + ret0, _ := ret[0].(error) + return ret0 +} + +// Add indicates an expected call of Add. +func (mr *MockDirectVolumeMockRecorder) Add(volumePath, mountInfo interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockDirectVolume)(nil).Add), volumePath, mountInfo) +} + +// GetSandboxIDForVolume mocks base method. +// nolint: var-naming +func (m *MockDirectVolume) GetSandboxIDForVolume(volumePath string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSandboxIDForVolume", volumePath) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSandboxIDForVolume indicates an expected call of GetSandboxIDForVolume. +func (mr *MockDirectVolumeMockRecorder) GetSandboxIDForVolume(volumePath interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSandboxIDForVolume", reflect.TypeOf((*MockDirectVolume)(nil).GetSandboxIDForVolume), volumePath) +} + +// RecordSandboxID mocks base method. +func (m *MockDirectVolume) RecordSandboxID(sandboxID, volumePath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecordSandboxID", sandboxID, volumePath) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecordSandboxID indicates an expected call of RecordSandboxID. +func (mr *MockDirectVolumeMockRecorder) RecordSandboxID(sandboxID, volumePath interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordSandboxID", reflect.TypeOf((*MockDirectVolume)(nil).RecordSandboxID), sandboxID, volumePath) +} + +// Remove mocks base method. +func (m *MockDirectVolume) Remove(volumePath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remove", volumePath) + ret0, _ := ret[0].(error) + return ret0 +} + +// Remove indicates an expected call of Remove. +func (mr *MockDirectVolumeMockRecorder) Remove(volumePath interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockDirectVolume)(nil).Remove), volumePath) +} + +// VolumeMountInfo mocks base method. +func (m *MockDirectVolume) VolumeMountInfo(volumePath string) (*volume.MountInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeMountInfo", volumePath) + ret0, _ := ret[0].(*volume.MountInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VolumeMountInfo indicates an expected call of VolumeMountInfo. +func (mr *MockDirectVolumeMockRecorder) VolumeMountInfo(volumePath interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeMountInfo", reflect.TypeOf((*MockDirectVolume)(nil).VolumeMountInfo), volumePath) +} diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 36c44adc5c..3e4c1d49fe 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -17,6 +17,7 @@ limitations under the License. package azurefile import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -26,6 +27,7 @@ import ( "time" "github.com/container-storage-interface/spec/lib/go/csi" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/volume/util" @@ -34,12 +36,14 @@ import ( "google.golang.org/grpc/status" "golang.org/x/net/context" - volumehelper "sigs.k8s.io/azurefile-csi-driver/pkg/util" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" "sigs.k8s.io/cloud-provider-azure/pkg/metrics" ) +var getRuntimeClassForPodFunc = getRuntimeClassForPod +var isConfidentialRuntimeClassFunc = isConfidentialRuntimeClass + // NodePublishVolume mount the volume from staging to target path func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { volCap := req.GetVolumeCapability() @@ -95,6 +99,42 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu return nil, status.Errorf(codes.InvalidArgument, "invalid mountPermissions %s", perm) } } + if d.enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" { + runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.cloud.KubeClient, context[podNameField], context[podNamespaceField]) + if err != nil { + klog.Errorf("failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) + return &csi.NodePublishVolumeResponse{}, nil + } + klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) + isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.cloud.KubeClient, runtimeClass) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err) + } + if isConfidentialRuntimeClass { + klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass %s is kata-cc", volumeID, runtimeClass) + source := req.GetStagingTargetPath() + if len(source) == 0 { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + // Load the mount info from staging area + mountInfo, err := d.directVolume.VolumeMountInfo(source) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err) + } + if mountInfo == nil { + return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID) + } + data, err := json.Marshal(mountInfo) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err) + } + if err = d.directVolume.Add(target, string(data)); err != nil { + return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err) + } + klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target) + return &csi.NodePublishVolumeResponse{}, nil + } + } } source := req.GetStagingTargetPath() @@ -147,6 +187,13 @@ func (d *Driver) NodeUnpublishVolume(_ context.Context, req *csi.NodeUnpublishVo if err := CleanupMountPoint(d.mounter, targetPath, true /*extensiveMountPointCheck*/); err != nil { return nil, status.Errorf(codes.Internal, "failed to unmount target %s: %v", targetPath, err) } + if d.enableKataCCMount { + // Remove deletes the direct volume path including all the files inside it. + // if there is no kata-cc mountinfo present on this path, it will return nil. + if err := d.directVolume.Remove(targetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to direct volume remove mount info %s: %v", targetPath, err) + } + } klog.V(2).Infof("NodeUnpublishVolume: unmount volume %s on %s successfully", volumeID, targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil @@ -367,6 +414,44 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe klog.V(2).Infof("volume(%s) mount %s on %s succeeded", volumeID, source, cifsMountPath) } + // If runtime OS is not windows and protocol is not nfs, save mountInfo.json + if d.enableKataCCMount { + if runtime.GOOS != "windows" && protocol != nfs { + // Check if mountInfo.json is already present at the targetPath + isMountInfoPresent, err := d.directVolume.VolumeMountInfo(cifsMountPath) + if err != nil && !os.IsNotExist(err) { + return nil, status.Errorf(codes.Internal, "Could not save direct volume mount info %s: %v", cifsMountPath, err) + } + if isMountInfoPresent != nil { + klog.V(2).Infof("NodeStageVolume: mount info for volume %s is already present on %s", volumeID, targetPath) + } else { + mountFsType := cifs + ipAddr, err := d.resolver.ResolveIPAddr("ip", server) + if err != nil { + klog.V(2).ErrorS(err, "Couldn't resolve IP") + return nil, err + } + mountOptions = append(mountOptions, "addr="+ipAddr.IP.String()) + mountInfo := volume.MountInfo{ + VolumeType: "azurefile", + Device: source, + FsType: mountFsType, + Metadata: map[string]string{ + "sensitiveMountOptions": strings.Join(sensitiveMountOptions, ","), + }, + Options: mountOptions, + } + data, _ := json.Marshal(mountInfo) + if err := d.directVolume.Add(cifsMountPath, string(data)); err != nil { + return nil, status.Errorf(codes.Internal, "Could not save direct volume mount info %s: %v", cifsMountPath, err) + } + klog.V(2).Infof("NodeStageVolume: mount info for volume %s saved on %s", volumeID, targetPath) + } + } else { + klog.V(2).Infof("NodeStageVolume: skip saving mount info for volume %s on %s, runtime OS: %s, protocol: %s", volumeID, targetPath, runtime.GOOS, protocol) + } + } + if isDiskMount { mnt, err := d.ensureMountPoint(targetPath, os.FileMode(mountPermissions)) if err != nil { @@ -440,6 +525,14 @@ func (d *Driver) NodeUnstageVolume(_ context.Context, req *csi.NodeUnstageVolume return nil, status.Errorf(codes.Internal, "failed to unmount staging target %s: %v", targetPath, err) } } + + if d.enableKataCCMount { + klog.V(2).Infof("NodeUnstageVolume:remove direct volume mount info %s from %s", volumeID, stagingTargetPath) + if err := d.directVolume.Remove(stagingTargetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to remove mount info %s: %v", stagingTargetPath, err) + } + } + klog.V(2).Infof("NodeUnstageVolume: unmount volume %s on %s successfully", volumeID, stagingTargetPath) isOperationSucceeded = true diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index 51fa6de733..d0d9a48fa4 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "net" "os" "path/filepath" "reflect" @@ -30,9 +31,12 @@ import ( azure2 "github.com/Azure/go-autorest/autorest/azure" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/stretchr/testify/assert" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + clientset "k8s.io/client-go/kubernetes" mount "k8s.io/mount-utils" "k8s.io/utils/exec" testingexec "k8s.io/utils/exec/testing" @@ -107,6 +111,14 @@ func TestNodeGetCapabilities(t *testing.T) { assert.NoError(t, err) } +func mockGetRuntimeClassForPod(_ context.Context, _ clientset.Interface, _, _ string) (string, error) { + return "mockRuntimeClass", nil +} + +func mockIsConfidentialRuntimeClass(_ context.Context, _ clientset.Interface, _ string) (bool, error) { + return true, nil +} + func TestNodePublishVolume(t *testing.T) { d := NewFakeDriver() d.cloud = &azure.Cloud{} @@ -120,6 +132,12 @@ func TestNodePublishVolume(t *testing.T) { targetTest = testutil.GetWorkDirPath("target_test", t) ) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockDirectVolume := NewMockDirectVolume(ctrl) + getRuntimeClassForPodFunc = mockGetRuntimeClassForPod + isConfidentialRuntimeClassFunc = mockIsConfidentialRuntimeClass + tests := []struct { desc string setup func() @@ -235,6 +253,25 @@ func TestNodePublishVolume(t *testing.T) { DefaultError: status.Error(codes.InvalidArgument, fmt.Sprintf("invalid mountPermissions %s", "07ab")), }, }, + { + desc: "[Success] Valid request with Kata CC Mount enabled", + req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap}, + VolumeId: "vol_1", + TargetPath: targetTest, + StagingTargetPath: sourceTest, + Readonly: true, + VolumeContext: map[string]string{mountPermissionsField: "0755", podNameField: "testPod", podNamespaceField: "testNamespace"}, + }, + setup: func() { + d.enableKataCCMount = true + d.directVolume = mockDirectVolume + mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(&volume.MountInfo{}, nil) + mockDirectVolume.EXPECT().Add(targetTest, gomock.Any()).Return(nil) + }, + cleanup: func() { + d.enableKataCCMount = false + }, + }, } // Setup @@ -272,6 +309,9 @@ func TestNodeUnpublishVolume(t *testing.T) { errorTarget := testutil.GetWorkDirPath("error_is_likely_target", t) targetFile := testutil.GetWorkDirPath("abc.go", t) d := NewFakeDriver() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockDirectVolume := NewMockDirectVolume(ctrl) tests := []struct { desc string @@ -308,6 +348,19 @@ func TestNodeUnpublishVolume(t *testing.T) { req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, expectedErr: testutil.TestError{}, }, + { + desc: "[Success] Valid request with Kata CC Mount enabled", + req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, + expectedErr: testutil.TestError{}, + setup: func() { + d.enableKataCCMount = true + d.directVolume = mockDirectVolume + mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) + }, + cleanup: func() { + d.enableKataCCMount = false + }, + }, } // Setup @@ -407,6 +460,11 @@ func TestNodeStageVolume(t *testing.T) { "accountkey": "testkey", } + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockResolver := NewMockResolver(ctrl) + mockDirectVolume := NewMockDirectVolume(ctrl) + tests := []struct { desc string setup func() @@ -701,6 +759,28 @@ func TestNodeStageVolume(t *testing.T) { DefaultError: status.Error(codes.InvalidArgument, fmt.Sprintf("invalid mountPermissions %s", "07ab")), }, }, + { + desc: "[Success] Valid request with Kata CC Mount enabled", + setup: func() { + d.resolver = mockResolver + d.directVolume = mockDirectVolume + d.enableKataCCMount = true + if runtime.GOOS != "windows" { + mockIPAddr := &net.IPAddr{IP: net.ParseIP("192.168.1.1")} + mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(nil, nil) + mockResolver.EXPECT().ResolveIPAddr("ip", "test_servername").Return(mockIPAddr, nil) + mockDirectVolume.EXPECT().Add(sourceTest, gomock.Any()).Return(nil) + } + }, + req: csi.NodeStageVolumeRequest{VolumeId: "vol_1##", StagingTargetPath: sourceTest, + VolumeCapability: &stdVolCap, + VolumeContext: volContext, + Secrets: secrets}, + skipOnWindows: true, + cleanup: func() { + d.enableKataCCMount = false + }, + }, } // Setup @@ -767,6 +847,9 @@ func TestNodeUnstageVolume(t *testing.T) { targetFile = testutil.GetWorkDirPath("abc.go", t) ) d := NewFakeDriver() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockDirectVolume := NewMockDirectVolume(ctrl) tests := []struct { desc string @@ -816,6 +899,19 @@ func TestNodeUnstageVolume(t *testing.T) { req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, expectedErr: testutil.TestError{}, }, + { + desc: "[Success] Valid request with Kata CC Mount enabled", + setup: func() { + d.enableKataCCMount = true + d.directVolume = mockDirectVolume + mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) + }, + req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, + expectedErr: testutil.TestError{}, + cleanup: func() { + d.enableKataCCMount = false + }, + }, } // Setup diff --git a/pkg/azurefile/resolver_interface.go b/pkg/azurefile/resolver_interface.go new file mode 100644 index 0000000000..368297cd78 --- /dev/null +++ b/pkg/azurefile/resolver_interface.go @@ -0,0 +1,32 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azurefile + +import "net" + +// Resolver is an interface for resolving IP addresses. +type Resolver interface { + ResolveIPAddr(network, address string) (*net.IPAddr, error) +} + +// NetResolver is the real implementation of the Resolver interface. +type NetResolver struct{} + +// ResolveIPAddr resolves the IP address using net.ResolveIPAddr. +func (r *NetResolver) ResolveIPAddr(network, address string) (*net.IPAddr, error) { + return net.ResolveIPAddr(network, address) +} diff --git a/pkg/azurefile/resolver_mock.go b/pkg/azurefile/resolver_mock.go new file mode 100644 index 0000000000..d8ee8e52f2 --- /dev/null +++ b/pkg/azurefile/resolver_mock.go @@ -0,0 +1,62 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azurefile + +import ( + net "net" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockResolver is a mock of Resolver interface. +type MockResolver struct { + ctrl *gomock.Controller + recorder *MockResolverMockRecorder +} + +// MockResolverMockRecorder is the mock recorder for MockResolver. +type MockResolverMockRecorder struct { + mock *MockResolver +} + +// NewMockResolver creates a new mock instance. +func NewMockResolver(ctrl *gomock.Controller) *MockResolver { + mock := &MockResolver{ctrl: ctrl} + mock.recorder = &MockResolverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockResolver) EXPECT() *MockResolverMockRecorder { + return m.recorder +} + +// ResolveIPAddr mocks base method. +func (m *MockResolver) ResolveIPAddr(network, address string) (*net.IPAddr, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveIPAddr", network, address) + ret0, _ := ret[0].(*net.IPAddr) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveIPAddr indicates an expected call of ResolveIPAddr. +func (mr *MockResolverMockRecorder) ResolveIPAddr(network, address interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveIPAddr", reflect.TypeOf((*MockResolver)(nil).ResolveIPAddr), network, address) +} diff --git a/pkg/azurefile/utils.go b/pkg/azurefile/utils.go index 5afd8908c1..0c2f077fa7 100644 --- a/pkg/azurefile/utils.go +++ b/pkg/azurefile/utils.go @@ -17,6 +17,7 @@ limitations under the License. package azurefile import ( + "context" "fmt" "os" "regexp" @@ -27,6 +28,8 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/volume" ) @@ -317,3 +320,25 @@ func isReadOnlyFromCapability(vc *csi.VolumeCapability) bool { return (mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY || mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY) } + +const confidentialRuntimeClassHandler = "kata-cc" + +// check if runtimeClass is confidential +func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interface, runtimeClassName string) (bool, error) { + // if runtimeClassName is empty, return false + if runtimeClassName == "" { + return false, nil + } + if kubeClient == nil { + klog.Warningf("kubeClient is nil") + return false, fmt.Errorf("kubeClient is nil") + } + runtimeClassClient := kubeClient.NodeV1().RuntimeClasses() + runtimeClass, err := runtimeClassClient.Get(ctx, runtimeClassName, metav1.GetOptions{}) + if err != nil { + klog.Warningf("Failed to get runtimeClass %s: %v", runtimeClassName, err) + return false, err + } + klog.Infof("runtimeClass %s handler: %s", runtimeClassName, runtimeClass.Handler) + return runtimeClass.Handler == confidentialRuntimeClassHandler, nil +} diff --git a/pkg/azurefile/utils_test.go b/pkg/azurefile/utils_test.go index 48109e65a5..d274651868 100644 --- a/pkg/azurefile/utils_test.go +++ b/pkg/azurefile/utils_test.go @@ -17,6 +17,7 @@ limitations under the License. package azurefile import ( + "context" "errors" "fmt" "os" @@ -26,6 +27,9 @@ import ( "time" "github.com/container-storage-interface/spec/lib/go/csi" + nodev1 "k8s.io/api/node/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fake "k8s.io/client-go/kubernetes/fake" utiltesting "k8s.io/client-go/util/testing" ) @@ -749,3 +753,64 @@ func TestIsReadOnlyFromCapability(t *testing.T) { } } } + +func TestIsConfidentialRuntimeClass(t *testing.T) { + ctx := context.TODO() + + // Test the case where kubeClient is nil + _, err := isConfidentialRuntimeClass(ctx, nil, "test-runtime-class") + if err == nil || err.Error() != "kubeClient is nil" { + t.Fatalf("expected error 'kubeClient is nil', got %v", err) + } + + // Create a fake clientset + clientset := fake.NewSimpleClientset() + + // Test the case where the runtime class exists and has the confidential handler + runtimeClass := &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-runtime-class", + }, + Handler: confidentialRuntimeClassHandler, + } + _, err = clientset.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + isConfidential, err := isConfidentialRuntimeClass(ctx, clientset, "test-runtime-class") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if !isConfidential { + t.Fatalf("expected runtime class to be confidential, got %v", isConfidential) + } + + // Test the case where the runtime class exists but does not have the confidential handler + nonConfidentialRuntimeClass := &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-runtime-class-non-confidential", + }, + Handler: "non-confidential-handler", + } + _, err = clientset.NodeV1().RuntimeClasses().Create(ctx, nonConfidentialRuntimeClass, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + isConfidential, err = isConfidentialRuntimeClass(ctx, clientset, "test-runtime-class-non-confidential") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if isConfidential { + t.Fatalf("expected runtime class to not be confidential, got %v", isConfidential) + } + + // Test the case where the runtime class does not exist + _, err = isConfidentialRuntimeClass(ctx, clientset, "nonexistent-runtime-class") + if err == nil { + t.Fatalf("expected an error, got nil") + } +} diff --git a/vendor/github.com/Microsoft/go-winio/.golangci.yml b/vendor/github.com/Microsoft/go-winio/.golangci.yml index af403bb13a..7b503d26a3 100644 --- a/vendor/github.com/Microsoft/go-winio/.golangci.yml +++ b/vendor/github.com/Microsoft/go-winio/.golangci.yml @@ -8,12 +8,8 @@ linters: - containedctx # struct contains a context - dupl # duplicate code - errname # erorrs are named correctly - - goconst # strings that should be constants - - godot # comments end in a period - - misspell - nolintlint # "//nolint" directives are properly explained - revive # golint replacement - - stylecheck # golint replacement, less configurable than revive - unconvert # unnecessary conversions - wastedassign @@ -23,10 +19,7 @@ linters: - exhaustive # check exhaustiveness of enum switch statements - gofmt # files are gofmt'ed - gosec # security - - nestif # deeply nested ifs - nilerr # returns nil even with non-nil error - - prealloc # slices that can be pre-allocated - - structcheck # unused struct fields - unparam # unused function params issues: @@ -42,6 +35,18 @@ issues: text: "^line-length-limit: " source: "^//(go:generate|sys) " + #TODO: remove after upgrading to go1.18 + # ignore comment spacing for nolint and sys directives + - linters: + - revive + text: "^comment-spacings: no space between comment delimiter and comment text" + source: "//(cspell:|nolint:|sys |todo)" + + # not on go 1.18 yet, so no any + - linters: + - revive + text: "^use-any: since GO 1.18 'interface{}' can be replaced by 'any'" + # allow unjustified ignores of error checks in defer statements - linters: - nolintlint @@ -56,6 +61,8 @@ issues: linters-settings: + exhaustive: + default-signifies-exhaustive: true govet: enable-all: true disable: @@ -98,6 +105,8 @@ linters-settings: disabled: true - name: flag-parameter # excessive, and a common idiom we use disabled: true + - name: unhandled-error # warns over common fmt.Print* and io.Close; rely on errcheck instead + disabled: true # general config - name: line-length-limit arguments: @@ -138,7 +147,3 @@ linters-settings: - VPCI - WCOW - WIM - stylecheck: - checks: - - "all" - - "-ST1003" # use revive's var naming diff --git a/vendor/github.com/Microsoft/go-winio/hvsock.go b/vendor/github.com/Microsoft/go-winio/hvsock.go index 52f1c280f6..c881916583 100644 --- a/vendor/github.com/Microsoft/go-winio/hvsock.go +++ b/vendor/github.com/Microsoft/go-winio/hvsock.go @@ -23,7 +23,7 @@ import ( const afHVSock = 34 // AF_HYPERV // Well known Service and VM IDs -//https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards +// https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards // HvsockGUIDWildcard is the wildcard VmId for accepting connections from all partitions. func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000 @@ -31,7 +31,7 @@ func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000 } // HvsockGUIDBroadcast is the wildcard VmId for broadcasting sends to all partitions. -func HvsockGUIDBroadcast() guid.GUID { //ffffffff-ffff-ffff-ffff-ffffffffffff +func HvsockGUIDBroadcast() guid.GUID { // ffffffff-ffff-ffff-ffff-ffffffffffff return guid.GUID{ Data1: 0xffffffff, Data2: 0xffff, @@ -246,7 +246,7 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) { var addrbuf [addrlen * 2]byte var bytes uint32 - err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /*rxdatalen*/, addrlen, addrlen, &bytes, &c.o) + err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /* rxdatalen */, addrlen, addrlen, &bytes, &c.o) if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil { return nil, l.opErr("accept", os.NewSyscallError("acceptex", err)) } diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/doc.go b/vendor/github.com/Microsoft/go-winio/internal/fs/doc.go new file mode 100644 index 0000000000..1f65388178 --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/doc.go @@ -0,0 +1,2 @@ +// This package contains Win32 filesystem functionality. +package fs diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go b/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go new file mode 100644 index 0000000000..509b3ec641 --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go @@ -0,0 +1,202 @@ +//go:build windows + +package fs + +import ( + "golang.org/x/sys/windows" + + "github.com/Microsoft/go-winio/internal/stringbuffer" +) + +//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go fs.go + +// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew +//sys CreateFile(name string, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateFileW + +const NullHandle windows.Handle = 0 + +// AccessMask defines standard, specific, and generic rights. +// +// Bitmask: +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +---------------+---------------+-------------------------------+ +// |G|G|G|G|Resvd|A| StandardRights| SpecificRights | +// |R|W|E|A| |S| | | +// +-+-------------+---------------+-------------------------------+ +// +// GR Generic Read +// GW Generic Write +// GE Generic Exectue +// GA Generic All +// Resvd Reserved +// AS Access Security System +// +// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-mask +// +// https://learn.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights +// +// https://learn.microsoft.com/en-us/windows/win32/fileio/file-access-rights-constants +type AccessMask = windows.ACCESS_MASK + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + // Not actually any. + // + // For CreateFile: "query certain metadata such as file, directory, or device attributes without accessing that file or device" + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew#parameters + FILE_ANY_ACCESS AccessMask = 0 + + // Specific Object Access + // from ntioapi.h + + FILE_READ_DATA AccessMask = (0x0001) // file & pipe + FILE_LIST_DIRECTORY AccessMask = (0x0001) // directory + + FILE_WRITE_DATA AccessMask = (0x0002) // file & pipe + FILE_ADD_FILE AccessMask = (0x0002) // directory + + FILE_APPEND_DATA AccessMask = (0x0004) // file + FILE_ADD_SUBDIRECTORY AccessMask = (0x0004) // directory + FILE_CREATE_PIPE_INSTANCE AccessMask = (0x0004) // named pipe + + FILE_READ_EA AccessMask = (0x0008) // file & directory + FILE_READ_PROPERTIES AccessMask = FILE_READ_EA + + FILE_WRITE_EA AccessMask = (0x0010) // file & directory + FILE_WRITE_PROPERTIES AccessMask = FILE_WRITE_EA + + FILE_EXECUTE AccessMask = (0x0020) // file + FILE_TRAVERSE AccessMask = (0x0020) // directory + + FILE_DELETE_CHILD AccessMask = (0x0040) // directory + + FILE_READ_ATTRIBUTES AccessMask = (0x0080) // all + + FILE_WRITE_ATTRIBUTES AccessMask = (0x0100) // all + + FILE_ALL_ACCESS AccessMask = (STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0x1FF) + FILE_GENERIC_READ AccessMask = (STANDARD_RIGHTS_READ | FILE_READ_DATA | FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE) + FILE_GENERIC_WRITE AccessMask = (STANDARD_RIGHTS_WRITE | FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | SYNCHRONIZE) + FILE_GENERIC_EXECUTE AccessMask = (STANDARD_RIGHTS_EXECUTE | FILE_READ_ATTRIBUTES | FILE_EXECUTE | SYNCHRONIZE) + + SPECIFIC_RIGHTS_ALL AccessMask = 0x0000FFFF + + // Standard Access + // from ntseapi.h + + DELETE AccessMask = 0x0001_0000 + READ_CONTROL AccessMask = 0x0002_0000 + WRITE_DAC AccessMask = 0x0004_0000 + WRITE_OWNER AccessMask = 0x0008_0000 + SYNCHRONIZE AccessMask = 0x0010_0000 + + STANDARD_RIGHTS_REQUIRED AccessMask = 0x000F_0000 + + STANDARD_RIGHTS_READ AccessMask = READ_CONTROL + STANDARD_RIGHTS_WRITE AccessMask = READ_CONTROL + STANDARD_RIGHTS_EXECUTE AccessMask = READ_CONTROL + + STANDARD_RIGHTS_ALL AccessMask = 0x001F_0000 +) + +type FileShareMode uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + FILE_SHARE_NONE FileShareMode = 0x00 + FILE_SHARE_READ FileShareMode = 0x01 + FILE_SHARE_WRITE FileShareMode = 0x02 + FILE_SHARE_DELETE FileShareMode = 0x04 + FILE_SHARE_VALID_FLAGS FileShareMode = 0x07 +) + +type FileCreationDisposition uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + // from winbase.h + + CREATE_NEW FileCreationDisposition = 0x01 + CREATE_ALWAYS FileCreationDisposition = 0x02 + OPEN_EXISTING FileCreationDisposition = 0x03 + OPEN_ALWAYS FileCreationDisposition = 0x04 + TRUNCATE_EXISTING FileCreationDisposition = 0x05 +) + +// CreateFile and co. take flags or attributes together as one parameter. +// Define alias until we can use generics to allow both + +// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants +type FileFlagOrAttribute uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( // from winnt.h + FILE_FLAG_WRITE_THROUGH FileFlagOrAttribute = 0x8000_0000 + FILE_FLAG_OVERLAPPED FileFlagOrAttribute = 0x4000_0000 + FILE_FLAG_NO_BUFFERING FileFlagOrAttribute = 0x2000_0000 + FILE_FLAG_RANDOM_ACCESS FileFlagOrAttribute = 0x1000_0000 + FILE_FLAG_SEQUENTIAL_SCAN FileFlagOrAttribute = 0x0800_0000 + FILE_FLAG_DELETE_ON_CLOSE FileFlagOrAttribute = 0x0400_0000 + FILE_FLAG_BACKUP_SEMANTICS FileFlagOrAttribute = 0x0200_0000 + FILE_FLAG_POSIX_SEMANTICS FileFlagOrAttribute = 0x0100_0000 + FILE_FLAG_OPEN_REPARSE_POINT FileFlagOrAttribute = 0x0020_0000 + FILE_FLAG_OPEN_NO_RECALL FileFlagOrAttribute = 0x0010_0000 + FILE_FLAG_FIRST_PIPE_INSTANCE FileFlagOrAttribute = 0x0008_0000 +) + +type FileSQSFlag = FileFlagOrAttribute + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( // from winbase.h + SECURITY_ANONYMOUS FileSQSFlag = FileSQSFlag(SecurityAnonymous << 16) + SECURITY_IDENTIFICATION FileSQSFlag = FileSQSFlag(SecurityIdentification << 16) + SECURITY_IMPERSONATION FileSQSFlag = FileSQSFlag(SecurityImpersonation << 16) + SECURITY_DELEGATION FileSQSFlag = FileSQSFlag(SecurityDelegation << 16) + + SECURITY_SQOS_PRESENT FileSQSFlag = 0x00100000 + SECURITY_VALID_SQOS_FLAGS FileSQSFlag = 0x001F0000 +) + +// GetFinalPathNameByHandle flags +// +// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew#parameters +type GetFinalPathFlag uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + GetFinalPathDefaultFlag GetFinalPathFlag = 0x0 + + FILE_NAME_NORMALIZED GetFinalPathFlag = 0x0 + FILE_NAME_OPENED GetFinalPathFlag = 0x8 + + VOLUME_NAME_DOS GetFinalPathFlag = 0x0 + VOLUME_NAME_GUID GetFinalPathFlag = 0x1 + VOLUME_NAME_NT GetFinalPathFlag = 0x2 + VOLUME_NAME_NONE GetFinalPathFlag = 0x4 +) + +// getFinalPathNameByHandle facilitates calling the Windows API GetFinalPathNameByHandle +// with the given handle and flags. It transparently takes care of creating a buffer of the +// correct size for the call. +// +// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew +func GetFinalPathNameByHandle(h windows.Handle, flags GetFinalPathFlag) (string, error) { + b := stringbuffer.NewWString() + //TODO: can loop infinitely if Win32 keeps returning the same (or a larger) n? + for { + n, err := windows.GetFinalPathNameByHandle(h, b.Pointer(), b.Cap(), uint32(flags)) + if err != nil { + return "", err + } + // If the buffer wasn't large enough, n will be the total size needed (including null terminator). + // Resize and try again. + if n > b.Cap() { + b.ResizeTo(n) + continue + } + // If the buffer is large enough, n will be the size not including the null terminator. + // Convert to a Go string and return. + return b.String(), nil + } +} diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/security.go b/vendor/github.com/Microsoft/go-winio/internal/fs/security.go new file mode 100644 index 0000000000..81760ac67e --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/security.go @@ -0,0 +1,12 @@ +package fs + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level +type SecurityImpersonationLevel int32 // C default enums underlying type is `int`, which is Go `int32` + +// Impersonation levels +const ( + SecurityAnonymous SecurityImpersonationLevel = 0 + SecurityIdentification SecurityImpersonationLevel = 1 + SecurityImpersonation SecurityImpersonationLevel = 2 + SecurityDelegation SecurityImpersonationLevel = 3 +) diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go new file mode 100644 index 0000000000..e2f7bb24e5 --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go @@ -0,0 +1,64 @@ +//go:build windows + +// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT. + +package fs + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) + errERROR_EINVAL error = syscall.EINVAL +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return errERROR_EINVAL + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + + procCreateFileW = modkernel32.NewProc("CreateFileW") +) + +func CreateFile(name string, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(name) + if err != nil { + return + } + return _CreateFile(_p0, access, mode, sa, createmode, attrs, templatefile) +} + +func _CreateFile(name *uint16, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) { + r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0) + handle = windows.Handle(r0) + if handle == windows.InvalidHandle { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go b/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go index 39e8c05f8f..aeb7b7250f 100644 --- a/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go +++ b/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go @@ -100,8 +100,8 @@ func (f *runtimeFunc) Load() error { (*byte)(unsafe.Pointer(&f.addr)), uint32(unsafe.Sizeof(f.addr)), &n, - nil, //overlapped - 0, //completionRoutine + nil, // overlapped + 0, // completionRoutine ) }) return f.err diff --git a/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go b/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go new file mode 100644 index 0000000000..7ad5057024 --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go @@ -0,0 +1,132 @@ +package stringbuffer + +import ( + "sync" + "unicode/utf16" +) + +// TODO: worth exporting and using in mkwinsyscall? + +// Uint16BufferSize is the buffer size in the pool, chosen somewhat arbitrarily to accommodate +// large path strings: +// MAX_PATH (260) + size of volume GUID prefix (49) + null terminator = 310. +const MinWStringCap = 310 + +// use *[]uint16 since []uint16 creates an extra allocation where the slice header +// is copied to heap and then referenced via pointer in the interface header that sync.Pool +// stores. +var pathPool = sync.Pool{ // if go1.18+ adds Pool[T], use that to store []uint16 directly + New: func() interface{} { + b := make([]uint16, MinWStringCap) + return &b + }, +} + +func newBuffer() []uint16 { return *(pathPool.Get().(*[]uint16)) } + +// freeBuffer copies the slice header data, and puts a pointer to that in the pool. +// This avoids taking a pointer to the slice header in WString, which can be set to nil. +func freeBuffer(b []uint16) { pathPool.Put(&b) } + +// WString is a wide string buffer ([]uint16) meant for storing UTF-16 encoded strings +// for interacting with Win32 APIs. +// Sizes are specified as uint32 and not int. +// +// It is not thread safe. +type WString struct { + // type-def allows casting to []uint16 directly, use struct to prevent that and allow adding fields in the future. + + // raw buffer + b []uint16 +} + +// NewWString returns a [WString] allocated from a shared pool with an +// initial capacity of at least [MinWStringCap]. +// Since the buffer may have been previously used, its contents are not guaranteed to be empty. +// +// The buffer should be freed via [WString.Free] +func NewWString() *WString { + return &WString{ + b: newBuffer(), + } +} + +func (b *WString) Free() { + if b.empty() { + return + } + freeBuffer(b.b) + b.b = nil +} + +// ResizeTo grows the buffer to at least c and returns the new capacity, freeing the +// previous buffer back into pool. +func (b *WString) ResizeTo(c uint32) uint32 { + // allready sufficient (or n is 0) + if c <= b.Cap() { + return b.Cap() + } + + if c <= MinWStringCap { + c = MinWStringCap + } + // allocate at-least double buffer size, as is done in [bytes.Buffer] and other places + if c <= 2*b.Cap() { + c = 2 * b.Cap() + } + + b2 := make([]uint16, c) + if !b.empty() { + copy(b2, b.b) + freeBuffer(b.b) + } + b.b = b2 + return c +} + +// Buffer returns the underlying []uint16 buffer. +func (b *WString) Buffer() []uint16 { + if b.empty() { + return nil + } + return b.b +} + +// Pointer returns a pointer to the first uint16 in the buffer. +// If the [WString.Free] has already been called, the pointer will be nil. +func (b *WString) Pointer() *uint16 { + if b.empty() { + return nil + } + return &b.b[0] +} + +// String returns the returns the UTF-8 encoding of the UTF-16 string in the buffer. +// +// It assumes that the data is null-terminated. +func (b *WString) String() string { + // Using [windows.UTF16ToString] would require importing "golang.org/x/sys/windows" + // and would make this code Windows-only, which makes no sense. + // So copy UTF16ToString code into here. + // If other windows-specific code is added, switch to [windows.UTF16ToString] + + s := b.b + for i, v := range s { + if v == 0 { + s = s[:i] + break + } + } + return string(utf16.Decode(s)) +} + +// Cap returns the underlying buffer capacity. +func (b *WString) Cap() uint32 { + if b.empty() { + return 0 + } + return b.cap() +} + +func (b *WString) cap() uint32 { return uint32(cap(b.b)) } +func (b *WString) empty() bool { return b == nil || b.cap() == 0 } diff --git a/vendor/github.com/Microsoft/go-winio/pipe.go b/vendor/github.com/Microsoft/go-winio/pipe.go index ca6e38fc00..25cc811031 100644 --- a/vendor/github.com/Microsoft/go-winio/pipe.go +++ b/vendor/github.com/Microsoft/go-winio/pipe.go @@ -16,11 +16,12 @@ import ( "unsafe" "golang.org/x/sys/windows" + + "github.com/Microsoft/go-winio/internal/fs" ) //sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe //sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateNamedPipeW -//sys createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateFileW //sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo //sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW //sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc @@ -163,19 +164,21 @@ func (s pipeAddress) String() string { } // tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout. -func tryDialPipe(ctx context.Context, path *string, access uint32) (syscall.Handle, error) { +func tryDialPipe(ctx context.Context, path *string, access fs.AccessMask) (syscall.Handle, error) { for { select { case <-ctx.Done(): return syscall.Handle(0), ctx.Err() default: - h, err := createFile(*path, + wh, err := fs.CreateFile(*path, access, - 0, - nil, - syscall.OPEN_EXISTING, - windows.FILE_FLAG_OVERLAPPED|windows.SECURITY_SQOS_PRESENT|windows.SECURITY_ANONYMOUS, - 0) + 0, // mode + nil, // security attributes + fs.OPEN_EXISTING, + fs.FILE_FLAG_OVERLAPPED|fs.SECURITY_SQOS_PRESENT|fs.SECURITY_ANONYMOUS, + 0, // template file handle + ) + h := syscall.Handle(wh) if err == nil { return h, nil } @@ -219,7 +222,7 @@ func DialPipeContext(ctx context.Context, path string) (net.Conn, error) { func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn, error) { var err error var h syscall.Handle - h, err = tryDialPipe(ctx, &path, access) + h, err = tryDialPipe(ctx, &path, fs.AccessMask(access)) if err != nil { return nil, err } @@ -279,6 +282,7 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy } defer localFree(ntPath.Buffer) oa.ObjectName = &ntPath + oa.Attributes = windows.OBJ_CASE_INSENSITIVE // The security descriptor is only needed for the first pipe. if first { diff --git a/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go index 83f45a1351..469b16f639 100644 --- a/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go @@ -63,7 +63,6 @@ var ( procBackupWrite = modkernel32.NewProc("BackupWrite") procCancelIoEx = modkernel32.NewProc("CancelIoEx") procConnectNamedPipe = modkernel32.NewProc("ConnectNamedPipe") - procCreateFileW = modkernel32.NewProc("CreateFileW") procCreateIoCompletionPort = modkernel32.NewProc("CreateIoCompletionPort") procCreateNamedPipeW = modkernel32.NewProc("CreateNamedPipeW") procGetCurrentThread = modkernel32.NewProc("GetCurrentThread") @@ -305,24 +304,6 @@ func connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) { return } -func createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) { - var _p0 *uint16 - _p0, err = syscall.UTF16PtrFromString(name) - if err != nil { - return - } - return _createFile(_p0, access, mode, sa, createmode, attrs, templatefile) -} - -func _createFile(name *uint16, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0) - handle = syscall.Handle(r0) - if handle == syscall.InvalidHandle { - err = errnoErr(e1) - } - return -} - func createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) { r0, _, e1 := syscall.Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount), 0, 0) newport = syscall.Handle(r0) diff --git a/vendor/github.com/golang/mock/AUTHORS b/vendor/github.com/golang/mock/AUTHORS new file mode 100644 index 0000000000..660b8ccc8a --- /dev/null +++ b/vendor/github.com/golang/mock/AUTHORS @@ -0,0 +1,12 @@ +# This is the official list of GoMock authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Alex Reece +Google Inc. diff --git a/vendor/github.com/golang/mock/CONTRIBUTORS b/vendor/github.com/golang/mock/CONTRIBUTORS new file mode 100644 index 0000000000..def849cab1 --- /dev/null +++ b/vendor/github.com/golang/mock/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This is the official list of people who can contribute (and typically +# have contributed) code to the gomock repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name +# +# An entry with two email addresses specifies that the +# first address should be used in the submit logs and +# that the second address should be recognized as the +# same person when interacting with Rietveld. + +# Please keep the list sorted. + +Aaron Jacobs +Alex Reece +David Symonds +Ryan Barrett diff --git a/vendor/github.com/golang/mock/LICENSE b/vendor/github.com/golang/mock/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/github.com/golang/mock/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/golang/mock/gomock/call.go b/vendor/github.com/golang/mock/gomock/call.go new file mode 100644 index 0000000000..13c9f44b1e --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/call.go @@ -0,0 +1,445 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +// Call represents an expected call to a mock. +type Call struct { + t TestHelper // for triggering test failures on invalid call setup + + receiver interface{} // the receiver of the method call + method string // the name of the method + methodType reflect.Type // the type of the method + args []Matcher // the args + origin string // file and line number of call setup + + preReqs []*Call // prerequisite calls + + // Expectations + minCalls, maxCalls int + + numCalls int // actual number made + + // actions are called when this Call is called. Each action gets the args and + // can set the return values by returning a non-nil slice. Actions run in the + // order they are created. + actions []func([]interface{}) []interface{} +} + +// newCall creates a *Call. It requires the method type in order to support +// unexported methods. +func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call { + t.Helper() + + // TODO: check arity, types. + mArgs := make([]Matcher, len(args)) + for i, arg := range args { + if m, ok := arg.(Matcher); ok { + mArgs[i] = m + } else if arg == nil { + // Handle nil specially so that passing a nil interface value + // will match the typed nils of concrete args. + mArgs[i] = Nil() + } else { + mArgs[i] = Eq(arg) + } + } + + // callerInfo's skip should be updated if the number of calls between the user's test + // and this line changes, i.e. this code is wrapped in another anonymous function. + // 0 is us, 1 is RecordCallWithMethodType(), 2 is the generated recorder, and 3 is the user's test. + origin := callerInfo(3) + actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} { + // Synthesize the zero value for each of the return args' types. + rets := make([]interface{}, methodType.NumOut()) + for i := 0; i < methodType.NumOut(); i++ { + rets[i] = reflect.Zero(methodType.Out(i)).Interface() + } + return rets + }} + return &Call{t: t, receiver: receiver, method: method, methodType: methodType, + args: mArgs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions} +} + +// AnyTimes allows the expectation to be called 0 or more times +func (c *Call) AnyTimes() *Call { + c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity + return c +} + +// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called or if MaxTimes +// was previously called with 1, MinTimes also sets the maximum number of calls to infinity. +func (c *Call) MinTimes(n int) *Call { + c.minCalls = n + if c.maxCalls == 1 { + c.maxCalls = 1e8 + } + return c +} + +// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called or if MinTimes was +// previously called with 1, MaxTimes also sets the minimum number of calls to 0. +func (c *Call) MaxTimes(n int) *Call { + c.maxCalls = n + if c.minCalls == 1 { + c.minCalls = 0 + } + return c +} + +// DoAndReturn declares the action to run when the call is matched. +// The return values from this function are returned by the mocked function. +// It takes an interface{} argument to support n-arity functions. +func (c *Call) DoAndReturn(f interface{}) *Call { + // TODO: Check arity and types here, rather than dying badly elsewhere. + v := reflect.ValueOf(f) + + c.addAction(func(args []interface{}) []interface{} { + c.t.Helper() + vArgs := make([]reflect.Value, len(args)) + ft := v.Type() + if c.methodType.NumIn() != ft.NumIn() { + c.t.Fatalf("wrong number of arguments in DoAndReturn func for %T.%v: got %d, want %d [%s]", + c.receiver, c.method, ft.NumIn(), c.methodType.NumIn(), c.origin) + return nil + } + for i := 0; i < len(args); i++ { + if args[i] != nil { + vArgs[i] = reflect.ValueOf(args[i]) + } else { + // Use the zero value for the arg. + vArgs[i] = reflect.Zero(ft.In(i)) + } + } + vRets := v.Call(vArgs) + rets := make([]interface{}, len(vRets)) + for i, ret := range vRets { + rets[i] = ret.Interface() + } + return rets + }) + return c +} + +// Do declares the action to run when the call is matched. The function's +// return values are ignored to retain backward compatibility. To use the +// return values call DoAndReturn. +// It takes an interface{} argument to support n-arity functions. +func (c *Call) Do(f interface{}) *Call { + // TODO: Check arity and types here, rather than dying badly elsewhere. + v := reflect.ValueOf(f) + + c.addAction(func(args []interface{}) []interface{} { + c.t.Helper() + if c.methodType.NumIn() != v.Type().NumIn() { + c.t.Fatalf("wrong number of arguments in Do func for %T.%v: got %d, want %d [%s]", + c.receiver, c.method, v.Type().NumIn(), c.methodType.NumIn(), c.origin) + return nil + } + vArgs := make([]reflect.Value, len(args)) + ft := v.Type() + for i := 0; i < len(args); i++ { + if args[i] != nil { + vArgs[i] = reflect.ValueOf(args[i]) + } else { + // Use the zero value for the arg. + vArgs[i] = reflect.Zero(ft.In(i)) + } + } + v.Call(vArgs) + return nil + }) + return c +} + +// Return declares the values to be returned by the mocked function call. +func (c *Call) Return(rets ...interface{}) *Call { + c.t.Helper() + + mt := c.methodType + if len(rets) != mt.NumOut() { + c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]", + c.receiver, c.method, len(rets), mt.NumOut(), c.origin) + } + for i, ret := range rets { + if got, want := reflect.TypeOf(ret), mt.Out(i); got == want { + // Identical types; nothing to do. + } else if got == nil { + // Nil needs special handling. + switch want.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + // ok + default: + c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]", + i, c.receiver, c.method, want, c.origin) + } + } else if got.AssignableTo(want) { + // Assignable type relation. Make the assignment now so that the generated code + // can return the values with a type assertion. + v := reflect.New(want).Elem() + v.Set(reflect.ValueOf(ret)) + rets[i] = v.Interface() + } else { + c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]", + i, c.receiver, c.method, got, want, c.origin) + } + } + + c.addAction(func([]interface{}) []interface{} { + return rets + }) + + return c +} + +// Times declares the exact number of times a function call is expected to be executed. +func (c *Call) Times(n int) *Call { + c.minCalls, c.maxCalls = n, n + return c +} + +// SetArg declares an action that will set the nth argument's value, +// indirected through a pointer. Or, in the case of a slice, SetArg +// will copy value's elements into the nth argument. +func (c *Call) SetArg(n int, value interface{}) *Call { + c.t.Helper() + + mt := c.methodType + // TODO: This will break on variadic methods. + // We will need to check those at invocation time. + if n < 0 || n >= mt.NumIn() { + c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]", + n, mt.NumIn(), c.origin) + } + // Permit setting argument through an interface. + // In the interface case, we don't (nay, can't) check the type here. + at := mt.In(n) + switch at.Kind() { + case reflect.Ptr: + dt := at.Elem() + if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) { + c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]", + n, vt, dt, c.origin) + } + case reflect.Interface: + // nothing to do + case reflect.Slice: + // nothing to do + default: + c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]", + n, at, c.origin) + } + + c.addAction(func(args []interface{}) []interface{} { + v := reflect.ValueOf(value) + switch reflect.TypeOf(args[n]).Kind() { + case reflect.Slice: + setSlice(args[n], v) + default: + reflect.ValueOf(args[n]).Elem().Set(v) + } + return nil + }) + return c +} + +// isPreReq returns true if other is a direct or indirect prerequisite to c. +func (c *Call) isPreReq(other *Call) bool { + for _, preReq := range c.preReqs { + if other == preReq || preReq.isPreReq(other) { + return true + } + } + return false +} + +// After declares that the call may only match after preReq has been exhausted. +func (c *Call) After(preReq *Call) *Call { + c.t.Helper() + + if c == preReq { + c.t.Fatalf("A call isn't allowed to be its own prerequisite") + } + if preReq.isPreReq(c) { + c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq) + } + + c.preReqs = append(c.preReqs, preReq) + return c +} + +// Returns true if the minimum number of calls have been made. +func (c *Call) satisfied() bool { + return c.numCalls >= c.minCalls +} + +// Returns true if the maximum number of calls have been made. +func (c *Call) exhausted() bool { + return c.numCalls >= c.maxCalls +} + +func (c *Call) String() string { + args := make([]string, len(c.args)) + for i, arg := range c.args { + args[i] = arg.String() + } + arguments := strings.Join(args, ", ") + return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin) +} + +// Tests if the given call matches the expected call. +// If yes, returns nil. If no, returns error with message explaining why it does not match. +func (c *Call) matches(args []interface{}) error { + if !c.methodType.IsVariadic() { + if len(args) != len(c.args) { + return fmt.Errorf("expected call at %s has the wrong number of arguments. Got: %d, want: %d", + c.origin, len(args), len(c.args)) + } + + for i, m := range c.args { + if !m.Matches(args[i]) { + return fmt.Errorf( + "expected call at %s doesn't match the argument at index %d.\nGot: %v\nWant: %v", + c.origin, i, formatGottenArg(m, args[i]), m, + ) + } + } + } else { + if len(c.args) < c.methodType.NumIn()-1 { + return fmt.Errorf("expected call at %s has the wrong number of matchers. Got: %d, want: %d", + c.origin, len(c.args), c.methodType.NumIn()-1) + } + if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) { + return fmt.Errorf("expected call at %s has the wrong number of arguments. Got: %d, want: %d", + c.origin, len(args), len(c.args)) + } + if len(args) < len(c.args)-1 { + return fmt.Errorf("expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d", + c.origin, len(args), len(c.args)-1) + } + + for i, m := range c.args { + if i < c.methodType.NumIn()-1 { + // Non-variadic args + if !m.Matches(args[i]) { + return fmt.Errorf("expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), formatGottenArg(m, args[i]), m) + } + continue + } + // The last arg has a possibility of a variadic argument, so let it branch + + // sample: Foo(a int, b int, c ...int) + if i < len(c.args) && i < len(args) { + if m.Matches(args[i]) { + // Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC) + // Got Foo(a, b) want Foo(matcherA, matcherB) + // Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD) + continue + } + } + + // The number of actual args don't match the number of matchers, + // or the last matcher is a slice and the last arg is not. + // If this function still matches it is because the last matcher + // matches all the remaining arguments or the lack of any. + // Convert the remaining arguments, if any, into a slice of the + // expected type. + vArgsType := c.methodType.In(c.methodType.NumIn() - 1) + vArgs := reflect.MakeSlice(vArgsType, 0, len(args)-i) + for _, arg := range args[i:] { + vArgs = reflect.Append(vArgs, reflect.ValueOf(arg)) + } + if m.Matches(vArgs.Interface()) { + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher) + // Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher) + break + } + // Wrong number of matchers or not match. Fail. + // Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE) + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c) want Foo(matcherA, matcherB) + + return fmt.Errorf("expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), formatGottenArg(m, args[i:]), c.args[i]) + } + } + + // Check that all prerequisite calls have been satisfied. + for _, preReqCall := range c.preReqs { + if !preReqCall.satisfied() { + return fmt.Errorf("expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v", + c.origin, preReqCall, c) + } + } + + // Check that the call is not exhausted. + if c.exhausted() { + return fmt.Errorf("expected call at %s has already been called the max number of times", c.origin) + } + + return nil +} + +// dropPrereqs tells the expected Call to not re-check prerequisite calls any +// longer, and to return its current set. +func (c *Call) dropPrereqs() (preReqs []*Call) { + preReqs = c.preReqs + c.preReqs = nil + return +} + +func (c *Call) call() []func([]interface{}) []interface{} { + c.numCalls++ + return c.actions +} + +// InOrder declares that the given calls should occur in order. +func InOrder(calls ...*Call) { + for i := 1; i < len(calls); i++ { + calls[i].After(calls[i-1]) + } +} + +func setSlice(arg interface{}, v reflect.Value) { + va := reflect.ValueOf(arg) + for i := 0; i < v.Len(); i++ { + va.Index(i).Set(v.Index(i)) + } +} + +func (c *Call) addAction(action func([]interface{}) []interface{}) { + c.actions = append(c.actions, action) +} + +func formatGottenArg(m Matcher, arg interface{}) string { + got := fmt.Sprintf("%v (%T)", arg, arg) + if gs, ok := m.(GotFormatter); ok { + got = gs.Got(arg) + } + return got +} diff --git a/vendor/github.com/golang/mock/gomock/callset.go b/vendor/github.com/golang/mock/gomock/callset.go new file mode 100644 index 0000000000..49dba787a4 --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/callset.go @@ -0,0 +1,113 @@ +// Copyright 2011 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "bytes" + "errors" + "fmt" +) + +// callSet represents a set of expected calls, indexed by receiver and method +// name. +type callSet struct { + // Calls that are still expected. + expected map[callSetKey][]*Call + // Calls that have been exhausted. + exhausted map[callSetKey][]*Call +} + +// callSetKey is the key in the maps in callSet +type callSetKey struct { + receiver interface{} + fname string +} + +func newCallSet() *callSet { + return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)} +} + +// Add adds a new expected call. +func (cs callSet) Add(call *Call) { + key := callSetKey{call.receiver, call.method} + m := cs.expected + if call.exhausted() { + m = cs.exhausted + } + m[key] = append(m[key], call) +} + +// Remove removes an expected call. +func (cs callSet) Remove(call *Call) { + key := callSetKey{call.receiver, call.method} + calls := cs.expected[key] + for i, c := range calls { + if c == call { + // maintain order for remaining calls + cs.expected[key] = append(calls[:i], calls[i+1:]...) + cs.exhausted[key] = append(cs.exhausted[key], call) + break + } + } +} + +// FindMatch searches for a matching call. Returns error with explanation message if no call matched. +func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) { + key := callSetKey{receiver, method} + + // Search through the expected calls. + expected := cs.expected[key] + var callsErrors bytes.Buffer + for _, call := range expected { + err := call.matches(args) + if err != nil { + _, _ = fmt.Fprintf(&callsErrors, "\n%v", err) + } else { + return call, nil + } + } + + // If we haven't found a match then search through the exhausted calls so we + // get useful error messages. + exhausted := cs.exhausted[key] + for _, call := range exhausted { + if err := call.matches(args); err != nil { + _, _ = fmt.Fprintf(&callsErrors, "\n%v", err) + continue + } + _, _ = fmt.Fprintf( + &callsErrors, "all expected calls for method %q have been exhausted", method, + ) + } + + if len(expected)+len(exhausted) == 0 { + _, _ = fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method) + } + + return nil, errors.New(callsErrors.String()) +} + +// Failures returns the calls that are not satisfied. +func (cs callSet) Failures() []*Call { + failures := make([]*Call, 0, len(cs.expected)) + for _, calls := range cs.expected { + for _, call := range calls { + if !call.satisfied() { + failures = append(failures, call) + } + } + } + return failures +} diff --git a/vendor/github.com/golang/mock/gomock/controller.go b/vendor/github.com/golang/mock/gomock/controller.go new file mode 100644 index 0000000000..f054200d56 --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/controller.go @@ -0,0 +1,336 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gomock is a mock framework for Go. +// +// Standard usage: +// (1) Define an interface that you wish to mock. +// type MyInterface interface { +// SomeMethod(x int64, y string) +// } +// (2) Use mockgen to generate a mock from the interface. +// (3) Use the mock in a test: +// func TestMyThing(t *testing.T) { +// mockCtrl := gomock.NewController(t) +// defer mockCtrl.Finish() +// +// mockObj := something.NewMockMyInterface(mockCtrl) +// mockObj.EXPECT().SomeMethod(4, "blah") +// // pass mockObj to a real object and play with it. +// } +// +// By default, expected calls are not enforced to run in any particular order. +// Call order dependency can be enforced by use of InOrder and/or Call.After. +// Call.After can create more varied call order dependencies, but InOrder is +// often more convenient. +// +// The following examples create equivalent call order dependencies. +// +// Example of using Call.After to chain expected call order: +// +// firstCall := mockObj.EXPECT().SomeMethod(1, "first") +// secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall) +// mockObj.EXPECT().SomeMethod(3, "third").After(secondCall) +// +// Example of using InOrder to declare expected call order: +// +// gomock.InOrder( +// mockObj.EXPECT().SomeMethod(1, "first"), +// mockObj.EXPECT().SomeMethod(2, "second"), +// mockObj.EXPECT().SomeMethod(3, "third"), +// ) +package gomock + +import ( + "context" + "fmt" + "reflect" + "runtime" + "sync" +) + +// A TestReporter is something that can be used to report test failures. It +// is satisfied by the standard library's *testing.T. +type TestReporter interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) +} + +// TestHelper is a TestReporter that has the Helper method. It is satisfied +// by the standard library's *testing.T. +type TestHelper interface { + TestReporter + Helper() +} + +// cleanuper is used to check if TestHelper also has the `Cleanup` method. A +// common pattern is to pass in a `*testing.T` to +// `NewController(t TestReporter)`. In Go 1.14+, `*testing.T` has a cleanup +// method. This can be utilized to call `Finish()` so the caller of this library +// does not have to. +type cleanuper interface { + Cleanup(func()) +} + +// A Controller represents the top-level control of a mock ecosystem. It +// defines the scope and lifetime of mock objects, as well as their +// expectations. It is safe to call Controller's methods from multiple +// goroutines. Each test should create a new Controller and invoke Finish via +// defer. +// +// func TestFoo(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() +// // .. +// } +// +// func TestBar(t *testing.T) { +// t.Run("Sub-Test-1", st) { +// ctrl := gomock.NewController(st) +// defer ctrl.Finish() +// // .. +// }) +// t.Run("Sub-Test-2", st) { +// ctrl := gomock.NewController(st) +// defer ctrl.Finish() +// // .. +// }) +// }) +type Controller struct { + // T should only be called within a generated mock. It is not intended to + // be used in user code and may be changed in future versions. T is the + // TestReporter passed in when creating the Controller via NewController. + // If the TestReporter does not implement a TestHelper it will be wrapped + // with a nopTestHelper. + T TestHelper + mu sync.Mutex + expectedCalls *callSet + finished bool +} + +// NewController returns a new Controller. It is the preferred way to create a +// Controller. +// +// New in go1.14+, if you are passing a *testing.T into this function you no +// longer need to call ctrl.Finish() in your test methods. +func NewController(t TestReporter) *Controller { + h, ok := t.(TestHelper) + if !ok { + h = &nopTestHelper{t} + } + ctrl := &Controller{ + T: h, + expectedCalls: newCallSet(), + } + if c, ok := isCleanuper(ctrl.T); ok { + c.Cleanup(func() { + ctrl.T.Helper() + ctrl.finish(true, nil) + }) + } + + return ctrl +} + +type cancelReporter struct { + t TestHelper + cancel func() +} + +func (r *cancelReporter) Errorf(format string, args ...interface{}) { + r.t.Errorf(format, args...) +} +func (r *cancelReporter) Fatalf(format string, args ...interface{}) { + defer r.cancel() + r.t.Fatalf(format, args...) +} + +func (r *cancelReporter) Helper() { + r.t.Helper() +} + +// WithContext returns a new Controller and a Context, which is cancelled on any +// fatal failure. +func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) { + h, ok := t.(TestHelper) + if !ok { + h = &nopTestHelper{t: t} + } + + ctx, cancel := context.WithCancel(ctx) + return NewController(&cancelReporter{t: h, cancel: cancel}), ctx +} + +type nopTestHelper struct { + t TestReporter +} + +func (h *nopTestHelper) Errorf(format string, args ...interface{}) { + h.t.Errorf(format, args...) +} +func (h *nopTestHelper) Fatalf(format string, args ...interface{}) { + h.t.Fatalf(format, args...) +} + +func (h nopTestHelper) Helper() {} + +// RecordCall is called by a mock. It should not be called by user code. +func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call { + ctrl.T.Helper() + + recv := reflect.ValueOf(receiver) + for i := 0; i < recv.Type().NumMethod(); i++ { + if recv.Type().Method(i).Name == method { + return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...) + } + } + ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver) + panic("unreachable") +} + +// RecordCallWithMethodType is called by a mock. It should not be called by user code. +func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call { + ctrl.T.Helper() + + call := newCall(ctrl.T, receiver, method, methodType, args...) + + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + ctrl.expectedCalls.Add(call) + + return call +} + +// Call is called by a mock. It should not be called by user code. +func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} { + ctrl.T.Helper() + + // Nest this code so we can use defer to make sure the lock is released. + actions := func() []func([]interface{}) []interface{} { + ctrl.T.Helper() + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + + expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args) + if err != nil { + // callerInfo's skip should be updated if the number of calls between the user's test + // and this line changes, i.e. this code is wrapped in another anonymous function. + // 0 is us, 1 is controller.Call(), 2 is the generated mock, and 3 is the user's test. + origin := callerInfo(3) + ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err) + } + + // Two things happen here: + // * the matching call no longer needs to check prerequite calls, + // * and the prerequite calls are no longer expected, so remove them. + preReqCalls := expected.dropPrereqs() + for _, preReqCall := range preReqCalls { + ctrl.expectedCalls.Remove(preReqCall) + } + + actions := expected.call() + if expected.exhausted() { + ctrl.expectedCalls.Remove(expected) + } + return actions + }() + + var rets []interface{} + for _, action := range actions { + if r := action(args); r != nil { + rets = r + } + } + + return rets +} + +// Finish checks to see if all the methods that were expected to be called +// were called. It should be invoked for each Controller. It is not idempotent +// and therefore can only be invoked once. +// +// New in go1.14+, if you are passing a *testing.T into NewController function you no +// longer need to call ctrl.Finish() in your test methods. +func (ctrl *Controller) Finish() { + // If we're currently panicking, probably because this is a deferred call. + // This must be recovered in the deferred function. + err := recover() + ctrl.finish(false, err) +} + +func (ctrl *Controller) finish(cleanup bool, panicErr interface{}) { + ctrl.T.Helper() + + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + + if ctrl.finished { + if _, ok := isCleanuper(ctrl.T); !ok { + ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.") + } + return + } + ctrl.finished = true + + // Short-circuit, pass through the panic. + if panicErr != nil { + panic(panicErr) + } + + // Check that all remaining expected calls are satisfied. + failures := ctrl.expectedCalls.Failures() + for _, call := range failures { + ctrl.T.Errorf("missing call(s) to %v", call) + } + if len(failures) != 0 { + if !cleanup { + ctrl.T.Fatalf("aborting test due to missing call(s)") + return + } + ctrl.T.Errorf("aborting test due to missing call(s)") + } +} + +// callerInfo returns the file:line of the call site. skip is the number +// of stack frames to skip when reporting. 0 is callerInfo's call site. +func callerInfo(skip int) string { + if _, file, line, ok := runtime.Caller(skip + 1); ok { + return fmt.Sprintf("%s:%d", file, line) + } + return "unknown file" +} + +// isCleanuper checks it if t's base TestReporter has a Cleanup method. +func isCleanuper(t TestReporter) (cleanuper, bool) { + tr := unwrapTestReporter(t) + c, ok := tr.(cleanuper) + return c, ok +} + +// unwrapTestReporter unwraps TestReporter to the base implementation. +func unwrapTestReporter(t TestReporter) TestReporter { + tr := t + switch nt := t.(type) { + case *cancelReporter: + tr = nt.t + if h, check := tr.(*nopTestHelper); check { + tr = h.t + } + case *nopTestHelper: + tr = nt.t + default: + // not wrapped + } + return tr +} diff --git a/vendor/github.com/golang/mock/gomock/matchers.go b/vendor/github.com/golang/mock/gomock/matchers.go new file mode 100644 index 0000000000..2822fb2c8c --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/matchers.go @@ -0,0 +1,341 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "fmt" + "reflect" + "strings" +) + +// A Matcher is a representation of a class of values. +// It is used to represent the valid or expected arguments to a mocked method. +type Matcher interface { + // Matches returns whether x is a match. + Matches(x interface{}) bool + + // String describes what the matcher matches. + String() string +} + +// WantFormatter modifies the given Matcher's String() method to the given +// Stringer. This allows for control on how the "Want" is formatted when +// printing . +func WantFormatter(s fmt.Stringer, m Matcher) Matcher { + type matcher interface { + Matches(x interface{}) bool + } + + return struct { + matcher + fmt.Stringer + }{ + matcher: m, + Stringer: s, + } +} + +// StringerFunc type is an adapter to allow the use of ordinary functions as +// a Stringer. If f is a function with the appropriate signature, +// StringerFunc(f) is a Stringer that calls f. +type StringerFunc func() string + +// String implements fmt.Stringer. +func (f StringerFunc) String() string { + return f() +} + +// GotFormatter is used to better print failure messages. If a matcher +// implements GotFormatter, it will use the result from Got when printing +// the failure message. +type GotFormatter interface { + // Got is invoked with the received value. The result is used when + // printing the failure message. + Got(got interface{}) string +} + +// GotFormatterFunc type is an adapter to allow the use of ordinary +// functions as a GotFormatter. If f is a function with the appropriate +// signature, GotFormatterFunc(f) is a GotFormatter that calls f. +type GotFormatterFunc func(got interface{}) string + +// Got implements GotFormatter. +func (f GotFormatterFunc) Got(got interface{}) string { + return f(got) +} + +// GotFormatterAdapter attaches a GotFormatter to a Matcher. +func GotFormatterAdapter(s GotFormatter, m Matcher) Matcher { + return struct { + GotFormatter + Matcher + }{ + GotFormatter: s, + Matcher: m, + } +} + +type anyMatcher struct{} + +func (anyMatcher) Matches(interface{}) bool { + return true +} + +func (anyMatcher) String() string { + return "is anything" +} + +type eqMatcher struct { + x interface{} +} + +func (e eqMatcher) Matches(x interface{}) bool { + // In case, some value is nil + if e.x == nil || x == nil { + return reflect.DeepEqual(e.x, x) + } + + // Check if types assignable and convert them to common type + x1Val := reflect.ValueOf(e.x) + x2Val := reflect.ValueOf(x) + + if x1Val.Type().AssignableTo(x2Val.Type()) { + x1ValConverted := x1Val.Convert(x2Val.Type()) + return reflect.DeepEqual(x1ValConverted.Interface(), x2Val.Interface()) + } + + return false +} + +func (e eqMatcher) String() string { + return fmt.Sprintf("is equal to %v (%T)", e.x, e.x) +} + +type nilMatcher struct{} + +func (nilMatcher) Matches(x interface{}) bool { + if x == nil { + return true + } + + v := reflect.ValueOf(x) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice: + return v.IsNil() + } + + return false +} + +func (nilMatcher) String() string { + return "is nil" +} + +type notMatcher struct { + m Matcher +} + +func (n notMatcher) Matches(x interface{}) bool { + return !n.m.Matches(x) +} + +func (n notMatcher) String() string { + return "not(" + n.m.String() + ")" +} + +type assignableToTypeOfMatcher struct { + targetType reflect.Type +} + +func (m assignableToTypeOfMatcher) Matches(x interface{}) bool { + return reflect.TypeOf(x).AssignableTo(m.targetType) +} + +func (m assignableToTypeOfMatcher) String() string { + return "is assignable to " + m.targetType.Name() +} + +type allMatcher struct { + matchers []Matcher +} + +func (am allMatcher) Matches(x interface{}) bool { + for _, m := range am.matchers { + if !m.Matches(x) { + return false + } + } + return true +} + +func (am allMatcher) String() string { + ss := make([]string, 0, len(am.matchers)) + for _, matcher := range am.matchers { + ss = append(ss, matcher.String()) + } + return strings.Join(ss, "; ") +} + +type lenMatcher struct { + i int +} + +func (m lenMatcher) Matches(x interface{}) bool { + v := reflect.ValueOf(x) + switch v.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == m.i + default: + return false + } +} + +func (m lenMatcher) String() string { + return fmt.Sprintf("has length %d", m.i) +} + +type inAnyOrderMatcher struct { + x interface{} +} + +func (m inAnyOrderMatcher) Matches(x interface{}) bool { + given, ok := m.prepareValue(x) + if !ok { + return false + } + wanted, ok := m.prepareValue(m.x) + if !ok { + return false + } + + if given.Len() != wanted.Len() { + return false + } + + usedFromGiven := make([]bool, given.Len()) + foundFromWanted := make([]bool, wanted.Len()) + for i := 0; i < wanted.Len(); i++ { + wantedMatcher := Eq(wanted.Index(i).Interface()) + for j := 0; j < given.Len(); j++ { + if usedFromGiven[j] { + continue + } + if wantedMatcher.Matches(given.Index(j).Interface()) { + foundFromWanted[i] = true + usedFromGiven[j] = true + break + } + } + } + + missingFromWanted := 0 + for _, found := range foundFromWanted { + if !found { + missingFromWanted++ + } + } + extraInGiven := 0 + for _, used := range usedFromGiven { + if !used { + extraInGiven++ + } + } + + return extraInGiven == 0 && missingFromWanted == 0 +} + +func (m inAnyOrderMatcher) prepareValue(x interface{}) (reflect.Value, bool) { + xValue := reflect.ValueOf(x) + switch xValue.Kind() { + case reflect.Slice, reflect.Array: + return xValue, true + default: + return reflect.Value{}, false + } +} + +func (m inAnyOrderMatcher) String() string { + return fmt.Sprintf("has the same elements as %v", m.x) +} + +// Constructors + +// All returns a composite Matcher that returns true if and only all of the +// matchers return true. +func All(ms ...Matcher) Matcher { return allMatcher{ms} } + +// Any returns a matcher that always matches. +func Any() Matcher { return anyMatcher{} } + +// Eq returns a matcher that matches on equality. +// +// Example usage: +// Eq(5).Matches(5) // returns true +// Eq(5).Matches(4) // returns false +func Eq(x interface{}) Matcher { return eqMatcher{x} } + +// Len returns a matcher that matches on length. This matcher returns false if +// is compared to a type that is not an array, chan, map, slice, or string. +func Len(i int) Matcher { + return lenMatcher{i} +} + +// Nil returns a matcher that matches if the received value is nil. +// +// Example usage: +// var x *bytes.Buffer +// Nil().Matches(x) // returns true +// x = &bytes.Buffer{} +// Nil().Matches(x) // returns false +func Nil() Matcher { return nilMatcher{} } + +// Not reverses the results of its given child matcher. +// +// Example usage: +// Not(Eq(5)).Matches(4) // returns true +// Not(Eq(5)).Matches(5) // returns false +func Not(x interface{}) Matcher { + if m, ok := x.(Matcher); ok { + return notMatcher{m} + } + return notMatcher{Eq(x)} +} + +// AssignableToTypeOf is a Matcher that matches if the parameter to the mock +// function is assignable to the type of the parameter to this function. +// +// Example usage: +// var s fmt.Stringer = &bytes.Buffer{} +// AssignableToTypeOf(s).Matches(time.Second) // returns true +// AssignableToTypeOf(s).Matches(99) // returns false +// +// var ctx = reflect.TypeOf((*context.Context)(nil)).Elem() +// AssignableToTypeOf(ctx).Matches(context.Background()) // returns true +func AssignableToTypeOf(x interface{}) Matcher { + if xt, ok := x.(reflect.Type); ok { + return assignableToTypeOfMatcher{xt} + } + return assignableToTypeOfMatcher{reflect.TypeOf(x)} +} + +// InAnyOrder is a Matcher that returns true for collections of the same elements ignoring the order. +// +// Example usage: +// InAnyOrder([]int{1, 2, 3}).Matches([]int{1, 3, 2}) // returns true +// InAnyOrder([]int{1, 2, 3}).Matches([]int{1, 2}) // returns false +func InAnyOrder(x interface{}) Matcher { + return inAnyOrderMatcher{x} +} diff --git a/vendor/github.com/kata-containers/kata-containers/src/runtime/LICENSE b/vendor/github.com/kata-containers/kata-containers/src/runtime/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/vendor/github.com/kata-containers/kata-containers/src/runtime/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume/utils.go b/vendor/github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume/utils.go new file mode 100644 index 0000000000..9e13a4d227 --- /dev/null +++ b/vendor/github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume/utils.go @@ -0,0 +1,126 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package volume + +import ( + b64 "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +const ( + mountInfoFileName = "mountInfo.json" + + FSGroupMetadataKey = "fsGroup" + FSGroupChangePolicyMetadataKey = "fsGroupChangePolicy" +) + +// FSGroupChangePolicy holds policies that will be used for applying fsGroup to a volume. +// This type and the allowed values are tracking the PodFSGroupChangePolicy defined in +// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/core/v1/types.go +// It is up to the client using the direct-assigned volume feature (e.g. CSI drivers) to determine +// the optimal setting for this change policy (i.e. from Pod spec or assuming volume ownership +// based on the storage offering). +type FSGroupChangePolicy string + +const ( + // FSGroupChangeAlways indicates that volume's ownership should always be changed. + FSGroupChangeAlways FSGroupChangePolicy = "Always" + // FSGroupChangeOnRootMismatch indicates that volume's ownership will be changed + // only when ownership of root directory does not match with the desired group id. + FSGroupChangeOnRootMismatch FSGroupChangePolicy = "OnRootMismatch" +) + +var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes" + +// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM. +type MountInfo struct { + // The type of the volume (ie. block) + VolumeType string `json:"volume-type"` + // The device backing the volume. + Device string `json:"device"` + // The filesystem type to be mounted on the volume. + FsType string `json:"fstype"` + // Additional metadata to pass to the agent regarding this volume. + Metadata map[string]string `json:"metadata,omitempty"` + // Additional mount options. + Options []string `json:"options,omitempty"` +} + +// Add writes the mount info of a direct volume into a filesystem path known to Kata Container. +func Add(volumePath string, mountInfo string) error { + volumeDir := filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath))) + stat, err := os.Stat(volumeDir) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.MkdirAll(volumeDir, 0700); err != nil { + return err + } + } + if stat != nil && !stat.IsDir() { + return fmt.Errorf("%s should be a directory", volumeDir) + } + + var deserialized MountInfo + if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil { + return err + } + + return os.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600) +} + +// Remove deletes the direct volume path including all the files inside it. +func Remove(volumePath string) error { + return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) +} + +// VolumeMountInfo retrieves the mount info of a direct volume. +func VolumeMountInfo(volumePath string) (*MountInfo, error) { + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)), mountInfoFileName) + if _, err := os.Stat(mountInfoFilePath); err != nil { + return nil, err + } + buf, err := os.ReadFile(mountInfoFilePath) + if err != nil { + return nil, err + } + var mountInfo MountInfo + if err := json.Unmarshal(buf, &mountInfo); err != nil { + return nil, err + } + return &mountInfo, nil +} + +// RecordSandboxId associates a sandbox id with a direct volume. +func RecordSandboxId(sandboxId string, volumePath string) error { + encodedPath := b64.URLEncoding.EncodeToString([]byte(volumePath)) + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, encodedPath, mountInfoFileName) + if _, err := os.Stat(mountInfoFilePath); err != nil { + return err + } + + return os.WriteFile(filepath.Join(kataDirectVolumeRootPath, encodedPath, sandboxId), []byte(""), 0600) +} + +func GetSandboxIdForVolume(volumePath string) (string, error) { + files, err := os.ReadDir(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) + if err != nil { + return "", err + } + // Find the id of the first sandbox. + // We expect a direct-assigned volume is associated with only a sandbox at a time. + for _, file := range files { + if file.Name() != mountInfoFileName { + return file.Name(), nil + } + } + return "", fmt.Errorf("no sandbox found for %s", volumePath) +} diff --git a/vendor/github.com/moby/sys/mountinfo/mountinfo_linux.go b/vendor/github.com/moby/sys/mountinfo/mountinfo_linux.go index 59332b07bf..b32b5c9b15 100644 --- a/vendor/github.com/moby/sys/mountinfo/mountinfo_linux.go +++ b/vendor/github.com/moby/sys/mountinfo/mountinfo_linux.go @@ -5,15 +5,19 @@ import ( "fmt" "io" "os" + "runtime" "strconv" "strings" + "sync" + + "golang.org/x/sys/unix" ) // GetMountsFromReader retrieves a list of mounts from the // reader provided, with an optional filter applied (use nil // for no filter). This can be useful in tests or benchmarks // that provide fake mountinfo data, or when a source other -// than /proc/self/mountinfo needs to be read from. +// than /proc/thread-self/mountinfo needs to be read from. // // This function is Linux-specific. func GetMountsFromReader(r io.Reader, filter FilterFunc) ([]*Info, error) { @@ -127,8 +131,40 @@ func GetMountsFromReader(r io.Reader, filter FilterFunc) ([]*Info, error) { return out, nil } -func parseMountTable(filter FilterFunc) ([]*Info, error) { - f, err := os.Open("/proc/self/mountinfo") +var ( + haveProcThreadSelf bool + haveProcThreadSelfOnce sync.Once +) + +func parseMountTable(filter FilterFunc) (_ []*Info, err error) { + haveProcThreadSelfOnce.Do(func() { + _, err := os.Stat("/proc/thread-self/mountinfo") + haveProcThreadSelf = err == nil + }) + + // We need to lock ourselves to the current OS thread in order to make sure + // that the thread referenced by /proc/thread-self stays alive until we + // finish parsing the file. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + var f *os.File + if haveProcThreadSelf { + f, err = os.Open("/proc/thread-self/mountinfo") + } else { + // On pre-3.17 kernels (such as CentOS 7), we don't have + // /proc/thread-self/ so we need to manually construct + // /proc/self/task// as a fallback. + f, err = os.Open("/proc/self/task/" + strconv.Itoa(unix.Gettid()) + "/mountinfo") + if os.IsNotExist(err) { + // If /proc/self/task/... failed, it means that our active pid + // namespace doesn't match the pid namespace of the /proc mount. In + // this case we just have to make do with /proc/self, since there + // is no other way of figuring out our tid in a parent pid + // namespace on pre-3.17 kernels. + f, err = os.Open("/proc/self/mountinfo") + } + } if err != nil { return nil, err } @@ -158,10 +194,10 @@ func PidMountInfo(pid int) ([]*Info, error) { // A few specific characters in mountinfo path entries (root and mountpoint) // are escaped using a backslash followed by a character's ascii code in octal. // -// space -- as \040 -// tab (aka \t) -- as \011 -// newline (aka \n) -- as \012 -// backslash (aka \\) -- as \134 +// space -- as \040 +// tab (aka \t) -- as \011 +// newline (aka \n) -- as \012 +// backslash (aka \\) -- as \134 // // This function converts path from mountinfo back, i.e. it unescapes the above sequences. func unescape(path string) (string, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 5eab9d7161..44ec41a4a5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -147,10 +147,12 @@ github.com/AzureAD/microsoft-authentication-library-for-go/apps/internal/options github.com/AzureAD/microsoft-authentication-library-for-go/apps/internal/shared github.com/AzureAD/microsoft-authentication-library-for-go/apps/internal/version github.com/AzureAD/microsoft-authentication-library-for-go/apps/public -# github.com/Microsoft/go-winio v0.6.0 +# github.com/Microsoft/go-winio v0.6.1 ## explicit; go 1.17 github.com/Microsoft/go-winio +github.com/Microsoft/go-winio/internal/fs github.com/Microsoft/go-winio/internal/socket +github.com/Microsoft/go-winio/internal/stringbuffer github.com/Microsoft/go-winio/pkg/guid # github.com/NYTimes/gziphandler v1.1.1 ## explicit; go 1.11 @@ -179,7 +181,7 @@ github.com/container-storage-interface/spec/lib/go/csi # github.com/coreos/go-semver v0.3.1 ## explicit; go 1.8 github.com/coreos/go-semver/semver -# github.com/coreos/go-systemd/v22 v22.5.0 +# github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 ## explicit; go 1.12 github.com/coreos/go-systemd/v22/daemon github.com/coreos/go-systemd/v22/journal @@ -237,6 +239,9 @@ github.com/golang-jwt/jwt/v5 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da ## explicit github.com/golang/groupcache/lru +# github.com/golang/mock v1.6.0 +## explicit; go 1.11 +github.com/golang/mock/gomock # github.com/golang/protobuf v1.5.4 ## explicit; go 1.17 github.com/golang/protobuf/descriptor @@ -333,6 +338,9 @@ github.com/klauspost/compress/internal/cpuinfo github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash +# github.com/kata-containers/kata-containers/src/runtime v0.0.0-20240702121346-ef3f6515cf8a +## explicit; go 1.21 +github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume # github.com/kubernetes-csi/csi-lib-utils v0.14.1 ## explicit; go 1.18 github.com/kubernetes-csi/csi-lib-utils/protosanitizer @@ -372,7 +380,7 @@ github.com/mattn/go-ieproxy ## explicit; go 1.13 github.com/moby/spdystream github.com/moby/spdystream/spdy -# github.com/moby/sys/mountinfo v0.6.2 +# github.com/moby/sys/mountinfo v0.7.1 ## explicit; go 1.16 github.com/moby/sys/mountinfo # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd From a82e15e93f1694cabaef6fd0c502de6e62ef3d04 Mon Sep 17 00:00:00 2001 From: Archana Choudhary Date: Wed, 9 Oct 2024 08:20:04 +0000 Subject: [PATCH 2/6] add SC/PV parameter for kata cc mount tar file update go mod vendor --- charts/latest/azurefile-csi-driver-v0.0.0.tgz | Bin 13611 -> 13612 bytes pkg/azurefile/azurefile.go | 3 +- pkg/azurefile/controllerserver.go | 5 ++ pkg/azurefile/nodeserver.go | 41 ++++++++------ pkg/azurefile/nodeserver_test.go | 51 ++++++------------ vendor/modules.txt | 6 +-- 6 files changed, 52 insertions(+), 54 deletions(-) diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz index bf65de26d8136f751dc741ac4012e47b7eb3dbd9..2d64209b31135f926d93aaa6a11988328cfffb1e 100644 GIT binary patch delta 12633 zcmV-fF{aL|YOHFIy?=Y%+uqxKy|cgfU2pr<&hFlKsJD{ZXFRcxn19z>x~_8LzLEz= z7OB%Aml~>!sjHC`nLlmF6ajm0y<<=VnTtV3&}Vh0y?1*_yGHW zUL0PWyi9*T3*#Z78x)VBk6Z$`F-ckIb$i|3#zXhYxAQpX|9=FBF^H8ZfadwXy|=r& zSDycSuV1g{|9w1UjEB(QK5a9GY$f%dg%*t#@U1@H2~aE3ln1N zI=%uH?1^pBGt}+2)OB&(JvxfCKdb zNro5fu=4oHB{K??Qr6`)u^fNn-WH8nzqNK2(qB!4&z`e-C&B>G!9R=b34178b0z{}}XbchuSmF&|7kQ^k^wH-p8wfo^ z*WU{dM}JYxI=CT#Plyi(zRzMR)rk2Vu!KI6JO=a({Y`a~`u8TXWf3-o1-c<&sQS}< z5}*O&Xv*R)x&WzI9Nyx!|a=s zYYtIJd@TBCM|GX#Si*36IgOx?&Y7NbAEE$DRd&RLJOz2f_;o6Eazq3YAf=j|K8wRZ z^?zAF1ilI(P$HEz5(NwdqD-OyBA@|c6#aa0l6<~_)C}Ro7iWyiKHB-u{p~%sh8zJ7 zBwv&8SUfS-n;tM+g<=mC;ELpijgj(0Vy=2`b1L>aHtaN~Hr+TNO zWOpzil*p--Nu^#dAOv4BZhw+21a1Xall9OwO#7&#RWWBFbc@UESWmW&Wr0QNYkx(p zb9_sL6v+!148WIt)H!7r{s@9Ngib5^R&zvZ-rjjF-V)Vp=ml)1{v6}m4-`)@36*j* zWBxE&U%x$}A))XK3*#~T1k(fG2O<7OV2p0|$Mt_tU5p@+6*YjBt zSz6y*D_nJrt6AkL>s)J1x4VR<+ZX_rF^5c`-hwnTeIcU*!Ld<@PC6LP-!)A zocw(e4+iA6k2>!!F1!(h<4&@Ve#hL#e_*#`&UtenFXkRGouEonk$?2u5e#q~N+;v?-6lE=W3_brK)5LVrTY3uV`pxnQ%1qA zWg*Bu+L~a#6_Tqh`-^Lx3)IGp^-%y<1XCv)=G7(|K)^Uw>P^YR8SpU?!bo)Q_3PJZ zuO$0qCGAGpCB1Y9)?q7CWmUFGtuNs2s+?&KC130!AfBI4rL~cR2m2s zo~%u$OrFrvTPyITUlqLx;WEadmrSIM8$ttiBV4K=7NBP28E48sNS*tlg|6^!2zBX; zbVffv?0=kK{xKw1AAjwS&t0?n(NDxz3kmDqG81ujmAg$k|RI|2S*4;+HzEHtgnlE zSO_wt5TG%pcnAR^mM$X3kwd8dUf?kx2^3R_u+TsGW>MD_%6}11q^7?qa&m-jh#cu+ z^VMprI5|>+Woa2hMoaS{2VD|}Kadd2p66#ArQ0bEfv0_Q4;@?i8Xjl-Pr^9TVVkYf z_m{^PU3n{)sbg0Be|LYo6#v_Qy}Q4T|J}!Psq{iFsDE*S%nGIE{?K}S1BgO4g#bxr zVg+{EcQ-aJ8GrIe;9u7eiC7?zkXXiI6Gb6l0f@)6zmY8eGDMz76aU7W&!18E9CW#E z>h(Zhz9?1Kc>U+?d3ujl`LJfA;r{Rf(mabK_8N)VLc@_%Nmy2T$sAN|MHmoFO|y4dk; zglV7`bZUcX+6K^QFm5Bd19z zvk5XJ-72z=K7U5ZEUyk#7dgKqR7gyH^L1In)-!K*ZH=6VIgV8cLj5of8XH%`qtC_> zquL#pi>p`5MVL#E4HoMNYq5G)(_&X7q2;6%gMUOM4pLa1n_!B6Q`9P-{DJ-%Gg;{5 zFJCrB5RSWIw58>!eN5sQX z#MuObqB`xPPJ$R45;=;mx;`71V|c0soPXjlA4=4Cv9106SDgg-Lm0gzV-OOLqduZ> z7^X3yi4HO)U+20`7}&Gzm<_Vj0F9 zhZXKjlyJnji6tjKx`};3=r9g3FMqayknmXqeU$FSf(?`yRGNy1a1apVXgX7-5r6PQ zMI-YA7VzUIG$6jxEK2CKPsvuLwwoMqFwJ8EB|{ukKLNs)wXXG#{X;N%hUKV_x@LRX z)n8w-4`cKp0WD$#|mxfhYpSHVr1qE^#7n6sI0M-99}F z(Ah7CXu#PR-HeDovUsY5{0$c91%KlSe(z;chpVj-_+3M@tM+9?NT4 z&|%^8Y>qT{z{EwprUOc)T6V^H!lG&Gm{2l~$4HRhb(;+Xsf*dj>u4HqWq*XJ;m(Od zZor#lSWEi_`MoT}<=Uw`tB~|V8{M(i+pVqTm6dcLY zX;dzmcDOC76cV*_m#Dof>!LH`hN8-(mLn{cJ|lzFJw4DRl^9}Q#yAY8DN#Gn!nnLt zkt-%gB^LCa)6KtR2&jm87JnGf-(wCaV#G`YED>OTgqR!w@1nn2UVi>?ajDdg;S~b4 z4`ti{c=52Lc8IX;ua~qd7Sq7VW&K#rHFeTu(;&OmQ0X>$gnE*Up{V{Oz6P3zVN3(y z;WV#E>|%)w@~OUA`9-_Xh{Ng&(}ENus-eLh!Vq$F-B|XJX&IOaw?q*8CXKRO5Vw;HZ zRPZHJML2l18X2L8vk15(#gJ9bOe_Sl3T=C?ns4{EW&GL>|p)%Ky@GicjT1KWZwI-%$g zk0Ct70`7bOR8I>}b!@vKNbLT)AyDrN;vay}R)jyo9QzWuntQ%56E>Z`>7rfgv#Y7q z5YCQpoSg9)^?xCfV$15*Eu#q1{g8O(+CyYdf~oHod&8|4bcbjmss^|V-C`w8b&gu+ zgjzkc`JXKpUep_dLy9$?D zS^I6S;U(kRNfeB2 z)P7qdD7dCQg~h!6t_;)ap3c>{%s!|NT-^}atz`{eGj4j=7!9X(>~P5xPy+W;rrL6q z*N|OuFn`TU(^Hj~jKceY3%|qD(?`E{YG;1uw;5m7ibCbfnhn|v&HLQ3Mbwf;svCVr zQAcOa+F!cJ=o*#faOF}XAq*s7y?+)N5%;hyjZXhBIQj*G0 z?0+JC630Q)+SnFNr;p{>y5P@ zbFS73WnA)c>M+5M{h@gd8CCXy+6~0QdRnIGKJyCXjT(%$4ix9!L#Tcgu|C%{h(q+^ zYN~cKB*50}!E!z~O{o^MS1GtHEuLm)gE$2u~OZP{f2FYP%r7kGa9ZiWtNu@|#53$?1GT?iOnY;Z+Mdnp@cCP=8s) zc-jmB&lgRkS8ds$p*|U( z)|oO>IWw4lQp9v8sp6ArTqpI<$e2^R>D5bJr?^IS95jdLXNSngVc11K=!Meh*=uOU zvV5v?n%8$k$b?hT$ZE1%Fn|Ok;(sV&T+IooO@*pTG0rBVRvd&=J(G@{_Cd315HhCM z)TY8HK4!%;K3=!PH3&a&&iD+e*JToM53x|1KmeaWU|$9LtU^wbY+E)R4JHGG>9nvN zOY=62AdJ*XDH)0*Wt@gEfiNeS;sxnK*F`oD!IYuI4hiU&xrUnK%|6~Yh9of zi?Zfwh%(=gxk!egIH^;2(uatp3zMBwX7uW|rK!Ol0$#ukH3lKl1J&iHXg1iQ1;1|q z5|c2Ez*3zP1$ZKJHQVSqNq)9h*}VLiVZFg3-)(NE!ff3!lj{c8Lxy~qp<%LuITNe}5X04l z4rqpJ@vL7e8>T{=#WS}AX?8zJ_9+(NT2aIFrelv%qZt)qfd~I)|E(3KKK09Y~kR>rt@sl8dQmfX3;24bl)}tOrK>oy^QA z<9AA|5 z_n*K0<;BOY{>Oj3{I8dPB!B$n<;xc@e*Nj^x0h$fzmb=J{C`U0@wNHuj~Bnf@oz1O zynOjzf2y!@>*b)EYRAX8XXbr+Fg!Opu4S_7FY|O4ZPb2c?CuIkEMGj#f>Q*-2FDX_QR)m*Z9v!zl z7H%cTw|^Dk+@3h9Z6z4CPk~SN_mcf**(PNJpJ+=a@*h-LL53@?l zW?_j%Vp&Aj6UT3@0yd%gTIW{yFE2;*AU5W%2cb$OI6geGQ0AU!cHDHxB+N9E(eed+oc~(N465k z^vcXSK}@fesQ|)lYitSKV_t|&16ldL<8kgb z);RB^Pw#q1aUVK-o?XML_6c{gGk;jcUf@Ys|Bu?x?-}nhf3HllcEB381(`ot~;)?RO+nR$4uM~fO-UblY6Vuj@d4* z7Hd=XG`e!y7*Y@WQ8Q%S<^M$OeV~mS z4@aBlS0wfnmCdc?Aep9kz&ZBPNL(g==q<>zFdh=BjLH`;4KjS-OckxCV_;9C*N;y_ zzb>30BstMla1*>>Jb&AnzH~5LGmR1Bm9&3Km;56ZGRu&Lb6&z)IW3uecb}ysY5$Mf zrQYJ)Qt^XQQVV@PASv;Q1fBfFv&E<8titRi&n-*pN%ELv{p{}5G<{^_V51Dxb{E-2 zT6cK9xU0L(d)wJH3L&d@X2zGijGHsPbvG!Ym0pCm{M~6q$$xBlVOmXLqchAJ^D_%( zshEpW@kEJ_)xV3!SZ+ov2N`hmQ*xGCTmNcAY)-X*|LcE#AR(Ap@b7>9uhKgI{@4Hf zHDvzvzyI~WtrKz=(I8ei8s?r0!*T9xOL2r<6w@&;2?M4GhtUYz92WDa^~Ey2%*yCe zmrd5usM`Fp=zlcLjcGJ+x(pCH+^nnGQG-31n%L<26mSy%*2Y;t1_RI;H3aB%dB!9+ zD+_j!6tO?r%pY#OQY*A3;)70Hp%o z@IyT>OqRfoD%3H|rS8%ELXt6QoCll?hv2^Z0e{(TlFsk+QmJ&JR^%KBYLoS(OLT?l zUw3}1<+#{iY8{7Lv}3mDM5tbRX=kLEQi*R>O}?@=>UvfudegIcq6`6LoIn_Q$=?44 zPgO4@1838@VlEq{HBIsN{^)(*aHa%mNRu2_1EO@FN%+-lS|H;VO=2Ye2-9JjZ*UOH zn191!DH^Jt;^c97jLVAS!F;YQ8%NvaDSV!fv^6%aDfcIv0G|-J*}7r;n$V%AMANgW zlD72vv-M2>XW_Q!1*~m@3ai~+wo26_wiaqs5?W}#jy=CfU`nNLOO{cpV{0Z85onA9 z$WrIru_&t!)oJ1Usb?`e9;Si%w@Fp%Pk$Hc!I-p~z8_AG?nl65vd~4c}*O5cGkPTj$zFnE;>ZB#Vk7P@e@RsVJt1P`SU?MenKpeltOgjg{f_-+zJo z8q>f0S^W9*zirnQ=bz!37HSnlnd964`MT%zw!PlAgJQ4O*Z&`VxIC=S4ySpoQEz$J zS@t~Gg{S!(Xm*>bQ>uM!`Os=s`1Q@3PbzmBe%$wNmKG()612X1^BcT-vu%2|MgX*{ zMHUjhU`95lLvL=0uB1tuDJTyh&wukXxl(CfD&Jde#B`|)a4vWG@;Tw0mSWp1an06~ z&0W|>y}1mlS@Ha)SU#x_;^%aRt0!loOBUk*o8=GJJo2M7t2;RJbdSj_bs%P$qc30QA(thOvpnXTYZpDouCKy_`^J=lYi~8+mV}E_x0jU;*9C z@2E?K_zFTh=W!IxCX0}o+=TjJ902O1lG!zsI;e~OK$Orx?LYU_H>$i|x*c@QN2Zki zQnJ6}aftpvWeQ52%pjJ#!^ZGhxYVX)1ql?6LlhDpi@F|3-hZ2#1cczrWV<`Y(jUFc z^=xgYCEvYsbl!f_;Vnl-NRA~8EhNq?8SZbAl9$uSew(Np%YRPT)Dsq(GM5B=pEaVj zfTdLo(vmlfUxy3_CyFSM)2=~Zo*Wg(D^Aj)U1G~_#RB@MBh?0=K);1ZMJ#xi7f~xc z-AsPIpF=<+7JnR+WmGd>lGE9>pb}_v#DqKrsoGxG!?_|^&B~4-J{-pw>C(q|Z=I#p;;K%v{ zMkUQ51h8N&-kU8$lZwj^4eO;`)qGikvS&Tbx>zf5#8G0FTk8fCZM_8OG z1$&D+TYsHm^<7_(v+N7CCNV)IUtGL&6bu0e zz&)+A2&1!0WY_M#sk6x(MAS@Vv3#?=)9VSxdw;l<9D4>e<|S+Qo8o$x7FN2)!x8cR z8HqQqdQBy+VlL#H?VhX90I-ZXctf>EdGjNqjGz5-4t|Kq*n|Nq3;>;!N5G!xGCf>| zovESjBenlg|JN1Fzb?y=nt%2j0GcHpQChYr855gPI;OHBA!GHcIiRL^C~;`GSA2o= z0)LWsg5Z)&8?;Go{M&RtF4B*9|cv?qN|4Wz3Rxmb8Xy|Jnq+h~vC&Y@1qO-Vf(>3_`J z70JNBKWF)F)Eq4GVknNJjJ)K8@|%$PI8>kY-QJ*LQ%8V9IXYuprW@x<7u#BNNiSkt zzFClRxhoCf$(xz`g6UYvY7K7Tzrdw+6z>2fuUL2}}YRJ-)L`v29;Zd=+;yP7?| z&**>*YZ%W8xCxjrrIM&T0ed>uI`dtwm5{|6XFVW8PiW7XS0<9OrHz~IbZ*L- zb=8u5gCoXdiSc3q+`~hl($RKP=7Ru!ph~z2WfVLS4+iAcK`2g0T<8*RLVq|6*^LL| zNKOkqP}5*t{8*Jf%$i=Jtbs6P5l{V=N~G!{3`MDPI!kPk(z}Q=$$S>}(dFS;qLwHu zwi32twP}%Z@oh@{K9lc?z9>{w?A3uWsxLgw>ba;MzbbNYwa`5ic4=3-AU3-`xICErEFjMbm9Q_1S=P_AF=w2^7ha7aQ z5{EyK5DpVE)!tLjHB9?j2psQNWR9lhl1i1^m|tkz+q0OG+y2&8y?^~$ouF1_>4Ph0 zoZU`Qr&FDohJhb1FV8MMot?k`|Nic-zp4szbW22Q1gMe2HVn?s-#qLULB zKX+nktAl%miZDQ-14EC%9jHU-qUc} zFkgiTz395!yX6X^YqI5IPx4sEEgcW?t2eqg)aDe_1@<|qL_NeF#|Wz+2j4vFbrIys{DfMW!z?vxhkxn+i`1CzDXzGppcz?|)v2JOd$zfG=>m_q0 zu^D2Tlfs7v<=x{m7yi@g{&}#U-_68-_O^SwuS@vPot>RmYy9VZJf7zjAfJ^ini~co zdU$bS6ZAHML5q9v62x;}>}Cm@rbzgL7&e;SEr}Ne<&Fc)6HHCMPf%`HO6-r$)61o6 zHn@dSrUelTPJifty{FoYNf$q}TD&6z@TY#Luar)ce4nYY5a|`L9y*RjFb1xHRRi&s zvp8~CRW_8;EtIeTOLh(P+~$Z(s=;wUfck3nh)92H>p=D1!O^8Edwgp$NPJ70@Y{Um z(*GJnctdEwZiJ`S+lVtCgqX1+H0%HEy}kYIlKy}7dVgnst^e=iv9=thK?Zj=1Kb6Q zuR^H)9t-$U*;;1o4^vQ~(lz!t0fsT77qGx-f~1l_oUZUNfs5)2&JrYKmp@35P+I9J z0+astqJYd-OBJER5BgPNDv$B)2Z|?{glfbKJ*TF8xA;Ekz6!`+m#xD4XC6;zH(JXa zW0>!RL4R9*=0A&2JMJrh`sYa95zc=;yuOl#Mn+eJ-8SE~Jqchdujgusg8+G4c-{QY zfyw7OVxZVMaN*7!>?h?is@px(0>`hH_Y#Qz5x~}0b&eo;X}juul-pP2TnJ`atGKCn zOb^5d0o-(1=kyXCe7OAa{rSn|-#>o5`1{4>@qf=rB&%>BCPzn&%3%WtU%met1LK)Z z7~Nltyjp45g*e*>UFUhWIc_Sk_K4j87b6J6F5Ef*?tbN`Q@HU?B#e>%U%e=%jy2?+ zVL3vF{f{3fn1B2jlBy*Y`|0BS;ZL8=4laLm(J&J4Byslc!`qWnP2%X} z-1Tu=9C+&xq{9HTjZ{-FnVt2Mvj%-VXhGPdKC4!tuX3P#`n% zf9-7dcFOs`cK3GI@xS|cTEzceP=6*{2LWHMqk!uu;5rJpjsmWufKMn2xY`ihBZw(B z1v5&A*0UzHJdbeL|5$e-!Fsp@(keKzJ`2C#S@^l57v%%-jv_T@&yByX{xsTsL@EyZ z+Sl)N_^;|L`@B1~M|9?V9h`i);N+u;E$T^E&g)v(=$z-M8>wx3mKJWy7k}8-ga`~b zTaA%f#|I1afRoCY1@OjAs79^b<@E^nZtcBXX2v&39M4e1vz-lrgh5Y|edqn+z=3?{}VdrS!6p z)g=FBoyz19#nzi`*maQni3iCaPvVuQWB1#%7kZvI5^(R3A-xn%$A6##G+Eli(tb17(v7^C4dxsp(aYeqP^W={Qd1rno)y+qyKG%SwV7@%fA^K(Q_R0>GtkVC^QOcpt%bg=z|UD(N%~hN`74t8c_e$I4PsKSzaMOvJjxhi+AUQ72xgY&V8yGoX(8H;2SpOqD} z+!Ie25ciEebL0Q!&i*tL|JU2!-Y?<*w)gkf_`iF3N+B_m$ju`ER)OTRnc-H7EDHmvQ zh0a{DtGm0`rr9fIrt!Q;z%^K}nX`ui>(yOITPe02L@g24hV95Bz~?y%KN@V_tg!B@ zIk9;QU81)L7Jsi||79+*E$b57s&$Dic9Y*~(Qpe~;#Jq}uw@tu;DrjZSNGEm$LfCgNm5LVp-S;FuB19kjTg3(1|CQuT_GR1^taSB=7Db|#z={}>@^4zaOR5j>? zCx6-8!yIC%THt?@tFpY@RabL?fM(qP_G*7`w|xKG>({gTFM9z2v$J+F5CVVx>vjC^ zKAsuzzk6MIbk2oGx8^+1N|W!5N43V}vuoYU>f#WK`PP;D7vhKo_rASjL5=u6nDnY% zSc`667~06K8m`u(Qe|s_apL}63ahpk>UX)^fH+%tM}J$gYZ8%PtcTWuoRgEba{SQN z&H9=3w6*`2IRrM#{(s$jRkDBocVG80|M zb&U0XW2~#MaHWWyg_&p_3cAxhl1r;SDr>U-1lUS#%LeWKa8VE!Q|1s$dWPi=SD*)s z1m3&ImwI(Q1Db^G-U25x?0?%&*!0taCxK?z|J%LSJ0<&ne|u-`|G$5ir=0(x6i>{I z{Zf`O=TvLX0nr?)oIm@+B554vxv-<%WgUg%+D)Gs@*W9__kcBVF^gjf6u05nl$%*~ zo}42=&4cHvSwE|PmeK!hBf&HE|9<8C=l<)Rwf?`4=fU)U=E7cU{`KM(ZGum9(d z1kBL?y}jMC{@?HQw$~~8|3031{Qvj2IMnUO?`Nr4>$&x_e%8;o@ciEZ00960%X;Ck H0O|n%b*0dY delta 12632 zcmV-eF{jR~YO89Hy?@((z4vN+|JCm9cfIXbJ9|6dq25YrpYg;(V*Xuk>AK2|`$`@h zkzard!e}2&wl{DTrGJK4f{+^l2%nQk>fa8OxS$_M2b+Xe|ND5z7!RSpfzTBea1Igcqp{DsW8!lr*g$rDHr_zltW`x3;t(-GB;(UQ z+PeN;Z25w06+}rFlX#e_5%qTzhv69ui9hY5lffyIXB-4j_0kvQjI#+*Y5=&S7AC~j zb$kV!f&`(eP=A$rQT9<*<>Xa&yW49~iI5!6(Mt8P)de0;J}q+R(qB#bJ11xap%`f7^J5JY*KC z@znO*^e32_BByu^8>)zh4r3wN7=`!>LfsA=1c*?{&=@2Ru*4g_F7i&7=%deHHV}G- zuD=%^j(?(>b#OxfpAa7oe4oWssuA-!UF&3cy=7`K12bQs_cjfc?$A|@#|FTVLC<2z(VnphPNbBnlV^M43bZL_h8*;sTsnFFU}a3eYEqR``deN4LJfF z%28HK@y=uVp8kx5RO77$I{DEG!dZwZWUsK4*dLw4KV!ln$bJvm@EuGb?4wueulyv% zlro8xy_@_Jv7j)GBNiM~3ULrD<~|6+ynky!Q>`2mDU!bs@}szqw$neyFxHCsRj>Cm zvA=ToXAGkHQEy!NY$y4uytLhC#TRx0%DF4wS+a762(_+zd&#;>2GTvXS?K`ZPW4Vl z$?jl4D3MbulS;i_KnT8M-2Nn42;2&;ChMVVnD$Xet76VV=oXjRv7T%l%L0qk*MEvy z=lGTgDUug37=SPPsB_9L{1F6k2%T2+t>%c-yuI^Uyd|pH&ziOwKcT4(oBUyu0aiDsQ z$ZAP<9Cg~FjfE<*&+J#mGW|X^{C^v*<3iPkg%!LF2M!x!5iT4ijQ%JyBI+tCuIIBP zvb4UrR=DaKSF_4h*16W2Zg&Yyw=n=LV-A@>y#;Ax`a)u5x?7PqFU^~um3jA;vp7m$ z4}|_Z8IN|U8H__h?&QDKH-)^Gg^Yi!T%4AdfqchHKr!Sd!2&PUp%&Sc_fPitV)SHrrGvH$)gpugp>({T- zUP<=JO4^OGOM2-Jtix8O%BpOWT3^83RXNifO1{`zsv?Im<6P4nc7Kq=QaVU}+DN!j zdxoQ!6FRhJ0HMQ^A!X^$$G70e+BI%j%-3`Ye4J~HhsId?qvP9%gO$Ma9m-4?sWcEO zJXxDgnLMGTw^rawzbbkY!exv@FPTUiH-rZ4Mz~Z#EI`f3GtQKOkUIB83ti#e5bDww z>5P7U*#9`e{9{P2K7ZODpSx!Dqo0Vc782IIWhUb6DtDWzY&{Vd+=%SePQ=U}FPY*@ zENbSjr?H*Jj9-Tg2dN3KHw^@+JUOyc=^6T!>6jf3e?un&Bu9V_4vr9xwB@MYSYH?S zun=TOAwXkH@el$;EL}v5BZpA^y})BY5-6q;VWEHW&7!U=lz$_jNKJoJoQ^&0M|L*>FDgL+rdT(zX|GSUpQt5?UQ2*itnH5US{h{^v1`vg83IUSL z#0u=R?`~{dGJoWcz`w2`60txaA+e0bCW=D90uYaBeK1t8kCuypxs{#;zX5vKLO53 z0O${8_ER72@9KY&vGJ&&j+YSw8t8v(1~(3Mtj;iGSXBI4aFH@ubfo!enz3!R6y1#m z?sd7W#(#(tDhH_ZXW{)>bc(Go|F!@s^oMrSDgD*?O9yp6brxw68;_Z?s*K7LjhrT> z%qGZ?bgRfd`urIsv%ES~UF7_dP$4n(&DUiOThF}NwKZ}c<~UX*2=&7_Xlz^!k3Jhm zjB0mWF0NiJ7hx_vHdw49ti|eGO^aQTgqD+541W@lI7nf2Zh|TPO;M|S@(227%w(aH zzkJykK{)P;(Uz9a+ICp!?dEy1lP!-=Ft3qhf5+pnp+s7HVYH2Uh~h9zksSlZQHsM5 zsA24*41tjofQ-#DOheUPfo|xJIHp5r{0^n)4%N{$O)9nSp*8f~J~P&Tp9fw(5U~&@ zuzzv=-`?Bb-F;nL|97@`U+=Hi|ND4KlPw)*>mJlI!1KI~JWd!#1a2iz?fmFoe{WRS z_Qo}#LEj>y&aIp62(&YXFuIWdqxvQ1or%X~Xq_QYtsbvp5(dBv`f<|hes4o-I3ga7 zBF-id6xC@TbrQtbkjPPd)%Dr99K%yB;C~c{`B0+9i*4=izv?8wAHwJ*8H12`9Q6^6 z!!V5rO>~ec`8wBi!oZ$w$83lQG>~X$B2jh43|ZS7rYCKH%YIN7I=yjeme2 zDjJy|uz(*wp#kxgW>G??eM+_}wcX@^gJ~WUC>i3Q`Uw!WtaYt_>>q;BGb~4a)HU17 zuKxOxeGrM;%?#6d`DdwPq*siEfT@$m0m6vFO~xAq2t*MmwrMa?c8L>tqd4{8>GtVa zfX;q7L<7#o=w?Lxk;PLb^}ltggsysuLVC!k;uf*1e}FHQb!v112u&H62hY)v`0r6BbQd$Apq`JVt{2uG?%FNL|cEUPse_D}N(Q4R=ly zas%EZ!&=%e$nRw#F8@A@^UKoEltv=NLL_efY)e!uRM-2HDyeFx=ijzu`Lv$hqTooD zPNQSr?rdHxyMSwH#ro^cfkX?&*Opsl*WbGR9#zO^Mop7RKeJ zid-={DzTvVoNoRlLqJ8$vwy&V{vLBc5hG?IV2J?xBgEtgco+TE^78YCi%X?`46hKV zeJJAwz>9|^wL^q$f4!t#v6u!fM881&Ru~oViE1*F2U|2)Untuoe1ta_sjB!5; zdeMWklV5f(3O}P__myOd2!wUm+xxmubjc4@SB7 zOKZ8vhCp=~p#$J%5ViTEmw6g!A+-TSlIfIGR^zKNk*TB$t+o&Ko{?=B?yAjLYL_A+~^q&YKUHe4qTLXlj#|hWaFYVxMLgV5*$=7JnrH(3LtrEw<#?I zrS{tzLBTccDJWE9>HT=*THo<91uQ#lvRun2<)@;ybXx`_JEuxk*Qr+k~ ziaI)T*8b8(M%Sn;hbxyF31J`+C83(c#JZBq9?mSFCbC%En1G>$__qA;Uu7%o=eUL6?CzKVe7FcDqjVt*;=y;NbDdF`4OxLEIIiG=ZUWXq%4 zq8i0)j>fu}j3v5BdJhZ1e4>}s0t&?F&V2v28~q$Tivz<`Uz$f&Xp)NUXa*3&Xg_nB89Z`5G4b)Y!+9zyl2i1oRqK^&qN zS5vi{Apy2#50>+}X-c)2y-L9?^AcUbKz|8VEKC&@hl~BR)_`nwzs%Cq&0n`r*v^J# zU7Gi>5#ct`oUXN#frDnI)d&Qeh{#lRi%fUB>lk4@WgJBHB+f=XoSH5*u8T$kX9}{5 zPTyZ1yVUk2LwLeSfFdRYQQHLpe#{LPR>UASk>4cJPEO|&a<^DJ2(Mbu(cHp5hkwc{ z#?xj9c)n;N&2ol6(WQ2C-m@A9W7)BYPACed6px9IMA@fYZ42_TfX$p1Pgk`83H8bN zw9b^7%9+9ZlOm=&Nfn<|<2tE-M#h}lO|M?+I>j}rwc;S0>X~%pv=5q9gOD-3 zrZyEu@i8l&@$tGPu0i;LbH-;#y)Ki8dx(Y71OoU30{be^XBBdiWZSaoXfPQdOs9qI zSemzC1Yx9BO36?hDdRMR34}Sp6fZ~@x-PPLAb(c=r6Lu8PXbW0Q@f<)2!EzIFRUaq z22^q0YFWZWOQNxda-&hq%yESU=wuRF8%!fTdb(qrTMs2K##!N!z94C@UR`EGMN6=v&>nOrxx9x~*^3=NYN%$Z;nW5zHrwBa1Drk~nCWVOrE4f@J(P4s;D6CIT(k{b^d11Bj|&p|)BvE% zLOquRbuJ0tTpFnP^+GaN0KQxvX1O$=@(!Vs4}w=*7M-{XEaLLW!=-VC%c2Tbh#|ZK z#Ne{{z~#|^-xBt3#c02~!uLI4MBnoGy%nJKR)Wo28j1I{irZtt)U66Zw<5gU^60qb zv2ZIvzJIL<=k~-=Z7adBEst1R9-p=XG}NeotmP$fX_VE9FjdPUsFuY`tG_Bcb>H{ zL%-CeF4QGvE?1!loKFf^NF}DF4$e zG|3T2;A0)$OLio6-jZa?AxE6W!y>HKw4vA_v1~pW&0})I!~UaF#&mD!b-6n65s!3{v;aXElw*e?BOIQqDP~t`5Yp&W`t3;ZE7YS{#VH!ByvstbcVK z6d^X*(yfQ`8m68Z$1#q~wzwpC$#Pt(E=Ne;YLG1HS?X_eIRXKBHN_(Q&luDaltWW2 zwiR>QTlz)bx?I5-mbRx>9*xcWt=hV4kZypnSx)dpBhnOPwg%W{2;CHvQm&`QeFlEIa)Ebz7EnpERTk&~|hY2A;Bg1tFtDu}pG<`heBqJhuU_m)+(Gmfea~ zyC$8(K;Q4$(4sjSV~)cu+oI)KqIqa&&S|O4*AAV!aour+ZPKF- z&GYjScp=JxgLTYv9kzLdTYtP6r_>9F)U(NC@T84Cp;BkfK4#)}0MsMco7`KKcFcBh zwOE_7r_q(u#*lK*T#0kc#p{aPW%G7J3kRBmt}bbrs*Ra2ODLmw!6qK z(z?U*#a-QP-rLTuQ3zSJGc&&AW!#+Ut-C=Ht@I+i;viTZ$v>qL_|(NfHiKG8ll)s3Ab7%QGgq zSy`})q=@~|X8v&Nm0F=S5g&A-1nE^z1us4>=ir!<4Y`ocVNlXvPIs8gP*}i0R?gRR z5rN;GmF_eD7(pJJHQe@Mig|JpdTx;~_9VD#S4IND!GAYVjeMfRD==9koagxFGj%r& z6FN~=JO5+7)u*L9unB2$?2d30L3``OL3vSorOw%wJO%kXI6EoB-0V;~cZ)MuZrxJt z1)W{@^OJ$8jAywl37DZseRqgZApr*sW)ej}CE<{cuF+k_gOy81fHmGq3laK%FW#SS zy=65d&3|}dod!T3xq~rKxvBMs$!MfA2VST*-7%(Q077MGI0X0I4}ZvRlXQNkmrA7*wIb(8P@AkLU7{;Y z|GM*AEyuKgP;$T+&b4b$^`gKCs|Y!f%+^kNkuV5gv$NBE_zS(_nQe~Y^)5Y|9=kL z*O>n0&*IOg|82XjIR6aKv{0)c${gSR&(}S#x9#<|9Ta=LzW)E{!{uRpb~w#zje5(& z&a&shEdj?X&5GwY#qvpg5I?6gTs=7xU9wm=*3a>gTHH-sRE?8~pxH5(#C_u{RR-IT zgAj8Bz1A0ou200weIjOsCt<$!6EN$OFZX)#Wqso1F`sx@XJ)yh%q&)9mVe64VrZ-^ zJIkD>lD?v|Nb8I!=2@qW_`1&&ea)wc9?0Zer_X75vS;aYJS#Z0vywA9bDYR&wBw%e z0i30e+bn;$=8+$zS>3^zr+Z9hsRJ>~9DVsZ54kLPoaHg!T)XHwhE< z?Wc(#ztg+7=R}9E@85n*_iwKR!@VqGdnNen^*!2O?LFFGDMET_Jb(1E=;z0Qb$+DR ze}De$A)o>BzI@sEZv8Yr`1hE@fP~=rf_O$FZAlnM8pO4we#{j~ujS}Y@i=T-N3YlG z?eFdB|Mq&l(tme$_jX==x4pOjdhgZt{;S>H?|M7C+q=E*Q15P=Zk||3%)jd`U01nr zU&&*p+7Uz{n~s6X(tp)1k=UAS7w&2?P}k3&QTG>v*~j>Z0cm0v6EC z{EoU*h_4{Da~?<0Y_bTc$xWyq#sQ#CDw$nFse`)c4@3zK)c$i%eWS|jrQ1Q*d}K=L zFD3gs9*5`;RHmTR$qZt-J8TTEg-dN(R**p9I7A`wv8d~jr_;tu|aH5D3Iqe$s<;hWjyy7G++9kH^RxF^8I#O)_3iMluRK$XJc@eeJ z)6L}P`#A(OVt>IwSw=PEB{`j43o3y&M@-05kgDx;JRj_k3vG@+s59>Q)=DaHS&_5s4;y{f5t)r&y`n|d7On%ys2ty ztP7@=Pfk?YS9WOrJ_|907F8w@h)9)htU2O@aU!P~%74WgyPj|Eck+BrWbVEo0e-AM zU{u1b%*IE6uXdGZoT-_cH_(mVIFG3$V>mL04;qI}vJiM#qH{mCl;}pqKG1iTO>;BG z{xZjOsSFz#;~|{IVR!*P2btr({H9P#-Pgs+D_X|f~m&`ThzEIERR#FJCI}nrQcDm6mvj zzI^GJF$e{;w4!K}5|&7Rxu=JH4K8ynlyV$+2fpV_vd$zbUSFX zFMl9;CrJJ($j@F`cNEM>Td>4#&V<_DDKcEE)YaqT2w#zqI8RP+7L|YV(80TRPF0l8 zgw?2u(uir=?HAdntI|nxRMKz#+1y&{?zw2Ex7uo``%zFOExOuy-ZVWMCZKxiXYXRu zGdE}tKRr1J@_TD5;oOv7xrN}Q>s+aml7EX^220kp+L`6F9PVSfJ@?Gb+@6^j9-*-S zPX|-qXxH!a$_F|7K~&!sSMqc4*#x+O9v|OESYImq114e2K`-ggA9hJk#&(hA{GRoN zW%XfoMMx$eXv@M(LlO7k&A^F)Eld+v5od9?i}i*+?3R#k$=w2 zU6Bk7{BxG?M$N$@FNWes%E(JjD8C7rk3;oY-|Y=5HgyCzl%q4oWx8>$bg`{Pm-HgW z<(ma5m%Gvsp1j$aOBEEWdFs8g${Gk$7V*??sYI$S!cdetr?bQsDZPt0lgwvfA6*`vC2EPn zVk==QR+|;XcFBse^R5Kk`*Y`Kqiix4DhI;F3-i?1W#gaz)8 z>vt{x|A+JAPlxBnN5`j^CkO97{c!T`*tLiGfip)o1vB-2#?enObsm#-gzojSbI3u* zDslJ&3E?myQ|&$VT*I`lg~0KSMdoN~E~!+xjroPfy*-O5x$SRl)qmTs)d^}`{LAsXwzX=}dCM-2etMT#8FSgtnay*r&hhfcoH4o8#lQP3nmVbNS2L)9 z^m8YswrU8)96Y~#w|`Vshw5l`Hy}cChB^ZX7{|2&BSbkDYMCz~!Yv=FseC!$;5`ke z4f9oq(2K6iy<4szx+Ysb_9Tym+|uzNzj~v4Lv2n$U0|P+O4LK_ag4A!BCAd*)JD@0 z;WqPY@e8NK&G^D0T53tHo>Gry3#^I4AL(QRj8ETlgQnidkAK&k66==MnH*L{w_Y-5 z5}P5WIVpT-P~JU0bKyU&?w<$y`Q1$XXK%Z=`?`ex+}YXPUE@FR^;?HOuG1))#4o)fIsy^eWi4oA|gx}^f zm;Tov!W%*Zb|XBs-bS4HAjFInp;`ZL@9phxm-PRu*MGZvYyE#8kG1764Kld18Q?BZ zd=*0V_gKJ>%GNStf0%*_m9DYJ2{4Qqy?_Ny6C{-c;&g?F30zcHaF!q;yZk|dgwje+ z5t#J97X@U#TB-;ge$cNHQ+bSUKTtfuBvd0_=s7jzyT$iO_fcTE$Jp zV|pMy2;ip6I;WTD;KSvQ@6S&z|Nimg#osS3kAHtoB3Xq4F*!PFR1OcF2vbB=sM4{&2dwSwMXm*xEMhgcH!0maQ7=eox+WGB4Ld5|LR3Cb*v%p z49gKZ?0@_?!TjULkX(KIs8m)6^5e%y=YbXrkDYYxMl z$A4i+lo8Mh9jZPw8~XF)v(?J<%nGW4=K5qo0LWd`UY7DYSyfv%=eU$bw+2;G1f$M^ zh*+^rYtaDH2+Pr%KfO3QKl$bO{L}gI+fNto4}bb}c5wNli-wVSCyBFnAKspvY7$2$ z=dO?2;=o&nARPvvZKRrd$?UA3oHgh>dwGb{4@u$;+pXZ2V>+~x%A?z&muC`3|Y{^~Gg<5HS*A%N;37W%DiC|nPwl3iyPgfU6C`J%X5G zQ!t}+XgzCE%kv0_{f~7g60C$C6+o`s(~dQmQAHHN2KDg zuYLVahySY1vd_Cydqii>*TKn$3r;?o*rJ|v<-D$ijm~+Fx{=zpXKCTKe1Cy`O^Cp7 zv(*@xb$qZu4>+leSpaX`q)IDY%_-RZ=2Nh{J+~0tt%iF{LDeS_@|>NvB$1)Uyd@oH zCzq#Y#V)F)=P;|1&kM3Vi%7ok5)0fq(EH7$R$0ep*Rk0r8k;qY5*iYkCilCBsb{W| zHurlbDec|NpXhH*ML)sxK!1PuHzG$#)_ixx%SU+UOc^t4!{*$#x5>bg{eI_ZS4uAn zSxxe9)~QS$QEa{0hFu5ApLme`@g!b(I(ENJd!grfBLVkr&E9!%JM{*_i%0}sWpHz3 zK7QExkP`V z4tLkr;AUy6r4^_!JAX`2v`&kuh$<|(SER+cnX7W=>a}!FG&modxT|Donz2YW@mX0h z%RTXg0de2hGdKQk?(9!9@qfMj?fnw|Z+rjs8vl1MPbnm361iFA-zt!NHZ$BxaeP&p zD?srXqRV6W44pNCuV$Pc4!`H-X7T$(%faOPyd>1LWd(4-lz-5DTn(D@p#@Yp{1bMF zl6UzO5Q|_A&9_V55o&n48$6)Z6?cUO%vgVnSiqXzSScR#i6af$WIkTw3?J#8JLLi` zuF#n)c6E36+BAE`%ru@C3AhI9HFNe*V7%#U zb0JyNcF4Io z;4H#Jy(#FUzmh=5=(S34vY@}whUDGdXjUW?z{L4+02AUvg^TrdXbW72R^H{!!l5~! zVl8ni^Ka3MYxoS%O2bT9oW+GWsF^5BcT@^=T8yn)e?c>2OWpU@Fx_X=TAurLh^hvi z@Z>L>dmLCSRSW!2a#fa>yXtB#5YUYK-(KzS?UwU@^?I}VFM9z2yR&vN5CVVeSL^uS zeLOSbfA_lb=$s3WZq0e1l_uXAk7|v{XV0#4a|mpf{r|f6s(gR`>veB`ZU5iLW9@%TgZeXI_W};A5wfqtOTLEst<*uj zjI#3iN=H zzu2@PGWx%5BzUI&->;nieBJA<_5Xc552pV!7xr57uQmVrd7x*0{Xc&s zV21wh?d_KJ|9-FcdaWq`@8g-r|9^jrL*0J-ewK>0o?AcbXZ?H&&;Jbo0RR7jJT`Rz G>Hz?B7Y1tp diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index b9b3dd7d29..e6d36d22f4 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -192,6 +192,7 @@ const ( FSGroupChangeNone = "None" // define tag value delimiter and default is comma tagValueDelimiterField = "tagValueDelimiter" + enableKataCCMountField = "enablekataccmount" ) var ( @@ -221,7 +222,6 @@ type Driver struct { allowInlineVolumeKeyAccessWithIdentity bool enableVHDDiskFeature bool enableGetVolumeStats bool - enableKataCCMount bool enableVolumeMountGroup bool appendMountErrorHelpLink bool mountPermissions uint64 @@ -292,7 +292,6 @@ func NewDriver(options *DriverOptions) *Driver { driver.allowEmptyCloudConfig = options.AllowEmptyCloudConfig driver.allowInlineVolumeKeyAccessWithIdentity = options.AllowInlineVolumeKeyAccessWithIdentity driver.enableVHDDiskFeature = options.EnableVHDDiskFeature - driver.enableKataCCMount = options.EnableKataCCMount driver.enableVolumeMountGroup = options.EnableVolumeMountGroup driver.enableGetVolumeStats = options.EnableGetVolumeStats driver.appendMountErrorHelpLink = options.AppendMountErrorHelpLink diff --git a/pkg/azurefile/controllerserver.go b/pkg/azurefile/controllerserver.go index 249eb2ccdc..b921140c89 100644 --- a/pkg/azurefile/controllerserver.go +++ b/pkg/azurefile/controllerserver.go @@ -278,6 +278,11 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) accountQuota = int32(value) case tagValueDelimiterField: tagValueDelimiter = v + case enableKataCCMountField: + _, err := strconv.ParseBool(v) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", enableKataCCMountField, v) + } default: return nil, status.Errorf(codes.InvalidArgument, "invalid parameter %q in storage class", k) } diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 3e4c1d49fe..8efe6ca583 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -99,7 +99,16 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu return nil, status.Errorf(codes.InvalidArgument, "invalid mountPermissions %s", perm) } } - if d.enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" { + + enableKataCCMount := getValueInMap(context, enableKataCCMountField) + if enableKataCCMount == "" { + enableKataCCMount = falseValue + } + enableKataCCMountVal, err := strconv.ParseBool(enableKataCCMount) + if err != nil { + return &csi.NodePublishVolumeResponse{}, err + } + if enableKataCCMountVal && context[podNameField] != "" && context[podNamespaceField] != "" { runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.cloud.KubeClient, context[podNameField], context[podNamespaceField]) if err != nil { klog.Errorf("failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) @@ -187,13 +196,13 @@ func (d *Driver) NodeUnpublishVolume(_ context.Context, req *csi.NodeUnpublishVo if err := CleanupMountPoint(d.mounter, targetPath, true /*extensiveMountPointCheck*/); err != nil { return nil, status.Errorf(codes.Internal, "failed to unmount target %s: %v", targetPath, err) } - if d.enableKataCCMount { - // Remove deletes the direct volume path including all the files inside it. - // if there is no kata-cc mountinfo present on this path, it will return nil. - if err := d.directVolume.Remove(targetPath); err != nil { - return nil, status.Errorf(codes.Internal, "failed to direct volume remove mount info %s: %v", targetPath, err) - } + + // Remove deletes the direct volume path including all the files inside it. + // if there is no kata-cc mountinfo present on this path, it will return nil. + if err := d.directVolume.Remove(targetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to direct volume remove mount info %s: %v", targetPath, err) } + klog.V(2).Infof("NodeUnpublishVolume: unmount volume %s on %s successfully", volumeID, targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil @@ -246,13 +255,12 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe // don't respect fsType from req.GetVolumeCapability().GetMount().GetFsType() // since it's ext4 by default on Linux var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName string - var ephemeralVol bool + var ephemeralVol, enableKataCCMount bool fileShareNameReplaceMap := map[string]string{} mountPermissions := d.mountPermissions performChmodOp := (mountPermissions > 0) fsGroupChangePolicy := d.fsGroupChangePolicy - for k, v := range context { switch strings.ToLower(k) { case fsTypeField: @@ -279,6 +287,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe fileShareNameReplaceMap[pvcNameMetadata] = v case pvNameKey: fileShareNameReplaceMap[pvNameMetadata] = v + case enableKataCCMountField: + enableKataCCMount, err = strconv.ParseBool(v) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", enableKataCCMountField, v) + } case mountPermissionsField: if v != "" { var err error @@ -415,7 +428,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } // If runtime OS is not windows and protocol is not nfs, save mountInfo.json - if d.enableKataCCMount { + if enableKataCCMount { if runtime.GOOS != "windows" && protocol != nfs { // Check if mountInfo.json is already present at the targetPath isMountInfoPresent, err := d.directVolume.VolumeMountInfo(cifsMountPath) @@ -526,11 +539,9 @@ func (d *Driver) NodeUnstageVolume(_ context.Context, req *csi.NodeUnstageVolume } } - if d.enableKataCCMount { - klog.V(2).Infof("NodeUnstageVolume:remove direct volume mount info %s from %s", volumeID, stagingTargetPath) - if err := d.directVolume.Remove(stagingTargetPath); err != nil { - return nil, status.Errorf(codes.Internal, "failed to remove mount info %s: %v", stagingTargetPath, err) - } + klog.V(2).Infof("NodeUnstageVolume:remove direct volume mount info %s from %s", volumeID, stagingTargetPath) + if err := d.directVolume.Remove(stagingTargetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to remove mount info %s: %v", stagingTargetPath, err) } klog.V(2).Infof("NodeUnstageVolume: unmount volume %s on %s successfully", volumeID, stagingTargetPath) diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index d0d9a48fa4..317fe9dc7b 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -260,16 +260,14 @@ func TestNodePublishVolume(t *testing.T) { TargetPath: targetTest, StagingTargetPath: sourceTest, Readonly: true, - VolumeContext: map[string]string{mountPermissionsField: "0755", podNameField: "testPod", podNamespaceField: "testNamespace"}, + VolumeContext: map[string]string{mountPermissionsField: "0755", podNameField: "testPod", podNamespaceField: "testNamespace", enableKataCCMountField: "true"}, }, setup: func() { - d.enableKataCCMount = true d.directVolume = mockDirectVolume mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(&volume.MountInfo{}, nil) mockDirectVolume.EXPECT().Add(targetTest, gomock.Any()).Return(nil) }, cleanup: func() { - d.enableKataCCMount = false }, }, } @@ -312,6 +310,7 @@ func TestNodeUnpublishVolume(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockDirectVolume := NewMockDirectVolume(ctrl) + d.directVolume = mockDirectVolume tests := []struct { desc string @@ -344,22 +343,12 @@ func TestNodeUnpublishVolume(t *testing.T) { }, }, { - desc: "[Success] Valid request", - req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, - expectedErr: testutil.TestError{}, - }, - { - desc: "[Success] Valid request with Kata CC Mount enabled", - req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, - expectedErr: testutil.TestError{}, + desc: "[Success] Valid request", + req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, setup: func() { - d.enableKataCCMount = true - d.directVolume = mockDirectVolume mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) }, - cleanup: func() { - d.enableKataCCMount = false - }, + expectedErr: testutil.TestError{}, }, } @@ -764,7 +753,6 @@ func TestNodeStageVolume(t *testing.T) { setup: func() { d.resolver = mockResolver d.directVolume = mockDirectVolume - d.enableKataCCMount = true if runtime.GOOS != "windows" { mockIPAddr := &net.IPAddr{IP: net.ParseIP("192.168.1.1")} mockDirectVolume.EXPECT().VolumeMountInfo(sourceTest).Return(nil, nil) @@ -774,12 +762,16 @@ func TestNodeStageVolume(t *testing.T) { }, req: csi.NodeStageVolumeRequest{VolumeId: "vol_1##", StagingTargetPath: sourceTest, VolumeCapability: &stdVolCap, - VolumeContext: volContext, - Secrets: secrets}, + VolumeContext: map[string]string{ + fsTypeField: "smb", + diskNameField: "test_disk.vhd", + shareNameField: "test_sharename", + serverNameField: "test_servername", + mountPermissionsField: "0755", + enableKataCCMountField: trueValue, + }, + Secrets: secrets}, skipOnWindows: true, - cleanup: func() { - d.enableKataCCMount = false - }, }, } @@ -850,6 +842,7 @@ func TestNodeUnstageVolume(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockDirectVolume := NewMockDirectVolume(ctrl) + d.directVolume = mockDirectVolume tests := []struct { desc string @@ -895,22 +888,12 @@ func TestNodeUnstageVolume(t *testing.T) { }, }, { - desc: "[Success] Valid request", - req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, - expectedErr: testutil.TestError{}, - }, - { - desc: "[Success] Valid request with Kata CC Mount enabled", + desc: "[Success] Valid request", + req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, setup: func() { - d.enableKataCCMount = true - d.directVolume = mockDirectVolume mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) }, - req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, expectedErr: testutil.TestError{}, - cleanup: func() { - d.enableKataCCMount = false - }, }, } diff --git a/vendor/modules.txt b/vendor/modules.txt index 44ec41a4a5..2dc270ea98 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -329,6 +329,9 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go +# github.com/kata-containers/kata-containers/src/runtime v0.0.0-20240702121346-ef3f6515cf8a +## explicit; go 1.21 +github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume # github.com/klauspost/compress v1.17.9 ## explicit; go 1.20 github.com/klauspost/compress @@ -338,9 +341,6 @@ github.com/klauspost/compress/internal/cpuinfo github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/kata-containers/kata-containers/src/runtime v0.0.0-20240702121346-ef3f6515cf8a -## explicit; go 1.21 -github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume # github.com/kubernetes-csi/csi-lib-utils v0.14.1 ## explicit; go 1.18 github.com/kubernetes-csi/csi-lib-utils/protosanitizer From f337f3fe04880c8a2c31ea49eea3abd518699455 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sun, 13 Oct 2024 02:44:01 +0000 Subject: [PATCH 3/6] feat: disable kataCCMount by default fix --- Makefile | 2 +- charts/latest/azurefile-csi-driver-v0.0.0.tgz | Bin 13612 -> 13685 bytes .../templates/csi-azurefile-node.yaml | 11 +++ .../templates/rbac-csi-azurefile-node.yaml | 35 ++----- .../latest/azurefile-csi-driver/values.yaml | 1 + deploy/csi-azurefile-node.yaml | 1 + deploy/rbac-csi-azurefile-controller.yaml | 22 ----- deploy/rbac-csi-azurefile-node.yaml | 35 ++----- hack/verify-yamllint.sh | 2 +- pkg/azurefile/azurefile.go | 2 + pkg/azurefile/azurefile_options.go | 2 +- pkg/azurefile/azurefile_test.go | 1 + pkg/azurefile/nodeserver.go | 91 ++++++++++-------- pkg/azurefile/nodeserver_test.go | 8 +- 14 files changed, 91 insertions(+), 122 deletions(-) diff --git a/Makefile b/Makefile index 1fd27ad06b..74856c5afe 100755 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ CSI_IMAGE_TAG ?= $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) CSI_IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGE_NAME):latest BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") LDFLAGS ?= "-X ${PKG}/pkg/azurefile.driverVersion=${IMAGE_VERSION} -X ${PKG}/pkg/azurefile.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/azurefile.buildDate=${BUILD_DATE} -s -w -extldflags '-static'" -E2E_HELM_OPTIONS ?= --set image.azurefile.repository=$(REGISTRY)/$(IMAGE_NAME) --set image.azurefile.tag=$(IMAGE_VERSION) --set linux.dnsPolicy=ClusterFirstWithHostNet --set driver.userAgentSuffix="e2e-test" +E2E_HELM_OPTIONS ?= --set image.azurefile.repository=$(REGISTRY)/$(IMAGE_NAME) --set image.azurefile.tag=$(IMAGE_VERSION) --set node.enableKataCCMount=true --set linux.dnsPolicy=ClusterFirstWithHostNet --set driver.userAgentSuffix="e2e-test" E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS} ifdef KUBERNETES_VERSION # disable kubelet-registration-probe on capz cluster testing E2E_HELM_OPTIONS += --set linux.enableRegistrationProbe=false --set windows.enableRegistrationProbe=false diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz index 2d64209b31135f926d93aaa6a11988328cfffb1e..aaf2ae0ae01033e25edbbf3eabba077395e3dc4d 100644 GIT binary patch delta 13205 zcmV;GGiuDNYV~T6Pk-_(PdGL-lG>wVx2%xlo-?Q2pBJi1k{A_%8URI!?RLaG$nML{ zlYD_o-BHEjMHkDg@7rmMg-RrVL?V&-LjwO6a~O~iJYNtm;A8@P>u`j*>`w7G{9zSO zuh;AC@9pXT_IkbYe|tMS`#)^&z3%Po@4ec7{pyF__N$%U*MC2t-b!kp@x(%6{zGr+ zy2_3FMjjlIpMeX)Xdg|sH*gfCe}-6skQ)IApOZ-H-wu?xpdU#H=#Wu~2?dTWB;$Ap z=!8n(1MCBOad>g^GX4E5jE977P&|e{atYkVBxRx3?R9$_58W%@&*Pl`6CB1MR;B=& z=l}NJ?*8lY{C|J7-CNK9`*_G0523$-&=nSN4iW34vCq3>;&Ue0Kz4mL-ay%`RYeiv z5HUd{|WQ@i0{*>hCBH!!s5Vf7(YUgHtBYI0&HXr7y@CXA`2- z0B}bwOo*-P_zE}$2|`t&D)pl5qpZrwtL}EU*P_aE5P#&id8*^)gIrbH-R+&(b%kUC z6ofeA?8?=Lj_Q(6BK1kG!kzB+-s}n~3*bo4=5rVlAvvC-mFi=w3p}2DUSwfrn>U_4 zL+3C64%7!E8DbO=<7mt{AOSuHDb&wbfMNjwx)}l0)Ce6SEtNu%;4tW;k(80>Z{=9+ z61EjEUw>>#EUv|t&**>*V-8-#0?$uYrY%L!)5N@plL?lPv}d}tp?^P%!|=jz(?`Gj zy73Ho$ShRjsqMMxJD8dxr+5q-s)&aUV0W-3}ZCh)~JU7$gp`#2daY@?Mwd zqc2}K5PF8L|13NlMK$Z-h5$YxJ{Xv2=6`d*68cE;7|=8HH`Pt*-QD1YfCh}CDT}-40;FPbfI|U@4AhV422=fGEI|Fu5xb#XRUf00voHkSS2M)R z@tMBy1V+!0>J7vO=)&@0nYM$PFu}MUUn`Pjnp)Bpq9!Pg#E8km62HmsOR~GcudrY4 ze}C18q7k#Pk?XJYbzQ$Hav_U@LoKeEOj&}SerZ%%P+wX^*oQt+lfe^HAwetXFl0B! z<48^qvu{$aIYc4xvFM{6)pe3%3B&2-G=e@lXL`6IVMshe<9>YaUX4`e}9f) ztQGUCUhgMjf93Gc7)14>-njDFPV!ZGX}iygFYE-Ab638zWaSPKYF+pCl6991qYa{~-NArRBBxd+m3qB^5PZqF{YkPAxD{MY)z z1s18V6}8UsEfG>AFJLeLU-nVwlz(0LBM9OUI<4qi%@L`2d*`)yOH{L=7qFT7bBu34 zQar&VRLaqe`NL>^{q}^0gu>4(jK}Z}rU$+cLi~-$(MbSQYEemaB2dNgJ1p_x;ZNHB zNMBV{_N#U(d$*)&HVLz+;@*Y> zhxxGx7Y^%2f0Wr5brltt`dPwSTKZgzUv>GbSpX|bU~46~yMz+l7yyJnu9GV13Sl&N_oUT&hJhjxoZ32_adtbk zOsF5Szc)fc|9R3LhW0t{nD1xVcqsx%cz&mF-4$A6)8GH%~(qQfv& ztICgri_%}Jum5UH&`os8DA;8#1ldPh6U?_la#?+Ox4f3gC)h>SV*b+C&2g z7{^MzDS0>pJ|;pKiSE6A{W|THWDBjN-6*@Hm+ruDY-Po)%51516Wm>uGcBd$i@l{P za@aY}<=$Z;IgF@-C zjHN$1zKu9o2~6Li%({_E1EIo`wds_}6Iyy}1-|sFqBkL2#yIqniL`M;XuxiSOBKWd z)QmjiOxX^p-(R%Q72XY@E`5>C=;w$1PZP{Ph2-j!{qebLR)0VGiTG+EVclD1BF?UI zx2ek36LG;k%3ke6%uMx?mCnSXJAm~xw$qsL>yY6fweIzPf&i5#M|LVbL%%Q`&cora z=wyK82++a75yFwS9Mv1^2IC$Uf($7HXpAWyLV$>+i-_^@5URfycnnAa#Z)3J^iRH7 z)OCe&1Qe<1Z-0uM9HAQ`N4nU2wc099j+9_oTE>vk(tOB4m&Dc-?r)dkfBUcZw%75$`*<$ZOQ>{s{c*8h;`Y3j`7p%UEopCBr3FM$st`e6zjl!4-6-9> zp83UR#{B<0f-nLux-wcE3NT~-@9gdEl;;2I{oTFw{J)Rq%a^VHKoc_V>*ZMqg0fED zj8(VzBY)_l|JeHabz?&pJHCxD4fLu_Z7@wQ^cpRu%YFIcp#TPi0_voa)m4Q%$jjbq z|3KHE%=859{%jB@s^o_Wa83e1zbo6G`e=Vw|C5Z3+XQvIj2O^B|64P-aj0W;h9SeE z;@5(UlwG1D%~#WmZL@FaZZvSO%Vjl2oKQJHoqsBb%#`V5RGw($G%00vI)&6JeaaWACw0zdK!%A;A&y$^Od3=I-jU@X!9)}Gj z(%K87ZPY^)hhd8B7%+}f9ELy*V;^M*jGO>uY?fgfs`d(WLx03E9YW)GC`EUuj;?7^ zsdW#nq3`yYvHtr!@bb}yg)o7Q>;LxN{#@>E3IDgVz59B1z5d_FQ<`k)I9nf}o&lcc zZRBynI3jQ>focy&_xjI9g>7$K6B_g_GV0v=(vCoTTnM8Z2{5W(^3a)}TZYyd0@dpA zDkfn7yr3T^z3!iF{0x(f3K$6kb*crN;xHdCldTFEe}SKgjwB^t=ekZ9*t6}J4H1C` z5)DnbsjiqIYkR}=q>WAgb zKifb^_$-1xO7~*H21*PnO~pev2#E0}ohj1@_@SbaIr|Fu@e>*lUuhO4blP!bt5Vxd z4mgyitHa6oF!!1`}nMIFUDsQ#YM%pPmKi?B_!?;B1U;M#LXkJXJ#e1`G6p z@dUs3GO5GW)(HHrq1jdYG9n~U8}2*|F&Up4fB$;c{tI3B?B~Nm^M&LFi{);8${G&M zpn&$lHZ)rQTbD}cs;4NVhm0z25xe>a_)=NNCf-fxu<&^{N18ie;-X&D0i{wcJL5cI z(X@3;C>h6NB*<^N&4z*0k8I?1G!3{i!qjl*L?JieO){*d{et{f7UJ^nvpByj4NYkz ze?lxo;>*vrM8!gNy+5gvs&;z*ZA+F<>)9;|j%4XHDwj+<+!j>|iQ2hK)ZUeK(V1~W zQDsug5td4ykwNN%9_W%v46!d`9EQ`Bs2ylwTwbck6_cY93wqD#=3g=dRKz^<{_F2C z2NW@4CIXfSus=dfj(~U3Uo0;_eZ06-f9l8Z3W3^(GHw98cvw<9MA-J%OWGBSY2f6t zek|vjI_a`$klkvibQ?WFJxRt;RDTj*15Ly*rUCG9npY%tu|x*>RNt)pqFrdjVfBS+ zK?;QN^5ha*rCYHA3RDkomV4yC-AHf*+v&a@bI6L`y_oDDKDt2E!q29K2eMj8Mc`1YDA0Tq!B!z&ISn6MBX*JTWvVSNTZZ}Mkm14 zTH~`JCBLP#wMtNx=8mUikG5kde-2ILZBtL_F`gn0TC2n~N6#LXLO&@ph0p=Z&MORw zOw41SjmI%1a%zx>&}A5N-)F^f32FM1n744qM8$)vQx(B6r8ZvotvUyufQ^ z8k$ENkPRcCY)B(i=2O#(>XLkZd~un^#r0s6d%v`ni);u~hY>meZU#}Ce?NMer-2qy z8zm%}PDy1oz8VvmO1jW$`%v#0v~8%-Ye5&CP;`jL5FTOycRm2Br-i3Fw%rgUc7NRv zsP_f&4?t)u!XIIdeF+m1cnDP)tHQSvvyQy)Hg-Htuwz}g-flh{WjO|l5y=MiqM&hQmut;4c2TY zQbRS@-Byui9o{uPt9CR>fd~$w%z-abW z1U!d{=o%MGN$;f!%gk%nw7|uBH%lappCely)fUw#W^**wf5l`h(M{5OSP145y`&aU zAVzoQ`?uZb=jd6m7nXU-38Q*cRG;XR!E&UGdD>RQw3^U6A#4XRSA(|N3qU*b;{hh2raS}++R>wwq%st{NT0-U(6lzTh0N(=Irez2Ochqi zj6zMSC{+t3e{`WSo*c$$MPr%ma)Wtl3gqm1W39)WtF=NImwcQ$Ot52rXr4nxm3^Rg z1F^84mT9`ryaIWn2BWP5#kuzos$WH{&ovF=5WTpXs@)6;ur+(IoX<^Ds>SS83T~N~ z=n4i(uwr4Vs5o5gr?m!Tv-@S1rf&Yag~E0=H0#p5e}|0-w~6L-t(^=UG&8M6AlO7i zrm9~8;qus}wl5jN6Gj3QF(HWBE(q{r zZUC<$2C<3!CXse>I-iic#o9r5)q;-Z7WO$*RxzG7L%{Pz6KR$+{E05LoAaL4I2g;0 zMRY<@e=wzZOnfBDKILj#kdFmy=CpXastrh}PsXQprp#0h2Iij>7Trmz_^cY&N&Pc2 z=G1O_^-|X%ro8d|X|pQ@bZ^*s?X;Z!uTn(P(~AOVRu ziWpaOLTXc?s#1)z$*2_v;Z)D0Bd2}PtQv%jf9W-~sW6I4$pB$GEo{fqybU7=BehaWhT=#Wry)!r%n7D= zLAua&keQX|F=FY$ zWapF_y}E5_YOsfZ7jQ$3L5TF!bonWo4Yp{(?;C)`Bupc)ROdtip2%FyHo8s{mDr{8 z(yp=zhxNNU^@MCVR8v_kRwVU0Q9EXse^4ZV3L?n_^46zg2|}|yGSayORXR*IFF$5j zZ?MRBo7<@{TX)Rly2163As=RFn5zB%gsnBNe%q>Bh-OrMJ zjs>_@m9}eV^GexZU^H0(HEbHDoSLh}(m8XXbgc+-(-?F&q{e0uCQVf5lE=_-f2^}k zqG$}*LP1mm2?bFOSv0MxRWwe%D^k=$Rj?Z;OG)ZeGUfg33_yol?#+UfihVLrd4Nab zOmNwg%b1Z`#GEz{ppI(U=c}m%(cXs4%>xJ7#carQ9yln@WOC*#a2r5%2BglRW~9Qz z%xeeICGvU{ti0r6DjJ}1`ay#*epsAAfrJ z^2Li^-u?9U^6dCm^78jzXgt0)fBpXA7dZa4C6Sje|LYGGR&KosB)dj@QsgH%jokghYP#P{=g|@}3}& zD}_F;4)VB4sN++DIDWIx#w(E6;2WPAFH_#kfTTcr`Z75zYVzi$kTN1rpfro0Y4>O7 z;Or!i{W8Oy5aQqinW$y&e+F}^<7zesg~pvJ(h;M=E{7TM+|q3sX7;41C`30Cs;LR8 zOn!}06KIU}6iC{Z=d)NHVf1f?P9Jq@j(lt-WU5%tjO+hLC+MxGQ2N+xe;*ET`h;Mn zs{xj-p``Ut(iMS6*KpA`aM61Jh(0bz=u-oLE(`Ts64bdQd~<1_f9AIf$y@>Wa(S5L z(tyf4gibyPUU6A;;wrF+%Oek$#u+Y)DqJCk@D31z%i;r@lOqR)7^-2{Pu9I$z=Je1*PoAl`Y_!VLXVm%31wf0&WK&~j_!%8|v;Pism4 zg67nQ$rm=i;bD4N#eB&PbGQndYS;+6>GGrePqWY@M<9WZb$Bn?k<@uhk}ZcEaTX7Y zuv(WFG;=!#5Q0o@TP9Li&x{L!pbI+SDEbXy)ZF(~Y_IE;quq};(U$G6G>{Wvj>l?X z?CO9Wo{zk1e^vWHl+`Ot7w2#il@QKBQjb65usg{Tms_vB_9#nX9D;;a-TX-$&OsRR zq%j9X8O?5B2p$3QqO``X6MQ-Bi;-wF_^lWn8YtqRv@J*ObWCYaVoA?8|fn zw6J#0LG9-|dc-Rn$0FB4x0Fq(5^rPv>Rsxyci&-a#>9sNyK)7v#mX4cZ0k(0<;YLT2ojR278gc$SQXrE8llK&fUfu=biNFUGFIFLx<0^YgpAj;ZAl2 ztJn)X3G4q++xzPy4Bv=N|IJ(R?PD$L=*q)Ke^-nSzlZQj&W%(XU@>K8g;*@h^)Z(0 zUo%20DNjoVSH8LQ2&FUfJ{w1BRV@U&Gqx$x!T1KY<~s?mx^QS!$?Sc$YiX^qJW}P9 zw?Gqa6)7Cxe(u?|Bl)7$eN>eg*p(tXm9Eck#}&3ok2*BZ&qv^eC*mP97{IF(voo@j1MnP8c0gu zNRR-0oW~s(t>TWY&`i>x{>CPsaTCu89H-We4^B1QiANTjI?;qnyfnx+IeVP%e}F=) zVjihV7RmZF4po0As`GiGI=?4shOE2%pQybLv~lC%X!HDv#GazExwRZ5(-aRl$6gwV z%j6He1$h?6Lqe5N`QoKPh7X*nqV;qP>}mA+@oDJSg%gA%C%Ou5f)|WuJJXjAhHIuV zV!V>}PwA3>#6o5n(s0g8SSzO`f3xrIvy>$5|EqSXw>Y;{{GgQ7LZ1&vN_-|kCx7v5 z@u@khFnh^!%aVGMJSJH`yL&ZFAK5tAC_}Z~MRt+a9iA`l>TdJic6N|o&qEtLl;$!vi;xU$+e-X<;1|0pA zoTb**zZwynQ|;gX`kx<32xb=i`(OX7w9dc(^*?_JnScH7fBkRkgxp0mh*ge;x#z-g zoIBf69AOv5bj(Y_fGNUZG{QEA#XM?#v5YUXGP=}dlXWzzHoq)7O><)!4V*3mgbp|B zs&>?1Po^d|x;_P*#J{z1e-@Cz0CYwT0XkitG0Dx!f?Xs(*slKyhK!(4{K0uHirzMhK+{O+uDpZUiK^4P55wii>(latVM zi+r&s!CkvD5)cl)iE88%9bSRSBH=v8H=n7yX_(N7vfB9{>#aU5f8BviNRwlCgrf-B zTPF_6i`px7&bH(!$lt-)Ng3v5htj!QoVjxAmTE8P?7E+y3`}J_%VkNx3`Od@Lxc(m zIA}1FC;}=8hjes}?lK;%Tsi`*@m5-h(Eoe!;dJXQt08H|3+prh`p6xOfyzy-KTJj= zojLGAz3GlIB?Az$e{1UiE`Hhhway$+!V9-}9EGr%1maQ?ESW<@nC?g^YK7<6P}v?p z5lfPW-{|J5IcAKR0JH1pYov0N#M}pd9J?Ro%VfvSyUu!uofA*JJ$Ke+3m@PUHs+V5c z87Zbz;#*adudI!_p4Exo^lY9eLqHiP5Qbi|_rJkY)eFhM*>tX$%SLHUQ~Z56`p`F= zDS;Z&B*)c&DBWigel?pG$hbw57>Pf^beQHF9K*z5K6|3@D$59_nTXp%%AJNE_r05?Mai)Qt?%Ca4)5Mn6EBbX#LGG}%N=EAu_CinZWcphW!YKgJeBkfokdz_L^01gZN#^Irs!Kf ze?{~_Cg(bRPRo-$OP}Le!Ks~Mb9y;`qaYGXB3{wiG+JNf3W;%gH@g_SoS2rs?H6pPYIMy z2`qIUU?s@^fsQzsAh3G4qe_ z{_oi?vhVf|@M2JL8PxjH81-cl>E9^+{K=us*SBv!O$7O!-n~61I(&Wq_G7w#dnFj| zWf9ve!Dp}U(f(%d(f&pe(o5r^mqkB64y^Mdz5e^l7Y_jqkoWcL#t-YK`N6-%e;fuR z1kV@5Ga6}2!Z^|(t}XRru1I<P)uV4Mp+u7aTdG!P8-EGs&6AOv?551-9DmU&MdCXKhf+%FuF;H2$+9eWOlkLJ? zEe7iPbv=^2H#G?e!I#N)cZ{V!dY|jr+D=Qpd++GH{iMTNe~yfh97`Bl zNSs+R+}|W6FQ<|HHc>a0|D3R?CoD8&E(!Q^)`->umR2!HOWrVk9Wor8D56A8y9RxE za#SF%I7y3ki7mSo3+SVcR2zT-{T3n>vEY4PM6L96Gx_=R90D4#;Gis{n(>mH&aMTO zK${~bCo1hLJ2Zcvg_uH%Dw7CA zq{=te9C5-pk<$$2VvSwTH}^YvJ|{AFUyuMl)*mn`;Z|njBfwX?f66n?)XdEr=tggx z$5fIr9GSxhjl(8c2)r!OxgT3fbfaP)=sU}%xfx@BnPa+ChK-Ez5YFN-ya1nr%yD0S zQ>dlx>tf{J+Q*`huKgU#K;S2_hMv z=BI{jPak!1GuwQBe}x5{L&VV6ua$RAwEMG4OFTqhzjn+RgaTUfv${Hq+5sWVhoT&| z>UfhkJDtjp+6>1*pmug|{_x`9=;-|T;^L*FU?rEJx7@b`ryLR_YolWK-qGlqC z<(uuDUQamQ!>#1lGpI2yS-am9*SoZ^(mfuIi1*J(ym{4Ye=2bmb0Obs_gsYrfMv|V z8>&6Zo4+#3_}S0r;D?xuO&GAk0MJQ!1nikE)5B%hnHuUoQu`nEe_g@+>#_`~`Df1o zpjqM(rDcnfF|iq?V=5~WGFHEu18Rzg5{HI+#TQ5~AbBT9{wl~XURie(%t%|X#BR=n z+TJNLT&vX8f8*l_Uy+bFPfl~Cqh2W&? zT&a_ii(3Xu*0tK1<+L2`W4b-}%+1`MnHV0Su>em8e^cLR*YESn2RZveRNod?@^kRn z1h|17AKyk;Un=|~CSlA$FX_)8c1cghc9G@$p7n)g^_0ZjrJ(+9O|Upl+>e<&dgnr3=I5pmhVQ*!6Gk);z-KKOHL@i z37L;WfAv}4?F}k6bp$w+qcg^3x^b>_v8_dy^diRPn*}MCyV4MzyxEyc6%?y^?AoBf z?ZYaB)L+|17i7-emFj^Sveg-x6vj1I`KhViU`n}7*ZDv8Pyu%~0K zGvDP}30bUh)&ny1g!Y_yWgrP?S2Sv&0rDy^A=L%x7UAT^^n#YKg*PD`6{En-(b--=@TGGx@IQi$X=k zUL6>t`f|iUj93`-(W}CHwV=NRS=Q>2S&|Ztma-d6;HZyMYoT1%f?Zf=K_nFO%scD> zLWv|eJc1BUFAQwCk9La?Bx^dQueXb@eaF~#(_MUpKVcOS1;CRO( zb2K%VRI1#@{6gd2p2d{h_P4g`?bqrAwJJ*=TshdZ6@{Pptk?Bes;fBA?1 z@9*yVtEw9 z-e*?ETsCxO^W3X*y!d;Bdm_ds#6NJ(R4((&HP&Y!YOexzHo?^T2ia0)T7w~ zYohQ+I@ti@(+}LBsWzb;#a_s=|@&~CJrImR&G34^x$ z%zqZ4cHCD0_0N&IBb@(yczq=ejf}2;2)k{*YkLyFR$kB55(feDxbV99odc84b;Lli zb>PCCJJ?UkWmLC&s0EH+FYhG~|095{uj(8@^3rzIhbXtN$hi>AvQ}|Z@t7Wnj{>;q zvd-xxI{0|`*AM3>mw*5C>EiDfm&ZRPk*vajm>eB7Du)dqeD(fY42)+sVRV0gG4g7q zWf$UXA9S7P+2**Z#M&cv16+(C47+ga0J!^=pHAV%JCQI(`hWGJm^#*wcZTH%9rizc znqdCvQ%J5peNrkb1o`Pxr1LKj} zOGOM{h2RZACMP|F#8SL}afBO8(YGLN!gy7#k!aq3>{$nL-FeX#z*FO(s)d|B5x7;- zYmPFjgk9Z<&9g>qtq8Go7Y0?{K|k5PD~oGY*3|w=uj7CB@wABly`W6C4g$VhM*-JSz;zUG9R*xR0iRG5aJ3vu>YyF{6G zS@wB%YLDp5`8qiHaKXt(6I;}iuAJAku+cfsQ8!ZC_AD*jmM^fc2@x1>wi+X|jt>^- z0VkC)3*e2LRB5GSt2qU`-+T&ox91juyVY=yDX97+LY}kJmLxLNn75?k?Bw#)tk^}h z^c-eY@_9j)XA#L4USfe;2YSD|)GF)P>^e64L}RmtQ9?sP)8u~FF!jt;(&m28B&EHZ z`4j!EspuV(;VfAP)2HqyRp?WbK`m+ljgzb`Vt<`IbL0Q!&i*tL|F`{WzgNQlZSU`| z@Bh4)rxX%1iQFvmZxu*Bn;CAUIKC>)6`=SG(d99GhRzznS2Iozhu?E^v-o|Y}<0xBH-2|Gl|yL<|WMKFiv+a>P^H9Xx79?jH~Fm= z4Y$B0UW-82wiW`em9_Cau(dU^>^nu4Wq4sG zzem8D;V%L{CPJuvVO4&O?#xO&IpkZhf3LB`p6{EUsc>ovy~7e89_j%r(ONxbZN}wZ zvk`5JZhY3LxAg^HYoyy6>DK0ou|_aX#L0w&FoeJ{BbGa~aX*)vHEpe&8x77PJb%<1 zj6V7c33R+)s{|(t`WtPO-renJMM42goF4};AwE>NSYNHytl?%_2n|EQW4dKbj?}@u zB}SM-uw~@N8dK%2vA7-eWkb0Iu7oS^bZ24l9B{do*q!;^6h+%(25_fghA+#RCdS%WTn@)z!v8rohjd9CzGBd4tu+5DFz z=A!P86)o2S|CL;o=H;$Bn+pUs@8vk_DdYofkVTp5QbBt90c`!M-nL4Q10%i&oD zBBl=rq*p$oOk<%NMxRS`1!#=dM-dqldaiiSM}PMa60HD&7cn0~AN8_d3Xnmw=dVBln})866kHCd*IGqbk%nxHBf!`aTEN8E*2vycpMIOdyN79AO`^F(_ z^Uc%7cyav|yu~GF6+z_r>*~+@_1xj+`{t>2V0y<9WC{TXz&))C2!Eq>3HwXMNMyNx)Pq|IoZ`@n zIGYef8o0-qHolj+KYyObp_y|Y!V%F31<^;pbUMFol(fw+ohZF@-Fka$p$YISU0?{Z zvxzz(5&FM3YP+NUVLD6wD@uL}AOs1TzydY;sC08ei@~m2uVZdZ`6$7ovtrfO6-rmp~a(yoXtI@J+ZCcT$`8l~( zsP&4fwNAZXmC8Yo-@ZK~bY8W(gkNK+8Wz_h_q7N;uz$X~Z*{dod4B?Ws?-|~qodTY z)DBtNImt=7dsb8p_E;vMET50l~@u=3Ae0H^aSzR1r(Pmvy zgJz|>`F}XU%H!NK8*_~%_WgU`0J5O1lS}th{mz6N;9U!k*4^zm+II4K?v%u>{oKh= zk82Cq&9MLWUiV&=&;Pvc?XK;=`*`dMiD^)O2IyYEVKqYbZFou7P_LCb zNY^p8`;D=!zQUCvHWp^0btvFYtyGs*dsJ3w{eKCdmD;Ec+Wq0Y;4Y@jA%^q}!yT?b z4;Tr$cabmk>Ust=3ERCn&S%*Fwx6)+rv*=f%&`Brd#`s&_W%C&?%Mx*FHiaWe<_}n z8T+NoV9u%5oCBaaR5kzU|3!}R&vRi%yURKX$90=NGvqxI6z>6R;$jxZ5-4uNu_-sR z>S{bWNBWXSp2=kWEb=U)|Jz0aXX^j`%K5+j*X#Sg?&Wzf{hztQ*P4H=`Pa__J@f1T z`6B@{^nY(}x2*s7d%eB2{=biB9{>OSEe>`2@%vdS)_QLJte^GsJv{$600960pBJFQ H0O|n%B`HP| delta 13192 zcmV;3Gk47OYOHFIPk(YRPdGL-lG?4)ZdoD8J!ek4KQC03Brz%iH2{hd+wF*XklmM? zC;0-Gx}%E4i!PQ~-?!5i3zbL!i9{mvhXnpT<}e^3c)lQBz{v#o*5L?q*`4BX_}wa= zUa!~N-`msw?e%))|Mqrv_P^WSd)?dK+kL&WzxQ2l`_<0w-hX$fx02dtJh70Nf7e^O zu5#nPk_SiR7vO?0+DDV^4ID-3pCOhY2ecO1&uiD64Yvs=M9owW#tO1b_K`p6a;yAXn9PcY9}cT_KqO z1tHEjyK?oRqq^jiNPUv4aHqSyH@ia00yxsM`5cBsNRH=drTWgHA06-OQldGI1Kt|BxNM}TRB#{ zglz@P7k^t4i)*pvGddu{n1dIw!1I%pX-m=bG%+vYWP&9m?U`X9OQ=ev)EJnZ(N8O@4`3P?*LM3l1uUI0zPVAB17vwVUkLe8+(+B#pMPT* zYsLJk*ZY~+Upf3U22uT}H?DlPlYCWP+U~RB3p)Yj+?DSvS-C@mTGze3WZfkL>7LrG zbbxQCdZ(jgcQ7E7$f=b{rCu)}1Ya_4f08T&ZUtA9_0Tm;`>3N;F=ruki_7d-PqvO_ zfko=p+CtwWuUIv7(n>wbRSHC4IV) ztih@{P`yTEwWK?aI&IO$LKWF(_A6tVexDltjn;9Y>chec-i8B*jj;$94u2Cyf0P*! zbrluY^H~yETHjnNTy>4BS>-D0Tx(6YyM(6O7yygBu&h9M23orP#Z^AYDCU?Vi}!_a6lC zgCS69HE^8#eGv}^ zXKtcXM!~LSA;>=3nqa;alB+HIi))<=)W(eUQ2 zNObS@>(^Q9~?MB%py>tiGVJlN*Rklg3FW~N~oM{dvU+gVak;9mAuIUat$YCiR zBtLB=+^9XnQOpS)T7NTu(Ba9Dvh?TUTkvD;8n-OwYq|tJ&NaqEV=VpA@omJxN?`g9 zWhRVN8VD7htWBp(p3u@;EAXXX6}<`JGRC2oOr(t)LIZXqT&f@zpl0M5XUafGo%^DN zuJCRMb?J+AMn6C7f1F_cF(g+X?T^o0v-;6b#8(Rm>)tXGaesD|yG>QLo`?%>MD}VY zVrGw*OmQX_HS^cg*iK`{uS14|)P&cY1_D%`9NDS#4E@S<%npaYp_2iUBR~fSM+isS za#U}uuZw$F2r{G)pfRR+2mvCNE+WQ}L#X~<;4vTx6jOo&>0@$(0Y6W zh(b1n07+(I1$NqZH#ROA@<-rb*AR(VAdrw)#$ppiA%9>2h{v?Qku3i*M4m?z|Hhlo zpHcT5bh&Qo^*~>~C{_$H&l6TJzvik)Fe-@o$E6l$wfPV`8q1|*!e|7%SL7h*XMOwthW2USs zqw+)}r%5TZ2{I(zDzcA0e@4kHuMSlgIlm-SNKAe6by>sKGjDcnjhu%$j#UXl{V)z1 z8&|`l&&Cm>+8vjRt5?fKm`jfh7V8LWv3gh2Vpk-g<)jsZL?jMUSe=_-ihonoDxds; z{(l)WS?J_1Up7V%j=N&CrRB4>9aeg~d7kWK%i|NwYb4p<@i=TKk=9-qZKEEdI1E!{ z$AEE^;xGhi82czgVB`cKW3vp?P_8= zftL?NEQAScT>rQC_IGz*7uWxt?cLYC^?&+*A5UqrrQ>YfgL(#dp0|<53FC;stpuu_ zAKmNkjSAb|xF$5{TV&L^b+a9TcE%7!Hxgh}zvR3#@wg1FGX$#D<5f(;0C+(^PI}$% zZD>T5D?>N zI#Z?*@Iyr-^8*&}<0mvAzS1m8=zp|N$yTMdn;dX3&0_*3LmX5;0m7EGuJw=oLoj-V z<*1LkW_#JyUth8hB5}K!VLC7WEOm_Zim?zdb@Dhs7*V*%c%uM;C<4Vc4JOJiaUyRN zrye}rK0OQ2*)NA^z}XnxjEFz7c&dc_4HoDH;|YH6Wm1Q$tr7TLL$j;)Wq(9Spf(bE z7-BL$HLmxp{TI6M*)NBM<_pOU7R%lGlr_WaZD_Rqw=R{?RZme!4;fY5B6jr; z@TIbjO(>kuVd3*^jx=|`#6`WP14^Y@cE)+aqG{`xP%@6kNRZ!kn+*f0i`mHQXc};3 zgsI`qi9&9`n`Br^`vv*EEPur1-)C`tSsI$sNQ793#Lb^=iHe2ldVf+SRqgcr+mI1HyL zQ9IDWxV%)6D<(%J7WAIe&A(&_sEBzM7|`Ei4k%*8Oav?uV1I;|9Df1tqQ6>Re*SQA zsnn0*6#}&nW!wOG@vx+Jh_LOim$WMu)4<7P{aDU5b<$;AJ;hGW1(VY5W=g?l zYll^0n~3pL@Fi14IC!-h8KH=?2)HE0kX6o1ECjL&ZF{boZ}+xlKWNmewJRX7J!X`F zHVmv9);dN`6miYk!rXD$N~F$sTRTP#l_| z+@_w=V?0G1v{s2}j-EX%g??6M3ZVm*omUtVnV82u8;@g3F7h3^LAlwpU z=wa@Uhy-889JZ8gt68VoMDC7*XKCE}d4bo;G&GMkAR9(N*^ox4%%`Rm)g}4-_~J4R znd`wQ_kL+D7k}9hs174^0Nf0sHh=UoPXjHaHh@Snos!CGd^ILAm2{!i_MzT0XxmT& z+k!4Sq395gAw0wa?tB1LPYX|VY`Y;y?EbnTQ11)kAArzSgg?R@`x3aCd%iFeHl4od zqFw5=Lobeg;A(CRt>eelz2-5wKc;?zeWPeYBsqYqh!>t!|hiDVQ!bp1yA`4cU235t5Nbgu<8drCJ`4|JsxcYuX6>lZsBemX zTW5Z|3YS`0`)#h_CF9yj6rnR0rCJNy8m!q)q=ss)yR9P4I=pLoR_$n%0udq}8`Jj+ zEKs0^sDFkJq@lh9K~P%g5ep@3bxTZaY#k~El4AbhK&egcgKBx{{-4NNWWer_3ZhF`l4X1YOaLE)< z0{2s<+H#fGkX>>x%}di$m6wdd`+*C;!_(78zkhaWXMX3m8DG|lLgmYv4cZLN``ocb z)RIQ38+}JnM`zC3U%JTX8kOa6gNTn*Z4FA&wEdIdny81w6j4Q1F)+PP^qjoBD!jt7{8n(`1R zXh)AylFCr*B7G9aLDSmU7EPy*<=Er7GJjQAB{K>&siIUZlz@lEcybu06^&)K%MIqK zDUh@4jkO+guGR`=T=H@1Fu{)fp?MA&RrZ0}4aCBFTBhkf^9tmR8jQ9M6zASUsD2f( zKG!seL-gWms&+FZz}D=+ay~arsTQ+WDY#``qAM6E!HR{cqT+C|pVk_X&F+_3nt!_a z>lO;z+0d*@^By)L+$NgSwRSRa(9E27x&Bdn*4gQ%Xw*{Fw8)1}6B z(P-dIL3YvU`^#gO+P-86PZ$YM#DpMfyCA@ixxvDU7{n&>n?%~l>3l-&7HbFLRSP7MBhseia*hN3+ zh0^KSYiPx?e5!Jq*LOt7gj3PTYO-4}fCMDsC}LdA38_tms!B1=CZkpygnv^#la8GB zL9=QQGN#wmrot#bX2ml;Ubno0G~i$Uj_QCLQaxwTQ(gH zCIf`&w6GmZ^EQkijMPdg8HyuioQ5!gFejMe1?fW9MK%xQ&&t14qyq3s0BUw>m$V$g zH0OntgvNj>&RZ=@m}p5f_J2@rG>VxyuCM@|OhRjeX{1L_cZ_rEq2$FlE1YxBi1;-? zq=c1gU7!_yslgrsUce1C1|iY|)#ayX zHrS#Czi$8%lQ50IQk@e8cp`H(+vqw;RAQITOS{S@9MqPCC zVM37rDu^T#$XlO|B?!&-$VlfBROv9;y!@D9y}=^iZEmN+Y~3-F>ju|DhJ2WzVX}fb z6RZUg!_|cjXohR?tY0b{rb3&=Gq(h3c0Wn>DHh;bRobqd%`0Vtfzf0E)Uauoa%!#? zOXtjm(zPPUO=HmAkbfGRMVK^Eol71=$Fa^jiJ~!J3k6XPBostBWYM&$R?#^5u1HZ2 zRl#nYEG4N=$&~lAGXNcKxiYXU#_4+v(hy^; z2S)py%*-g`cS@`hJxg*XZ8}qP-y1!edZCgT>QYRzw%fV^P4hCsQUa%a^nYG_{L3HS z%l~@u;@6({pTGU(#mBDx$A7&1ua|!$fBfa;%NH+x{pshomuJVnk(Yn`O5^dh`Rk7t zzryivEs4B*`F~%3s<3kF<)E8t$H%y5=6!lFJU2S7WwPoo^K=(&)P7~`?i_u)Pt5-k zUsY0}Sht!vi@b(Lx>17PBqZ{iheEC)koN?ETq*Q%b&$tZLLHwH#PO?zHeP|m2H*J1 zc$xBM1|$X2)0fF%QIj_}g_IG20;O5}OuIir2WKaF?0=UT?t~BrAIL;4dpDR<9appI zDKzd(k&YM@b~((5=az2EFtaC3MIpMGP)$uxW%6s3nm}W$r$ExSJfFqt2%~>9bo!`M zbL3+yAydVAW?cV2Izeweh0@1n`}=T!(=aQh#CE=S(12w;1NahN_m&?N}mj+bcA$0OV@QTZ#6IX#nTpoG2G|q5Y zRN)FSgm-`#Toxa=JR0y@!v3uo?RQuBz9)?6TOPl+0<_*ruz5=(@xE4ZdrX+RRUznB zgqK?$9k)CdZY9XK72({TII3+W7`EjRYs=%)R)2s-TLt#)aU;!^!IZ7hV~-gvr5cnVTnazSwzgG@4a1O$dCyhBE%4l{2L+}WY7o|0Bo#4x1Cl_a7 z!&npxs$FX zZtFyw8mECk(P%1&kHflyzl_$qJvDFF5Xo-JRIAEMRn)RFyYiC4P98V70cbof2apWg zr5{a4wi3wn%FH@JOs|!x0K#o+Yzf_CBTR!(tBp#^IfvBMfjHLL@g6JODO*^J1Cckl z>YS0au7e`PCR@7oP+r5-GvheMk$>41mjo|aj!V_$2 zgIa=eXo|(QVorNYzsOscD>%c__SDLwv3b8$TX&7Tz8mDV5ujz@*P6nrFxZRiMOL{3 zS^2)>aqc$OIPauS?|MgZA3A)VUBjyO33swHSjAr8Nm&1n+TLFuVfadH`hTz9if
    %aZPshI9eijxNH$Q?{=lWOOK&Np4Uduv(nwHsJNL z+dRRtTd``_q;nYP`&}DaG)H61akyn$v|LLx4-L&ZEtUDYVzRb}O4u)xq5^do^#UTU zJFc)zdeotLem(*(L^*JkxR`%xxJI^2gVcluuhqz zHlf7md%TXcc#Cg=Ufl^*1*8jGK5) z;5fBzd~mAaPCT;M)PIR4T;ioczRB6+d>Tqb|$Ey%Ml9ulgI$`>yUGJN1n6|JXZ zU{9mhk55CtE}S4FInh;c6TDzN+nK&}FkCZ@5#yD#e@d79BYze$%aDe1Ucy>AEt!3H zpQR*e|Bu?G-s0R+@qwkVAA(&b4?|=QT z(mMbC*Z=%AWd8NP|MkDE6LJ^PAXYgV=AH|~aqetOafDqI(=jgz1EvUv(Fofd7W1g} z#WKFk%IH#;P1ez<+WfNUG|i1^G;q2M5IWqftJ+b6J%5>+*y#Eca1#I4##ulH1JD^Y z1n6{m#w0f@3wDtdu|L|(A8x%;E3_u!gHDtnz3Qpp#mD6w98l1b6MqNI*FFCaRH7ba(|Oi-hwW z-+ZR-rhj2VC(3H)f2_Cqv~&kHAx)0m5so5gZ=Ey32U5a_I=L##?D2 zLjUi@`_rwrtcIi+FRar5=p%P91}ZnT{xBJhbbsc+3-zWu#*_>|$gZsexcGJFw>on` z2`}8@aTLO45{OGpuw)JqVY(xws1=@LLuGpaMJ!1gexsYKmPchLV(Y;R$u;y*M?1hX zHa{!r|KhsvLp?7{mcWiG)G^GZ?$P{0k}+wV2b>It;J*6-*=>@}@AOisbfQ+|90_WZ z^?#&GbcN|(cYdqoxY%E69fw=AW47o-s9t($XQY@?iEmX+zOpvzdR8ZT)3bS^3;|`F zKp1+--v0(qRWBq1XVbZ2E*qsaP4V~s=zZUCrUYt8lN?tAqI920_| zYbFyBXp95MQs>;UD60rTz1&X1OD@^FUVu*BgJ|`2C=V7bkf~zKh+8Ykz%5=v=weQpZS* z`@5J$t>_~Gn}sP2-)C+R^nsFF=h{Y@0H5h3i;5ypp9LnVD5i)|xxd#%@2UQNGeL}v zmErW?f%_WMzx-MJ`SibS*A?fV;eVMHY86D8kA}VveBK`ohrliI}-h#H{cn%-4PbW_|MIUQfQPPrN+l6EEw`EO(Td#fr>Q zxmgU2m1Sp{^HkDTbQWoy5r4%z>$DMH_nD%v`4rIunVjqNIW152EPak=1*djaaz4v-EMB-nkI5``AZD4PFJI>&mnDz0Jm#Bg7d^+Y>Qf6# zpHX-!Clc=A{K4|44OV%!VA+!dt2#HZJ|$2-C9u?afR!Nsm&5oki+|={8n?d^g#I#E z{AE!0%i!zRNcy=Ow?@*hk@RcK{2DXA#>_vy`@d(q$iCS-z>7h}Wl-x&W7L;Lq<^LO z^CyQkU*Ep{G!f)?diVC6=TkUawdB@6PVt&a3aX_g?q5_jX_J?C*Wo+u7aT+4&Cj?zZXXiG{@c zyWY}ul^ge!JZ7pLK@_s-7^o~=?GlNt$#&tc76Wzt{26tBF@Ko7oKR@z`j;;od2ggH zdJZ9A0o}~+s7rKAeOts#_(FW)TU(x2^5Y)6cQhcx*kd1o0 zmjryDHKMhErBw{lk~fTBhYSZNiYSrOu0da(92LkbPST=XV#{vD0{W;U)drwIzlBIe zEO?g}Q7b*&On$zfLqH=I9F%2LGhUL@*|ne&XmiAbJbwkL+FsYgxguH3%8npD9Qb+% zep*&6#n(A%83oD$xF|@!HXjPEYbN?ABs9Kl?43NN*4j}quxRo4x1~uj-YxkStdY2Yf zx_`&R5%K;Ri8rr$O(m{kF65i-o~zIRu#7o)L$ybF^CP2-pZ#(Ueu&A~gaIoI0G*Ua zz@F(cJzR#JsiE#8wf|B7*A>jaF3XUbfA$;znk61lTDB+|6Pr;wrm`X-WA&>!pr&{z zacH<#e1Y@=l6QjSuY&ySm32qKjI;$y?0@D=sO_C1!?j9XJwA@`6$y#+M8Vm4rF!hag{XVaJkh32|^=)w_KL?*pfE(!X@oj|lrNTd862=_#lK%W*m-J+8 z7g^5lSzlOIA68d{WC9d~P+N;DXrnHYFV55ltJN3{q^lFTSa?Cbv8o!|XpiE~p-#$8 zNj)0r%-j{pz`#Fe`EJx4Eb?L~j(?<#yyS%Pn~?c9RG;8|O+F z+gfx6VPmMAQ?61HNsX_0dAZA$z;lkbYYC{$GJ)qydpFGn22h=oBPy(+v{3;J7-Wvw2W zB`M)(DZ9Z0j`}FI7Rq%k*oAc#L_#soyu%(Klt_ZZBM9;I!oZgMXn(f|L9(V(`g*(g zIs#5u;QqLN*Yf{=I6wY$cz%3ze0q6u@b1$OC-073dzc?Mb7WI6Q}1US{RC6zF@>s|%9S`!WH@Y{}<`mQg_Bp9UJ;WZz2&*Hq z>XbrlG#wFcGrtzUa7x^aFC3z!melGg^=P)hnkf8{PBy^!^gTCd>W%z(%_*^NX`RVo zRdnkmb0)DFVj-H7!iNUs-QzPC{?qFId9a_~&BTB9wtKs;OZd;7ot;;cD=1G3y{FoY zNf$q}TD&8Zg(yn|OPcW8lhi09e`kNK|L^0mwj8EG26r|C+y#oSLa6>83;0plT4wAI zQ&6GOHTF0GhB2cTu)t}8q>@0KuJAB{i|Pu_5+r1oKS+>JTIneQlm7RjfXr7*6`{iq z`c+~okMZpXiYJ(aYQzgYr>1%y^px8Qa;m#fG zC*?A#+db3*$FG<75{Um1z}8oFjv#qyyXt+E+gIdV2xeKUxT$zd55xxn+;myz^b#F> zxcu?``N`$qKYqOU`^Dw)f6qxIt8gGDM@NmyVFL(Xz5f~mOPpDYLfxr^G%QeG#kYU}14m$K;Aph}8h)L9S_E4FDZ8eke>IePP_ z7f0tOzZ{=`IzN8<>Eiw2PoK^XE`M~JGqcPc75>sr|8oad+;scn0f7H-QIf7sWA2n;t{jgeW$2MhFolggL{ z@WxH5w9?g_g57UE1-sjG3&GuLxW^P!eG(zh*=b7>8EVX1(s6cjd1_YdqFQJWn|-3OS;HuyA)#q|CQuT_GR1^taSB=7DEuGnSYC3T*GI8RvKo?;w&!2LCr*2x}#E{(_(DZ z`U{#FTk5{IhUq?|*7Dr1LsT{BgeQO5+{0q2THt?@tFpY@RabL?fM(qP_G*7`w|xKG z>(}f1-|popIo#80(URm}UIu>xf+%FuG0+82Kq=YeABUMJslqD)2DPeY1z4e`dX`5J zHBnzFu82bv5+94!eVBZ(ARegY@GJun(g5-F%0HB8EOf)@bBV41jWNH50AZnRuoMG6 z*+z%wM+Q_LXCNs}lTh^RRd26A+so4?VTjNf6%5G{5oT+J#E8XVpx7PiwU{G0!c_f_A~GiQT=AZd?jR&u0mLq1K7>B% zWxo_4V`l$YfrK>;T@@*|98jyZn5-c4T2oysfLF~R|0puFRasjUJ7tZaS~QU##zF$W zH$YL&URe>UsHuuPjLUzHqzjw%l|z>1o2QNO!ul&%i%ZTbf{63i)t~q4xxww`Ad@plooXchzi5_{vag+2EFKf(cN$Ko{E9q@B4?vwT66-MZ2SEk!U}xMuQh zh0lJ_SmP=yQaXLsh&k%>8RG$=`I+ykOE3C$v{NZay3x*_f;5yT*>4-3=WSHA(_aY< z2p!&qlCnpnQ~h*ht6EM)iTLUt;7hBeOnWX0`!h|pYI}bd_i%n?@7Bhdrc|trDOdIU zGDm^QQALkcWhNGaynn6fHX@g^9-U^e@r}8r*-CA!<;?7r+jd#M z%~oWYYN>zMtIP6Nqh;6Hw4zP(b8@Xv>lIaNoqE42m4hI^e|<*iylQm`zs6EEEUriH zYY}>2eRbdJYK8Lt1oTv?Hy%busbQ%dvb1xOlT`Pts2c3COs0RU>6I18@_e21tt`L` ziA)>m{`@%!O+Iuio*D7KdtG^S&V@&} z<~-0!lkbd2wZ`PLYu(H0;t-4Z)|LAg;)n(JzP)2Xjrcy8^r~K1i*8;R+Q_XMuGXVc zWov!)Sm&n7jR&WkbNCq@-@_Nr4I6SjP-tFtgEkZrHGw{ znP?pfy3;+9ORGI9YqI_X*h+272JQZEQ4kkX<`7GIhUE@dpa+Zu-n+<`dUZVmnuP7% z0w*)Hz>a $LOG diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index e6d36d22f4..7372f228c2 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -234,6 +234,7 @@ type Driver struct { appendNoResvPortOption bool appendActimeoOption bool printVolumeStatsCallLogs bool + enableKataCCMount bool mounter *mount.SafeFormatAndMount server *grpc.Server // lock per volume attach (only for vhd disk feature) @@ -294,6 +295,7 @@ func NewDriver(options *DriverOptions) *Driver { driver.enableVHDDiskFeature = options.EnableVHDDiskFeature driver.enableVolumeMountGroup = options.EnableVolumeMountGroup driver.enableGetVolumeStats = options.EnableGetVolumeStats + driver.enableKataCCMount = options.EnableKataCCMount driver.appendMountErrorHelpLink = options.AppendMountErrorHelpLink driver.mountPermissions = options.MountPermissions driver.fsGroupChangePolicy = options.FSGroupChangePolicy diff --git a/pkg/azurefile/azurefile_options.go b/pkg/azurefile/azurefile_options.go index c331a90aff..52b24e467c 100644 --- a/pkg/azurefile/azurefile_options.go +++ b/pkg/azurefile/azurefile_options.go @@ -68,7 +68,7 @@ func (o *DriverOptions) AddFlags() *flag.FlagSet { fs.BoolVar(&o.EnableVHDDiskFeature, "enable-vhd", true, "enable VHD disk feature (experimental)") fs.BoolVar(&o.EnableVolumeMountGroup, "enable-volume-mount-group", true, "indicates whether enabling VOLUME_MOUNT_GROUP") fs.BoolVar(&o.EnableGetVolumeStats, "enable-get-volume-stats", true, "allow GET_VOLUME_STATS on agent node") - fs.BoolVar(&o.EnableKataCCMount, "enable-kata-cc-mount", true, "enable Kata Confidential Containers mount") + fs.BoolVar(&o.EnableKataCCMount, "enable-kata-cc-mount", false, "enable Kata Confidential Containers mount") fs.BoolVar(&o.AppendMountErrorHelpLink, "append-mount-error-help-link", true, "Whether to include a link for help with mount errors when a mount error occurs.") fs.Uint64Var(&o.MountPermissions, "mount-permissions", 0777, "mounted folder permissions") fs.StringVar(&o.FSGroupChangePolicy, "fsgroup-change-policy", "", "indicates how the volume's ownership will be changed by the driver, OnRootMismatch is the default value") diff --git a/pkg/azurefile/azurefile_test.go b/pkg/azurefile/azurefile_test.go index 103c81273a..a727fc4f08 100644 --- a/pkg/azurefile/azurefile_test.go +++ b/pkg/azurefile/azurefile_test.go @@ -63,6 +63,7 @@ func NewFakeDriver() *Driver { KubeConfig: "", Endpoint: "tcp://127.0.0.1:0", WaitForAzCopyTimeoutMinutes: 1, + EnableKataCCMount: true, } driver := NewDriver(&driverOptions) driver.Name = fakeDriverName diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 8efe6ca583..ab55f1afbd 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -100,48 +100,50 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu } } - enableKataCCMount := getValueInMap(context, enableKataCCMountField) - if enableKataCCMount == "" { - enableKataCCMount = falseValue - } - enableKataCCMountVal, err := strconv.ParseBool(enableKataCCMount) - if err != nil { - return &csi.NodePublishVolumeResponse{}, err - } - if enableKataCCMountVal && context[podNameField] != "" && context[podNamespaceField] != "" { - runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.cloud.KubeClient, context[podNameField], context[podNamespaceField]) - if err != nil { - klog.Errorf("failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) - return &csi.NodePublishVolumeResponse{}, nil + if d.enableKataCCMount { + enableKataCCMount := getValueInMap(context, enableKataCCMountField) + if enableKataCCMount == "" { + enableKataCCMount = falseValue } - klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) - isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.cloud.KubeClient, runtimeClass) + enableKataCCMountVal, err := strconv.ParseBool(enableKataCCMount) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err) + return &csi.NodePublishVolumeResponse{}, err } - if isConfidentialRuntimeClass { - klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass %s is kata-cc", volumeID, runtimeClass) - source := req.GetStagingTargetPath() - if len(source) == 0 { - return nil, status.Error(codes.InvalidArgument, "Staging target not provided") - } - // Load the mount info from staging area - mountInfo, err := d.directVolume.VolumeMountInfo(source) + if enableKataCCMountVal && context[podNameField] != "" && context[podNamespaceField] != "" { + runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.cloud.KubeClient, context[podNameField], context[podNamespaceField]) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err) - } - if mountInfo == nil { - return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID) + klog.Errorf("failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) + return &csi.NodePublishVolumeResponse{}, nil } - data, err := json.Marshal(mountInfo) + klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) + isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.cloud.KubeClient, runtimeClass) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err) + return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err) } - if err = d.directVolume.Add(target, string(data)); err != nil { - return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err) + if isConfidentialRuntimeClass { + klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass %s is kata-cc", volumeID, runtimeClass) + source := req.GetStagingTargetPath() + if len(source) == 0 { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + // Load the mount info from staging area + mountInfo, err := d.directVolume.VolumeMountInfo(source) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err) + } + if mountInfo == nil { + return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID) + } + data, err := json.Marshal(mountInfo) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err) + } + if err = d.directVolume.Add(target, string(data)); err != nil { + return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err) + } + klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target) + return &csi.NodePublishVolumeResponse{}, nil } - klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target) - return &csi.NodePublishVolumeResponse{}, nil } } } @@ -197,10 +199,13 @@ func (d *Driver) NodeUnpublishVolume(_ context.Context, req *csi.NodeUnpublishVo return nil, status.Errorf(codes.Internal, "failed to unmount target %s: %v", targetPath, err) } - // Remove deletes the direct volume path including all the files inside it. - // if there is no kata-cc mountinfo present on this path, it will return nil. - if err := d.directVolume.Remove(targetPath); err != nil { - return nil, status.Errorf(codes.Internal, "failed to direct volume remove mount info %s: %v", targetPath, err) + if d.enableKataCCMount { + klog.V(2).Infof("NodeUnstageVolume: remove direct volume mount info %s from %s", volumeID, targetPath) + // Remove deletes the direct volume path including all the files inside it. + // if there is no kata-cc mountinfo present on this path, it will return nil. + if err := d.directVolume.Remove(targetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to direct volume remove mount info %s: %v", targetPath, err) + } } klog.V(2).Infof("NodeUnpublishVolume: unmount volume %s on %s successfully", volumeID, targetPath) @@ -428,7 +433,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } // If runtime OS is not windows and protocol is not nfs, save mountInfo.json - if enableKataCCMount { + if d.enableKataCCMount && enableKataCCMount { if runtime.GOOS != "windows" && protocol != nfs { // Check if mountInfo.json is already present at the targetPath isMountInfoPresent, err := d.directVolume.VolumeMountInfo(cifsMountPath) @@ -539,9 +544,11 @@ func (d *Driver) NodeUnstageVolume(_ context.Context, req *csi.NodeUnstageVolume } } - klog.V(2).Infof("NodeUnstageVolume:remove direct volume mount info %s from %s", volumeID, stagingTargetPath) - if err := d.directVolume.Remove(stagingTargetPath); err != nil { - return nil, status.Errorf(codes.Internal, "failed to remove mount info %s: %v", stagingTargetPath, err) + if d.enableKataCCMount { + klog.V(2).Infof("NodeUnstageVolume: remove direct volume mount info %s from %s", volumeID, stagingTargetPath) + if err := d.directVolume.Remove(stagingTargetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to remove mount info %s: %v", stagingTargetPath, err) + } } klog.V(2).Infof("NodeUnstageVolume: unmount volume %s on %s successfully", volumeID, stagingTargetPath) diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index 317fe9dc7b..ab9d348cf1 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -346,7 +346,9 @@ func TestNodeUnpublishVolume(t *testing.T) { desc: "[Success] Valid request", req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, setup: func() { - mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) + if runtime.GOOS != "windows" { + mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) + } }, expectedErr: testutil.TestError{}, }, @@ -891,7 +893,9 @@ func TestNodeUnstageVolume(t *testing.T) { desc: "[Success] Valid request", req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, setup: func() { - mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) + if runtime.GOOS != "windows" { + mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) + } }, expectedErr: testutil.TestError{}, }, From 3ddde6bcef7921c269bb56af02768dd0e38e0993 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sun, 13 Oct 2024 03:38:06 +0000 Subject: [PATCH 4/6] test: fix ut and refine examples fix fix --- charts/latest/azurefile-csi-driver-v0.0.0.tgz | Bin 13685 -> 13683 bytes .../templates/rbac-csi-azurefile-node.yaml | 2 +- deploy/example-cc/cc-deployment.yaml | 53 ------------- deploy/example-cc/cc-nginx-pod-azurefile.yaml | 24 ------ .../shared-pvc-across-runtimes.yaml | 72 ------------------ .../kata-cc/statefulset.yaml} | 2 +- .../storageclass-azurefile-kata-cc.yaml | 20 +++++ deploy/rbac-csi-azurefile-node.yaml | 14 ++-- hack/verify-yamllint.sh | 2 +- pkg/azurefile/nodeserver_test.go | 18 +++-- 10 files changed, 40 insertions(+), 167 deletions(-) delete mode 100644 deploy/example-cc/cc-deployment.yaml delete mode 100644 deploy/example-cc/cc-nginx-pod-azurefile.yaml delete mode 100644 deploy/example-cc/shared-pvc-across-runtimes.yaml rename deploy/{example-cc/cc-statefulset.yaml => example/kata-cc/statefulset.yaml} (95%) create mode 100644 deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz index aaf2ae0ae01033e25edbbf3eabba077395e3dc4d..396e7a37f189ba21afe34d3d2ed0875f90bba339 100644 GIT binary patch delta 13674 zcmV-wHI>TsYV&H4O@EXu%e_3|*w9F7w~lRDA;~>wPP;!ZRFxz#DgreCiW1xHhPs4af=my1O=p&cFZA?-Ydfi^Pw{hRS^8Gx{`9Hy73}R&p zpn3jp@4nc3RezrUyRTlX=l@+iWQ>Q<-$3XR3pj&__0ibp-7)bw6Ko*6J{xbKY}Trx z2yuv*Ad>NEA8lRzD7Jh-whE%8i%C38)rk5#io@`fg~Xrs(edDf$x{vjsCwxOa?06+ zC^Z1wQ414d>ngqkPCG!9 zR=b341%J#JTM~;avE?&5Aj6o07qP(ela*;p(epGhFXCi^B_!>cZf)q_PvbB=H{A5m zufJ_PK^`&-)p%-qZu%CcrpO5%!-gv2p@UdRHbx=7giyBw2LU2fGBgH>11#}|uZz6X zCHm;g*A0Z8psOE+hoh)w9o!JWC&Y(+-)AwEYJbFh4p>4TNge}wg8rtuN&R~h*|G?m z!UA2BFjW0%J_*o(aWrLd7oCGtEDmre0Fi3 zcsV}PH=e-g2~xd**Z`edJ}lFAP!lE?*W+tNvP@G;+CtO>#gP~>c~Igv`F%-tH~1y? z%YXf^8c{T2Ha2qomAw4=ICax7suy_iPOM`ujWxsOqRr7AmOLY{!UX8bCZIyoW& z36N4vPM^hLp!zHz0$+v@D3Qtic@x>|QvX6HD zb8mastszH%LpjQdDc*TZKhXDBNHyMCppzf1Ae@GnLiP$viT%+T{4*vTg6#K@4d1~8 z!ajPb{>o2MOevFC*}KUv5eo{_IAXzmr4R?fV(x=5%)1se)ygrEBKZp;KZ^TkJAeIi z3}daBU-o+MiT#zsKVuNpk9yj-Bk-UJx0DRd;oqrQ{?vEgdL+G@kZ#74x=IyQ5;w@3lhF-vC>d!I0 z`AG2ulTax~Gv@cB_4V6h8WIY>urMCOTbSF(TPA6$8WL32M6!9 z{gJ+^sO*>RRQ8LKs@+JI=YCpWPAR9Gkvc`v#&{LkXZ9;&wEmdx2^tOOLVwim7wY2oP7QgE9SF-?CmcZ6Z@Wm2JaAN>i#vC$%dIQpk z`?6pHZgfm3Y314v+r&@y#YWKKbdxCi>g{*~#(An@x0dcJ|>+S#xUZbjsPy z)H0!d$o}3435{QGS(VvR>n6CpDrZ_s$rrmz zRphX9oXfq#L~9uLWkBuAar;j@DuUnLc+Sc%tV}B z<%^~&TTjFV_b9uy6EQQ@OIA7)i|zo{)7Va9#;-z#gVeg$`w0S69v|AN^aTCNbT|)( zzoFv+k|RL-`-cce+HzEHtQ(AbSO_wt5TG%pcnAR^mM$X3$3v+8Uf?kx2^3R_u+TsG zW>MD_$`MecrhmUFa(sxci5%%-^VMprI6hQ@Woa2hMoaSn2VD|}Kamj3cIZ8h(w!EE zz|(HK`wqK2n8z9alQ52S*kX;S(f3de+ivR7s+Sy&l|L)?sP?J9w z)IUE)W*Jfwd}uws0Yo92LVzSQu>w1{yBix94EZDQuYYQYL@W?UNGxNqiJ}m&0K{Y3 z-$>SY86wZ4iGSnumoKP$2D)4~^?IPMUll8cnCA(rd|!8P6qObbMX5pref`=&o_DQu z^LpkNpBeN2^9aHSxai7gaVWry`M^z;JYT+S{Rf(mabGXbN)VKF z@_MYg#eW|`AN|MH*RLBJy4cZ8glV8xZEAyQdZE{7FlC)e_H?*`a}ETl>X}c zrGq-3JBzf4jmJ!xPDbU4MoyDbW~XCFx>aN!effftSzaBgE^>ZJsF0ZY=IgSCt!LhB z%^Eoma~!J@g!*9|G&Zh=N1u%&Mzzl^7gw*Ai!hfS8!Xll)-3fdr^T*FLd!`j28l@Q zr+=_IH^CJDrl?gu`2+nkX0p)9U%zgQARKqaXiLjyZ9A;=cJn;h$(F|_nAb?MzvFS( zP$I3pFxo~vL~$6V$c_QyD8*q2)G+o@hQP=PK*nYnrlD%DKsWS99Md5*euq+YhwA8> zCY4%u-x~UMpBd}F&jT+XZCD5s*tq_0?|<&Scu~Ut?QFkzxwl^b@8T&k1U=tcsJ>X$rpCg_%-b%sE- zdc2HD7yvKm$4RgIqm7^8hs;3f1ADd|vmqkTK%$`uH`Nt0WNmMl zp0rUc`+@aI3i;OY@*PPamSN0sSmDlu0!NISSaRZso7fkG4&xB>@@E?e37x|nGQ>gk z6Ci9^>slw+KLn$vSdRLrYqpnN{q+U=C=%bA8K$%H&r%0MFBuB~QzwrDgb{`7j5i7p zh$2vI(_o_P5-0LTaq6bi?bEXWo&Iuw2Aqx2^@#W*i>FG+Ut@uuF`nS}o_{BGxY`yf9p~S zUG)@&^ng*tEn=7d0ADKW*u=XD9Tq;%=16k~OkC7!I-pdlWoMivESk2C2_@rrj0E{z zx7jd|`jL&ij-~-uMwlAzoPQ|f2E0jzwX~m;-^)T={(Tnb7p0*ojYNorNPPL(mZ(^$ zuJ&J4gsgo|72HCBK zO1IG?)RSZkMfE4~70^TsV;TStr+GzU7fWQ2PxZ~pFWQAh98_PJ7NkHJFHbJ9Rk{@` zpg?thSVPR32nOmR{C^RQaX*V}(f!loUtXLSen!RaE6EfQ2 z_Jc;fTDt-Q+hax6BDf4B-J5aO(r0dRlm@W7`cuV)xe#fqGvM{{V!xBK#5N z*q6Z7-1CK*u<7(o7wuA?T~4iraCU^_W0W}Eojbb)OV_i(f5?v>~hlOB1(MxIp1!8n(zJJ?|evY06 zdtsTUoG_|KMfHh387xQ2n5S(;OsfgK6T)^7b2VtIy+Blt>Lmb0W6ZBAHk4sIY3HWd zG-hL@IUZmVYRW^PpdCF*Nh(9Ji}Xnx2Y*d#V_V3aK9*yT=gL%JmCPvAq>56tP(l|P z4$-sAsoKqu09&&M%lX_irCQ8hrQnu%jxJ%K1S=M%ihqj3 z#eQ0AKsLKyW@+l?uUjZ=XG60t&3o90aGPjO*V^&GK{L~81cFUOWU9JFrn}vBjIf?E z4x)MzXQLiYO_v(iMWcZ;1=&R>A1;nuYWspAJYgh25fg%_?ScS5<_7R8Vi23iZxU%I zr}GK9TdW;~S1sshZegE8WfkLTGk*j;Uo??sIm4gmQoA|tS&f6S>{vv{6a`a?$HYgX z>{G6`1^HOOW=@NztJ;8s`eb}sXUa_FU|{}9VbPtWiqEQXozy=gV@~a+S1)y);u_U) z&>Wtf9v~lwVHf?R7fPpRub~yo@~O&cUf&TR6HY}VtI2M`01}Xhqlj@eCx4_i6{;%5 zIGc=GaS%@ROgeJf2hFNM$e3PHn+l`&m=(|Xc-<1$ApF2N<1?gQmr2Au#6oET0ek|1 zeHG{f2{}o!ZP|1*m<$l6)53Nv&D$`7Fj6a}WGIf5aT>w|!kl1=XQT^V7uh^oKP&%S zkqW>k0jSxjUD9#{)0`Jp5`P*4syJ`8EMcN0(bz+|(I{r-xWocrjZ^!-7(Is zhmsfLtZ>dfBjQ&8krGy}b%9na%9^Vo%6vcOA{mC_q)y#QA0w77Om;mEfhpGkWdii zkVVs~T1DgJyCOwBR0X?nvXrDgB~#wd&H!|{<=!kvsn{n2l?Qk<&IFe|xr`a9Ma*gQ z0P3igeZHJZ5bbTq+&plQUCf3|=YfObOeSZ}0=EHFXF%#4YJWy5Ow7D?AYCG_N5RTV zE~cUZ8Ydq#NJEUV9vJO+GBcx$-zl+5^eoAlwCPOEeQ)$=>V-;Xs7o=;+HUIxG|kHh zO9`Cz(f@h&=`VkH&;RS$vtN7OfByEDXP>(IAOG?Ezn=e*{PCCP&!0W}_3isN7pF(R zk>`K>O5^dB`G4z=XTQSHZ!L*DfBs*8s<3k7<)E8t$H%y5=6!nlJ2yJ6WwPoo^K=(& z)P7~`?i_u)Pt5-kUsm#+Sht!vaJ+^_x>17PB_#5@heEC)kaq-uTq*Q%b&$tZLLDCy z#POSjHeQ0n2H*I^c$xBM1|$X2)0fF%QIj_}g_IG20)M4h{7kz)LHno2dF+=N?t~Br zAIL;4d)Js#9apnCC^YU&k&YM@b~((5=az2EFtaC3MIpMGP)$uxW%6s3nm}W$r$ExS zJfFqt2%~>9bo!`MbL3+yAydVAW?cV2Izeweh0@1n`+I+Y(?gG@4a1O$dCyhBE%4l{2L+}8Q z7o|0Bo#4x1Cl_a7!&npxs$F z(FzATWdkiOlYS0au7e`PCVyMH^-x~H)HCBa#*x_;mjo|aj!V_$2gIa=eXo|(QVorNYzsOscD>%c__SDLQv3b8&TX&7Tz8&PX5ujz@ z*P6nrFxZRiMOL{3S^2)>VeU58IPauSZ+k~^7dm{NUBjyO3AeH{SjAr8QGZzf58B>e zA7S`LZ2E8Bifa5wvOxzBDdIWoud#lop*)FaYYg6_#x^mhWQVyCcagMooU6H$N-fn2&Ky#2>kxR`% zxxJI^C&m-_uuhqzHlf7md%TXcc#C zg=Ufl^*1*8jGK5);D0!^ZhUa6;Z8iV*wl$8T;ioczRB6+d>Tqb|$4an0l9ulgI z%4g3FGJN1n6|JXZU{9mhk55CtE}S4FInh;c6Fg%)+nGLhFn?S#jS=IOw0}yM{38}J z%aDe1Ucy>AEt!3{pQR*e|IgZ`-s0R+@qEaDr`o^& z^*=w65X>z2_rLyEX`O%n>wo?lGXLt||N7t73Au}C5UU&wb5DigICr+CIKnQ9>6n*< z0aJv-XoPJJi+R-gVi{j%Wpt^_ChKTaZGKsFlIF%V8hZAO!JbS_Y;=7J zIEjC2<18S90qBex0(81OW0ISd1-nR!*dJ}?54T>b6`ehq(-e1sr7Md_Ct8_}y9QKJ$+eyiX9+=8_mdlcW8H&_*hX@rCaL`~TQ3O;H4(aF` z-DNyjxpV|r%!^zegRzuQ^7uIP2^nZ~%7z34?T7Q^~Mmlrgxq8zbV@d`f zWY^XKT>QH8Tb((egcok`I0|7i3B;u)STcu*Fx`<-)C$kBp|U-IB9p`K?ZOJGM8>KNuy_h^10$(S_G15SoRaNqra>^4c~ zcYk`WR60>Ba*hPG$$HWyy2A9YJHOR(T8SwFDfF{hJZ3oAPl`^?|+S_suz-hv*}zhmyObzruh4C_@Qq&Qvx-lNsg-lQM%70 z{AxBWka3G9F%o}-=`hVVIEZD;VX+hqRew)$@;E%kWyNuCKG&9wqwVq(KFvqk8XMP? z`;$$8PY7IZT{C_~=+IN5>Dg3CTYCN3dZPcca9i{O*0w=~)$TT1rRot|3pFYUEwo?9 zo}MQ#rP8-0%P7^cHIs=5G{ymBsdH{wlvRi7v~d2^vzQ$Z(?I>(q$>5N3-w@3T7OO7 zkH?31Bj7PvXs*_7dFzt+)u6v)mEdxu+|E>y5wf{C?1b^W(fD-^Fgl zwZFb2bgtZKsbi$Z{awtWR`ij8&BBz1?=v?D`bf!*b8VwcfX{T2MMV*)&jOQF6jMZ~ z+}rJ<4^)4@nIOi-%5eJcz<1Dd+zPR(|iszyG_+8)xNcSXf-SR`sU3?l{*bT?0Yv$ zi;`mrTHn3-9p1g!Ha%M-0NT|e3yEGZBb(EqH@8Gr(xlB4lzWip`I%g)G=DFZ@2xgs zx>N=@m%DuVoN!J{v2B*PX6wo3F6^V;T!z)Gcz#nXpVSBOb2`J-lQYpJi*;lD93QF0 z-NZ%JIGG5V9dk+CH@;G3unjo~F-OpAePQVOM9kbLVpez(=374jvp)H9rzc<5Cte=% ziI;U|mRriqVnt@D+$@I1%73!6%y}y58#;@$&WK{3b=rt;`%KZde2VCvOwM)soR%kh zmOjU`f>S#yIioYjiJV3|?hzlrS^Bum@`r04_)(hG9h`Z*$7Gf|5VOqDmv8fs%aX@g z9`en#i=Jax^{IuW&nP^W6A5>4{$Tmj2CF<mitTO;Y$NcuHqevO%5W9A>;{ok`)WZ&%_;KiWg zGN|>XG3v`A(!Wvs`J+RduW#ReoCxw;y?c93bol!I?T2*#_DV3^%ObW{g3n&xqy5d^ zqy3E{q?g7+FN=PD7=Kvj2YUVYmoFXy8X)iM*Nq?6PxFI+k2wrT2%ay9XEf55gmI)n zTwChLT#@uzj_wqX!?tzwdcEG>?ymlCuh%R6cjv|K&dVRRcVG2(UhVdFc6vYbc3y06 z_kKXV+ikjeVj(gAp|^Bh<;Hy@kC|$R5QS_y1}aNeyF_AZvVUE;tHnTFzkEU6Ukqk1 zCluPb{`Kod-W#cloQW)TgwW1;97VIqBBUlap?(+#fI6vUb`7Nt>Y_go zB{Wd`&t3J6DzBGr2VL`#DW$)Z?C*FSqCZfXf>I|lh~@6EF}xNowP{&F0)^ucg~Z3A zu1AvhrX~R)_LckdjXx0iHy%aIY1V+lhGi8D)v`_KXjc6@kX%&OCPBWB? zHFiDU+<))n`JBkyeL({JSbxB%gd3TSj{sloDo;66GdFLb8@+KBQ%S~fXbvAV4x3~l z@Uleberzexjf#Ds?<||?!03x9H&eWBJQCWvHwnx7iBJ$=;4&201i zB^Gc75kp_UR^Bzy?$0VM@eqCe+A(7g3TVmC>gp_N2ZS&migMVh<4s=gbSgh;GaLtj z+S$GS)3g1aVGr*#%#basjC+TGW6HkpHnnu#oyueW!4J>hr{Hg?zo;a}^o@mN5r!sP-tYe`b{N(_hZO z4>1{=FkpoNpp)_l*fU+Ghs&@tHPn5i_CMqvrkv0ljf+T-}guqDZO$F!AaM-QYR%Bw+xo7Yk##f z%V{~>#dLe>nVY#iGci0uV*#EHroPs$-=~!ia`uC$zAdigXW+95a05L)x{0v9RQM-M z!kB|z(x2Y%lAet1BFp(5>kG^3{pyO4On`zAYHM)`ZPZ2b#i{yWwHl*=baf&Z3oocQ zR#jsg?NQu0)JeH1sYfH7nJ+{#Fn{pRS-u-J2aCKIiX$l_FFB$7I%GZ$)n|RTH>lXu z5#UgcP8pZ!#<|kPwiaE|ix`)$7o=S7N<(<^dS@Jn)mbTNbW{>YPIv~Ru#ZWiR8G@#Se68<7PXVn{sAdwItu*h%s4WyjTGD@DQkUwB3~XD1h&)5^h2n1y96- z0l9GyiW3s&x`dk$4nubB!GAcC(?So_G*}luQl$^Frk5ydAWT`rQ@^DWsk#V5QRqdBc43_bkxIkG93 zsrMd7Z(-^@ChG{@>t*MFgN{|=@Fx<&K|-e5d+ND@XG|i=vk(8@-`(|BRbh^9iD-=g zHFDU7!TI@{hrOb_z4`F@{P6APcOTw-es}cC(Yv;_YSDSa&JW+d%dCvKZ0OA9xl`wO z`D4zQ-0I@reHKlfRLrXxR6zQ<6H{9?gklb!U%p$aszY_Ox_=uGAvr^x0R)WWT7eOw zoC~$gmk{BW57kt@9B}Y~hSP@mDn#f-SLNOKIr=Tvd z&q*cfKK3|9SRIj7rxa?V>5y=n`L+0kQ{rZP;Sepgq*hOwioRtD;*knKOya5YwC#-Zv=k4xhR3pH}zJgZ=zrCjPUx-Fxw>g#X;x z*?zIcf8NF8d0qkXS;?ZgVGyDR=f^fdZzCAAxCbvmJmWBJD=`_jr zsTvEBUIFW&qi6(U;2Kyp5N|k(BZpOGLn+-t2@9}fS3u8fj>x1M90vrbuU3zU^tZP5 zRqyQ|UZ}E1HztF`_oNBG&u1?EuR(;@ga+(dcxt_kIP*b>87o4w{@>o+-P1r;h?V~-PH7&Cef3!ElMDhb5N z3J(*wsIK5FK|*%3`1)$b7X_5jy;+UnQpU7~g!Pc!Ei&M!e8-YRY$u z@00GUfc$mYD!hN@@q~7xwahVw`A!(LtTe6Aw~imiPY?%ctCQZA#q-9s&K{CasWf%qQ)Y<*Sd z2$Gk!t3E`zeMQcNV3xItn~KMDUwjn6O_y~}F3|qRi=RK79bf$Y)2H*lpI;okPa;`` z1Aj3&I%-r78bJ8!{kIqx&uqfz?qcNCO3N<9**@qx&$G>OQ;D@l>>9WjK^S)7#sP5m zD?gpUwRbFGjP(ENMKN`(A@3B+5jyC9`ZU4()2EPJe)^>75+0mQN=N}Hp5 z_}N9nNW7E8>AR0_j!!g+!{alP+bS`C?Z>|!eLnecc=Y*X|NR`1Y@L3kCWM`(-qn_= zo-Mg6x=<^v@0wzDD?xJ@DiMq<2IZi=M%+C%!=g^*rj*PKX{l%n@IAeUUKwytQ6s_Y zKRqkX#X1FEg>U6G-#sDnbdTHy7O$T0qeJ28rnbS~r6PteL-2+mlan4oVkusKJHid7 z=o^qWVZ18WNHp(1_N)WB?!0IV;Hhy?)k01m3EV2_HAk6M!me(`=2;`QR)pBP4TCE0 zpr35tmc_Ly>*}5mK94w~*uwF@IZz-o@_+4Y_jbzpzh1o9UdR9L;%O28dq$aT9Rz&7 zjsmWufa@sWItsXs0zRTB;A%sEa1S7+*c8kt9a_(t)bc#SLH|?Ti3IE64oIuu$oee& zf@k69j$V`x#5;=AoIN-Gy86>-cM+*L=xblU)8W6Wv+VQk)E>~8^L23Y{(_SaCbp<2 zT{*97VWV@Fqi&?O?O9s5DPLe;6CyC&Y&Aw^9Um;v15PSq7QkyasnSY+S91z>ulW@0 zi=JBuZdb!Sq@e1f2zky#-Go<$^Ic!>pW9q9e; zQmd?Ev+LOGBaO`(MhOiGO_Tdw!PGNXNt^pUla%&u=8yHarlPkn-Pa%fjmS}wHQ!zF z@)6!SQ^w5NusQebZ8ETbWWV2f+LhAFLRORfn{_Ia2NYXxwqe&n@<$#de>jO(9*^B` z(_ZL#-blc`TeEi_+)TZJ@FEg{R~g(KnNOd(pH!hwxz9)l)@xq+vRZ{0UoKRK`mXfJ z6C_6gSR8WxBaYYcA1WsKed$HMm)tYH0;3%nlP2t+i{O#I*W%e`I+|F^yO za((~jojj$Gm`UVjk$$ex>2~mdR#)5>8Zcx1 zAz}e*dSj({&_|9mY?JwTjWc|pckYx6w75cNuGrP>-D}hA6*JR#UL@cetk=xheS!7r zE~Kp#TMnX@2y4T35m} z_nM7pTXf^IM!l^s@LD6?)=0NDSBy1+aUxD8B!nS<1dbW8+@X!Tx!kO2YvtT%a2DaA z-eC06UrC_j{aPhBSF9Wgsw! zLN*-(T>u4^l1=_`kcpBiydq#&t7=w&6>F-0XL$r!6ZMti$~Z(J@v&&#hsg&E;=x)D z&oU4(eLx_+@)2bk3tcn%RH92jW6ZB0Kv-zIIXwd~*+vIvhvra0oPp#tO+wL=m%ZHr zZ7)xogdu_(A`u=);0SXT(*V8dzEm4bMgyU0AyO0bv0`LEXi%tEFeFDrnC&nUBNm5$ zfns;4mlclW2vhYxipZGIGsSy8x`&Wx1rR)s`4IZ3m;F+J44OTE1rpdabXBC_azMS- zVzPqFdrft%0A4qP{Da8QS7mKc?3^{LYSCnV7z+vf&H!aOdu2tavZgBXFg`nyE^OX6 z4q2OTo;Jpd>#yJ~E;*|RBF|q}f8MWu=MFdDHBYSr(>snJQwTT!?rB{>7_Cd#Un*7# z!n#blPHP8Gvt?qxY&dw?+9ceZy+`@h}Z z_B#Lfojj%GMvZsZ*CuCf>vv!l?lTrbVeh9)<5-RuC%@~3G80z|+hYq&fM4nY zLy(S?XU=@>2jINYDfpsM!xO8Mm(9oDFi-RlLdHRkJPOD?_bi zgIl%#y94gW-GO^mNTfNeT4ubrD{_Po|^QzS){2EKuu(%$%t3~LZ_0?Ugs};)oBhXW&-nbtf zrG}+;$kNV9PSV}6qH3_mGMWCZrdL)V%ky>4_p$&lBrrVi!)I@F2?)T>fcQIuSF{Ec0ZgB;= z$4Jnfi+rh9*E66=*zU}6KEwXE{e(?FEqD@ShW)?Yd$m)t|M#}HU#{){yLigy|4Z?t z%-Anw26Ile<{SXcp{n^$|1WZkf1V3F+FjOBIIi3DnIZ3ek)U`FSQ8hsIF>+h8;(u6 znN{b>IntLr@JuG_XOU+a{oghcI8*=cRnGtI?XB+SFD>Hqe6z4Cv1J3IS7Z1279?dwoh6f3@9P&;R>)$QTcyzk$#d7H|#`>!Y#HyJO;WCfGoBeKy`e*{oGX z5#kUrK_uhTKH9qev)J+l*(!*VE++9XRU_)}C=SCj77~BjM<;_*CeJtspz5VB$Qfr7 zqSOFzM=eZ^=PgbTaMbFd3yoi$tmXNe(y0xKyKa0cg!f?|^ zzx=xK40*^bRO6}bx#>HYnj)uo3>&J5hYn*Q*%*cR3PRlu90Z6^$)?g}J|R9F_&$rNRDUDpbHEb%Nb(raGxRsrP3qs9$d*Oe z6c*@)grVwB^GSdPjH4-wyXXR>VsU^&0f-FLkLU(d{bMXZ{ml`(p|YzbSGdi-SWgu9-|(f}Vb9R9aA9T141~K2npx6H_5UE9fv} zH^<{hP7kwhQm#2fA@Q;3qaD?Cl4A+O>E$$nK00T5&V7smELGVN6Y><~4dd6T)X5PM zNPv`Ta{4R|1J!2%5%?;EK#5e=NE9#-h<`GP0*HVHh*9*@#Yyt{22wMG6JMM$F8gTb zKliuy+!}HOIFzHTnBtws^aK3~3#rCi3v}|M6@;@8Q^;OnDX~8~hkwR|Ly-L*vf+D} zK-fpG)L;2YiYa9hD|TRs1)KLSj>G8hI!Y5rdl~BQY3#Nq_L=?47_EO!_XLfGbAO@g!@}a; zh69KBu?QCq>qdW+*%x&c6_@&1!dhDTT#H|I`Kws~D@$N&CAhnU65JR7mNAD+px%Ns z;(j5qvg++@FfYv;E!}T9i=*`QK!G8e9p>#5C-)*A9 zFjlL|kA#cTU#hSFYD~~gbjm2$WiABSM_Uujw?cBYWq)xkfq~kyu|5jmieTzw!@Sx= z0|*$$O1&w0I0HT=LKuney?*^V?UiH;t)$&3yQG)yz;JA3#jMI~sdW?FU6nH}rR0mf zr7CjRInL$YVInzl*toXdTRx~^sAybAza2d^pc6RaYJapZiGt} z!~)ceJmXB+4yoT?w9pma4WTZ5kXZHPxqoX`Kl+LIY9V3WTV^88 zu5!1j%GMKc!9B`e?L^E>^^%p&#G*TZ^)$BAnDOh7;UKl{^?rf?l_y7bDm_ELFdfdr z;jidqfaD0!!NC#2k+vMw8|wz+9u|TODFkSYDIP+Ah^32&@$nFnQGsekEjikuvw8zM)#*nG9xDo&1+U|CwmkkQh7$U&FH;g2K)vmN>gN9j(B zL*Qw*-9v|69?j#7|4A4}I&8Cb`r-2UqAPFZGIh*~|L^W^m*Ri>ulKgs@xS|cF4g4E z1@$jZkXeS*1Rq+DZvauqrVt>>Osv4p?e50LB}4uQ{D12jA`uG&5)#W;Y@#RxECBJC z_BWC>UWUl?XyV^^^W_Wbo`WveO}!rI>sQ5!A?A6)D&IF997UxCL{X{`L0`Xikmubf z-MpUp#b?I+|2%>)0xr5TS{w>6WB%{#?d_E2|LgtTz4iRRkLSymt^Yt1GVbf;SqXx& zPTq`Fw}1E}=%fGG`ucTaLl--~jW7-Ls!eS$O)vBsEvCzT`Qo7f2806Yq>|NDg*(X0 z-fRCr*PzVw1nvH85GShShY4^_0zkhj+n)Mpe^>vLjE&m_b-aui&_MrNGq`c6V|9ih z!=mEXf{T=0q9e^$(~NDiZ|H6`aIec{HAb9JIe$Q%KML=UqEl>z`L_j7q2IMHPU)}C zpE{`XxwA-%*m%s8>10%%Xyi00Wp+A-q+3Py(U&hMndQ}?>LTZtgbInNZ@w;T*m~y8 z)~u29FvqbfL8u?bL1W`;c=XvgVpRLwa&h%)xd?OVvB6>;Va-zSYFg}yB($8gVvvZ$ zL4OLXa}!MQZ;D#wli$%lVhUTjVF0|KA1A%;pKbgMN5sQX#MuObqB`xPPJ$R45;=;mx;`7119hqeoZ>JaFMrf{ zv9106SDgg-Lm0gzV-OOLqduZ>7^Z=riH;;CU+20`7}&Gzm<hI0%UGCY>qM2>79*k$*Y+3i$C88W3M;7A17rab&Af+f5EQnC3Bok|7SN zp8#RYTGu+k{vjAW!*bL|U9-LH>aQ=^N0Io}%rKpof0jBBdc{}>m^yhJAdD#7WV}&; zKoo&un+6kQmpG9(ic>e8Zl9h7=28-oxeaadR&7gqx!8SBn|67+z z=&GkEq=$?uZV|is2l!H1$0puQ=&YH{eY&tfl>e{8kp?^6#@azbp++X(U1{MB>ZOwnW83 zb-h2SlB#xk{%uQ^PwUw&3XWvyG%A-&JKPpk3W?geOVr+#bE>TD1XRR4^Zx7aF@FaXF=8eHmI$yv zLQIZ;chO%gFF$>}xK!%L@Ct$2hca#eym(krJ4D#_*Gt+Ji)rBGvVJV*nmXySX^`D& zsB{}WLOn^wP*i^sUjt3VFs1?UaGF;ncCkbT`BdMm{GwfG#9{S?X+a8v@$%#nTcumE z0t!?QhBd^jiC~~E!havZ827Wt7CksS`FZ!E@G~lQUrDBjKv>5{o~P4RZ?S+VF|oFa zlAdBG=ORXFcQd8nv$ex2u}#EyD)^GAA{@M0jf_ylSp-~?Vq7X`CKjSqg|C6pN+>cC30$z zh~z4kjEk)RCJ=53GW0O_M?``zV-8!&w$-dtZ6bHa!Lu|t{k*_yWg41C8;}hnplnDZ zROVCDit3Vlet&#%na0KSV3d2mw3dr(2vmm=Isk44QJX({nWupkQX3^CnNCS%HNF}X znM%6QYWq;{8MJMv(Q82$olta$#}FQ30e3zCs;7mgI=0;qBzAw@5UBSB@ee>~E5aXP zj(rJS%{^b337byebkQ#L+11o)2xmt)PR{s@`VdL6Wq)<+mQe)hen>oX?IE%!!PIw) zz2Vjix1kzAn0uv}Lbcr6%jh^A4hUgXOz(sjCnVvyOHZDqo zJGOBy!9n%H<1SvK0FtM6o6e}W*<}su5O6z*0P4K z88PFvD)X|x<_LnX)x<+L=T)EUp2m^^I3DqPf)|F)TaApBD zk;URJHcD9BRRlbTiRc;^OG)pg z3V+MYYuB{E#d0>$ec&_P)O{yqW3ng@+ zF`gX8X+>k1?Q(;8Y6|4+dSk7}oU64$8JB#VI!v%*e`uaVMwNY_b_21no|b94&%6S8 zqXwg`1I4-b5UO8Atj{$K;t;*KnyTFl39vPLu$<3LQ>w-6RSIsIm*@%xO0Z&Ks(+|B zTkC4ZqYpo;TW%MvD95{*5S8;xRSjw>ucCzH_HU>fPs(;eg7 zdMJ4@&I;$;Ga`Ns5Gi5hS{G==qO7?ZqRjVWE|Os=PU_U1^f6-T!er-^8NIq~X=<>C zfERE>jX{X?)O7hNnhmyS!S5S@#3W23uvF(n0iMWQ%{IDD5|!Ac^MBH=vI&RvyE^rR zY&cX?SuIv1^*T{IW|&YUfC?hX1oGCWV+lgDJu=d{1XVgrHZMPBSZ}b%cbnU(Fk5%b zE&ID@##Bg60|^CD z4p}sjzAIAHLshUFCre4{Q!?fK>aPw(dJCo?n3_?;4~M9-3(Nt@2p-1kP0re3IIhPo8ftnIdLK-0X8 zu#~`QAN`*fpZ@f__wv79y!fT({pYWLdhw~N|M4F$|Lf)N$sd1u`SQh!U*7%n_VVob zSMu`rUuZnOHh+Kp{^A!n{3QWI#5^%O|j zmglor9bxouhE5-KYL0wtC1k2t&y4H;MmZX`tq}3&~so_;Pue<z9B~#8i?CXk7c_G_2M~fx zZd)c&SkH_LfuIXI;3)bHVbt9BRcx>8m4Bn%k2lel?XNVD6Jd_WYGCZ@fE}KXylYka zK$O)hOc&>H5|t3nK~j%D-Az4jlR^9wb9L_-)@}w~bL>bL)Ul)#@o`vp@R!kgx2NXq8Y0;QP z*vaDtHvo;t!$W{WG-iui$i0QR56+pOcjV+;jY=mhLYPC^GIp>hNIuOS? zJKkf3J7o)NaUk*rSDiDm)^$*X*neb8w;sxCn0jU$$2cT@JkH(58t0w#>0R$A?n8&qvujw@KH*Mw2CLW$Jbwx6|54lf z>mv-`h)w^^Tk-8|ZlND=AM)23Nkh z^a!Oh@;)0!YE>-+yEC>a(!ux!w&pttuexw(Rmtpqwrgpvu{=`cl(#?{QK0R;!1L_J z_0jtkY{_en+#gFAR!r9R zPzn2GQdFQ0qh3J7b;lL9Nsl@-&(BBTg(wFO)-lg@*ya&#@n)P-FMk|T&nA<>lQ#N< zN}V*15a&J}IG26w}Vr|NvMpsT7L&`yOCC)JyuPbtw&D#wv9B2-bD{`qh zD7SaA{m6I%AJ!?8)FzY|o$sKV`WvdCT^vg`#?q2;A&d_%P8vu`;7E`Fe4NJ}7p>xs zt=BqzEGZh{w#XFJoE4u6JgrZHl?lJ-yOl7GZP zW*O3O&P!M;rzNxR?z5C6?fhh)*YTN?&@yy-gb74LddF}neinr_mYWgFK?WTCl$@p3*1sANn^W!I z|N5UFNeE^Z{QF=3tF+F)|MfqA37LQW?|=Po>xA4zG>BD>hPmg$aGX2aQXF9y#dOR| z!hk8lVKl-vhs8W7hCN{c0 z1)RjcwQ&}Z!2ond4FNh`o-xVI%7R@aMeL6@^M_ln)C#SM_@EOdNUwS-c=2&L2gj6b z$c1zcgOdJoy2D(C!U7Jma=xC62>kA>bf5Xh2=dsh;kFl3%#)MQbBlbjC&692G7=CD zzKLq&6Mr3EfypA_JjXYmsk>>I(226z`5)`8J}upWO-PetcZ8z|+FK_M%8S}7by~OS=pllPCfz35Rra zjqWlYtXw(*tnpS_h|vFg@!@pqEvq4E#tZ8-0Dt<(9gKm>O|3snMkAd$@It-mjxi+z z5VC9Q04{#n`L)g*P{IqhcpQbWnFQie6D*lSM40YKDQbo1*ihLXKoLulhTrJss^wAH ziP(DZLUIj#)X@&`jLk1f`oFp^{7}yelO?dD3Uv%~se3fPkYr36=K&|fA-L~;Kz5s? z^M5TF2oQ?U*e(5vrG7+8HUPRN`Azldr6e zx}MdE-t=sqC__LQClH2SviHBiQ`HN}z}a-Jn9D|KO;h}RIQq~xoGF1C(j>>#fGFK( z5`HzC7Rb0olNgCV!gQGC8yv(k=CD|bhJUK3IC&f%RHT=hiRbxZBmu`(}j94CV#D_ z@5hs)`w{S%EHqbZx4d;p{Ay5Q$b}R`b!q>+s#)#`?L5$x!1c!8cYZ(U;l)Ybk?&%+ z;@V%|5jt1ywA3+D^ZZP%RDYV6%J)_q zFv0rPwx0T(k9La~JkeZ!W`XRy@BcmQU(~_&J^7>dBetlEu2QevXgS z;%?%iYMe|2&5pSw?i*jJGT4S3gqS1fwZ1TPeIjP=6EQ113G=O=fLWh>x!02~>k}`J z`NYdQGs_)iX0al(RBjeSV}E7YS>`;I^bMUwT4zKt&pK_yw|%DQTRuhfKqlupeNM}h zJxibCS;48Dm7LL;<3vuQ9ruI};4FRIX8FT4kNhaj>JH94-D5IK9f(=x=*zcx$Ysgn zERXr-+C|SXtoqc#(q|N&%87(~IDfGGX@gasEm-y>!K%&;tWOD)Pk#w4bsk_P$p7Uq z{>!5Im&Wa{1fjnS7JnJk{WAFaHIjbr#;uX`Yb5;|Grz{nuQBtF@BZ)EF0$|T4)9`7 zaT(P5(irt+5$WG3{`|?I&DXbYKTQPro!-4YCpvt6|Mp|Le|seu?qw0%E5T>4@6rBd z@6rB75z!fLSA%@YfW`47FN>nbiXpi>i%pn zdpV)d&h@WfH}c*{UGyA6zyi9N-%*ze@fC!2&f_SWO%@?Fxe4{dH~`d1C9`WNbx;@m zjwqpl+JEk;Z&Z1`bUWypk4!23rDT7{;}HFh$`q73nL#XfhmGO2aH&np3KA$BhbSaI z7Ii(6yf-xo2!Fws$#!>)r9XO~>)F~)OTK&W=)C=;!&{DwkQ_@GT1cE(GTh%JB`>Fu z{WeiImj9fvsV6KnWiAQ$bJmE~0+v=WNK4)@ejPF#oG7A1PP+zud2&=BuQ*AIc8M*! z6$|L2j#L|f0{s>u6|vxbUPP_*bTj$+^Be*ivEZOAqko$5lAO-21(iUXBPQf2NY(bb z9?liXYF2gx@!`PNJMh!8Vky4PQOhV$7QjV8`nCB`a9uOeM3zX5%BkSG&qH&eY7!8|X%FoX1p>F&vr02aUrf zSqQu=(YYU6N_3-QALu*Frnwnof0<*tRECX=@et19FuVYtgUoSXep9HW?(1UZ6)j_a z$YR`iTi-0`%FxSzdwj#PJHp~jDcD=o+3FOl?|=G&oMm6AHHir#8K35-hHXzDb#gP? ze1C-noI}LW*RPd#O|<*7N=rOMU%z(D7=!{^^0T@+i`oGp%!i^Jw(5A3H#?omkJ=2! zL7;YaZ~pM&;OOZ5_~PQFqhJU)0Pbm>MHro3BD;3?O`T2VAfjd>i{+c`onB8k-ovfr z*ncyqF)vxW-xSxow6M}W9*&6j&q%y^)oUto6>}lqZ1-G+27qPE!5gYQ%A3D3%J|vO z=irB!j7=D@!T``oc?9g4F4Mzh*qIvYK2rN1^?zN#{Ohs|srhHm0iapp5v65|k}FCcj*NPqq+$S+=5cNEM>Td>4#&V<_DDKcEE)YaqT z2w#zqI8RP+7L|YV(82rnPF0l8gw?2u(uir=?HAdntI|nxRMKz#+1y&{?zw2Ex7uo` z`%zFOExOuy-ZVWMCZKxiXYXRuGdE}tKRr1J@>^>w;oOv7xrN}Q>s+aml8ajgOMlk2 z+L`6F9PVSfJ@?Gb+@6^j9-*-SPX|-qXxH!a$_F|7K~&!sSMqc4*#x+O9v|OESYImq zBPLeXv@M(LlO7k&A^F z)Eld+v5od9?i}i*+?3R#k$bg`{Pm-HgW<(ma5m%Gvsp1j$aOBEEWdFs8g${Gk$7V*??sYI$S!cdet zr?bQsDZPt0lgwvfA6*`vC2EPnVk==QR+|=!-%{#avt{x|Ht#=&xhy7N5`j^CkO98|9JBL*tLiGkuyg& z1vB-2!qGdJI*-XZLic*vIpm;Yl{oy7gm9RUsrH_Fu3_5OLg0AEB6Bn~msG0U#{5F# z-k!yj-1fJ&>h0I+1b?+EOCMZ0@0KfwuE~~*!93!lb$f{Ecwb67$xXt`p{K6@5Grn+$mReG)r_`g_0&Al1M>^R6}Cm@rbzgL7&e;SEr}Ne z<&Fc)6HHCMPf%`HO6-r$)61o6Hn@dSrUelTPUwJrpnuwoNf$q}TD&I%@TY#Luar)c ze4nYY5a|`L9y*RjFb1xHRRi&svp8~CRW_8;EtIeTOLh(P+~$Z(s=;wUfck3nh)92H z>p=D1!O^8Edwgp$NPJJ4@cVq`(*GJnctdEwZiJ`S+lVtCgqX1+H0%HEy}kYIlKy}7 zdUt!R|9|h}v9=thK?Zj=1Kb6QuR^H)9t-%ZvbD_EAEux}rEBbQ0t{nDFJOVw1W6@< zI9=gk0vFX4oFz!eE`N|9p|sLd1Sb9OMFE+wmMTJrAN8xmR378oj}%WZ3Dt-fdQMIG zZt;E6eHD^1 z>KsAx(stE{D7UZ3xe&~!6a0=Vh2&gmsO_;~r(59cSBfB*F9;_nxi$3G>J ztbf9Rm>eB7Du)dqeD(fY42)+sVRU~n@@l1J7vgLmbe-qf=D4ZE+9P%YT#O(LyKw6O zxcil#PT|HokuXO3fAyl6I@XYPhUEwy_CI}^VE*Y-NUlD8QYtG1`RP-n^FWJFpK7<) zpFSb)N5=gEBX{QDY&t!nHHTr&<1i%32xDl44pkqT4gGoY*=l8aW(8G2bA7TP0OT%e zFH3oytg5Y>b6m=zTZ1Yof>CEdM6B4RwP=88gyrbXA6^`tpZt7${`vg)?dOXRhwnb0 z9bEp^MZ-wElf>Ekk8e*-HHo8>bCcOBF@N31zZ`!){cv>r`SjqYIU?CQ{Yp&;J4?N* zEmJ*Pa#wVrR$AXR#p+gq<}g$u7+DO;L3@q3duoP7oytuqnHkbj(H7u)dJnxa;GUvJ zf;WG7QJjl)3cL#6%4@y{LgeWlxeF{_J>e&Z!qZJ{gTG5f3}1!d4M8R+J%q$kynk_o z8%)u+AZ@~URj!d}-hb>_2Xftc(H6i{?(MJRfA{gUi2uEyOtuaJ zzFbEE*HOTA6mT5{Tt@+)P!w>rA%D0>5L0XlW|R)CXH9B(9^tV6sqRFA^>7EIRd8f| z7Jk9A@N-8m$_L^dMQYBT8-HE>X|(%@R2=rTuixqLU)5Rmd3S1$=*;;#IQekF$ww1g z)RV59*R`tbu&)Ub7;d&2BeRYV7U%&dl`#w8jhj?yrGKkA1-svT z3U;^W7J|FgaE~dd`XoZ0v(uI&GSry2q~q-5^3<%@MYZ%CW>xZeL6&C`$roN?fm;W9 zzq`~b>)7l%Hv2?lvxZSZLqgN!e%CPd%vI9ne$OPOy_@+H{jI6!9ZV1Ohkqk-lw{3! zSG;_Lcg~bCvo>tbeS4b>EPvVWcb;~o^s!fe%H$En)|+kEb&&jt2gx5#;+3ak z_uI4=dY(5DaPQXaod>s5Zy>ygMBr5hH%I2vr|u_J=u_@95`y)bm%gl4A;wn=)uFyC zeew*+5eU%AOCZc4y%bKzpaL{tT;JP>CDHo|eH|@KpL265Xo(o3;eRx_l2C_hMmV`< zPW^NR5~GQ|MBUyrczAJwG<&W4RZ&@@bE}^>_VE0*nQktB_l@9F%)e_h(9DqYro<_& zg}$x8&skYX`d23TE0X$oBz~E0F4CGy^cU)IcYO_RmZn--feN$31V!t#n2M;vl6ys3 zoSV5ScdlMb_e6v9v44rXN|vS>i)0g@l@+tx6Hgcr_nkd+^nu4W%F>2EBoGXWfuIU6^}RMERO1tWj|-S zQRf`LN5GolF9JR$La2RVRep``%t|~tH#a!T0LfM z#^qkK5p9cZeAcM9^#xvQq}v+l*5-<_Mlepq$%KS3gnz&>BbGa~aX*)vHEpe&8x77P zJk%SEKKcs@bi7}y1Sbpn8*P-{-R)>aLIF&i9|tfYK2*3^U#-@x;bvM04MV|Wx@Am` z)WN+aMwmmeW#q;hQ{}F)xE=LnL%9X6ge&iKXJPRiaJiP)o%!7qMcZQraHnC0FV5mZ z9MnwkrGGmr1$!;VR;|yZnX#qrylYTXGiojCtU6R#gD!jW7w(lB+Fmbtt@KGFr>zy) z{Ffx=qVA6sE!P77m0XqP<*quL3j{Xf{=ZlId%NZQZ|e0L|8);fX~w13;w8!7ybJ^e zQOKrapbMbDQnJZE4l_|wg;xX&YgNq(uwqU1EPsz6YofkVTp5QbBt90c`!M-nK|EN? z;aLVErVj|DS3aUlW1$;HpG$NFXpH$a1PBXlH>YO+Cfn%n{KydM-dqldaiiSNB0mCtpI`-F&{!7^|D_IkU_KOuRsEuhOUYfTn?z$ zT1-}ud9SIi6~OCekbe{z`l_rgik-7&RV|v#4`U&L-y5JTXRoXXRn}BR9>!-!(uK|Y z#vyC-&C|wsas3s%#U*DILFD=C>d*W2+<)Qb`{t>2V0y<9WC{TXz&))C2%~ig`%A@2 zL0Feb*J(>`oA}7yQBVLI!pa4N`49;1PPkJ0yX;~CgaxCo3lZ#x{5dXyK1&2d}XM$ zY;emq!33yWpbKql(#~9{S-v33Zhu{AgO(zgEnG8sx58&XXsmIS6)BxQYs4J&`Hb;^ z(EQAI)uk8xI@+leB;9CdPeB^WlkC?G&+|5_+UYNZ280gpLP^;p@~M8hvQ;goqC|Z4 z5AdbcQl>o@h5eZ(TeZE5dpN(ccWdKJQ!3Wt0IGU^8Ks>FNkivv)QsSru759BO6-rm zp~a6?+P8$nGF~MU-<#r0C*E3X)~KS# zsxlJ`LEgXCbQ_V&S&vRL*!ad=(`=-)1YaOtsYO)n)ms(Xwl8TG6KY zIk{G-^@^&sPQ721%0ZCdzJEOWO=^M`Cb;_g+!)}bbtAhgeD(47V}gY4V-%sDIX&e0H^aSzR1r z(PmvygJz|>`8dJKbFbB!hT{d?a4vY@S#OZQa$&V(D_T?>!a-R(HqcJg}el*Fz5 z+{sXnYYXkEK5gy4Wex$&u>ba6_g z9j-tR7zw&}kuUY?dImHJ+r2r?XW0L?pRnnt1y6#^u>ZGvuXjrJ|Ni#w+W&hmPx<_R zDV~%W`=!ia&Z*X%1E4unHUH`VMUL^$b74oj%Q_0jb(=mj6R;$jxZ5-4uN zu_-sR>O46|`jSVU$z=U3@+_nO+eQLs>i_-9`M>?w>-)d%<#{mupSi-Hz>XzB*a} diff --git a/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml b/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml index 107b421bd9..fc78df006c 100644 --- a/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml +++ b/charts/latest/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml @@ -27,7 +27,7 @@ roleRef: name: csi-{{ .Values.rbac.name }}-node-secret-role apiGroup: rbac.authorization.k8s.io --- -{{- if Values.node.enableKataCCMount -}} +{{- if .Values.node.enableKataCCMount -}} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/example-cc/cc-deployment.yaml b/deploy/example-cc/cc-deployment.yaml deleted file mode 100644 index 8c246691b1..0000000000 --- a/deploy/example-cc/cc-deployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: pvc-azurefile -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - storageClassName: azurefile-csi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: nginx - name: cc-deployment-azurefile -spec: - replicas: 1 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - name: cc-deployment-azurefile - spec: - runtimeClassName: kata-cc - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: cc-deployment-azurefile - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 - command: - - "/bin/bash" - - "-c" - - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata-cc.txt; sleep 1; done - volumeMounts: - - name: cc-azurefile - mountPath: "/mnt/azurefile" - readOnly: false - volumes: - - name: cc-azurefile - persistentVolumeClaim: - claimName: pvc-azurefile - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate diff --git a/deploy/example-cc/cc-nginx-pod-azurefile.yaml b/deploy/example-cc/cc-nginx-pod-azurefile.yaml deleted file mode 100644 index 1a03193afe..0000000000 --- a/deploy/example-cc/cc-nginx-pod-azurefile.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -kind: Pod -apiVersion: v1 -metadata: - name: cc-nginx-azurefile -spec: - runtimeClassName: kata-cc - nodeSelector: - "kubernetes.io/os": linux - containers: - - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 - name: nginx-azurefile - command: - - "/bin/bash" - - "-c" - - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata-cc.txt; sleep 1; done - volumeMounts: - - name: persistent-storage - mountPath: "/mnt/azurefile" - readOnly: false - volumes: - - name: persistent-storage - persistentVolumeClaim: - claimName: pvc-azurefile diff --git a/deploy/example-cc/shared-pvc-across-runtimes.yaml b/deploy/example-cc/shared-pvc-across-runtimes.yaml deleted file mode 100644 index efee0e14f4..0000000000 --- a/deploy/example-cc/shared-pvc-across-runtimes.yaml +++ /dev/null @@ -1,72 +0,0 @@ ---- -kind: Pod -apiVersion: v1 -metadata: - name: nginx-azurefile -spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 - name: nginx-azurefile - command: - - "/bin/bash" - - "-c" - - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/runc.txt; sleep 1; done - volumeMounts: - - name: persistent-storage - mountPath: "/mnt/azurefile" - readOnly: false - volumes: - - name: persistent-storage - persistentVolumeClaim: - claimName: pvc-azurefile ---- -kind: Pod -apiVersion: v1 -metadata: - name: kata-nginx-azurefile -spec: - runtimeClassName: kata - nodeSelector: - "kubernetes.io/os": linux - containers: - - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 - name: nginx-azurefile - command: - - "/bin/bash" - - "-c" - - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata.txt; sleep 1; done - volumeMounts: - - name: persistent-storage - mountPath: "/mnt/azurefile" - readOnly: false - volumes: - - name: persistent-storage - persistentVolumeClaim: - claimName: pvc-azurefile ---- -kind: Pod -apiVersion: v1 -metadata: - name: kata-cc-nginx-azurefile -spec: - runtimeClassName: kata-cc - nodeSelector: - "kubernetes.io/os": linux - containers: - - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 - name: nginx-azurefile - command: - - "/bin/bash" - - "-c" - - set -euo pipefail; while true; do echo $(date) >> /mnt/azurefile/kata-cc.txt; sleep 1; done - volumeMounts: - - name: persistent-storage - mountPath: "/mnt/azurefile" - readOnly: false - volumes: - - name: persistent-storage - persistentVolumeClaim: - claimName: pvc-azurefile ---- diff --git a/deploy/example-cc/cc-statefulset.yaml b/deploy/example/kata-cc/statefulset.yaml similarity index 95% rename from deploy/example-cc/cc-statefulset.yaml rename to deploy/example/kata-cc/statefulset.yaml index 61d2c35b40..ded4bdefcb 100644 --- a/deploy/example-cc/cc-statefulset.yaml +++ b/deploy/example/kata-cc/statefulset.yaml @@ -37,7 +37,7 @@ spec: - metadata: name: persistent-storage spec: - storageClassName: azurefile-csi + storageClassName: azurefile-csi-kata-cc accessModes: ["ReadWriteMany"] resources: requests: diff --git a/deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml b/deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml new file mode 100644 index 0000000000..a66859f0cb --- /dev/null +++ b/deploy/example/kata-cc/storageclass-azurefile-kata-cc.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azurefile-csi-kata-cc +provisioner: file.csi.azure.com +allowVolumeExpansion: true +parameters: + skuName: Premium_LRS # available values: Premium_LRS, Premium_ZRS, Standard_LRS, Standard_GRS, Standard_ZRS, Standard_RAGRS, Standard_RAGZRS + enableKataCCMount: "true" +reclaimPolicy: Delete +volumeBindingMode: Immediate +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - mfsymlinks + - cache=strict # https://linux.die.net/man/8/mount.cifs + - nosharesock # reduce probability of reconnect race + - actimeo=30 # reduce latency for metadata-heavy workload + - nobrl # disable sending byte range lock requests to the server and for applications which have challenges with posix locks diff --git a/deploy/rbac-csi-azurefile-node.yaml b/deploy/rbac-csi-azurefile-node.yaml index a2c47d2882..61f0f0c8a4 100644 --- a/deploy/rbac-csi-azurefile-node.yaml +++ b/deploy/rbac-csi-azurefile-node.yaml @@ -32,9 +32,7 @@ roleRef: kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-{{ .Values.rbac.name }}-node-katacc-role - labels: - {{- include "azurefile.labels" . | nindent 4 }} + name: csi-azurefile-node-katacc-role rules: - apiGroups: [""] resources: ["pods"] @@ -46,15 +44,13 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-{{ .Values.rbac.name }}-node-katacc-binding - labels: - {{- include "azurefile.labels" . | nindent 4 }} + name: csi-azurefile-node-katacc-binding subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.node }} - namespace: {{ .Release.Namespace }} + name: csi-azurefile-node-sa + namespace: kube-system roleRef: kind: ClusterRole - name: csi-{{ .Values.rbac.name }}-node-katacc-role + name: csi-azurefile-node-katacc-role apiGroup: rbac.authorization.k8s.io --- diff --git a/hack/verify-yamllint.sh b/hack/verify-yamllint.sh index fd1d9521ee..e45794b212 100755 --- a/hack/verify-yamllint.sh +++ b/hack/verify-yamllint.sh @@ -21,7 +21,7 @@ fi LOG=/tmp/yamllint.log helmPath=charts/latest/azurefile-csi-driver/templates -for path in "deploy/*.yaml" "deploy/example/*.yaml" "deploy/example/snapshot/*.yaml" "deploy/example/disk/*.yaml" "deploy/example/windows/*.yaml" "deploy/example/metrics/*.yaml" "deploy/example/largeFileShares/*.yaml" "deploy/example/smb-provisioner/*.yaml" "deploy/example/cloning/*.yaml" "deploy/example-cc/*.yaml" +for path in "deploy/*.yaml" "deploy/example/*.yaml" "deploy/example/snapshot/*.yaml" "deploy/example/disk/*.yaml" "deploy/example/windows/*.yaml" "deploy/example/metrics/*.yaml" "deploy/example/largeFileShares/*.yaml" "deploy/example/smb-provisioner/*.yaml" "deploy/example/cloning/*.yaml" "deploy/example/kata-cc/*.yaml" do echo "checking yamllint under path: $path ..." yamllint -f parsable $path | grep -v "line too long" > $LOG diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index ab9d348cf1..9cfdbc2865 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -341,14 +341,17 @@ func TestNodeUnpublishVolume(t *testing.T) { expectedErr: testutil.TestError{ DefaultError: status.Error(codes.Internal, fmt.Sprintf("failed to unmount target %s: fake IsLikelyNotMountPoint: fake error", errorTarget)), }, + setup: func() { + if runtime.GOOS == "windows" { + mockDirectVolume.EXPECT().Remove(errorTarget).Return(nil) + } + }, }, { desc: "[Success] Valid request", req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, setup: func() { - if runtime.GOOS != "windows" { - mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) - } + mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) }, expectedErr: testutil.TestError{}, }, @@ -888,14 +891,17 @@ func TestNodeUnstageVolume(t *testing.T) { expectedErr: testutil.TestError{ DefaultError: status.Error(codes.Internal, fmt.Sprintf("failed to unmount staging target %v: fake IsLikelyNotMountPoint: fake error", errorTarget)), }, + setup: func() { + if runtime.GOOS == "windows" { + mockDirectVolume.EXPECT().Remove(errorTarget).Return(nil) + } + }, }, { desc: "[Success] Valid request", req: csi.NodeUnstageVolumeRequest{StagingTargetPath: targetFile, VolumeId: "vol_1"}, setup: func() { - if runtime.GOOS != "windows" { - mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) - } + mockDirectVolume.EXPECT().Remove(targetFile).Return(nil) }, expectedErr: testutil.TestError{}, }, From 6e32cc608ffaad259bbf6dfa772af564d5c3b831 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Mon, 14 Oct 2024 08:28:30 +0000 Subject: [PATCH 5/6] chore: refine kata-cc mount --- pkg/azurefile/azure.go | 5 +---- pkg/azurefile/controllerserver.go | 3 +-- pkg/azurefile/nodeserver.go | 12 ++---------- pkg/azurefile/utils.go | 2 -- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/pkg/azurefile/azure.go b/pkg/azurefile/azure.go index f48b877264..2f8131a57b 100644 --- a/pkg/azurefile/azure.go +++ b/pkg/azurefile/azure.go @@ -59,10 +59,7 @@ func getRuntimeClassForPod(ctx context.Context, kubeClient clientset.Interface, if err != nil { return "", err } - if pod.Spec.RuntimeClassName != nil { - return *pod.Spec.RuntimeClassName, nil - } - return "", nil + return ptr.Deref(pod.Spec.RuntimeClassName, ""), nil } // getCloudProvider get Azure Cloud Provider diff --git a/pkg/azurefile/controllerserver.go b/pkg/azurefile/controllerserver.go index b921140c89..42646912a2 100644 --- a/pkg/azurefile/controllerserver.go +++ b/pkg/azurefile/controllerserver.go @@ -279,8 +279,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) case tagValueDelimiterField: tagValueDelimiter = v case enableKataCCMountField: - _, err := strconv.ParseBool(v) - if err != nil { + if _, err := strconv.ParseBool(v); err != nil { return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", enableKataCCMountField, v) } default: diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index ab55f1afbd..d1e3a0d424 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -102,18 +102,10 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu if d.enableKataCCMount { enableKataCCMount := getValueInMap(context, enableKataCCMountField) - if enableKataCCMount == "" { - enableKataCCMount = falseValue - } - enableKataCCMountVal, err := strconv.ParseBool(enableKataCCMount) - if err != nil { - return &csi.NodePublishVolumeResponse{}, err - } - if enableKataCCMountVal && context[podNameField] != "" && context[podNamespaceField] != "" { + if strings.EqualFold(enableKataCCMount, trueValue) && context[podNameField] != "" && context[podNamespaceField] != "" { runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.cloud.KubeClient, context[podNameField], context[podNamespaceField]) if err != nil { - klog.Errorf("failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) - return &csi.NodePublishVolumeResponse{}, nil + return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) } klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.cloud.KubeClient, runtimeClass) diff --git a/pkg/azurefile/utils.go b/pkg/azurefile/utils.go index 0c2f077fa7..331e200548 100644 --- a/pkg/azurefile/utils.go +++ b/pkg/azurefile/utils.go @@ -330,13 +330,11 @@ func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interf return false, nil } if kubeClient == nil { - klog.Warningf("kubeClient is nil") return false, fmt.Errorf("kubeClient is nil") } runtimeClassClient := kubeClient.NodeV1().RuntimeClasses() runtimeClass, err := runtimeClassClient.Get(ctx, runtimeClassName, metav1.GetOptions{}) if err != nil { - klog.Warningf("Failed to get runtimeClass %s: %v", runtimeClassName, err) return false, err } klog.Infof("runtimeClass %s handler: %s", runtimeClassName, runtimeClass.Handler) From bb47dce21029f1c62807fb507fdca10d3392552e Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Mon, 14 Oct 2024 08:28:30 +0000 Subject: [PATCH 6/6] chore: refine kata-cc mount fix --- deploy/csi-azurefile-controller.yaml | 6 ------ pkg/azurefile/azure.go | 5 +---- pkg/azurefile/controllerserver.go | 3 +-- pkg/azurefile/nodeserver.go | 12 ++---------- pkg/azurefile/utils.go | 2 -- 5 files changed, 4 insertions(+), 24 deletions(-) diff --git a/deploy/csi-azurefile-controller.yaml b/deploy/csi-azurefile-controller.yaml index b4bcea5f67..eaf570e9d0 100644 --- a/deploy/csi-azurefile-controller.yaml +++ b/deploy/csi-azurefile-controller.yaml @@ -161,8 +161,6 @@ spec: - name: CSI_ENDPOINT value: unix:///csi/csi.sock volumeMounts: - - mountPath: /run/kata-containers/shared/direct-volumes - name: kata-direct-volumes - mountPath: /csi name: socket-dir - mountPath: /root/.azcopy @@ -188,7 +186,3 @@ spec: hostPath: path: /etc/kubernetes/ type: DirectoryOrCreate - - name: kata-direct-volumes - hostPath: - path: /run/kata-containers/shared/direct-volumes/ - type: DirectoryOrCreate diff --git a/pkg/azurefile/azure.go b/pkg/azurefile/azure.go index f48b877264..2f8131a57b 100644 --- a/pkg/azurefile/azure.go +++ b/pkg/azurefile/azure.go @@ -59,10 +59,7 @@ func getRuntimeClassForPod(ctx context.Context, kubeClient clientset.Interface, if err != nil { return "", err } - if pod.Spec.RuntimeClassName != nil { - return *pod.Spec.RuntimeClassName, nil - } - return "", nil + return ptr.Deref(pod.Spec.RuntimeClassName, ""), nil } // getCloudProvider get Azure Cloud Provider diff --git a/pkg/azurefile/controllerserver.go b/pkg/azurefile/controllerserver.go index b921140c89..42646912a2 100644 --- a/pkg/azurefile/controllerserver.go +++ b/pkg/azurefile/controllerserver.go @@ -279,8 +279,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) case tagValueDelimiterField: tagValueDelimiter = v case enableKataCCMountField: - _, err := strconv.ParseBool(v) - if err != nil { + if _, err := strconv.ParseBool(v); err != nil { return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", enableKataCCMountField, v) } default: diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index ab55f1afbd..d1e3a0d424 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -102,18 +102,10 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu if d.enableKataCCMount { enableKataCCMount := getValueInMap(context, enableKataCCMountField) - if enableKataCCMount == "" { - enableKataCCMount = falseValue - } - enableKataCCMountVal, err := strconv.ParseBool(enableKataCCMount) - if err != nil { - return &csi.NodePublishVolumeResponse{}, err - } - if enableKataCCMountVal && context[podNameField] != "" && context[podNamespaceField] != "" { + if strings.EqualFold(enableKataCCMount, trueValue) && context[podNameField] != "" && context[podNamespaceField] != "" { runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.cloud.KubeClient, context[podNameField], context[podNamespaceField]) if err != nil { - klog.Errorf("failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) - return &csi.NodePublishVolumeResponse{}, nil + return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err) } klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass) isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.cloud.KubeClient, runtimeClass) diff --git a/pkg/azurefile/utils.go b/pkg/azurefile/utils.go index 0c2f077fa7..331e200548 100644 --- a/pkg/azurefile/utils.go +++ b/pkg/azurefile/utils.go @@ -330,13 +330,11 @@ func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interf return false, nil } if kubeClient == nil { - klog.Warningf("kubeClient is nil") return false, fmt.Errorf("kubeClient is nil") } runtimeClassClient := kubeClient.NodeV1().RuntimeClasses() runtimeClass, err := runtimeClassClient.Get(ctx, runtimeClassName, metav1.GetOptions{}) if err != nil { - klog.Warningf("Failed to get runtimeClass %s: %v", runtimeClassName, err) return false, err } klog.Infof("runtimeClass %s handler: %s", runtimeClassName, runtimeClass.Handler)