From c76483a30f63dda6cfd0bed4b317aa01eb7a01e9 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Fri, 29 Oct 2021 17:48:50 -0500 Subject: [PATCH 1/8] Added md for Log Ingestion Demo Guide Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- ...ngestionFluentBit_DataPrepper_OpenSearch.jpg | Bin 0 -> 27674 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/images/LogIngestionFluentBit_DataPrepper_OpenSearch.jpg diff --git a/docs/images/LogIngestionFluentBit_DataPrepper_OpenSearch.jpg b/docs/images/LogIngestionFluentBit_DataPrepper_OpenSearch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bf49c8798c6f6197898231ca79848a9900666b20 GIT binary patch literal 27674 zcmeFZ2T)Yawl6+_q9CATNg_E3NY0>Sh9m<+4oaRONzULWIi~?6ry0@^lrUsa$vMq{ zC9oFjI-K)FzTI<*Q=G)C8;1N(!MG=67 z1pr{({s1>K0672-_MhR;E6#1WgL~)Ca32@k>!q__8v{$zsnCne6E`?rLT9^gH=P5nO%H=hB-_W_qUzi_Y^0ocS? zIK)^t9e^i)J}Nd2)@=a%HQc+4d*?nL77jN4ZNAAP0N^eT7B(*7Jv>~TyLa(!+l7sD z=PvF&;`=0|lni>wV-%11L|$s?>U+XcP~&8bZ#-&8F?i(s!p?srq}DJAh{}S{<}U8c z;KaO7RIG|2`D}9X+B%l+gTCCdr~RL`|If++0GvBm*thKO-{uMv-}WIk_Pu*JcW%4x zFAw-<<-~VMNXfK7c?^6n-z0om!sT~n6!_!I=)pa6awcZkn`r>SZCPw$9AbbB;8^YN zod5Sb_c%McO5-ou`BN$GtSrFwWLiY?>flNoVKP;s5Kt|~PIoIh$}2oPeAq-exbl9$ zE8qM&pYh%3PYQ6>4Bf==ax`SOJXzk^S_PO1Ov%^w?J`WWg)j-MjlZBfzzS{LOD9iu z)NyRe&Y#6M;SNzwyts^b>2*H}hPKHgQH2fkL}~NWJuAD`W~qByB7FT!6)h;1$7v-$ z13Q$#Lpkz3tqGE3# zh>q)0Xnps;H}e09l>6&rm+~MFpPGCOU15M*879k!tcUeGZbr(w@&FS`P=B!e!mHa>B!G%X#~V8kdW-T!)0x6HC=9mw& zL*dNf(Wf;}$>{r-j}4k}=*ad+SNCS0kIHy9e15A*<>XJxA#i#0#5zu!an3(=djHKF za;yG2&7i?h&3dz@@5@y&x}1=f3CN_Xkr=@#o46c~R><>uSb1F8i|}P#)b`ku&ufs= z4r@G`$d*TqE;2db2|nv@sJKu*nv|WCCDL2%cPZ7QJL-p+t&fM`X}VmE`DMBW{kkmN zKcM^dXh2HtOOsH0Tv)yt+S0fh4JxFaauDFO>tke2%N<^)NGGA0G!f#T7XZbs|9>nL z;@@qT2)GlghMX!^)fBAez^uICTGVMqx$>Ep_N?FXO1N3NGe8k{5asfTQbLAtUtdc+ z#s*JkRdr`*aGnTf9mv&6Sj7ovf_EGw7T=Y=OZF(T|Fk|H*%DLa_h?sR%3`pH+?Ph< z>?vn1i%H4MGw+(^N(;!s9SLoe{!d7!Y1eq`ef(lPJ*Y?j69-Y`w`tEWMCjcV$!TZ3 zPc{4Ru}lA7sVq+H2EZW&ajB}UeZjIdC;bkj7Jg1r2Nzc~^m^zHqu*zkmHzF)1*3{y zmuK;aS8c^}yBiL2;3=(NLW}WqCcg0ZhzgF;0FVH7WKm?s^=Dn`gnqkVg%cD`z8JTa_5mano^3woqP4z+hq;SA|dWq1n=>;Ua6FDn;k$R+*@ z;=33cRY4ZZWMa1U?5&aVhp7t8m!Wb^^hqA%?Xp<`AUhpsFvXgXX5w&Pe*GwQyc5Ex zFrQIoT%+ydJJ0BUn^Lw^RkXn5@LBv|MJt75v$`q&unZQ|HWb;x!k{9pz}zzs9T(s8 z*=dnAQ^TAGx^o>Ag(hO-qZIN8eA&4LK)g$L77K4cR$mbDz7oysFvd*%x<^@zoq`SN zSZ7?X7~T+Vd9mzD?5byTL~ELhua8$m`XMU4NhWvs7$;&tVbCOJ#M$Q^$Ls%Q|KFYS z{}aiBLYdI{!!GGX>8@K{_sDj6$K z`XN;N`{^USDe19UkL=S`UEP%`uIB@rs1xLh`WqG*?XQN7s^{L=*8s(JbZr`Epek%S zdwCzWLGR=(p-n!L|CYG|dza98tv2Q^qoy;l4I z3&+?fP$eO+o1%b`d+vUCINFWygH!+VruvkFWn}H=<1a6giJe|nj&1%Z{OTwtv8hR@ zVVDoJ0?LKXgfk=EY7D3ns6xK|ZS3wK2~RtR=HnSIkuS-$INSYyxYoh%2W*Wf==@t5 z2vyjzH1C9}=We)#C$-|6Uz$>F3Fz44IksR`)>0~L#j6&s{ zAW*(WQz_#tX*i!auLwUUUr1pF;`^)1*(H$(RtSreT@l+`cO{;c zvKxS($rO&Qgn1lZK@}2qJdHLu5(_iTp?K#gx+H6MABGSxNz!tV`BoVMaY$F|RpFo5 zc=ocP%E7VHMxQeAkedH8anv+PIeS)LcV$h{)MS@8toT^j8ajYt|Qc^@3w2rG&H781~Iqo5E6{cD~7|MC*vCC1h zrGzK7(GQ+t$BHD@+HYItg937pmwCM9@bV5vYLM#7bV8O!Ma!%Q!%f1&X|6j<8j@mB zPA(-nZ~Z|m+rsKke6)>=rJhiiY`oDZ^6)Lp_jlk9G7#=N72;l2(XH6#QCXuNloMQ` zncwPmnsmfy-@d><+iV2SUVVFOb@1#dCO=)i>sw;rRL%O2{ikT)(>*nN-#7AMPwp5! zS+ITv1a9zsbNtX}Wc{IKZbMmxO}<;I13K#0KC8*bKqyXFUEMkAKgv^T6=4VF@v(Ew z18v*S^t$)3sQcx&ezhi5Od;z>3Xe^dbFR9+jgD^-Pw*e(zvI`UT%XacXNjL&>ujKy z#)*O+_aSR^$2B?jl1-AYB0^O-VdJBQkZw~W$NGFm-IA3QmNthiseswe4~HAMeIfK* zB3wV+S=D%G;d+obS$Ug54%yj&4_Q5X!Uex@fSTTjM7g$4<-SmEyna*eCN=BGpN=(a zV)ZeFoHK4{NBA`3EWbgyb^f@5)04(63wle7C>*MB+Psw}#t?h$EZ0yMA;xU9;>3Ee zQJ*S=QENlqFo!2tVU}TBm=DajT_&T}sXu>^XATVMw?eH^ZAk%Bzp6lbOG{N2>48ri zlC0jQd(Yt!aXi2%T3Y+ZPUQ~=(vR{QNlW2dJqgihDTUPKwa$b?N?peKBiL8lu2+JM zW}4ar8efO6_giOz`i#jiQ9e;fya9mLy zkSX~QP-WC+FNc*001z9mWmX`m^FY-<2H*V}HY%DPtMEw*FRrHiX)l;QV`Zxvp+|H$ z>oC_9=g#)I5=_bPBt~iGak2J~-2yMrCWvJ=p}g3|TFhDAx`@+C>%%HYACbpe+of8C z$0KSz#oTLDti`7Fs-H$?%%%J_GQ{6Rx2%NBb9XC$gIDn-dNUU3t(7G*LVKE~s^&ZN@U~mOs$HHsC#97^Cpy6zIk*?(jHRx#NAM^8C z!itub)Cd_n2UM_d1?e9m zWDX>C+D(%l9UbTaW8_JeZp5qd(joI>>oJbL2LX{lMhnmxRs`8Sd7fF zlS~UNGbZ9V8e@F4f#LP|!WO!gBCLMG=pAqgVadrLB31iUd4c{rnGjN}rHa zc3C>CF)Dh7E*1nwXF=))VNWxxxSUZVS)gf9*6x_V9Fvr$_VcfLsG_EF5!M+`R+9L+ zb@Tc8H}-d&I_STS|as|c`akY7IjSEWLM+5-32$jNgcRO zvTkYHqa5MQDEC*(d!Oh~5wMa|*sZ<6FjFi2cw7^WaxIi?E~PY~z5z5Xj&lmx@UZQA z9o|m#W3cV0W{Lc8pE|x4X{1f)BCIXbRvLsQ7K3UreIgN+xOc_H6Q@=&<>GYP*6kRe zPyVdS8;veQ(3lq`A<3(iz(UoR?8DCTwp6O^_AD0c;BYEE`gRnLmkawwkv81;M;{pq zY8M~AFCo^!P`~-|Yic_*nM9O6-O-87grBw3lSJ4t2CBPHJr{5i5<4`G_)@G9t#iSY zNF8_BwOn5KsFoz@oNZ7;)Dwvqvtr#+@^EqkvYB+jbob{a z$fArx8dz|;6@MX=n;SP6p;$hFhNYD#@8qT%eB(@j8tbTbIed4U1#bx&Hd|>RRUz#u z?vxDm7b(Gl!NMw{1S~cD>L^(HH6(3I=i}ce^Z$BBEv!6vR!!VO93AFSt>R+$5a_^0 z_dSMhS}>`y;R}N?DTh#QOy^#1jJ)BN1@1Na8@^zAv&f+&v&4&{?oxF4it?Bsgiy&v z9Om~Vy)n)COpthA6p{W42+~oxwFyu;2*eikq3CjT43bsK!JLA9nNwZ6Vr~lUMN(qz zi5;P1D>n6G?yAA_5zbs|+(NV2eZtJfE@id+jam+ZFRs>0VIPM1VIAWjw~-ps;3RH> zI8Rt~^sRjk7mck)X}jZO7V*R?%OvU*@9;<4B$~|GQr|M#d;v%YYK3NxWK6Me*qysx z-bYm4Fm7vfIta&%uW+c2|Cxxn_Qsy^Ltql!S;A2IeDDrj{L)q~1_foVhf~)JQL*?I z^v!s2YVfvCbImlBKJKo{y&~oxCwYLhB|NlC2|6csDAJ@EN z`$e_tb@HbVzePUMeE9c@|1}Lymu5dItL3TOsqTBu{NLW{Z*4(;DH05Vh%qs*X2Sc$ z4v%XgsPa*g2=&5Ijw6=mGx&L+bq=6TpIaELR!V3^KtyodEj;SwCuflaza5uK#p$*n zz^zH?x9B}mZ_1%=&Laop*>n8jjxp|1!n=(@f<8vX#NZFdvJhcI(nGrOU}v)*T(W3e zTJ=>k30mYRs&es5$GpPNPkk*k&}#zh5XTv`k;WhDwdH-Tf*0T?OwXToSk^d7dc-R` z)TF*kge6e!Q1NjtJ29SrTQiNR4a6m4J><=b1#isg9lR!gMN;uw^tV!2V66W#5q%e7 zvB`Jc-ifRke@w+tMux4cWxwP4OF#d^ z<{I()sO0`7ALqBtm8O17;Ng~^Sy{^d%%jgXf64z(T-=rH@J}R0&2L^oX$Q>*US)&&@@<1+Rqfm zcGQ`$5^f0zCywP0fQ$rQa(^QsQGD;}mMPm0CWbeG0OBz|JI*dPVu{-Z4%6yXGl|~= z>cBy=$X?e~0+E+jb9d4L|FB&Z4~w7KUXXoc=^LsFR8CGDI`dh{sy112n=kf2B$-sx z!8NYN^^04SQ#P3szKUpq62DOQL|7XW9jXIWI1HWWA)oK*Z`Jeio1Vg=3;gK{luO7i zA?yecl;5hGoh-FTvW{@A+jim-nc31oW2p>OjfZWOuNN^EtnK+Uo<`_#zYSrU(-Xm!EiJKF(Y~eX?$MO%T zZi5#=v^w<4ILja!A%q-tt!^~_2p}!QrT@)FZ2b?hf3p~4Ygaj)>9+^O?e159Ab-O# zC35xbO`Fom>C^!JzbPXRgJ1ZZ-7`hy%&^h#Oi`8Ee;d+sjEFh-R6;z<>{tMc=uWZp zqgLzbX&@N53@I!$;VKRQNAK)R(iIt??7d12G91J^!YE*8_a`)pv@hIvzwy2~a%3Xa zs3o3SUte~spn0H+=xg$8GA&EV2B5uXVgk9ylvep3aIIK%$wn!$jhZbm>czctg+(>Z z8sv;vG>eXNugrdNVqEXmlNaw`S~a`VLThEV>0CXktT&R+Kz>DjC9;Bdm zFlR9QkYW{pUA6eKiv(mOaB8>oO5&o?vw4t@=Kl&#??m7=<$+Io_L$Q zRC`R`^1<6hyqqwF%Px!f(8|P}a07~tas&4}@aa-{BMwlMNw-9Kd66>p81&zcg>qY{{0)-$_5}}HUhVi?Ff4Ia6&F*(}YGq2lJF}KIE|iycD)^;v^x>7?U1 z)#pFyZ?QV1mUpQ$@;YVqR>2@u-X|*$#dEGvV}bQ=!}zbSPiAibNUjSGrckj{8IQ`- zpq#){W&nM3Zvw{;`W$I(z5+!DP064A%);XkZXE>sIY}a;fPl0tBQ;~dXTir*!W;GZ z`Q;Xi#t|-~IoxL$MN-wGYL2CPfl+ zXa+Q*j|%N;#qImo4F0&~;f{s)jVSClUiQev+yC4Ph{Pa0`^10EPYB|zl(jqwVqn=>1R2p0j}pXqE0DVN5GKZw*gRGCP=2D5!< zfCw>I^(C{1vPPEdey{^KmPD`9JOU6>zx&I`{QdI0&S62W3L=xuRoSfUyuREOM-|eO z#4i_CVE??|(#{=2&%nTJ?N>C*#Q%lyq>q<5E2`>5)jc_-`0aP2==7-1zOfik&2=Yp zT$j9XBqy?wMq2^wDd+i5+dgAuhCB2BM(_SUyjKftRV-2nwX&Vzq};Ezb=mPDBi}h{ z&U(Y$DdA$5G!DQYkNzCC?Zx>Q7UkQa5gu9$vmR?IX-8ALNuO2)Z)f@*J;PzbZGiX0 zuqrCbTUzon3ckMWX+q#}FX}jnhlT3Q-Dl7TK0OyCpRb*jB@HAlJnTjFE6#AM>d^{5 zjZ5-1efc3GyN8k0hTS1okd@zRoFQ~UWB6R*&N7~5Dn|lIKDu_KXLU>I=;3&@5%L3V z*67F$U^eE&{lMu~;JmR4oMYTw?nM3{tRd(}?Pi-y&EI@BS1~*iObZ z#q))NOW&RKyZS&6^&7y0$4;ZaT25TC%W%Fkq&}tGdH86OS9L7TLOUPL^^lnMA{uC{ zdpt6iz<@>tr|#?iI7>{D0SuCXO|;bcGW&tsg^|{!MeLzHG9tR_g0-6$972KE^ixmZ z(j3e+={kl9DWQCXMW`otztb$2B=WCKd4uPjCril=o7fe*EbTGr)s^2`igX>>?iu=mt%}^3e9qBwk!=dn-xmD+_tk3xu^9; zFymT{;U>00=afC879Hc**`bZOA5YBN-o`9Y|NO|MAqN`^&#@dBjBv0kso9c?I^@SST~1?_fsNodq#e9eH&wNU(yAhW0~u565Zz$!P()u4Y?9#mruqPq~;X=Itu6 zOVOsvDX0?9D>^!|%DCmr^2FDMMr-OJbSPX6#4aLN3YK_g(=2BxNECloCEK z!uHDVfaQO8eZIo_2j3^~kV`1buxCur+4<;PICHJtMc;mxrltwQEnB)FY@12sZ>QP*>iBB}InL%<%dU zib!DiIYD%EPmlL#&CpQoGbvM(UPr;76Ilu?!W6Ld3U5aLftUDVefkGCIOLlF(gsiI05m2Aa|+3<)g3_1WXnP+#2${H#ANPBJ9yQ6D(2 z7l-$A=Z9B0@eP1MJNFY`P`4V!x?*!bJ#fHRWUfLuywDSveA|FT_bAEw2@nG(n6qP2@H+0;F^F-ELOx-LIA{a#$J>7@pjhBP zh47yWDe+&j0f65f^mAPa9|EVYqiJj(G(truFr|r_Kga0S>om1Qcpsw(%Lqo;Dq6ydd*ak_5T6Ae$z9Bx7WE^ z0M>}xJ1oiE@Aft(&2}jK3R|-N&@erhm~&Ktumr!2=&UYt`M1Hhp>@>EDk{`<$~|K( zsgc9C$3x61z_CTf=TwCQ1e>s%Vqsv}dZz;~obQ`sQgw&urEQpoN#uTJX6v zTYG!_nX6f+g%tdHt$hpCk+2v}M#2r;S^pQF5)Do$0_m9M1}c|r`)^mnhns zGD-!cIw=_w;qgw30Gy>$<(G#>1Yf7k`;*D<%p^yVEXl7?y05kugNXRJx*HBReZ1MeymEbkRb)C9NMqqMhyO>wT>J*Vqk?3l zMN(aQ?9Z4zCz1BFw-B?Fh*3^0>SpTb(*&wh5n}#+8kIMh&&J06O+QX$t6&7{iw+}@ zKLX8T)b9k`<#m@){#K0Foe z)>`fNIrvUSY?(em8K9QgV^~1NiI#&}USuifhPKy1OA=XQkNuR+f&8^_`*9b49_}>_ zl?@k=N`w^N4Im9~adx}|PMi~BN{ohP@f0a!|KhCHNXE9eI&kVr>`Qk?1R|8Rh~d{k z3};pSW2U#RiInY%;`m?&Rs^@A4>9rWXyv>!m>N$p;I(&HOVPCcL^N?GAo!M;8`)4y z?pebK2Z!hP&&wRbqX&s{c%5t6^}7=vXS~k*p7?0auuTKno0s-BR;@fC@A{#-+9=J@ zWoDa9kVg8|(x&dC4wC1X;wY=DC|8qE`J7!le!LEWBGzrZaoy1y0A7h%T|L)qVY)8@ zxJ#yW(iaK9W_}77In;gIr5{k_G+qVniG1KG5cdh!uQ26R#6$$HL-RSs3ae!c=oSuQ z8FF|?&|xzxuXZXH{Tw3)X$~YY$D4RPt*34-u45cC;HdC~pN{t|BCRT+G6Lh({W_{% z{8ToTB(G*JLY$xw-jxvNSZCm!oH(*fl;=dn!O0`X!~^1|3+7pDvfBxh?cQzQ0IaB{ zQs2>AAA+I1MrX7uB@cV=JK+6Aak@n|4Gf&$4=yo|r?&HsBW_XI3CTp-RHRny_iGzR zIv+32x@wRHdYW{Znt2`>8k~|?n^jM#(iK)}*D;x*IclfnS#(%N^U)t%a%b=qug^cK zE->=*j|_x2sHVbjzv1Y7Iut?%j@r8W2q~a)Dqk(ls%bMzPTL6K1M$@BS2#d1EI;ei zdIlvrEP+z@4x*7U3#Dp#+WU=nWvlTeBQ!yI7^Wt{j5k2JqR)w^*U%C86SzE(iD{(g zv%_b>CV^+)PFteXdY;=kZys%Bv1S<>f$^Tz98s`Vg;~I0iG$o6Dh3o8U5Y*%HgmKM zRGgp9Yk4NQrr+&)5C=LaD}ZGBK^24=B`6rU{2^L!5VHyq4)l*+I$ox1JmPD@rwllEt@w2a z0K`0gE{aVn=KYXwR-+1mwwOI&SN3?KH9|R*GNx4mQpicw0;J8 z)dPM8!pz|z8)51tTk=W5%%UNvR1y?uiIN}-Ls9DF6$OiXnK3nfTqN`>FlKL%K>rhc ziN!-Iy_qji04;TkdD4p_<+4d=!mUDMD`Kgda$#TvqjXm_j806vQ0uIn6va)PH~VPW zq>`Z7P@-*H8)>}NB(#Ciw<}_mRC+8I>iBl1k4X&gd;=Jt)k%U55+phM>FOb(N;V}Z zkwVxt2waY{%YcbRtEm|b?c0V8^4P}AjnDIdDG)!^aIbLaRLZrGqqC`U$gjm8`}aDX zza>68eDo74SU;7n){~M1{46W(FkZ?GX3d6k4ZN>Dv$pBTjbsBX8_>d`f_g# z2b6`1IeOTg)#ft1LGzT5hU_~5I0M2n;hemQTNB>5xYi{b)9^NviC<8L&QhbQT)AHX zUOS%j>ajyo;MXpiqd+JB%+E+Xjel_W1hU-#iZ{a62>QUBL@z!%^cj_^jHrt+h0jZ2 zTG}0}OZ;g`bIt}l+v3~a{vRgok#eUP+eR)kuQXZeU{|LqYYztsKy(P)EO zT-yGGbgqCcb2^o;+*?*J6`laW%eN4#w&@_gC)pQ9l=)6zFxjAPGiU50FffZXQAAq-@qX= z?r;8)T;kp+uUe+{W>AYVrVCVX0~qLz^i^KuJQ=2vwF=R3TkCLUVNDhBluahLwoIn- zB2!dMri_mJk{4(Du^d|+8McFTgsSs%cwF7DNI0VieU!wzEbA|1Lg`>skv1%Os5e5f zXVBy_=bU0-b8b&kVdp|_F4e~0qG?{l$69}2oEY4CsA{54n6}T}@UY$#he~#J?MED+ zYm1_lk>g1czIGwX5y-4d$R%YY2y@}rjQZ|Gzt6YX=NoL^nmkHez^z|tCf1+^=1cWV z)l*W0sEkqaY9Uh)1_6Exxd0-8kN?z^So-dtni2mBmiQN>0KmgFlcDv3KlI_CZS)jb z`T#DgENlvO`7k#n1bzv(^a~a1*J5CW>V~Wk(|*ZbHu^!IXD~+039m_~134yDO4dDe3FG@3$KY8xH*;UtTYPrhNA9|eF+2y&lsDGTgTBR%`g*UcvjvuE^ z!xlX-yL!N0s~L=G=`Em%K}eP|{vcR$J||^v;;|UIHA;%mIzxpxpaSTDfy<)eDd*UG z&4)E7lXxNH8`BAGApdICp4vTOrM&uzeb@MAR^#-GUz9~$83qfP%aVQpW?4ty$BrP) zewd(qNP1BfQ=^n`^Fk|b=Pf309_?NemG_oKxC%1^9wQ?ok1QkO-lNa@DRgzQAe~cZMxB4*H*_jfCA~HPe&C*4jtb;TBnPqnfmBQ07Mp3 zx9ZL9IjLnX&~?AT*H6Z!^P`pmezMUpEnX;wD(IyVnGkWw2mUmOD#oUCkhPjog({7O z*(CaRrR(0%M5gB&EBM;p#4J3_PDA1_iSq^^%v0H+@TXha(0Kutrg#3yJ%Ev}Ty(!r z4SDsgv5olFm5tfauHKChzZw`dtF(mWXZdzU_f{wIDC)xL>J;1tG5OvcYy0B)EcH-> zr}6KlIv6+*Cp9Z=9Q>6(&9|=P{eMSZ-}=jo=+0G=12ZV~mKmnUW3S+SZG%zBAH;w9 z*#EN+D7aHnUHb>_Wb{v=t8@fs+e_Os`#e|4EDXjxPpPqCT|S?M3SkIMwuPs$hxdQ{ z2UI8gq;6$_y(;ym?#9&%38NqTFay0ryH4G$T2{R4 z<)dfsN&t9Hb^%g{Q5=Cnj&U70wQWz%6<}@YbxOr-KzKc3hcDM?=XB()s~+qY97*8 zWLkOpK4kMlg^AD5j9_U?c!ixWl}6LLv3q!62BU~kugHUVP{adOP4PoYK3iw*sygFY z5JmDHiTVSkInhwwZxmtG3j8x|Tp}2aI*$gQDd%iQ8f26jGrv&7Bf+i4%4Oz2+dyFW_hl#)GT)25UdCDxw-m`kL21s*bqV%B_F9*jqlRIH;U; z=H9x#ThL-RQhq|6-=^&1*ZVv>rM6KX))%Wa7J&z;HdMi|Y4g7%QTA{&7Oj08_cDt- zjStSCF4j2bUzIbuX##Gov%L`Gag?>j<7dB7hn-d+o&y}!dLr% z$L_p2FlOq{$$c2_RD#NMV0XyIQ{eVVSM>`OuM9?oqCOp;LtcGcExj5Di(xTT=aDkc z?01`rD4Ce&h0#=zmKvKEkHr_BUi*0J)f*N!%{`%2zQyWV@b|&Rog^J{6CFh@z44Z)O#OgGa)oJ)dEr7-U8F7`#*OMJt+=)eb)lT&o1DH! z?GR5e)OR&;9(^U*82cIk*!CFF^=CXDJ{wEYPcx40_i&$M;DYN}kC_O;rm>Y_MkI+NE0;9>1zAoLdjGm=j9O8vw-CmhsEDipCg*t#3=9l!N^9Mn9=Aa5 z3xLY)D7qSvBukv|#t>bk>nu;7b!KQn;doDEw*O(ol1K9T(7QAnR1v!4`;{JQK18%rk6XX2Pw<>vAQ3tLxdB|!ax#W#7a z)Qe97GB~%U3fL>w5s0;tGiNj46T`7U(X7a&k}#N|t+i5HMI&>VkBg}o+)4tZo@a#A z;~0fsCms1D#V_y{c;Cw(@9#@aObp;w4pTxlk?pd)<8(6d6u`NxTo)%&msRqKQ_Gnp@LW z6>J@K*?+PUs#M7+5iBiGvp1yw@-WvvBP4RA-c%YZ{a7nZX9X#xT+gptrK?mC`hHRQ zN^U2n(l!;ppL(~U`hul5(wJ%(id~={tJK@_<=I$`U7t0|SOYPu5?8is5%NQ#q()fc z^G1$giXV=JqxV88UU9Hq)*51$jjTIqURKv6YfDJ!mbj?_8kF_;rue#h z3)o`X3fUj^N+v0_K1h?(ZLbHUV=|}NK7AEfP0rd?ooo2qaLEoF6F$1}B@g&(9LlqT6wHkhYX*nidKmu1jg~I!x+k@~%=|qnawP9B%mXoc6-EZCUD; z<7OUiw1g28+g;+mxTlBW`#Qyi-IWG4t(h!M*mk4l*777M;WJ4i$wdpVDCajjAieFo zg(I}k@3kQgb}x}$;gfuwC)}I95+p(;;lPwR(q{|c^U3POr-^LcI*lCy(bJ&)KK1Vu zTaUFwr;S*rSe&T#5K*iOxgkED@r9q-w)X22tkGI#OqS1+)g5f~4)pw4QUTsyrR66-<3pEhTbm&v~;X zXFQt~_i2U#OjO`HUS;xA*klsB1Z&cU;VcXQlKe| z%NOPG2T3_&3rU!!c?tKO*@*HkSu^juv%p!#c`X-8h7UyPS|^5WN#Q(^R(dpFU3QYp z^=qJpa|wjqNy2I5cNuCVzEX*NkxY`>T?R&nk3m=Y<`!>X6|vC4=9kcuL4yjUL8-~; z0`x}^9Xs5V9IDFt9#ve*r2TxgW;(Sr4xE0A+e{P~$y$W&Xz!@oL$xmeFyE&M_!nPGJm`m-yEiRsrpM0qXoDFG~r`(byV^beYc zS+uFUYJaF=71k6?q{{xJT)k=)2pwT@(cz6al8_D|Vphg@AG4sh$;sI+t!U(ZNZEn2 zMwWKEaL594F-*ey(^~#q=!AX^Rplg?!`S|Zfxq-23sTP9#=r(Wq1JbDqD3W8b+PtB(tmr+O#YlCkdVfGh zyKZv=18QPgtC;<`L0&Q*jb+OGc!*>Ks8dsJUg|X za}Bs_Mc;I$g#9axeR^0+QAgqN3*jHwbmb>Myv#0kMwx{s>aG_mW)1GP&+R<|eAwB1 z(YSLZ19DD%SVPfwJZuHP`5<@v3lie zR%A$^FppYsZ<^Os8O^APu&I5j%eZ#)^mt!LN@HRaEIPa^tphBQS(dtM-}-R@RRm0p zG$$c}^0(|HjYOHsHf)C4T-_4{cc^uIEo8N3CDmXoXRBlC$}}_2bSu0B@SN;fik|waX8v+=DwWJLk|rr5cif{;S8l8YDV$M zde|M0#TJ&*NKE-Q1pelr(!1&5)*$bN?q|D}O1D;bmr9iDEgJV!^}8PDrHw!tXYTEn z8vYg&w|dvc%Dp#XP;4_$muRQ!uaT#lq!S}wYxh8BJz`SbH?h?0yRXyr#YGgama#$EF&8+8oNkixFnssj5H z3J*x(A4Q%i`E@y;$A0swD@lFI0+fwV)`IufTlzd6UeJ~5jCE`{ai7G}f(m|DN5F3Y z3Ra7TnqQMCLY0>J-tlPD=j|F75Ib_^yGHCrtUL^vbZjCYBv;g?+cJr(33ceU4mJ`a zD*mv$p+KS9#{N3I`A(Qfc+`olgh*r?2kk;~wOao~^*y>y60kM`nk;M_(u~rvX*L-b zP)Qw0qN9?Md>vT*va9=-NP%0l#x&YSnr}+_)33%4bW(Aj)$A)YjM-39EPLFaTwm?u z6L6)>mI%EnL{qaB^jH}pATqxQnnIQO6cv^F6ETA+{J?jgp7i?PmN=+TeqE6 zpaQI8g;a)#S-v67*5wbr7RcP+MIkQ3z6w_03wj8zQ?Q8^x(xS!Iq|28$_yZxXSCob z73iIyD?9PG5x1O`9#5!S(S{~wBdLmXvZU}FIuR!A=Y6t9yu?k#J577tL4@6NhKA-6 z)(8%zd=M;^Bj@&8J^@{cLbqR!;mvh#ca$9H66mOMDD|sS_t*jEJ`YBIuY25o=^1Ff zxku@m?34Po>+G4-E?d9^Q5;pDWOeq`HAA$?jJPmn*2GvKXIX*)9ZeDDsynT<7fYxb z-MHF#IIQkTWs64DUWi1%uYYX4%+5?FO1jn{03CDwVbJfc+!e-k?;Mp5Fz@aLvm0UKKTHPwyQuQ&9QoD`D@f^z;=?5q_5x`ri z=r@NxG5B|dAN=A!Iyd|;Nbj}pJ*|9Xc1pJEsPdc^-f#-HOB1qS*$24a$b zktMX#&K$Ek8WkG)O;<%9OUYc(<}8cKWx1Z-no;~1{&X%A`rH6&Xj*qUCi+TwwX?mZ z5{O*cS$Jr>=;VIDS=f3C9M&@oh!3Ca1--!v&un@_reX4NGS|j0=XUd^^cXtjHiQok z_&6ZJa0$(LtKP4|+Us5ZReGMz%2IP~3#J*}eY+8jl9Mhu6*Dr?1oe1zkxBbPd~~1m zl+sUhj%O@>IG>_8H-4J-wKm;C>-ay@_@vE456ykeE8>K9klUzRG~Eojn|Gy^nF&gU zlIk^Ki{ekcB+}o~h`u2U27Ng%c5g9u{Q!q8k@V&d*}tWEbecS|o6?(fi0kaq$8pkX z#;kUn{5`FE=(n={@BWS6@mo}J|N2e~toFc30~OvDknAcY;~XtSLcZi_rfFsWhkA@6 z3?rmt$U+#Qz`l7fv1Q`loz2Kxews2))2lQftM9;!$UYo)!7A4)t>8~dUMQ{adjTM3 z{8I#EqafdEXk)yl>XMNL>O4Vi;~NB%x!R{C2cyEYEZygJrJi0X4g8skKWzmOtP|IJ zO??CM!>)X!Qhg!EF=fv(RK!_s01$1ikLRAm-0wl@0(%*DzgRK7b1DhHnCN2OqMmg; z*B}zFg4FmQ?KLo<23<^fG!$B^MJSB3+O^^*_UH<@BOQ_{whKnN$SF&Q^%%6x8B)gT zEXPK$^19)`qv{62;mBuM2rN2NG=;g0&}RwY4{$({VZeffKUoJ) z9)&O8D2qcgtll9pKx6p{8%eE5z0dxGM-%22EoB|uUH}||+b^lTYTnS}a^J#L`XsHm z+iiD+C|pu^S63?ek_aRFVCT*ON*AJ06Ai=TWUl`zk_b-^x6-cRo)0Er-uaR^IL1xk zT&lY~EpV{M;jY8p-(ek`Ga@I5ZODU>x5Be+0TaXig%k?UpclN6y%D>Sx3XJRSusN& z6=n|}U2jSD_xN^YZ1qm~m}byaP4l-jiN8zi$}_CG;{QV_rvIIu=dbh<=8ic+vnAW8 z;T4?2KwFwaIiPi1HTN01lOl%mlU}Z3DDdEIg;KGyJIBSVzTpkNOu#&+NP2p!Ol@h( zDknmAt(Z@GxdG~=yA5K>9r@Dd%=+wG(xXJG8btr4aOmvyrGWS0oX0a;usU#Og8p!nxc{b^35s zFzPHbM^KFCar{=jMH80`WzuSK7^l#23uUIO+Wkon^WkB;%u5E$U_Q?m1(DD2EbyHx z0uW$Ms23lKvX$yPd0xPTF6D8Y+DEWiZ@*5R2)-RP(V?XRhOWc0>Q8z3Yx@ zD(m)f9I=cjMQ{KCsS46NYCt+ks3FJ@nivQq1eB&25iEeTC`5`0O{INNAI;3!I;*k`V)n| zbzUC}1!j6#d{m9->7{@085h&P{W`>MMCc;CxAG-=!U@gD1yf*?3JFg`I9R2u_+P#^ z8PODrFlj>?4S-3V1BRvV+Q?%U?N^hqWG}EsQtoR49%7qSR`L;tC@ohQGEFowsy-XZ z#g7qQUOeNMD8%s^UWT{yP4=&+so$MaO||;ji%_;K&{8GhR6=Eo(vBe0HPS_5$$f!y zp0l}B*n8?TpUR1X9ThRMmwLd^2;L>Mxl~t~fJ@j7gV%mqEK4f|d{PPtiRI;)YeEir z@s2f#T{A8$3+^|MPuxR%EtkS<|2W5{@WE5%4qb3nQ<{Fq7E~qr2*8F&=wfn5z?&JK zReRS4?|7ibFh!1ZyG?y*PQJsY^=*nI9qFnazuX)rDr>1_MlE&eousqEpXRP_YxBzH zz5T{DWADL*x-0=%sLRc5U{HM?flRU2m7g#WS~??EQ1Ct_7pT#19pBXH&+)`uQ8ew|1wzid_+FRhMIU)KxSUU0@=z(%aZMQ%(ug;*jsM~MTDGt$b zl^`WUWuaNGxd-poZ#FXD=hV9Rr+Q&*`9FSG7jrwd0kC%~C`C2e>b_lNs(#e^;L6Gy zeEU&v;Eu5c`jXmF$z~Mrakfl$Snn&sngw!igXr_os7#3$Xo?R7O$j7i#G+y)lQ9SWjB&rM#Wd9A0?~YA0dL;PN5r=AR-6oHlxjgy zHo2UBI6f&^`f2>9t)*Gax9SX|@H&R!tuX_;hT45#drr20r8KJg#*X&$K^1n{sBCZ7 zgQBNT_MFexOG8&y*PT&?R1?c{K!ZEnesyfP=}kO{n1Av6qRE#)Zq`sE=z;F~@- z1C|def4tSfm~~?!F6xoVkaeQ{O#_GuN#0quduhij5>RHEE5=WnDfj&tENaD34jHj( zsCDr1tGn9+zPCW2RfF`U*nWL!xuzB?4<6C#_f7EWSgvz7^VZcB&oW#ZRhx3$cWe5R zN=F{VEbKxn$>g_HOLj|X5oTr1B-VmVZ+PELcVbQFsxM~VIr=Qv{>?Gnt0PURTo`hP z-<8B^C2gD2$-6}GxmI}$QJ6zQ)at&7Dglp7%b<%>!buSdd4eW+=jaNa$shQdl2SSq z>mL)D672gSOkT-4B}qhKOx~>p;)vOiWp7LTuo$pg^L{Bk)aPtk?uIEvYd;J^L z;cG!!kmt69S?ny9^X7A4N*V1zi4;=q9n{Y6%`)y2R+=Ma zfV`FS&7nPRpoE7;1hG&g`^njq-bk`|M6HMCi%)}-Y^e#6SN-S|OFc}1yv9rbDo_kK z1PkzXrN}EEXeKc+w;4?%}Lue zFwB1I|1jX#uHp;u%$wFfeGxR30y>QlfvJPdTUzGjtJr8P5P0f~z^G~ma^pJOybf%a zx<{ExCQmP~T>BzGrxcd8v~@52>yP~lN-HiqKU^w+AFV)+tZJ%FQf)e} zqLcC;q51eltHKYDK>R2_n2x5&BD07YBbH_tz7*|u!TmOA{=6MfD9!;<4b1RIc(xZ( z5)h!a*AAF&`TYv-?NtTekqr0TW6+DT5(0K-ys%eAUW0fur9EgIf~fUHTH;K<7IgB?+slRooeRQRjuOXr#k&nzdO$(HBAWlg6l)R5gCH+Qm9__L#Zb zcOJ3po|)VJyTFy-;W&-)JNxgbPv2JKzfhjPrl^-E&Ivd#l>x;v?|l)_&g{&nrm4N( zYFavfs=e(V{ni_RIK100b-+`jF>~?j0Z&jbRUVzKwkz3_8Dx#)cn%`o6jb7t7|=l*~XpbL~{zd zU&uW5D9l)gDD>*3441drA-P!(lkB}ZZmdpBTil{qG~2RbU;h4WVl2$F0}pcGrCzPW zY_?cd__V(AEfup`3oWzp5s9I0VF&Kq(}fSNljb{vn4@!i9Kt=3aB+Kevy>GIcTiQ6jWYTFhvN*BlW!@#IptbyJM#wWh9#{3Q8j@EVGL#fbQV&y6(+7m; znk7-Hsh>SFEc`GjIu;e5BYpzlUippQ?b9&vx2Yfh+%dAd?@w;6KT=wp*1vr~;60<) z0943SybWdRS4L!uELO5oXbcHCMe9U4&gvft-2XYo;mxJpM4ilA<@nRvM@@_h?~uus zJ8*jc1jQ*~n%;Ek-uNsSS^yi816v}KBxG|HgA5@LPiU!pQqVsQKmHko^jUwmyc@mUpdBD8aH*}=Nyu!7PtUzhtYy*W$C_QX&iwY^0H0X! z15JS=5%0uXpFIgp!dsr%0HlS7Pwg_s<*U(zOQ1DM@`|J)L)?m&SEkc<0s0be>J zPdWUm6r@utS+nwDi%d~U#?C{AxKa{#WPt4GO>_sum$3l=YMLIMSg{o;JR3FDmS~|} zZ<-on^s|&~)&4gvF>X2tOkFYtEVJU$(tm_ceuWr0q%|Ee$dpau5)GD)%7F2opP=Ab znxNyp!Y%Z!b#%~5^ZtjlKg9`wH@10K%^m&Luh7SsXEi4mg0PK4{Xp^^7x`q zL44~P6%4Wz;)5m2AbjeVxVr4uDvHI(ee-R=ar0C(?4Yh0)jT(+J{zxSl4dc_!aDrPGBKe8dB@-8{yDWOko)?7O>=L?fL5pXBWW*Q_p~oVq2nDsU=} zaQrn5Guie@>6#dj;Ij*-Y8}HKxDbL^Q)(_vPc&ne__d6bQP#mz*^0^KV^+tCswCJl zeYhW(&J^GeCuV=Kyo0{2mvxr)%3D_~E4aO=OD=&18F_bJ&6|CUjqSk6O5yVvQi{Cd z-AP4#R;ty;g9=Z}cRD$-)yw7Pwa?hioR?tC@gy(Jsg3!w=pmL({#7b7J5lOz&xVvF zoi}YS2H?AKRxhO;odyHBMY+gEX~sn`XPW*V3s#xoSYbPrHXRGisU%*zOR+qiF7b|`IA+v!wxj%f+&)r}txmL*xq#ri_KwZP(sND^$dR@z=N%7F{7 z)bZl_gY$Ig=q=ZZnv&XqICRsjeYAtBWu{q--Ud-oK?v;6brcG#sA%W)sZ z9jGaXT=a52h8ePl93^-Y$pX|cMcPq~>Iky&ioB}%;n%?zo(Mx(p7lvYizE;f;=*&) zdc}KwHMWQhW8s7&fvjZN95!6jFdLqs=hl|WZG-dEmta@U?DHh&Wuu`s8QSLY(GSCJ z^GozCr6?7+tZMAAp^7V#dnk$qn(1efPh?5*Q+>ki`O(gviqg62lD52rnZlfS?kumN za{~M&?a;I(;qZ1@hK`a@bNB+^M2)=RYC1IeP39%$OmSpgjyc}gB$Hz|zF~X53NHH% z0r^#{^Nl<|r}_7@XFMO&znf_1>dYP}f8Jd-i4t5;Qjd`UT>0%KB$A&B>oUTF+e!~ao>SI#Q*&y8poY*NJMG1 z4D+Q>zv9US8mU4rsIbUPPeA3gp&=iqhuSI4kgXLa5aXNHzTpx4`oH^CCHShl!1*?x zw}1R2{MEew#S4ETE%~n~OumxucNc;%?=HWLou`sk9=+WbR;JMfMQcKk`>L!LB@UZ@ zDAvlS2JZkv>?XkGaS72j55K2w3a)T{Q-}ls4y<^St0S=~9s`G88CNh0C;J)Y^a@>Vyh)Y=XV_u#nU{)*}S Kd^&t%@IL?ojJ^N> literal 0 HcmV?d00001 From 0e8bf7f273c2d411d170c471829284731ac43b81 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Tue, 2 Nov 2021 16:12:44 -0500 Subject: [PATCH 2/8] More fixes Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- ...ngestionFluentBit_DataPrepper_OpenSearch.jpg | Bin 27674 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/images/LogIngestionFluentBit_DataPrepper_OpenSearch.jpg diff --git a/docs/images/LogIngestionFluentBit_DataPrepper_OpenSearch.jpg b/docs/images/LogIngestionFluentBit_DataPrepper_OpenSearch.jpg deleted file mode 100644 index bf49c8798c6f6197898231ca79848a9900666b20..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27674 zcmeFZ2T)Yawl6+_q9CATNg_E3NY0>Sh9m<+4oaRONzULWIi~?6ry0@^lrUsa$vMq{ zC9oFjI-K)FzTI<*Q=G)C8;1N(!MG=67 z1pr{({s1>K0672-_MhR;E6#1WgL~)Ca32@k>!q__8v{$zsnCne6E`?rLT9^gH=P5nO%H=hB-_W_qUzi_Y^0ocS? zIK)^t9e^i)J}Nd2)@=a%HQc+4d*?nL77jN4ZNAAP0N^eT7B(*7Jv>~TyLa(!+l7sD z=PvF&;`=0|lni>wV-%11L|$s?>U+XcP~&8bZ#-&8F?i(s!p?srq}DJAh{}S{<}U8c z;KaO7RIG|2`D}9X+B%l+gTCCdr~RL`|If++0GvBm*thKO-{uMv-}WIk_Pu*JcW%4x zFAw-<<-~VMNXfK7c?^6n-z0om!sT~n6!_!I=)pa6awcZkn`r>SZCPw$9AbbB;8^YN zod5Sb_c%McO5-ou`BN$GtSrFwWLiY?>flNoVKP;s5Kt|~PIoIh$}2oPeAq-exbl9$ zE8qM&pYh%3PYQ6>4Bf==ax`SOJXzk^S_PO1Ov%^w?J`WWg)j-MjlZBfzzS{LOD9iu z)NyRe&Y#6M;SNzwyts^b>2*H}hPKHgQH2fkL}~NWJuAD`W~qByB7FT!6)h;1$7v-$ z13Q$#Lpkz3tqGE3# zh>q)0Xnps;H}e09l>6&rm+~MFpPGCOU15M*879k!tcUeGZbr(w@&FS`P=B!e!mHa>B!G%X#~V8kdW-T!)0x6HC=9mw& zL*dNf(Wf;}$>{r-j}4k}=*ad+SNCS0kIHy9e15A*<>XJxA#i#0#5zu!an3(=djHKF za;yG2&7i?h&3dz@@5@y&x}1=f3CN_Xkr=@#o46c~R><>uSb1F8i|}P#)b`ku&ufs= z4r@G`$d*TqE;2db2|nv@sJKu*nv|WCCDL2%cPZ7QJL-p+t&fM`X}VmE`DMBW{kkmN zKcM^dXh2HtOOsH0Tv)yt+S0fh4JxFaauDFO>tke2%N<^)NGGA0G!f#T7XZbs|9>nL z;@@qT2)GlghMX!^)fBAez^uICTGVMqx$>Ep_N?FXO1N3NGe8k{5asfTQbLAtUtdc+ z#s*JkRdr`*aGnTf9mv&6Sj7ovf_EGw7T=Y=OZF(T|Fk|H*%DLa_h?sR%3`pH+?Ph< z>?vn1i%H4MGw+(^N(;!s9SLoe{!d7!Y1eq`ef(lPJ*Y?j69-Y`w`tEWMCjcV$!TZ3 zPc{4Ru}lA7sVq+H2EZW&ajB}UeZjIdC;bkj7Jg1r2Nzc~^m^zHqu*zkmHzF)1*3{y zmuK;aS8c^}yBiL2;3=(NLW}WqCcg0ZhzgF;0FVH7WKm?s^=Dn`gnqkVg%cD`z8JTa_5mano^3woqP4z+hq;SA|dWq1n=>;Ua6FDn;k$R+*@ z;=33cRY4ZZWMa1U?5&aVhp7t8m!Wb^^hqA%?Xp<`AUhpsFvXgXX5w&Pe*GwQyc5Ex zFrQIoT%+ydJJ0BUn^Lw^RkXn5@LBv|MJt75v$`q&unZQ|HWb;x!k{9pz}zzs9T(s8 z*=dnAQ^TAGx^o>Ag(hO-qZIN8eA&4LK)g$L77K4cR$mbDz7oysFvd*%x<^@zoq`SN zSZ7?X7~T+Vd9mzD?5byTL~ELhua8$m`XMU4NhWvs7$;&tVbCOJ#M$Q^$Ls%Q|KFYS z{}aiBLYdI{!!GGX>8@K{_sDj6$K z`XN;N`{^USDe19UkL=S`UEP%`uIB@rs1xLh`WqG*?XQN7s^{L=*8s(JbZr`Epek%S zdwCzWLGR=(p-n!L|CYG|dza98tv2Q^qoy;l4I z3&+?fP$eO+o1%b`d+vUCINFWygH!+VruvkFWn}H=<1a6giJe|nj&1%Z{OTwtv8hR@ zVVDoJ0?LKXgfk=EY7D3ns6xK|ZS3wK2~RtR=HnSIkuS-$INSYyxYoh%2W*Wf==@t5 z2vyjzH1C9}=We)#C$-|6Uz$>F3Fz44IksR`)>0~L#j6&s{ zAW*(WQz_#tX*i!auLwUUUr1pF;`^)1*(H$(RtSreT@l+`cO{;c zvKxS($rO&Qgn1lZK@}2qJdHLu5(_iTp?K#gx+H6MABGSxNz!tV`BoVMaY$F|RpFo5 zc=ocP%E7VHMxQeAkedH8anv+PIeS)LcV$h{)MS@8toT^j8ajYt|Qc^@3w2rG&H781~Iqo5E6{cD~7|MC*vCC1h zrGzK7(GQ+t$BHD@+HYItg937pmwCM9@bV5vYLM#7bV8O!Ma!%Q!%f1&X|6j<8j@mB zPA(-nZ~Z|m+rsKke6)>=rJhiiY`oDZ^6)Lp_jlk9G7#=N72;l2(XH6#QCXuNloMQ` zncwPmnsmfy-@d><+iV2SUVVFOb@1#dCO=)i>sw;rRL%O2{ikT)(>*nN-#7AMPwp5! zS+ITv1a9zsbNtX}Wc{IKZbMmxO}<;I13K#0KC8*bKqyXFUEMkAKgv^T6=4VF@v(Ew z18v*S^t$)3sQcx&ezhi5Od;z>3Xe^dbFR9+jgD^-Pw*e(zvI`UT%XacXNjL&>ujKy z#)*O+_aSR^$2B?jl1-AYB0^O-VdJBQkZw~W$NGFm-IA3QmNthiseswe4~HAMeIfK* zB3wV+S=D%G;d+obS$Ug54%yj&4_Q5X!Uex@fSTTjM7g$4<-SmEyna*eCN=BGpN=(a zV)ZeFoHK4{NBA`3EWbgyb^f@5)04(63wle7C>*MB+Psw}#t?h$EZ0yMA;xU9;>3Ee zQJ*S=QENlqFo!2tVU}TBm=DajT_&T}sXu>^XATVMw?eH^ZAk%Bzp6lbOG{N2>48ri zlC0jQd(Yt!aXi2%T3Y+ZPUQ~=(vR{QNlW2dJqgihDTUPKwa$b?N?peKBiL8lu2+JM zW}4ar8efO6_giOz`i#jiQ9e;fya9mLy zkSX~QP-WC+FNc*001z9mWmX`m^FY-<2H*V}HY%DPtMEw*FRrHiX)l;QV`Zxvp+|H$ z>oC_9=g#)I5=_bPBt~iGak2J~-2yMrCWvJ=p}g3|TFhDAx`@+C>%%HYACbpe+of8C z$0KSz#oTLDti`7Fs-H$?%%%J_GQ{6Rx2%NBb9XC$gIDn-dNUU3t(7G*LVKE~s^&ZN@U~mOs$HHsC#97^Cpy6zIk*?(jHRx#NAM^8C z!itub)Cd_n2UM_d1?e9m zWDX>C+D(%l9UbTaW8_JeZp5qd(joI>>oJbL2LX{lMhnmxRs`8Sd7fF zlS~UNGbZ9V8e@F4f#LP|!WO!gBCLMG=pAqgVadrLB31iUd4c{rnGjN}rHa zc3C>CF)Dh7E*1nwXF=))VNWxxxSUZVS)gf9*6x_V9Fvr$_VcfLsG_EF5!M+`R+9L+ zb@Tc8H}-d&I_STS|as|c`akY7IjSEWLM+5-32$jNgcRO zvTkYHqa5MQDEC*(d!Oh~5wMa|*sZ<6FjFi2cw7^WaxIi?E~PY~z5z5Xj&lmx@UZQA z9o|m#W3cV0W{Lc8pE|x4X{1f)BCIXbRvLsQ7K3UreIgN+xOc_H6Q@=&<>GYP*6kRe zPyVdS8;veQ(3lq`A<3(iz(UoR?8DCTwp6O^_AD0c;BYEE`gRnLmkawwkv81;M;{pq zY8M~AFCo^!P`~-|Yic_*nM9O6-O-87grBw3lSJ4t2CBPHJr{5i5<4`G_)@G9t#iSY zNF8_BwOn5KsFoz@oNZ7;)Dwvqvtr#+@^EqkvYB+jbob{a z$fArx8dz|;6@MX=n;SP6p;$hFhNYD#@8qT%eB(@j8tbTbIed4U1#bx&Hd|>RRUz#u z?vxDm7b(Gl!NMw{1S~cD>L^(HH6(3I=i}ce^Z$BBEv!6vR!!VO93AFSt>R+$5a_^0 z_dSMhS}>`y;R}N?DTh#QOy^#1jJ)BN1@1Na8@^zAv&f+&v&4&{?oxF4it?Bsgiy&v z9Om~Vy)n)COpthA6p{W42+~oxwFyu;2*eikq3CjT43bsK!JLA9nNwZ6Vr~lUMN(qz zi5;P1D>n6G?yAA_5zbs|+(NV2eZtJfE@id+jam+ZFRs>0VIPM1VIAWjw~-ps;3RH> zI8Rt~^sRjk7mck)X}jZO7V*R?%OvU*@9;<4B$~|GQr|M#d;v%YYK3NxWK6Me*qysx z-bYm4Fm7vfIta&%uW+c2|Cxxn_Qsy^Ltql!S;A2IeDDrj{L)q~1_foVhf~)JQL*?I z^v!s2YVfvCbImlBKJKo{y&~oxCwYLhB|NlC2|6csDAJ@EN z`$e_tb@HbVzePUMeE9c@|1}Lymu5dItL3TOsqTBu{NLW{Z*4(;DH05Vh%qs*X2Sc$ z4v%XgsPa*g2=&5Ijw6=mGx&L+bq=6TpIaELR!V3^KtyodEj;SwCuflaza5uK#p$*n zz^zH?x9B}mZ_1%=&Laop*>n8jjxp|1!n=(@f<8vX#NZFdvJhcI(nGrOU}v)*T(W3e zTJ=>k30mYRs&es5$GpPNPkk*k&}#zh5XTv`k;WhDwdH-Tf*0T?OwXToSk^d7dc-R` z)TF*kge6e!Q1NjtJ29SrTQiNR4a6m4J><=b1#isg9lR!gMN;uw^tV!2V66W#5q%e7 zvB`Jc-ifRke@w+tMux4cWxwP4OF#d^ z<{I()sO0`7ALqBtm8O17;Ng~^Sy{^d%%jgXf64z(T-=rH@J}R0&2L^oX$Q>*US)&&@@<1+Rqfm zcGQ`$5^f0zCywP0fQ$rQa(^QsQGD;}mMPm0CWbeG0OBz|JI*dPVu{-Z4%6yXGl|~= z>cBy=$X?e~0+E+jb9d4L|FB&Z4~w7KUXXoc=^LsFR8CGDI`dh{sy112n=kf2B$-sx z!8NYN^^04SQ#P3szKUpq62DOQL|7XW9jXIWI1HWWA)oK*Z`Jeio1Vg=3;gK{luO7i zA?yecl;5hGoh-FTvW{@A+jim-nc31oW2p>OjfZWOuNN^EtnK+Uo<`_#zYSrU(-Xm!EiJKF(Y~eX?$MO%T zZi5#=v^w<4ILja!A%q-tt!^~_2p}!QrT@)FZ2b?hf3p~4Ygaj)>9+^O?e159Ab-O# zC35xbO`Fom>C^!JzbPXRgJ1ZZ-7`hy%&^h#Oi`8Ee;d+sjEFh-R6;z<>{tMc=uWZp zqgLzbX&@N53@I!$;VKRQNAK)R(iIt??7d12G91J^!YE*8_a`)pv@hIvzwy2~a%3Xa zs3o3SUte~spn0H+=xg$8GA&EV2B5uXVgk9ylvep3aIIK%$wn!$jhZbm>czctg+(>Z z8sv;vG>eXNugrdNVqEXmlNaw`S~a`VLThEV>0CXktT&R+Kz>DjC9;Bdm zFlR9QkYW{pUA6eKiv(mOaB8>oO5&o?vw4t@=Kl&#??m7=<$+Io_L$Q zRC`R`^1<6hyqqwF%Px!f(8|P}a07~tas&4}@aa-{BMwlMNw-9Kd66>p81&zcg>qY{{0)-$_5}}HUhVi?Ff4Ia6&F*(}YGq2lJF}KIE|iycD)^;v^x>7?U1 z)#pFyZ?QV1mUpQ$@;YVqR>2@u-X|*$#dEGvV}bQ=!}zbSPiAibNUjSGrckj{8IQ`- zpq#){W&nM3Zvw{;`W$I(z5+!DP064A%);XkZXE>sIY}a;fPl0tBQ;~dXTir*!W;GZ z`Q;Xi#t|-~IoxL$MN-wGYL2CPfl+ zXa+Q*j|%N;#qImo4F0&~;f{s)jVSClUiQev+yC4Ph{Pa0`^10EPYB|zl(jqwVqn=>1R2p0j}pXqE0DVN5GKZw*gRGCP=2D5!< zfCw>I^(C{1vPPEdey{^KmPD`9JOU6>zx&I`{QdI0&S62W3L=xuRoSfUyuREOM-|eO z#4i_CVE??|(#{=2&%nTJ?N>C*#Q%lyq>q<5E2`>5)jc_-`0aP2==7-1zOfik&2=Yp zT$j9XBqy?wMq2^wDd+i5+dgAuhCB2BM(_SUyjKftRV-2nwX&Vzq};Ezb=mPDBi}h{ z&U(Y$DdA$5G!DQYkNzCC?Zx>Q7UkQa5gu9$vmR?IX-8ALNuO2)Z)f@*J;PzbZGiX0 zuqrCbTUzon3ckMWX+q#}FX}jnhlT3Q-Dl7TK0OyCpRb*jB@HAlJnTjFE6#AM>d^{5 zjZ5-1efc3GyN8k0hTS1okd@zRoFQ~UWB6R*&N7~5Dn|lIKDu_KXLU>I=;3&@5%L3V z*67F$U^eE&{lMu~;JmR4oMYTw?nM3{tRd(}?Pi-y&EI@BS1~*iObZ z#q))NOW&RKyZS&6^&7y0$4;ZaT25TC%W%Fkq&}tGdH86OS9L7TLOUPL^^lnMA{uC{ zdpt6iz<@>tr|#?iI7>{D0SuCXO|;bcGW&tsg^|{!MeLzHG9tR_g0-6$972KE^ixmZ z(j3e+={kl9DWQCXMW`otztb$2B=WCKd4uPjCril=o7fe*EbTGr)s^2`igX>>?iu=mt%}^3e9qBwk!=dn-xmD+_tk3xu^9; zFymT{;U>00=afC879Hc**`bZOA5YBN-o`9Y|NO|MAqN`^&#@dBjBv0kso9c?I^@SST~1?_fsNodq#e9eH&wNU(yAhW0~u565Zz$!P()u4Y?9#mruqPq~;X=Itu6 zOVOsvDX0?9D>^!|%DCmr^2FDMMr-OJbSPX6#4aLN3YK_g(=2BxNECloCEK z!uHDVfaQO8eZIo_2j3^~kV`1buxCur+4<;PICHJtMc;mxrltwQEnB)FY@12sZ>QP*>iBB}InL%<%dU zib!DiIYD%EPmlL#&CpQoGbvM(UPr;76Ilu?!W6Ld3U5aLftUDVefkGCIOLlF(gsiI05m2Aa|+3<)g3_1WXnP+#2${H#ANPBJ9yQ6D(2 z7l-$A=Z9B0@eP1MJNFY`P`4V!x?*!bJ#fHRWUfLuywDSveA|FT_bAEw2@nG(n6qP2@H+0;F^F-ELOx-LIA{a#$J>7@pjhBP zh47yWDe+&j0f65f^mAPa9|EVYqiJj(G(truFr|r_Kga0S>om1Qcpsw(%Lqo;Dq6ydd*ak_5T6Ae$z9Bx7WE^ z0M>}xJ1oiE@Aft(&2}jK3R|-N&@erhm~&Ktumr!2=&UYt`M1Hhp>@>EDk{`<$~|K( zsgc9C$3x61z_CTf=TwCQ1e>s%Vqsv}dZz;~obQ`sQgw&urEQpoN#uTJX6v zTYG!_nX6f+g%tdHt$hpCk+2v}M#2r;S^pQF5)Do$0_m9M1}c|r`)^mnhns zGD-!cIw=_w;qgw30Gy>$<(G#>1Yf7k`;*D<%p^yVEXl7?y05kugNXRJx*HBReZ1MeymEbkRb)C9NMqqMhyO>wT>J*Vqk?3l zMN(aQ?9Z4zCz1BFw-B?Fh*3^0>SpTb(*&wh5n}#+8kIMh&&J06O+QX$t6&7{iw+}@ zKLX8T)b9k`<#m@){#K0Foe z)>`fNIrvUSY?(em8K9QgV^~1NiI#&}USuifhPKy1OA=XQkNuR+f&8^_`*9b49_}>_ zl?@k=N`w^N4Im9~adx}|PMi~BN{ohP@f0a!|KhCHNXE9eI&kVr>`Qk?1R|8Rh~d{k z3};pSW2U#RiInY%;`m?&Rs^@A4>9rWXyv>!m>N$p;I(&HOVPCcL^N?GAo!M;8`)4y z?pebK2Z!hP&&wRbqX&s{c%5t6^}7=vXS~k*p7?0auuTKno0s-BR;@fC@A{#-+9=J@ zWoDa9kVg8|(x&dC4wC1X;wY=DC|8qE`J7!le!LEWBGzrZaoy1y0A7h%T|L)qVY)8@ zxJ#yW(iaK9W_}77In;gIr5{k_G+qVniG1KG5cdh!uQ26R#6$$HL-RSs3ae!c=oSuQ z8FF|?&|xzxuXZXH{Tw3)X$~YY$D4RPt*34-u45cC;HdC~pN{t|BCRT+G6Lh({W_{% z{8ToTB(G*JLY$xw-jxvNSZCm!oH(*fl;=dn!O0`X!~^1|3+7pDvfBxh?cQzQ0IaB{ zQs2>AAA+I1MrX7uB@cV=JK+6Aak@n|4Gf&$4=yo|r?&HsBW_XI3CTp-RHRny_iGzR zIv+32x@wRHdYW{Znt2`>8k~|?n^jM#(iK)}*D;x*IclfnS#(%N^U)t%a%b=qug^cK zE->=*j|_x2sHVbjzv1Y7Iut?%j@r8W2q~a)Dqk(ls%bMzPTL6K1M$@BS2#d1EI;ei zdIlvrEP+z@4x*7U3#Dp#+WU=nWvlTeBQ!yI7^Wt{j5k2JqR)w^*U%C86SzE(iD{(g zv%_b>CV^+)PFteXdY;=kZys%Bv1S<>f$^Tz98s`Vg;~I0iG$o6Dh3o8U5Y*%HgmKM zRGgp9Yk4NQrr+&)5C=LaD}ZGBK^24=B`6rU{2^L!5VHyq4)l*+I$ox1JmPD@rwllEt@w2a z0K`0gE{aVn=KYXwR-+1mwwOI&SN3?KH9|R*GNx4mQpicw0;J8 z)dPM8!pz|z8)51tTk=W5%%UNvR1y?uiIN}-Ls9DF6$OiXnK3nfTqN`>FlKL%K>rhc ziN!-Iy_qji04;TkdD4p_<+4d=!mUDMD`Kgda$#TvqjXm_j806vQ0uIn6va)PH~VPW zq>`Z7P@-*H8)>}NB(#Ciw<}_mRC+8I>iBl1k4X&gd;=Jt)k%U55+phM>FOb(N;V}Z zkwVxt2waY{%YcbRtEm|b?c0V8^4P}AjnDIdDG)!^aIbLaRLZrGqqC`U$gjm8`}aDX zza>68eDo74SU;7n){~M1{46W(FkZ?GX3d6k4ZN>Dv$pBTjbsBX8_>d`f_g# z2b6`1IeOTg)#ft1LGzT5hU_~5I0M2n;hemQTNB>5xYi{b)9^NviC<8L&QhbQT)AHX zUOS%j>ajyo;MXpiqd+JB%+E+Xjel_W1hU-#iZ{a62>QUBL@z!%^cj_^jHrt+h0jZ2 zTG}0}OZ;g`bIt}l+v3~a{vRgok#eUP+eR)kuQXZeU{|LqYYztsKy(P)EO zT-yGGbgqCcb2^o;+*?*J6`laW%eN4#w&@_gC)pQ9l=)6zFxjAPGiU50FffZXQAAq-@qX= z?r;8)T;kp+uUe+{W>AYVrVCVX0~qLz^i^KuJQ=2vwF=R3TkCLUVNDhBluahLwoIn- zB2!dMri_mJk{4(Du^d|+8McFTgsSs%cwF7DNI0VieU!wzEbA|1Lg`>skv1%Os5e5f zXVBy_=bU0-b8b&kVdp|_F4e~0qG?{l$69}2oEY4CsA{54n6}T}@UY$#he~#J?MED+ zYm1_lk>g1czIGwX5y-4d$R%YY2y@}rjQZ|Gzt6YX=NoL^nmkHez^z|tCf1+^=1cWV z)l*W0sEkqaY9Uh)1_6Exxd0-8kN?z^So-dtni2mBmiQN>0KmgFlcDv3KlI_CZS)jb z`T#DgENlvO`7k#n1bzv(^a~a1*J5CW>V~Wk(|*ZbHu^!IXD~+039m_~134yDO4dDe3FG@3$KY8xH*;UtTYPrhNA9|eF+2y&lsDGTgTBR%`g*UcvjvuE^ z!xlX-yL!N0s~L=G=`Em%K}eP|{vcR$J||^v;;|UIHA;%mIzxpxpaSTDfy<)eDd*UG z&4)E7lXxNH8`BAGApdICp4vTOrM&uzeb@MAR^#-GUz9~$83qfP%aVQpW?4ty$BrP) zewd(qNP1BfQ=^n`^Fk|b=Pf309_?NemG_oKxC%1^9wQ?ok1QkO-lNa@DRgzQAe~cZMxB4*H*_jfCA~HPe&C*4jtb;TBnPqnfmBQ07Mp3 zx9ZL9IjLnX&~?AT*H6Z!^P`pmezMUpEnX;wD(IyVnGkWw2mUmOD#oUCkhPjog({7O z*(CaRrR(0%M5gB&EBM;p#4J3_PDA1_iSq^^%v0H+@TXha(0Kutrg#3yJ%Ev}Ty(!r z4SDsgv5olFm5tfauHKChzZw`dtF(mWXZdzU_f{wIDC)xL>J;1tG5OvcYy0B)EcH-> zr}6KlIv6+*Cp9Z=9Q>6(&9|=P{eMSZ-}=jo=+0G=12ZV~mKmnUW3S+SZG%zBAH;w9 z*#EN+D7aHnUHb>_Wb{v=t8@fs+e_Os`#e|4EDXjxPpPqCT|S?M3SkIMwuPs$hxdQ{ z2UI8gq;6$_y(;ym?#9&%38NqTFay0ryH4G$T2{R4 z<)dfsN&t9Hb^%g{Q5=Cnj&U70wQWz%6<}@YbxOr-KzKc3hcDM?=XB()s~+qY97*8 zWLkOpK4kMlg^AD5j9_U?c!ixWl}6LLv3q!62BU~kugHUVP{adOP4PoYK3iw*sygFY z5JmDHiTVSkInhwwZxmtG3j8x|Tp}2aI*$gQDd%iQ8f26jGrv&7Bf+i4%4Oz2+dyFW_hl#)GT)25UdCDxw-m`kL21s*bqV%B_F9*jqlRIH;U; z=H9x#ThL-RQhq|6-=^&1*ZVv>rM6KX))%Wa7J&z;HdMi|Y4g7%QTA{&7Oj08_cDt- zjStSCF4j2bUzIbuX##Gov%L`Gag?>j<7dB7hn-d+o&y}!dLr% z$L_p2FlOq{$$c2_RD#NMV0XyIQ{eVVSM>`OuM9?oqCOp;LtcGcExj5Di(xTT=aDkc z?01`rD4Ce&h0#=zmKvKEkHr_BUi*0J)f*N!%{`%2zQyWV@b|&Rog^J{6CFh@z44Z)O#OgGa)oJ)dEr7-U8F7`#*OMJt+=)eb)lT&o1DH! z?GR5e)OR&;9(^U*82cIk*!CFF^=CXDJ{wEYPcx40_i&$M;DYN}kC_O;rm>Y_MkI+NE0;9>1zAoLdjGm=j9O8vw-CmhsEDipCg*t#3=9l!N^9Mn9=Aa5 z3xLY)D7qSvBukv|#t>bk>nu;7b!KQn;doDEw*O(ol1K9T(7QAnR1v!4`;{JQK18%rk6XX2Pw<>vAQ3tLxdB|!ax#W#7a z)Qe97GB~%U3fL>w5s0;tGiNj46T`7U(X7a&k}#N|t+i5HMI&>VkBg}o+)4tZo@a#A z;~0fsCms1D#V_y{c;Cw(@9#@aObp;w4pTxlk?pd)<8(6d6u`NxTo)%&msRqKQ_Gnp@LW z6>J@K*?+PUs#M7+5iBiGvp1yw@-WvvBP4RA-c%YZ{a7nZX9X#xT+gptrK?mC`hHRQ zN^U2n(l!;ppL(~U`hul5(wJ%(id~={tJK@_<=I$`U7t0|SOYPu5?8is5%NQ#q()fc z^G1$giXV=JqxV88UU9Hq)*51$jjTIqURKv6YfDJ!mbj?_8kF_;rue#h z3)o`X3fUj^N+v0_K1h?(ZLbHUV=|}NK7AEfP0rd?ooo2qaLEoF6F$1}B@g&(9LlqT6wHkhYX*nidKmu1jg~I!x+k@~%=|qnawP9B%mXoc6-EZCUD; z<7OUiw1g28+g;+mxTlBW`#Qyi-IWG4t(h!M*mk4l*777M;WJ4i$wdpVDCajjAieFo zg(I}k@3kQgb}x}$;gfuwC)}I95+p(;;lPwR(q{|c^U3POr-^LcI*lCy(bJ&)KK1Vu zTaUFwr;S*rSe&T#5K*iOxgkED@r9q-w)X22tkGI#OqS1+)g5f~4)pw4QUTsyrR66-<3pEhTbm&v~;X zXFQt~_i2U#OjO`HUS;xA*klsB1Z&cU;VcXQlKe| z%NOPG2T3_&3rU!!c?tKO*@*HkSu^juv%p!#c`X-8h7UyPS|^5WN#Q(^R(dpFU3QYp z^=qJpa|wjqNy2I5cNuCVzEX*NkxY`>T?R&nk3m=Y<`!>X6|vC4=9kcuL4yjUL8-~; z0`x}^9Xs5V9IDFt9#ve*r2TxgW;(Sr4xE0A+e{P~$y$W&Xz!@oL$xmeFyE&M_!nPGJm`m-yEiRsrpM0qXoDFG~r`(byV^beYc zS+uFUYJaF=71k6?q{{xJT)k=)2pwT@(cz6al8_D|Vphg@AG4sh$;sI+t!U(ZNZEn2 zMwWKEaL594F-*ey(^~#q=!AX^Rplg?!`S|Zfxq-23sTP9#=r(Wq1JbDqD3W8b+PtB(tmr+O#YlCkdVfGh zyKZv=18QPgtC;<`L0&Q*jb+OGc!*>Ks8dsJUg|X za}Bs_Mc;I$g#9axeR^0+QAgqN3*jHwbmb>Myv#0kMwx{s>aG_mW)1GP&+R<|eAwB1 z(YSLZ19DD%SVPfwJZuHP`5<@v3lie zR%A$^FppYsZ<^Os8O^APu&I5j%eZ#)^mt!LN@HRaEIPa^tphBQS(dtM-}-R@RRm0p zG$$c}^0(|HjYOHsHf)C4T-_4{cc^uIEo8N3CDmXoXRBlC$}}_2bSu0B@SN;fik|waX8v+=DwWJLk|rr5cif{;S8l8YDV$M zde|M0#TJ&*NKE-Q1pelr(!1&5)*$bN?q|D}O1D;bmr9iDEgJV!^}8PDrHw!tXYTEn z8vYg&w|dvc%Dp#XP;4_$muRQ!uaT#lq!S}wYxh8BJz`SbH?h?0yRXyr#YGgama#$EF&8+8oNkixFnssj5H z3J*x(A4Q%i`E@y;$A0swD@lFI0+fwV)`IufTlzd6UeJ~5jCE`{ai7G}f(m|DN5F3Y z3Ra7TnqQMCLY0>J-tlPD=j|F75Ib_^yGHCrtUL^vbZjCYBv;g?+cJr(33ceU4mJ`a zD*mv$p+KS9#{N3I`A(Qfc+`olgh*r?2kk;~wOao~^*y>y60kM`nk;M_(u~rvX*L-b zP)Qw0qN9?Md>vT*va9=-NP%0l#x&YSnr}+_)33%4bW(Aj)$A)YjM-39EPLFaTwm?u z6L6)>mI%EnL{qaB^jH}pATqxQnnIQO6cv^F6ETA+{J?jgp7i?PmN=+TeqE6 zpaQI8g;a)#S-v67*5wbr7RcP+MIkQ3z6w_03wj8zQ?Q8^x(xS!Iq|28$_yZxXSCob z73iIyD?9PG5x1O`9#5!S(S{~wBdLmXvZU}FIuR!A=Y6t9yu?k#J577tL4@6NhKA-6 z)(8%zd=M;^Bj@&8J^@{cLbqR!;mvh#ca$9H66mOMDD|sS_t*jEJ`YBIuY25o=^1Ff zxku@m?34Po>+G4-E?d9^Q5;pDWOeq`HAA$?jJPmn*2GvKXIX*)9ZeDDsynT<7fYxb z-MHF#IIQkTWs64DUWi1%uYYX4%+5?FO1jn{03CDwVbJfc+!e-k?;Mp5Fz@aLvm0UKKTHPwyQuQ&9QoD`D@f^z;=?5q_5x`ri z=r@NxG5B|dAN=A!Iyd|;Nbj}pJ*|9Xc1pJEsPdc^-f#-HOB1qS*$24a$b zktMX#&K$Ek8WkG)O;<%9OUYc(<}8cKWx1Z-no;~1{&X%A`rH6&Xj*qUCi+TwwX?mZ z5{O*cS$Jr>=;VIDS=f3C9M&@oh!3Ca1--!v&un@_reX4NGS|j0=XUd^^cXtjHiQok z_&6ZJa0$(LtKP4|+Us5ZReGMz%2IP~3#J*}eY+8jl9Mhu6*Dr?1oe1zkxBbPd~~1m zl+sUhj%O@>IG>_8H-4J-wKm;C>-ay@_@vE456ykeE8>K9klUzRG~Eojn|Gy^nF&gU zlIk^Ki{ekcB+}o~h`u2U27Ng%c5g9u{Q!q8k@V&d*}tWEbecS|o6?(fi0kaq$8pkX z#;kUn{5`FE=(n={@BWS6@mo}J|N2e~toFc30~OvDknAcY;~XtSLcZi_rfFsWhkA@6 z3?rmt$U+#Qz`l7fv1Q`loz2Kxews2))2lQftM9;!$UYo)!7A4)t>8~dUMQ{adjTM3 z{8I#EqafdEXk)yl>XMNL>O4Vi;~NB%x!R{C2cyEYEZygJrJi0X4g8skKWzmOtP|IJ zO??CM!>)X!Qhg!EF=fv(RK!_s01$1ikLRAm-0wl@0(%*DzgRK7b1DhHnCN2OqMmg; z*B}zFg4FmQ?KLo<23<^fG!$B^MJSB3+O^^*_UH<@BOQ_{whKnN$SF&Q^%%6x8B)gT zEXPK$^19)`qv{62;mBuM2rN2NG=;g0&}RwY4{$({VZeffKUoJ) z9)&O8D2qcgtll9pKx6p{8%eE5z0dxGM-%22EoB|uUH}||+b^lTYTnS}a^J#L`XsHm z+iiD+C|pu^S63?ek_aRFVCT*ON*AJ06Ai=TWUl`zk_b-^x6-cRo)0Er-uaR^IL1xk zT&lY~EpV{M;jY8p-(ek`Ga@I5ZODU>x5Be+0TaXig%k?UpclN6y%D>Sx3XJRSusN& z6=n|}U2jSD_xN^YZ1qm~m}byaP4l-jiN8zi$}_CG;{QV_rvIIu=dbh<=8ic+vnAW8 z;T4?2KwFwaIiPi1HTN01lOl%mlU}Z3DDdEIg;KGyJIBSVzTpkNOu#&+NP2p!Ol@h( zDknmAt(Z@GxdG~=yA5K>9r@Dd%=+wG(xXJG8btr4aOmvyrGWS0oX0a;usU#Og8p!nxc{b^35s zFzPHbM^KFCar{=jMH80`WzuSK7^l#23uUIO+Wkon^WkB;%u5E$U_Q?m1(DD2EbyHx z0uW$Ms23lKvX$yPd0xPTF6D8Y+DEWiZ@*5R2)-RP(V?XRhOWc0>Q8z3Yx@ zD(m)f9I=cjMQ{KCsS46NYCt+ks3FJ@nivQq1eB&25iEeTC`5`0O{INNAI;3!I;*k`V)n| zbzUC}1!j6#d{m9->7{@085h&P{W`>MMCc;CxAG-=!U@gD1yf*?3JFg`I9R2u_+P#^ z8PODrFlj>?4S-3V1BRvV+Q?%U?N^hqWG}EsQtoR49%7qSR`L;tC@ohQGEFowsy-XZ z#g7qQUOeNMD8%s^UWT{yP4=&+so$MaO||;ji%_;K&{8GhR6=Eo(vBe0HPS_5$$f!y zp0l}B*n8?TpUR1X9ThRMmwLd^2;L>Mxl~t~fJ@j7gV%mqEK4f|d{PPtiRI;)YeEir z@s2f#T{A8$3+^|MPuxR%EtkS<|2W5{@WE5%4qb3nQ<{Fq7E~qr2*8F&=wfn5z?&JK zReRS4?|7ibFh!1ZyG?y*PQJsY^=*nI9qFnazuX)rDr>1_MlE&eousqEpXRP_YxBzH zz5T{DWADL*x-0=%sLRc5U{HM?flRU2m7g#WS~??EQ1Ct_7pT#19pBXH&+)`uQ8ew|1wzid_+FRhMIU)KxSUU0@=z(%aZMQ%(ug;*jsM~MTDGt$b zl^`WUWuaNGxd-poZ#FXD=hV9Rr+Q&*`9FSG7jrwd0kC%~C`C2e>b_lNs(#e^;L6Gy zeEU&v;Eu5c`jXmF$z~Mrakfl$Snn&sngw!igXr_os7#3$Xo?R7O$j7i#G+y)lQ9SWjB&rM#Wd9A0?~YA0dL;PN5r=AR-6oHlxjgy zHo2UBI6f&^`f2>9t)*Gax9SX|@H&R!tuX_;hT45#drr20r8KJg#*X&$K^1n{sBCZ7 zgQBNT_MFexOG8&y*PT&?R1?c{K!ZEnesyfP=}kO{n1Av6qRE#)Zq`sE=z;F~@- z1C|def4tSfm~~?!F6xoVkaeQ{O#_GuN#0quduhij5>RHEE5=WnDfj&tENaD34jHj( zsCDr1tGn9+zPCW2RfF`U*nWL!xuzB?4<6C#_f7EWSgvz7^VZcB&oW#ZRhx3$cWe5R zN=F{VEbKxn$>g_HOLj|X5oTr1B-VmVZ+PELcVbQFsxM~VIr=Qv{>?Gnt0PURTo`hP z-<8B^C2gD2$-6}GxmI}$QJ6zQ)at&7Dglp7%b<%>!buSdd4eW+=jaNa$shQdl2SSq z>mL)D672gSOkT-4B}qhKOx~>p;)vOiWp7LTuo$pg^L{Bk)aPtk?uIEvYd;J^L z;cG!!kmt69S?ny9^X7A4N*V1zi4;=q9n{Y6%`)y2R+=Ma zfV`FS&7nPRpoE7;1hG&g`^njq-bk`|M6HMCi%)}-Y^e#6SN-S|OFc}1yv9rbDo_kK z1PkzXrN}EEXeKc+w;4?%}Lue zFwB1I|1jX#uHp;u%$wFfeGxR30y>QlfvJPdTUzGjtJr8P5P0f~z^G~ma^pJOybf%a zx<{ExCQmP~T>BzGrxcd8v~@52>yP~lN-HiqKU^w+AFV)+tZJ%FQf)e} zqLcC;q51eltHKYDK>R2_n2x5&BD07YBbH_tz7*|u!TmOA{=6MfD9!;<4b1RIc(xZ( z5)h!a*AAF&`TYv-?NtTekqr0TW6+DT5(0K-ys%eAUW0fur9EgIf~fUHTH;K<7IgB?+slRooeRQRjuOXr#k&nzdO$(HBAWlg6l)R5gCH+Qm9__L#Zb zcOJ3po|)VJyTFy-;W&-)JNxgbPv2JKzfhjPrl^-E&Ivd#l>x;v?|l)_&g{&nrm4N( zYFavfs=e(V{ni_RIK100b-+`jF>~?j0Z&jbRUVzKwkz3_8Dx#)cn%`o6jb7t7|=l*~XpbL~{zd zU&uW5D9l)gDD>*3441drA-P!(lkB}ZZmdpBTil{qG~2RbU;h4WVl2$F0}pcGrCzPW zY_?cd__V(AEfup`3oWzp5s9I0VF&Kq(}fSNljb{vn4@!i9Kt=3aB+Kevy>GIcTiQ6jWYTFhvN*BlW!@#IptbyJM#wWh9#{3Q8j@EVGL#fbQV&y6(+7m; znk7-Hsh>SFEc`GjIu;e5BYpzlUippQ?b9&vx2Yfh+%dAd?@w;6KT=wp*1vr~;60<) z0943SybWdRS4L!uELO5oXbcHCMe9U4&gvft-2XYo;mxJpM4ilA<@nRvM@@_h?~uus zJ8*jc1jQ*~n%;Ek-uNsSS^yi816v}KBxG|HgA5@LPiU!pQqVsQKmHko^jUwmyc@mUpdBD8aH*}=Nyu!7PtUzhtYy*W$C_QX&iwY^0H0X! z15JS=5%0uXpFIgp!dsr%0HlS7Pwg_s<*U(zOQ1DM@`|J)L)?m&SEkc<0s0be>J zPdWUm6r@utS+nwDi%d~U#?C{AxKa{#WPt4GO>_sum$3l=YMLIMSg{o;JR3FDmS~|} zZ<-on^s|&~)&4gvF>X2tOkFYtEVJU$(tm_ceuWr0q%|Ee$dpau5)GD)%7F2opP=Ab znxNyp!Y%Z!b#%~5^ZtjlKg9`wH@10K%^m&Luh7SsXEi4mg0PK4{Xp^^7x`q zL44~P6%4Wz;)5m2AbjeVxVr4uDvHI(ee-R=ar0C(?4Yh0)jT(+J{zxSl4dc_!aDrPGBKe8dB@-8{yDWOko)?7O>=L?fL5pXBWW*Q_p~oVq2nDsU=} zaQrn5Guie@>6#dj;Ij*-Y8}HKxDbL^Q)(_vPc&ne__d6bQP#mz*^0^KV^+tCswCJl zeYhW(&J^GeCuV=Kyo0{2mvxr)%3D_~E4aO=OD=&18F_bJ&6|CUjqSk6O5yVvQi{Cd z-AP4#R;ty;g9=Z}cRD$-)yw7Pwa?hioR?tC@gy(Jsg3!w=pmL({#7b7J5lOz&xVvF zoi}YS2H?AKRxhO;odyHBMY+gEX~sn`XPW*V3s#xoSYbPrHXRGisU%*zOR+qiF7b|`IA+v!wxj%f+&)r}txmL*xq#ri_KwZP(sND^$dR@z=N%7F{7 z)bZl_gY$Ig=q=ZZnv&XqICRsjeYAtBWu{q--Ud-oK?v;6brcG#sA%W)sZ z9jGaXT=a52h8ePl93^-Y$pX|cMcPq~>Iky&ioB}%;n%?zo(Mx(p7lvYizE;f;=*&) zdc}KwHMWQhW8s7&fvjZN95!6jFdLqs=hl|WZG-dEmta@U?DHh&Wuu`s8QSLY(GSCJ z^GozCr6?7+tZMAAp^7V#dnk$qn(1efPh?5*Q+>ki`O(gviqg62lD52rnZlfS?kumN za{~M&?a;I(;qZ1@hK`a@bNB+^M2)=RYC1IeP39%$OmSpgjyc}gB$Hz|zF~X53NHH% z0r^#{^Nl<|r}_7@XFMO&znf_1>dYP}f8Jd-i4t5;Qjd`UT>0%KB$A&B>oUTF+e!~ao>SI#Q*&y8poY*NJMG1 z4D+Q>zv9US8mU4rsIbUPPeA3gp&=iqhuSI4kgXLa5aXNHzTpx4`oH^CCHShl!1*?x zw}1R2{MEew#S4ETE%~n~OumxucNc;%?=HWLou`sk9=+WbR;JMfMQcKk`>L!LB@UZ@ zDAvlS2JZkv>?XkGaS72j55K2w3a)T{Q-}ls4y<^St0S=~9s`G88CNh0C;J)Y^a@>Vyh)Y=XV_u#nU{)*}S Kd^&t%@IL?ojJ^N> From bf8d63820052c55336b607071f3720c19c1d1c5e Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Mon, 15 Nov 2021 19:53:48 -0600 Subject: [PATCH 3/8] Refactored file source, added type and format configuration options for json and plaintext Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- data-prepper-plugins/common/README.md | 24 +- data-prepper-plugins/common/build.gradle | 4 +- .../plugins/source/FileSource.java | 91 ------- .../plugins/source/file/FileFormat.java | 37 +++ .../plugins/source/file/FileSource.java | 114 +++++++++ .../plugins/source/file/FileSourceConfig.java | 42 ++++ .../plugins/source/FileSourceTests.java | 155 ------------ .../plugins/source/file/FileSourceTests.java | 224 ++++++++++++++++++ .../test-file-source-invalid-json.tst | 2 + .../test/resources/test-file-source-json.tst | 2 + .../test/resources/test-file-source-plain.tst | 2 + .../src/test/resources/test-file-source.tst | 1 - 12 files changed, 447 insertions(+), 251 deletions(-) delete mode 100644 data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/FileSource.java create mode 100644 data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java create mode 100644 data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java create mode 100644 data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java delete mode 100644 data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/FileSourceTests.java create mode 100644 data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java create mode 100644 data-prepper-plugins/common/src/test/resources/test-file-source-invalid-json.tst create mode 100644 data-prepper-plugins/common/src/test/resources/test-file-source-json.tst create mode 100644 data-prepper-plugins/common/src/test/resources/test-file-source-plain.tst delete mode 100644 data-prepper-plugins/common/src/test/resources/test-file-source.tst diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index 86da8ea331..b85fc561dd 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -8,9 +8,27 @@ A prepper plugin to generate new string records with upper or lower case convers ## `file` (source) -A source plugin to read input data from the specified file path. - -- path (String): absolute input data file path +A source plugin to read input data from the specified file path. The file source creates a new Record for eadh line of data in the file. + +* `path` (String): absolute input data file path. It is required + +* `write_timeout` (int): The amount of time to attempt writing a Record to the Buffer before timing out. Unit is milliseconds and default is `5,000` + +* `format` (String): The format of each line of the file. Valid options are `json` or `plain`. Default is `plain`. +

+ * `plain`: Reads plaintext data from files. Internally, a plain text line from a file will be given a key of `message` as shown below. + ``` + Example log line in file + ``` + becomes + ``` + { "message": "Example log line in file" } + ``` + + * `json`: Reads data that is in the form of a JSON string from a file. If the json string is unable to be parsed, the file source will treat it as a plaintext line. + + +* type (String): The Event type that will be stored in the metadata of the Event. Default is `event`. ## `file` (sink) diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index 22e498359c..6b62e66279 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -14,6 +14,7 @@ plugins { } dependencies { api project(':data-prepper-api') + implementation project(':data-prepper-plugins:blocking-buffer') implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation "commons-io:commons-io:2.11.0" @@ -25,6 +26,7 @@ dependencies { implementation 'org.reflections:reflections:0.10.2' testImplementation 'commons-io:commons-io:2.11.0' testImplementation "org.hamcrest:hamcrest:2.2" + testImplementation "org.mockito:mockito-inline:${versionMap.mockito}" } jacocoTestCoverageVerification { @@ -32,7 +34,7 @@ jacocoTestCoverageVerification { violationRules { rule { //in addition to core projects rule limit { - minimum = 0.90 + minimum = 0.89 } } } diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/FileSource.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/FileSource.java deleted file mode 100644 index a2b75aed09..0000000000 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/FileSource.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package com.amazon.dataprepper.plugins.source; - -import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; -import com.amazon.dataprepper.model.buffer.Buffer; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.model.source.Source; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.concurrent.TimeoutException; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.lang.String.format; - -@DataPrepperPlugin(name = "file", pluginType = Source.class) -public class FileSource implements Source> { - private static final Logger LOG = LoggerFactory.getLogger(FileSource.class); - private static final String ATTRIBUTE_PATH = "path"; - private static final String ATTRIBUTE_TIMEOUT = "write_timeout"; - private static final int WRITE_TIMEOUT = 5_000; - - private final String filePathToRead; - private final int writeTimeout; - private final String pipelineName; - private boolean isStopRequested; - - - /** - * Mandatory constructor for Data Prepper Component - This constructor is used by Data Prepper - * runtime engine to construct an instance of {@link FileSource} using an instance of {@link PluginSetting} which - * has access to pluginSetting metadata from pipeline - * pluginSetting file. - * - * @param pluginSetting instance with metadata information from pipeline pluginSetting file. - */ - public FileSource(final PluginSetting pluginSetting) { - this((String) checkNotNull(pluginSetting, "PluginSetting cannot be null") - .getAttributeFromSettings(ATTRIBUTE_PATH), - pluginSetting.getIntegerOrDefault(ATTRIBUTE_TIMEOUT, WRITE_TIMEOUT), - pluginSetting.getPipelineName()); - } - - public FileSource(final String filePath, final int writeTimeout, final String pipelineName) { - if (filePath == null || filePath.isEmpty()) { - throw new RuntimeException(format("Pipeline [%s] - path is a required attribute for file source", - pipelineName)); - } - this.filePathToRead = filePath; - this.writeTimeout = writeTimeout; - this.pipelineName = checkNotNull(pipelineName, "Pipeline name cannot be null"); - isStopRequested = false; - } - - - @Override - public void start(final Buffer> buffer) { - checkNotNull(buffer, format("Pipeline [%s] - buffer cannot be null for file source to start", pipelineName)); - try (BufferedReader reader = Files.newBufferedReader(Paths.get(filePathToRead), StandardCharsets.UTF_8)) { - String line; - while ((line = reader.readLine()) != null && !isStopRequested) { - buffer.write(new Record<>(line), writeTimeout); - } - } catch (IOException | TimeoutException ex) { - LOG.error("Pipeline [{}] - Error processing the input file [{}]", pipelineName, filePathToRead, ex); - throw new RuntimeException(format("Pipeline [%s] - Error processing the input file %s", pipelineName, - filePathToRead), ex); - } - } - - @Override - public void stop() { - isStopRequested = true; - } -} \ No newline at end of file diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java new file mode 100644 index 0000000000..56f49127b6 --- /dev/null +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java @@ -0,0 +1,37 @@ +package com.amazon.dataprepper.plugins.source.file; + +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * An enumm to represent the file data types supported in Data Prepper. + * @since 1.2 + */ +@JsonFormat(shape = JsonFormat.Shape.STRING) +public enum FileFormat { + + PLAIN("plain"), + JSON("json"); + + private static final Map NAMES_MAP = Arrays.stream(FileFormat.values()) + .collect(Collectors.toMap(FileFormat::toString, Function.identity())); + + private final String name; + + FileFormat(final String name) { + this.name = name; + } + + public String toString() { + return this.name; + } + + @JsonValue + public static FileFormat getByName(final String name) { + return NAMES_MAP.get(name.toLowerCase()); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java new file mode 100644 index 0000000000..57a8b89aba --- /dev/null +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package com.amazon.dataprepper.plugins.source.file; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; +import com.amazon.dataprepper.model.annotations.DataPrepperPluginConstructor; +import com.amazon.dataprepper.model.buffer.Buffer; +import com.amazon.dataprepper.model.event.Event; +import com.amazon.dataprepper.model.event.JacksonEvent; +import com.amazon.dataprepper.model.plugin.PluginFactory; +import com.amazon.dataprepper.model.record.Record; +import com.amazon.dataprepper.model.source.Source; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeoutException; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; + +@DataPrepperPlugin(name = "file", pluginType = Source.class, pluginConfigurationType = FileSourceConfig.class) +public class FileSource implements Source> { + + static final String MESSAGE_KEY = "message"; + private static final Logger LOG = LoggerFactory.getLogger(FileSource.class); + private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final FileSourceConfig fileSourceConfig; + + private boolean isStopRequested; + + @DataPrepperPluginConstructor + public FileSource(final FileSourceConfig fileSourceConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory) { + Objects.requireNonNull(fileSourceConfig.getFilePathToRead(), "File path is required"); + Objects.requireNonNull(fileSourceConfig.getFormat(), "Invalid file format. Options are [json] and [plain]"); + this.fileSourceConfig = fileSourceConfig; + this.isStopRequested = false; + } + + + @Override + public void start(final Buffer> buffer) { + checkNotNull(buffer, "Buffer cannot be null for file source to start"); + try (BufferedReader reader = Files.newBufferedReader(Paths.get(fileSourceConfig.getFilePathToRead()), StandardCharsets.UTF_8)) { + String line; + while ((line = reader.readLine()) != null && !isStopRequested) { + buffer.write(getEventRecordFromLine(line), fileSourceConfig.getWriteTimeout()); + } + } catch (IOException | TimeoutException ex) { + LOG.error("Error processing the input file path [{}]", fileSourceConfig.getFilePathToRead(), ex); + throw new RuntimeException(format("Error processing the input file %s", + fileSourceConfig.getFilePathToRead()), ex); + } + } + + @Override + public void stop() { + isStopRequested = true; + } + + private Record getEventRecordFromLine(final String line) { + Map structuredLine = new HashMap<>(); + + switch(fileSourceConfig.getFormat()) { + case JSON: + structuredLine = parseJson(line); + break; + case PLAIN: + structuredLine.put(MESSAGE_KEY, line); + break; + default: + LOG.error("The file source type is [{}]. It must be \"json\" or \"plain\"", fileSourceConfig.getType().toString()); + } + + return new Record<>(JacksonEvent + .builder() + .withEventType(fileSourceConfig.getType()) + .withData(structuredLine) + .build()); + } + + private Map parseJson(final String jsonString) { + try { + return OBJECT_MAPPER.readValue(jsonString, MAP_TYPE_REFERENCE); + } catch (JsonProcessingException e) { + LOG.error("Unable to parse json data [{}], assuming plain text", jsonString, e); + final Map plainMap = new HashMap<>(); + plainMap.put(MESSAGE_KEY, jsonString); + return plainMap; + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java new file mode 100644 index 0000000000..689321d94b --- /dev/null +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java @@ -0,0 +1,42 @@ +package com.amazon.dataprepper.plugins.source.file; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class FileSourceConfig { + static final String ATTRIBUTE_PATH = "path"; + static final String ATTRIBUTE_TIMEOUT = "write_timeout"; + static final String ATTRIBUTE_TYPE = "type"; + static final String ATTRIBUTE_FORMAT = "format"; + static final int DEFAULT_TIMEOUT = 5_000; + static final String DEFAULT_TYPE = "event"; + + + + @JsonProperty(ATTRIBUTE_PATH) + private String filePathToRead; + + @JsonProperty(ATTRIBUTE_TIMEOUT) + private int writeTimeout = DEFAULT_TIMEOUT; + + @JsonProperty(ATTRIBUTE_FORMAT) + private FileFormat format = FileFormat.PLAIN; + + @JsonProperty(ATTRIBUTE_TYPE) + private String type = DEFAULT_TYPE; + + public String getFilePathToRead() { + return filePathToRead; + } + + public int getWriteTimeout() { + return writeTimeout; + } + + public FileFormat getFormat() { + return format; + } + + public String getType() { + return type; + } +} diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/FileSourceTests.java deleted file mode 100644 index 42981af9b3..0000000000 --- a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/FileSourceTests.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package com.amazon.dataprepper.plugins.source; - -import com.amazon.dataprepper.model.CheckpointState; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.plugins.buffer.TestBuffer; -import com.google.common.collect.ImmutableMap; -import org.junit.Test; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.Map; -import java.util.Queue; - -import static java.lang.String.format; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.MatcherAssert.assertThat; - -public class FileSourceTests { - private static final String TEST_PIPELINE_NAME = "test-pipeline"; - private static final int TEST_WRITE_TIMEOUT = 100; - private static final String TEST_FILE_PATH = "src/test/resources/test-file-source.tst"; - private static final String FILE_DOES_NOT_EXIST = "file_does_not_exist"; - private static final String FILE_CONTENT = "THIS IS A TEST"; - - - @Test - public void testFileSourceWithEmptyFilePath() { - try { - new FileSource("", TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - } catch (RuntimeException ex) { - assertThat(ex.getMessage(), is(equalTo(format("Pipeline [%s] - path is a required attribute for file " + - "source", TEST_PIPELINE_NAME)))); - } - } - - @Test - public void testFileSourceWithNullFilePath() { - try { - new FileSource(null, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - } catch (RuntimeException ex) { - assertThat(ex.getMessage(), is(equalTo(format("Pipeline [%s] - path is a required attribute for file " + - "source", TEST_PIPELINE_NAME)))); - } - } - - @Test - public void testFileSourceCreationWithValues() { - final FileSource fileSource = new FileSource(TEST_FILE_PATH, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - assertThat(fileSource, notNullValue()); - } - - @Test - public void testFileSourceCreationWithNullPipelineName() { - try { - new FileSource(TEST_FILE_PATH, TEST_WRITE_TIMEOUT, null); - } catch (NullPointerException ex) { - assertThat(ex.getMessage(), is(equalTo("Pipeline name cannot be null"))); - } - } - - @Test - public void testFileSourceCreationUsingPluginSettings() { - final Map settingMap = ImmutableMap.of( - "path", TEST_FILE_PATH); - final PluginSetting pluginSetting = new PluginSetting("file", settingMap); - pluginSetting.setPipelineName(TEST_PIPELINE_NAME); - final FileSource fileSource = new FileSource(pluginSetting); - assertThat(fileSource, notNullValue()); - } - - @Test - public void testFileSourceCreationWithNullPluginSetting() { - try { - new FileSource(null); - } catch (NullPointerException ex) { - assertThat(ex.getMessage(), is(equalTo("PluginSetting cannot be null"))); - } - } - - @Test - public void testFileSourceWithNonExistentFile() { - final Queue> bufferQueue = new LinkedList<>(); - final FileSource fileSource = new FileSource(FILE_DOES_NOT_EXIST, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - try { - fileSource.start(new TestBuffer(bufferQueue, 1, true)); - } catch (RuntimeException ex) { - assertThat(ex.getMessage(), is(equalTo(format("Pipeline [%s] - Error processing the input file %s", - TEST_PIPELINE_NAME, FILE_DOES_NOT_EXIST)))); - } - } - - @Test - public void testFileSourceWriteTimeoutException() { - final Queue> bufferQueue = new LinkedList<>(); - final TestBuffer buffer = new TestBuffer(bufferQueue, 1, true); - final FileSource fileSource = new FileSource(TEST_FILE_PATH, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - try { - fileSource.start(buffer); - } catch (RuntimeException ex) { - assertThat(ex.getMessage(), is(equalTo(format("Pipeline [%s] - Error processing the input file %s", - TEST_PIPELINE_NAME, TEST_FILE_PATH)))); - } - } - - @Test - public void testFileSourceWritingToBuffer() { - final Queue> bufferQueue = new LinkedList<>(); - final TestBuffer buffer = new TestBuffer(bufferQueue, 1); - final FileSource fileSource = new FileSource(TEST_FILE_PATH, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - assertThat(buffer.size(), is(equalTo(0))); - fileSource.start(buffer); - assertThat(buffer.size(), is(equalTo(1))); - final Map.Entry>, CheckpointState> readResult = buffer.read(TEST_WRITE_TIMEOUT); - final Collection> recordsFromBuffer = readResult.getKey(); - assertThat(recordsFromBuffer.size(), is(equalTo(1))); - recordsFromBuffer.forEach(actualRecord -> assertThat(actualRecord.getData(), is(equalTo(FILE_CONTENT)))); - } - - @Test - public void testFileSourceStartAfterStop() { - final Queue> bufferQueue = new LinkedList<>(); - final TestBuffer buffer = new TestBuffer(bufferQueue, 1); - final FileSource fileSource = new FileSource(TEST_FILE_PATH, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - assertThat(buffer.size(), is(equalTo(0))); - fileSource.stop(); - fileSource.start(buffer); //should not write any records to buffer - assertThat(buffer.size(), is(equalTo(0))); - } - - @Test - public void testFileSourceWithNullBuffer() { - final FileSource fileSource = new FileSource(TEST_FILE_PATH, TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME); - try { - fileSource.start(null); - } catch (NullPointerException ex) { - assertThat(ex.getMessage(), is(equalTo(format("Pipeline [%s] - buffer cannot be null for file source to start", - TEST_PIPELINE_NAME)))); - } - } - -} diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java new file mode 100644 index 0000000000..653741af23 --- /dev/null +++ b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java @@ -0,0 +1,224 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package com.amazon.dataprepper.plugins.source.file; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.buffer.Buffer; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.event.Event; +import com.amazon.dataprepper.model.event.JacksonEvent; +import com.amazon.dataprepper.model.plugin.PluginFactory; +import com.amazon.dataprepper.model.record.Record; +import com.amazon.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@ExtendWith(MockitoExtension.class) +public class FileSourceTests { + private static final Logger LOG = LoggerFactory.getLogger(FileSourceTests.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; + + private static final String TEST_PIPELINE_NAME = "pipeline"; + private static final int TEST_WRITE_TIMEOUT = 100; + private static final String TEST_FILE_PATH_PLAIN = "src/test/resources/test-file-source-plain.tst"; + private static final String TEST_FILE_PATH_JSON = "src/test/resources/test-file-source-json.tst"; + private static final String TEST_FILE_PATH_INVALID_JSON = "src/test/resources/test-file-source-invalid-json.tst"; + private static final String FILE_DOES_NOT_EXIST = "file_does_not_exist"; + + private FileSourceConfig fileSourceConfig; + private FileSource fileSource; + + @Mock + private PluginMetrics pluginMetrics; + + @Mock + private PluginFactory pluginFactory; + + private Buffer> buffer; + + final Map pluginSettings = new HashMap<>(); + + final List> expectedEventsPlain = new ArrayList<>(); + final List> expectedEventsJson = new ArrayList<>(); + final List> expectedEventsInvalidJson = new ArrayList<>(); + + + @BeforeEach + public void setup() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_PLAIN); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, "new_type"); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_TIMEOUT, TEST_WRITE_TIMEOUT); + + // plain + final String expectedPlainFirstLine = "THIS IS A PLAINTEXT LINE"; + final String expectedPlainSecondLine = "THIS IS ANOTHER PLAINTEXT LINE"; + + + final Record firstEventPlain = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedPlainFirstLine); + final Record secondEventPlain = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedPlainSecondLine); + + expectedEventsPlain.add(firstEventPlain); + expectedEventsPlain.add(secondEventPlain); + + //json + final String expectedJsonFirstLine = "{\"test_key: \"test_value\"}"; + final String expectedJsonSecondLine = "{\"second_test_key\": \"second_test_value\"}"; + + final Record firstEventJson = createRecordEventWithKeyValuePair("test_key", "test_value"); + final Record secondEventJson = createRecordEventWithKeyValuePair("second_test_key", "second_test_value"); + + expectedEventsJson.add(firstEventJson); + expectedEventsJson.add(secondEventJson); + + // invalid json + final String expectedInvalidJsonFirstLine = "{\"test_key: test_value\"}"; + final String expectedInvalidJsonSecondLine = "{\"second_test_key\": \"second_test_value\""; + + + final Record firstEventInvalidJson = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedInvalidJsonFirstLine); + final Record secondEventInvalidJson = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedInvalidJsonSecondLine); + + expectedEventsInvalidJson.add(firstEventInvalidJson); + expectedEventsInvalidJson.add(secondEventInvalidJson); + + + + buffer = getBuffer(); + } + + private FileSource createObjectUnderTest() { + fileSourceConfig = OBJECT_MAPPER.convertValue(pluginSettings, FileSourceConfig.class); + return new FileSource(fileSourceConfig, pluginMetrics, pluginFactory); + } + + private BlockingBuffer> getBuffer() { + final HashMap integerHashMap = new HashMap<>(); + integerHashMap.put("buffer_size", 2); + integerHashMap.put("batch_size", 2); + final PluginSetting pluginSetting = new PluginSetting("blocking_buffer", integerHashMap) {{ + setPipelineName(TEST_PIPELINE_NAME); + }}; + return new BlockingBuffer<>(pluginSetting); + } + + @Test + public void testFileSourceWithEmptyFilePathThrowsRuntimeException() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, ""); + fileSource = createObjectUnderTest(); + assertThrows(RuntimeException.class, () -> fileSource.start(buffer)); + } + + @Test + public void testFileSourceWithNonexistentFilePathThrowsRuntimeException() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, FILE_DOES_NOT_EXIST); + fileSource = createObjectUnderTest(); + assertThrows(RuntimeException.class, () -> fileSource.start(buffer)); + } + + @Test + public void testFileSourceWithNullFilePathThrowsNullPointerException() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, null); + assertThrows(NullPointerException.class, this::createObjectUnderTest); + } + + @Test + public void testFileWithPlainTextAddsEventsToBufferCorrectly() { + fileSource = createObjectUnderTest(); + fileSource.start(buffer); + + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + + assertThat(bufferEvents.size(), equalTo(expectedEventsPlain.size())); + assertExpectedRecordsAreEqual(expectedEventsPlain, bufferEvents); + } + + @Test + public void testFileWithJSONAddsEventsToBufferCorrectly() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_JSON); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, FileFormat.getByName("json")); + fileSource = createObjectUnderTest(); + fileSource.start(buffer); + + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + + assertThat(bufferEvents.size(), equalTo(expectedEventsJson.size())); + assertExpectedRecordsAreEqual(expectedEventsJson, bufferEvents); + } + + @Test + public void testFileWithInvalidJSONAddsEventsToBufferAsPlainText() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_INVALID_JSON); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, FileFormat.getByName("json")); + fileSource = createObjectUnderTest(); + fileSource.start(buffer); + + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + + assertThat(bufferEvents.size(), equalTo(expectedEventsInvalidJson.size())); + assertExpectedRecordsAreEqual(expectedEventsInvalidJson, bufferEvents); + } + + @Test + public void testNonSupportedFileFormatThrowsNullPointerException() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, FileFormat.getByName("unsupported")); + assertThrows(NullPointerException.class, this::createObjectUnderTest); + } + + static void assertExpectedRecordsAreEqual(final List> expectedEvents, final List> actualEvents) { + for (int i = 0; i < expectedEvents.size(); i++) { + assertThat(actualEvents.get(i), notNullValue()); + assertThat(actualEvents.get(i).getData(), notNullValue()); + assertRecordsAreEqual(actualEvents.get(i), expectedEvents.get(i)); + } + } + + static void assertRecordsAreEqual(final Record first, final Record second) { + try { + final Map recordMapFirst = OBJECT_MAPPER.readValue(first.getData().toJsonString(), MAP_TYPE_REFERENCE); + final Map recordMapSecond = OBJECT_MAPPER.readValue(second.getData().toJsonString(), MAP_TYPE_REFERENCE); + assertThat(recordMapFirst, is(equalTo(recordMapSecond))); + } catch (JsonProcessingException e) { + LOG.error("Unable to parse Event as JSON"); + } + } + + private Record createRecordEventWithKeyValuePair(final String key, final String value) { + final Map eventData = new HashMap<>(); + eventData.put(key, value); + + return new Record<>(JacksonEvent + .builder() + .withEventType("event") + .withData(eventData) + .build()); + } +} diff --git a/data-prepper-plugins/common/src/test/resources/test-file-source-invalid-json.tst b/data-prepper-plugins/common/src/test/resources/test-file-source-invalid-json.tst new file mode 100644 index 0000000000..5cdfe97552 --- /dev/null +++ b/data-prepper-plugins/common/src/test/resources/test-file-source-invalid-json.tst @@ -0,0 +1,2 @@ +{"test_key: test_value"} +{"second_test_key": "second_test_value" \ No newline at end of file diff --git a/data-prepper-plugins/common/src/test/resources/test-file-source-json.tst b/data-prepper-plugins/common/src/test/resources/test-file-source-json.tst new file mode 100644 index 0000000000..0d0d8037ab --- /dev/null +++ b/data-prepper-plugins/common/src/test/resources/test-file-source-json.tst @@ -0,0 +1,2 @@ +{"test_key": "test_value"} +{"second_test_key": "second_test_value"} \ No newline at end of file diff --git a/data-prepper-plugins/common/src/test/resources/test-file-source-plain.tst b/data-prepper-plugins/common/src/test/resources/test-file-source-plain.tst new file mode 100644 index 0000000000..226beb530d --- /dev/null +++ b/data-prepper-plugins/common/src/test/resources/test-file-source-plain.tst @@ -0,0 +1,2 @@ +THIS IS A PLAINTEXT LINE +THIS IS ANOTHER PLAINTEXT LINE \ No newline at end of file diff --git a/data-prepper-plugins/common/src/test/resources/test-file-source.tst b/data-prepper-plugins/common/src/test/resources/test-file-source.tst deleted file mode 100644 index ca80c316c9..0000000000 --- a/data-prepper-plugins/common/src/test/resources/test-file-source.tst +++ /dev/null @@ -1 +0,0 @@ -THIS IS A TEST \ No newline at end of file From 7fb93292793f673dd8532c27d0d864de5c1bb2b9 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Tue, 16 Nov 2021 10:31:29 -0600 Subject: [PATCH 4/8] Address PR comments Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- data-prepper-plugins/common/README.md | 11 +++++++++-- .../dataprepper/plugins/source/file/FileSource.java | 3 +++ .../plugins/source/file/FileSourceTests.java | 3 +-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index b85fc561dd..60eb35a603 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -8,7 +8,7 @@ A prepper plugin to generate new string records with upper or lower case convers ## `file` (source) -A source plugin to read input data from the specified file path. The file source creates a new Record for eadh line of data in the file. +A source plugin to read input data from the specified file path. The file source creates a new Record for each line of data in the file. * `path` (String): absolute input data file path. It is required @@ -26,9 +26,16 @@ A source plugin to read input data from the specified file path. The file source ``` * `json`: Reads data that is in the form of a JSON string from a file. If the json string is unable to be parsed, the file source will treat it as a plaintext line. + Expects json lines as follows: + ``` + { "key1": "val1" } + { "key2": "val2" } + { "key3": "val3" } + ``` + -* type (String): The Event type that will be stored in the metadata of the Event. Default is `event`. +* `type` (String): The Event type that will be stored in the metadata of the Event. Default is `event`. ## `file` (sink) diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java index 57a8b89aba..468806b478 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java @@ -54,6 +54,9 @@ public class FileSource implements Source> { @DataPrepperPluginConstructor public FileSource(final FileSourceConfig fileSourceConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory) { Objects.requireNonNull(fileSourceConfig.getFilePathToRead(), "File path is required"); + if (!Files.exists(Paths.get(fileSourceConfig.getFilePathToRead()))) { + throw new IllegalArgumentException("File path does not exist."); + } Objects.requireNonNull(fileSourceConfig.getFormat(), "Invalid file format. Options are [json] and [plain]"); this.fileSourceConfig = fileSourceConfig; this.isStopRequested = false; diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java index 653741af23..4cdfccd94e 100644 --- a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java +++ b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java @@ -140,8 +140,7 @@ public void testFileSourceWithEmptyFilePathThrowsRuntimeException() { @Test public void testFileSourceWithNonexistentFilePathThrowsRuntimeException() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, FILE_DOES_NOT_EXIST); - fileSource = createObjectUnderTest(); - assertThrows(RuntimeException.class, () -> fileSource.start(buffer)); + assertThrows(IllegalArgumentException.class, this::createObjectUnderTest); } @Test From 3b580ec5174405ffd125d2c2badad4e0c488ceb0 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Tue, 16 Nov 2021 12:20:15 -0600 Subject: [PATCH 5/8] Changed format JsonProperty to String, removed file source from DataPrepperTest Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- .../resources/valid_multiple_pipeline_configuration.yml | 3 +-- .../amazon/dataprepper/plugins/source/file/FileFormat.java | 6 +----- .../dataprepper/plugins/source/file/FileSourceConfig.java | 7 +++++-- .../dataprepper/plugins/source/file/FileSourceTests.java | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml b/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml index 6c35d72f70..4aa2943282 100644 --- a/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml +++ b/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml @@ -1,8 +1,7 @@ # this configuration file is solely for testing formatting test-pipeline-1: source: - file: - path: "/tmp/file-source.tmp" + stdin: buffer: bounded_blocking: #to check non object nodes for plugins sink: diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java index 56f49127b6..a969ea7a31 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileFormat.java @@ -4,14 +4,11 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonValue; /** - * An enumm to represent the file data types supported in Data Prepper. + * An enumm to represent the file formats supported in Data Prepper's file source. * @since 1.2 */ -@JsonFormat(shape = JsonFormat.Shape.STRING) public enum FileFormat { PLAIN("plain"), @@ -30,7 +27,6 @@ public String toString() { return this.name; } - @JsonValue public static FileFormat getByName(final String name) { return NAMES_MAP.get(name.toLowerCase()); } diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java index 689321d94b..505ace2ee1 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java @@ -1,5 +1,6 @@ package com.amazon.dataprepper.plugins.source.file; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; public class FileSourceConfig { @@ -9,6 +10,7 @@ public class FileSourceConfig { static final String ATTRIBUTE_FORMAT = "format"; static final int DEFAULT_TIMEOUT = 5_000; static final String DEFAULT_TYPE = "event"; + static final String DEFAULT_FORMAT = "plain"; @@ -19,7 +21,7 @@ public class FileSourceConfig { private int writeTimeout = DEFAULT_TIMEOUT; @JsonProperty(ATTRIBUTE_FORMAT) - private FileFormat format = FileFormat.PLAIN; + private String format = DEFAULT_FORMAT; @JsonProperty(ATTRIBUTE_TYPE) private String type = DEFAULT_TYPE; @@ -32,8 +34,9 @@ public int getWriteTimeout() { return writeTimeout; } + @JsonIgnore public FileFormat getFormat() { - return format; + return FileFormat.getByName(format); } public String getType() { diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java index 4cdfccd94e..1d17bbfa89 100644 --- a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java +++ b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java @@ -163,7 +163,7 @@ public void testFileWithPlainTextAddsEventsToBufferCorrectly() { @Test public void testFileWithJSONAddsEventsToBufferCorrectly() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_JSON); - pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, FileFormat.getByName("json")); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, "json"); fileSource = createObjectUnderTest(); fileSource.start(buffer); @@ -176,7 +176,7 @@ public void testFileWithJSONAddsEventsToBufferCorrectly() { @Test public void testFileWithInvalidJSONAddsEventsToBufferAsPlainText() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_INVALID_JSON); - pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, FileFormat.getByName("json")); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, "json"); fileSource = createObjectUnderTest(); fileSource.start(buffer); @@ -188,7 +188,7 @@ public void testFileWithInvalidJSONAddsEventsToBufferAsPlainText() { @Test public void testNonSupportedFileFormatThrowsNullPointerException() { - pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, FileFormat.getByName("unsupported")); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, "unsupported"); assertThrows(NullPointerException.class, this::createObjectUnderTest); } From b76ed510e67428ba905518a52dc6a5c053add7a0 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Tue, 16 Nov 2021 17:37:18 -0600 Subject: [PATCH 6/8] Allow FileSource to support both Event and String, removed Paths.exists call Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- .../valid_multiple_pipeline_configuration.yml | 3 +- data-prepper-plugins/common/README.md | 2 + .../plugins/source/file/FileSource.java | 26 ++++--- .../plugins/source/file/FileSourceConfig.java | 4 +- .../plugins/source/file/FileSourceTests.java | 67 +++++++++++++------ 5 files changed, 68 insertions(+), 34 deletions(-) diff --git a/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml b/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml index 4aa2943282..6c35d72f70 100644 --- a/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml +++ b/data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml @@ -1,7 +1,8 @@ # this configuration file is solely for testing formatting test-pipeline-1: source: - stdin: + file: + path: "/tmp/file-source.tmp" buffer: bounded_blocking: #to check non object nodes for plugins sink: diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index 60eb35a603..d4a519c14d 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -36,6 +36,8 @@ A source plugin to read input data from the specified file path. The file source * `type` (String): The Event type that will be stored in the metadata of the Event. Default is `event`. +Temporarily, `type` can either be `event` or `string`. If you would like to use the file source for trace analytics use cases, + change this to `string`. ## `file` (sink) diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java index 468806b478..14f00ae7cf 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java @@ -15,7 +15,6 @@ import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; import com.amazon.dataprepper.model.annotations.DataPrepperPluginConstructor; import com.amazon.dataprepper.model.buffer.Buffer; -import com.amazon.dataprepper.model.event.Event; import com.amazon.dataprepper.model.event.JacksonEvent; import com.amazon.dataprepper.model.plugin.PluginFactory; import com.amazon.dataprepper.model.record.Record; @@ -40,7 +39,7 @@ import static java.lang.String.format; @DataPrepperPlugin(name = "file", pluginType = Source.class, pluginConfigurationType = FileSourceConfig.class) -public class FileSource implements Source> { +public class FileSource implements Source> { static final String MESSAGE_KEY = "message"; private static final Logger LOG = LoggerFactory.getLogger(FileSource.class); @@ -54,9 +53,6 @@ public class FileSource implements Source> { @DataPrepperPluginConstructor public FileSource(final FileSourceConfig fileSourceConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory) { Objects.requireNonNull(fileSourceConfig.getFilePathToRead(), "File path is required"); - if (!Files.exists(Paths.get(fileSourceConfig.getFilePathToRead()))) { - throw new IllegalArgumentException("File path does not exist."); - } Objects.requireNonNull(fileSourceConfig.getFormat(), "Invalid file format. Options are [json] and [plain]"); this.fileSourceConfig = fileSourceConfig; this.isStopRequested = false; @@ -64,14 +60,14 @@ public FileSource(final FileSourceConfig fileSourceConfig, final PluginMetrics p @Override - public void start(final Buffer> buffer) { + public void start(final Buffer> buffer) { checkNotNull(buffer, "Buffer cannot be null for file source to start"); try (BufferedReader reader = Files.newBufferedReader(Paths.get(fileSourceConfig.getFilePathToRead()), StandardCharsets.UTF_8)) { String line; while ((line = reader.readLine()) != null && !isStopRequested) { - buffer.write(getEventRecordFromLine(line), fileSourceConfig.getWriteTimeout()); + writeLineAsEventOrString(line, buffer); } - } catch (IOException | TimeoutException ex) { + } catch (IOException | TimeoutException | IllegalArgumentException ex) { LOG.error("Error processing the input file path [{}]", fileSourceConfig.getFilePathToRead(), ex); throw new RuntimeException(format("Error processing the input file %s", fileSourceConfig.getFilePathToRead()), ex); @@ -83,7 +79,7 @@ public void stop() { isStopRequested = true; } - private Record getEventRecordFromLine(final String line) { + private Record getEventRecordFromLine(final String line) { Map structuredLine = new HashMap<>(); switch(fileSourceConfig.getFormat()) { @@ -114,4 +110,16 @@ private Map parseJson(final String jsonString) { return plainMap; } } + + // Temporary function to support both trace and log ingestion pipelines. + // TODO: This function should be removed with the completion of: https://github.com/opensearch-project/data-prepper/issues/546 + private void writeLineAsEventOrString(final String line, final Buffer> buffer) throws TimeoutException, IllegalArgumentException { + if (fileSourceConfig.getType().equals(FileSourceConfig.DEFAULT_TYPE)) { + buffer.write(getEventRecordFromLine(line), fileSourceConfig.getWriteTimeout()); + } else if (fileSourceConfig.getType().equals(FileSourceConfig.STRING_TYPE)) { + buffer.write(new Record<>(line), fileSourceConfig.getWriteTimeout()); + } else { + throw new IllegalArgumentException("Invalid type: must be either [event] or [string]"); + } + } } \ No newline at end of file diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java index 505ace2ee1..c355e33a71 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java @@ -1,6 +1,6 @@ package com.amazon.dataprepper.plugins.source.file; -import com.fasterxml.jackson.annotation.JsonIgnore; +import :wqcom.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; public class FileSourceConfig { @@ -11,7 +11,7 @@ public class FileSourceConfig { static final int DEFAULT_TIMEOUT = 5_000; static final String DEFAULT_TYPE = "event"; static final String DEFAULT_FORMAT = "plain"; - + static final String STRING_TYPE = "string"; @JsonProperty(ATTRIBUTE_PATH) diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java index 1d17bbfa89..e892dcb696 100644 --- a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java +++ b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java @@ -63,19 +63,18 @@ public class FileSourceTests { @Mock private PluginFactory pluginFactory; - private Buffer> buffer; + private Buffer> buffer; final Map pluginSettings = new HashMap<>(); - final List> expectedEventsPlain = new ArrayList<>(); - final List> expectedEventsJson = new ArrayList<>(); - final List> expectedEventsInvalidJson = new ArrayList<>(); + final List> expectedEventsPlain = new ArrayList<>(); + final List> expectedEventsJson = new ArrayList<>(); + final List> expectedEventsInvalidJson = new ArrayList<>(); @BeforeEach public void setup() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_PLAIN); - pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, "new_type"); pluginSettings.put(FileSourceConfig.ATTRIBUTE_TIMEOUT, TEST_WRITE_TIMEOUT); // plain @@ -83,8 +82,8 @@ public void setup() { final String expectedPlainSecondLine = "THIS IS ANOTHER PLAINTEXT LINE"; - final Record firstEventPlain = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedPlainFirstLine); - final Record secondEventPlain = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedPlainSecondLine); + final Record firstEventPlain = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedPlainFirstLine); + final Record secondEventPlain = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedPlainSecondLine); expectedEventsPlain.add(firstEventPlain); expectedEventsPlain.add(secondEventPlain); @@ -93,8 +92,8 @@ public void setup() { final String expectedJsonFirstLine = "{\"test_key: \"test_value\"}"; final String expectedJsonSecondLine = "{\"second_test_key\": \"second_test_value\"}"; - final Record firstEventJson = createRecordEventWithKeyValuePair("test_key", "test_value"); - final Record secondEventJson = createRecordEventWithKeyValuePair("second_test_key", "second_test_value"); + final Record firstEventJson = createRecordEventWithKeyValuePair("test_key", "test_value"); + final Record secondEventJson = createRecordEventWithKeyValuePair("second_test_key", "second_test_value"); expectedEventsJson.add(firstEventJson); expectedEventsJson.add(secondEventJson); @@ -104,8 +103,8 @@ public void setup() { final String expectedInvalidJsonSecondLine = "{\"second_test_key\": \"second_test_value\""; - final Record firstEventInvalidJson = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedInvalidJsonFirstLine); - final Record secondEventInvalidJson = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedInvalidJsonSecondLine); + final Record firstEventInvalidJson = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedInvalidJsonFirstLine); + final Record secondEventInvalidJson = createRecordEventWithKeyValuePair(FileSource.MESSAGE_KEY, expectedInvalidJsonSecondLine); expectedEventsInvalidJson.add(firstEventInvalidJson); expectedEventsInvalidJson.add(secondEventInvalidJson); @@ -120,7 +119,7 @@ private FileSource createObjectUnderTest() { return new FileSource(fileSourceConfig, pluginMetrics, pluginFactory); } - private BlockingBuffer> getBuffer() { + private BlockingBuffer> getBuffer() { final HashMap integerHashMap = new HashMap<>(); integerHashMap.put("buffer_size", 2); integerHashMap.put("batch_size", 2); @@ -140,7 +139,8 @@ public void testFileSourceWithEmptyFilePathThrowsRuntimeException() { @Test public void testFileSourceWithNonexistentFilePathThrowsRuntimeException() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, FILE_DOES_NOT_EXIST); - assertThrows(IllegalArgumentException.class, this::createObjectUnderTest); + fileSource = createObjectUnderTest(); + assertThrows(RuntimeException.class, () -> fileSource.start(buffer)); } @Test @@ -154,7 +154,7 @@ public void testFileWithPlainTextAddsEventsToBufferCorrectly() { fileSource = createObjectUnderTest(); fileSource.start(buffer); - final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); assertThat(bufferEvents.size(), equalTo(expectedEventsPlain.size())); assertExpectedRecordsAreEqual(expectedEventsPlain, bufferEvents); @@ -167,7 +167,7 @@ public void testFileWithJSONAddsEventsToBufferCorrectly() { fileSource = createObjectUnderTest(); fileSource.start(buffer); - final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); assertThat(bufferEvents.size(), equalTo(expectedEventsJson.size())); assertExpectedRecordsAreEqual(expectedEventsJson, bufferEvents); @@ -180,37 +180,60 @@ public void testFileWithInvalidJSONAddsEventsToBufferAsPlainText() { fileSource = createObjectUnderTest(); fileSource.start(buffer); - final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); assertThat(bufferEvents.size(), equalTo(expectedEventsInvalidJson.size())); assertExpectedRecordsAreEqual(expectedEventsInvalidJson, bufferEvents); } + @Test + public void testStringTypeAddsStringsToBufferCorrectly() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, FileSourceConfig.STRING_TYPE); + fileSource = createObjectUnderTest(); + fileSource.start(buffer); + + final List> bufferEvents = new ArrayList<>(buffer.read(1000).getKey()); + + assertThat(bufferEvents.size(), equalTo(expectedEventsPlain.size())); + assertThat(bufferEvents.get(0).getData(), equalTo("THIS IS A PLAINTEXT LINE")); + assertThat(bufferEvents.get(1).getData(), equalTo("THIS IS ANOTHER PLAINTEXT LINE")); + + } + @Test public void testNonSupportedFileFormatThrowsNullPointerException() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, "unsupported"); assertThrows(NullPointerException.class, this::createObjectUnderTest); } - static void assertExpectedRecordsAreEqual(final List> expectedEvents, final List> actualEvents) { + @Test + public void testNonSupportedFileTypeThrowsRuntimeException() { + pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, "bad_type"); + fileSource = createObjectUnderTest(); + assertThrows(RuntimeException.class, () -> { fileSource.start(buffer); }); + } + + static void assertExpectedRecordsAreEqual(final List> expectedEvents, final List> actualEvents) { for (int i = 0; i < expectedEvents.size(); i++) { assertThat(actualEvents.get(i), notNullValue()); assertThat(actualEvents.get(i).getData(), notNullValue()); - assertRecordsAreEqual(actualEvents.get(i), expectedEvents.get(i)); + assertEventRecordsAreEqual(actualEvents.get(i), expectedEvents.get(i)); } } - static void assertRecordsAreEqual(final Record first, final Record second) { + static void assertEventRecordsAreEqual(final Record first, final Record second) { try { - final Map recordMapFirst = OBJECT_MAPPER.readValue(first.getData().toJsonString(), MAP_TYPE_REFERENCE); - final Map recordMapSecond = OBJECT_MAPPER.readValue(second.getData().toJsonString(), MAP_TYPE_REFERENCE); + final Event firstEvent = (Event) first.getData(); + final Event secondEvent = (Event) second.getData(); + final Map recordMapFirst = OBJECT_MAPPER.readValue(firstEvent.toJsonString(), MAP_TYPE_REFERENCE); + final Map recordMapSecond = OBJECT_MAPPER.readValue(secondEvent.toJsonString(), MAP_TYPE_REFERENCE); assertThat(recordMapFirst, is(equalTo(recordMapSecond))); } catch (JsonProcessingException e) { LOG.error("Unable to parse Event as JSON"); } } - private Record createRecordEventWithKeyValuePair(final String key, final String value) { + private Record createRecordEventWithKeyValuePair(final String key, final String value) { final Map eventData = new HashMap<>(); eventData.put(key, value); From 187a3f34e72a826153385dc21485b1396a672641 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Tue, 16 Nov 2021 18:31:35 -0600 Subject: [PATCH 7/8] Address PR comments Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- data-prepper-plugins/common/README.md | 7 ++---- data-prepper-plugins/common/build.gradle | 2 +- .../plugins/source/file/FileSource.java | 14 ++++------- .../plugins/source/file/FileSourceConfig.java | 21 ++++++++-------- .../plugins/source/file/FileSourceTests.java | 24 ++++++++++--------- 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index d4a519c14d..b72c27dd04 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -11,9 +11,7 @@ A prepper plugin to generate new string records with upper or lower case convers A source plugin to read input data from the specified file path. The file source creates a new Record for each line of data in the file. * `path` (String): absolute input data file path. It is required - -* `write_timeout` (int): The amount of time to attempt writing a Record to the Buffer before timing out. Unit is milliseconds and default is `5,000` - + * `format` (String): The format of each line of the file. Valid options are `json` or `plain`. Default is `plain`.

* `plain`: Reads plaintext data from files. Internally, a plain text line from a file will be given a key of `message` as shown below. @@ -34,8 +32,7 @@ A source plugin to read input data from the specified file path. The file source ``` - -* `type` (String): The Event type that will be stored in the metadata of the Event. Default is `event`. +* `record_type` (String): The Event type that will be stored in the metadata of the Event. Default is `event`. Temporarily, `type` can either be `event` or `string`. If you would like to use the file source for trace analytics use cases, change this to `string`. diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index 6b62e66279..91e1d6d606 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -34,7 +34,7 @@ jacocoTestCoverageVerification { violationRules { rule { //in addition to core projects rule limit { - minimum = 0.89 + minimum = 0.90 } } } diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java index 14f00ae7cf..2a5a589dc6 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java @@ -32,7 +32,6 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import java.util.concurrent.TimeoutException; import static com.google.common.base.Preconditions.checkNotNull; @@ -49,13 +48,14 @@ public class FileSource implements Source> { private final FileSourceConfig fileSourceConfig; private boolean isStopRequested; + private final int writeTimeout; @DataPrepperPluginConstructor public FileSource(final FileSourceConfig fileSourceConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory) { - Objects.requireNonNull(fileSourceConfig.getFilePathToRead(), "File path is required"); - Objects.requireNonNull(fileSourceConfig.getFormat(), "Invalid file format. Options are [json] and [plain]"); + fileSourceConfig.validate(); this.fileSourceConfig = fileSourceConfig; this.isStopRequested = false; + this.writeTimeout = FileSourceConfig.DEFAULT_TIMEOUT; } @@ -89,8 +89,6 @@ private Record getEventRecordFromLine(final String line) { case PLAIN: structuredLine.put(MESSAGE_KEY, line); break; - default: - LOG.error("The file source type is [{}]. It must be \"json\" or \"plain\"", fileSourceConfig.getType().toString()); } return new Record<>(JacksonEvent @@ -115,11 +113,9 @@ private Map parseJson(final String jsonString) { // TODO: This function should be removed with the completion of: https://github.com/opensearch-project/data-prepper/issues/546 private void writeLineAsEventOrString(final String line, final Buffer> buffer) throws TimeoutException, IllegalArgumentException { if (fileSourceConfig.getType().equals(FileSourceConfig.DEFAULT_TYPE)) { - buffer.write(getEventRecordFromLine(line), fileSourceConfig.getWriteTimeout()); + buffer.write(getEventRecordFromLine(line), writeTimeout); } else if (fileSourceConfig.getType().equals(FileSourceConfig.STRING_TYPE)) { - buffer.write(new Record<>(line), fileSourceConfig.getWriteTimeout()); - } else { - throw new IllegalArgumentException("Invalid type: must be either [event] or [string]"); + buffer.write(new Record<>(line), writeTimeout); } } } \ No newline at end of file diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java index c355e33a71..c6e212537e 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java @@ -1,12 +1,14 @@ package com.amazon.dataprepper.plugins.source.file; -import :wqcom.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import java.util.Objects; public class FileSourceConfig { static final String ATTRIBUTE_PATH = "path"; - static final String ATTRIBUTE_TIMEOUT = "write_timeout"; - static final String ATTRIBUTE_TYPE = "type"; + static final String ATTRIBUTE_TYPE = "record_type"; static final String ATTRIBUTE_FORMAT = "format"; static final int DEFAULT_TIMEOUT = 5_000; static final String DEFAULT_TYPE = "event"; @@ -17,9 +19,6 @@ public class FileSourceConfig { @JsonProperty(ATTRIBUTE_PATH) private String filePathToRead; - @JsonProperty(ATTRIBUTE_TIMEOUT) - private int writeTimeout = DEFAULT_TIMEOUT; - @JsonProperty(ATTRIBUTE_FORMAT) private String format = DEFAULT_FORMAT; @@ -30,10 +29,6 @@ public String getFilePathToRead() { return filePathToRead; } - public int getWriteTimeout() { - return writeTimeout; - } - @JsonIgnore public FileFormat getFormat() { return FileFormat.getByName(format); @@ -42,4 +37,10 @@ public FileFormat getFormat() { public String getType() { return type; } + + void validate() { + Objects.requireNonNull(filePathToRead, "File path is required"); + Preconditions.checkArgument(type.equals(STRING_TYPE) || type.equals(DEFAULT_TYPE), "Invalid type: must be either [event] or [string]"); + Preconditions.checkArgument(format.equals(DEFAULT_FORMAT) || format.equals("json"), "Invalid file format. Options are [json] and [plain]"); + } } diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java index e892dcb696..4532bd5f93 100644 --- a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java +++ b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java @@ -48,7 +48,6 @@ public class FileSourceTests { private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; private static final String TEST_PIPELINE_NAME = "pipeline"; - private static final int TEST_WRITE_TIMEOUT = 100; private static final String TEST_FILE_PATH_PLAIN = "src/test/resources/test-file-source-plain.tst"; private static final String TEST_FILE_PATH_JSON = "src/test/resources/test-file-source-json.tst"; private static final String TEST_FILE_PATH_INVALID_JSON = "src/test/resources/test-file-source-invalid-json.tst"; @@ -65,17 +64,21 @@ public class FileSourceTests { private Buffer> buffer; - final Map pluginSettings = new HashMap<>(); + private Map pluginSettings; - final List> expectedEventsPlain = new ArrayList<>(); - final List> expectedEventsJson = new ArrayList<>(); - final List> expectedEventsInvalidJson = new ArrayList<>(); + private List> expectedEventsPlain; + private List> expectedEventsJson; + private List> expectedEventsInvalidJson; @BeforeEach public void setup() { + pluginSettings = new HashMap<>(); + expectedEventsPlain = new ArrayList<>(); + expectedEventsJson = new ArrayList<>(); + expectedEventsInvalidJson = new ArrayList<>(); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_PLAIN); - pluginSettings.put(FileSourceConfig.ATTRIBUTE_TIMEOUT, TEST_WRITE_TIMEOUT); // plain final String expectedPlainFirstLine = "THIS IS A PLAINTEXT LINE"; @@ -201,16 +204,15 @@ public void testStringTypeAddsStringsToBufferCorrectly() { } @Test - public void testNonSupportedFileFormatThrowsNullPointerException() { + public void testNonSupportedFileFormatThrowsIllegalArgumentException() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, "unsupported"); - assertThrows(NullPointerException.class, this::createObjectUnderTest); + assertThrows(IllegalArgumentException.class, this::createObjectUnderTest); } @Test - public void testNonSupportedFileTypeThrowsRuntimeException() { + public void testNonSupportedFileTypeThrowsIllegalArgumentException() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, "bad_type"); - fileSource = createObjectUnderTest(); - assertThrows(RuntimeException.class, () -> { fileSource.start(buffer); }); + assertThrows(IllegalArgumentException.class, this::createObjectUnderTest); } static void assertExpectedRecordsAreEqual(final List> expectedEvents, final List> actualEvents) { From 9d5050fa68c30c61cc94b46c140631a74b949d70 Mon Sep 17 00:00:00 2001 From: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> Date: Wed, 17 Nov 2021 11:43:31 -0600 Subject: [PATCH 8/8] Address PR comments, make string the default record_type Signed-off-by: Taylor Gray <33740195+graytaylor0@users.noreply.github.com> --- data-prepper-plugins/common/README.md | 6 +++--- data-prepper-plugins/common/build.gradle | 2 +- .../dataprepper/plugins/source/file/FileSource.java | 6 +++--- .../plugins/source/file/FileSourceConfig.java | 12 ++++++------ .../plugins/source/file/FileSourceTests.java | 9 +++++---- data-prepper-plugins/grok-prepper/README.md | 2 ++ 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index b72c27dd04..9be29adec5 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -32,9 +32,9 @@ A source plugin to read input data from the specified file path. The file source ``` -* `record_type` (String): The Event type that will be stored in the metadata of the Event. Default is `event`. -Temporarily, `type` can either be `event` or `string`. If you would like to use the file source for trace analytics use cases, - change this to `string`. +* `record_type` (String): The Event type that will be stored in the metadata of the Event. Default is `string`. +Temporarily, `type` can either be `event` or `string`. If you would like to use the file source for log analytics use cases like grok, + change this to `event`. ## `file` (sink) diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index 91e1d6d606..7d07857328 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -14,7 +14,6 @@ plugins { } dependencies { api project(':data-prepper-api') - implementation project(':data-prepper-plugins:blocking-buffer') implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation "commons-io:commons-io:2.11.0" @@ -24,6 +23,7 @@ dependencies { implementation "org.bouncycastle:bcprov-jdk15on:1.69" implementation "org.bouncycastle:bcpkix-jdk15on:1.69" implementation 'org.reflections:reflections:0.10.2' + testImplementation project(':data-prepper-plugins:blocking-buffer') testImplementation 'commons-io:commons-io:2.11.0' testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-inline:${versionMap.mockito}" diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java index 2a5a589dc6..0023069451 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSource.java @@ -93,7 +93,7 @@ private Record getEventRecordFromLine(final String line) { return new Record<>(JacksonEvent .builder() - .withEventType(fileSourceConfig.getType()) + .withEventType(fileSourceConfig.getRecordType()) .withData(structuredLine) .build()); } @@ -112,9 +112,9 @@ private Map parseJson(final String jsonString) { // Temporary function to support both trace and log ingestion pipelines. // TODO: This function should be removed with the completion of: https://github.com/opensearch-project/data-prepper/issues/546 private void writeLineAsEventOrString(final String line, final Buffer> buffer) throws TimeoutException, IllegalArgumentException { - if (fileSourceConfig.getType().equals(FileSourceConfig.DEFAULT_TYPE)) { + if (fileSourceConfig.getRecordType().equals(FileSourceConfig.EVENT_TYPE)) { buffer.write(getEventRecordFromLine(line), writeTimeout); - } else if (fileSourceConfig.getType().equals(FileSourceConfig.STRING_TYPE)) { + } else if (fileSourceConfig.getRecordType().equals(FileSourceConfig.DEFAULT_TYPE)) { buffer.write(new Record<>(line), writeTimeout); } } diff --git a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java index c6e212537e..2d0e846e07 100644 --- a/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java +++ b/data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/file/FileSourceConfig.java @@ -11,9 +11,9 @@ public class FileSourceConfig { static final String ATTRIBUTE_TYPE = "record_type"; static final String ATTRIBUTE_FORMAT = "format"; static final int DEFAULT_TIMEOUT = 5_000; - static final String DEFAULT_TYPE = "event"; + static final String DEFAULT_TYPE = "string"; static final String DEFAULT_FORMAT = "plain"; - static final String STRING_TYPE = "string"; + static final String EVENT_TYPE = "event"; @JsonProperty(ATTRIBUTE_PATH) @@ -23,7 +23,7 @@ public class FileSourceConfig { private String format = DEFAULT_FORMAT; @JsonProperty(ATTRIBUTE_TYPE) - private String type = DEFAULT_TYPE; + private String recordType = DEFAULT_TYPE; public String getFilePathToRead() { return filePathToRead; @@ -34,13 +34,13 @@ public FileFormat getFormat() { return FileFormat.getByName(format); } - public String getType() { - return type; + public String getRecordType() { + return recordType; } void validate() { Objects.requireNonNull(filePathToRead, "File path is required"); - Preconditions.checkArgument(type.equals(STRING_TYPE) || type.equals(DEFAULT_TYPE), "Invalid type: must be either [event] or [string]"); + Preconditions.checkArgument(recordType.equals(EVENT_TYPE) || recordType.equals(DEFAULT_TYPE), "Invalid type: must be either [event] or [string]"); Preconditions.checkArgument(format.equals(DEFAULT_FORMAT) || format.equals("json"), "Invalid file format. Options are [json] and [plain]"); } } diff --git a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java index 4532bd5f93..b5e5459fe6 100644 --- a/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java +++ b/data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/file/FileSourceTests.java @@ -78,6 +78,7 @@ public void setup() { expectedEventsJson = new ArrayList<>(); expectedEventsInvalidJson = new ArrayList<>(); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, FileSourceConfig.EVENT_TYPE); pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_PLAIN); // plain @@ -126,9 +127,8 @@ private BlockingBuffer> getBuffer() { final HashMap integerHashMap = new HashMap<>(); integerHashMap.put("buffer_size", 2); integerHashMap.put("batch_size", 2); - final PluginSetting pluginSetting = new PluginSetting("blocking_buffer", integerHashMap) {{ - setPipelineName(TEST_PIPELINE_NAME); - }}; + final PluginSetting pluginSetting = new PluginSetting("blocking_buffer", integerHashMap); + pluginSetting.setPipelineName(TEST_PIPELINE_NAME); return new BlockingBuffer<>(pluginSetting); } @@ -167,6 +167,7 @@ public void testFileWithPlainTextAddsEventsToBufferCorrectly() { public void testFileWithJSONAddsEventsToBufferCorrectly() { pluginSettings.put(FileSourceConfig.ATTRIBUTE_PATH, TEST_FILE_PATH_JSON); pluginSettings.put(FileSourceConfig.ATTRIBUTE_FORMAT, "json"); + fileSource = createObjectUnderTest(); fileSource.start(buffer); @@ -191,7 +192,7 @@ public void testFileWithInvalidJSONAddsEventsToBufferAsPlainText() { @Test public void testStringTypeAddsStringsToBufferCorrectly() { - pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, FileSourceConfig.STRING_TYPE); + pluginSettings.put(FileSourceConfig.ATTRIBUTE_TYPE, FileSourceConfig.DEFAULT_TYPE); fileSource = createObjectUnderTest(); fileSource.start(buffer); diff --git a/data-prepper-plugins/grok-prepper/README.md b/data-prepper-plugins/grok-prepper/README.md index 7e71d9ba6b..86999decb5 100644 --- a/data-prepper-plugins/grok-prepper/README.md +++ b/data-prepper-plugins/grok-prepper/README.md @@ -17,6 +17,8 @@ grok-pipeline: source: file: path: "/full/path/to/grok_logs_json.log" + record_type: "event" + format: "json" prepper: - grok: match: