From 797da3a8a50cbd9b3f09677fc3e1639ffbad7187 Mon Sep 17 00:00:00 2001 From: Jim Park Date: Tue, 31 Mar 2020 16:36:14 -0700 Subject: [PATCH] feat: Update AWS DD Forwarder to v3.5.0 COREINF-1665 --- README.md | 18 +- files/aws-dd-forwarder-3.5.0.zip | Bin 0 -> 24469 bytes files/dd_log_lambda.zip | Bin 8063 -> 0 bytes files/lambda_function.py | 865 ------------------------------- logshipping.tf | 6 +- 5 files changed, 15 insertions(+), 874 deletions(-) create mode 100644 files/aws-dd-forwarder-3.5.0.zip delete mode 100644 files/dd_log_lambda.zip delete mode 100644 files/lambda_function.py diff --git a/README.md b/README.md index e368c51..3e93d3f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,17 @@ # terraform-aws-datadog -Terraform module which sets up various AWS / Datadog integrations including: - -- Configure Datadog's builtin AWS integration -- Configure Cloudtrail logshipping -- Create ELB S3 bucket for logs and logshipping -- Sync Cloudwatch logs for a given list of log groups +This module configures the AWS / Datadog integration. + +There are two main components: + +1. Datadog core integration, enabling [datadog's AWS integration](https://docs.datadoghq.com/integrations/amazon_web_services/) +2. Datadog log forwarder, enabling [logshipping watched S3 buckets](https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring) + - Forward CloudWatch, ELB, S3, CloudTrail, VPC and CloudFront logs to Datadog + - Forward S3 events to Datadog + - Forward Kinesis data stream events to Datadog, only CloudWatch logs are supported + - Forward custom metrics from AWS Lambda functions via CloudWatch logs + - Forward traces from AWS Lambda functions via CloudWatch logs + - Generate and submit enhanced Lambda metrics (aws.lambda.enhanced.*) parsed from the AWS REPORT log: duration, billed_duration, max_memory_used, and estimated_cost ## Usage diff --git a/files/aws-dd-forwarder-3.5.0.zip b/files/aws-dd-forwarder-3.5.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..0befd7cf78e3649198197f02d4beef1ef950f355 GIT binary patch literal 24469 zcmZ^~LwGJ+)NT33wsT_J#))m)wr$(CZQHhO+fGiB`tJW!)os+R=5FlHZmegnvB#8` z0s%z@0058x7H21UY3QzpePjT@Q3e2j1KFF~4V+ErJPd5CRg|FsAfR1Nx~bz% zx-RZ803gs8AOPThuHW!jIc;$y?tN1;Y*sXS6Dzc)7?V{yu23;0nTxfmBxY}HMMr@p z#1AuySVA(j9=`bX+4>j111KjZW_H7zZcL@~7tS#{|ICzn^~sc4x42r2?|uzLhUTgF z^|5@Y61YbUK0?bR?wMEp*so#fy(H(We>Dz4rHh8%^jklq*T_{%L zhNN-;ASEnrEbCg9ijd$atU-ey?JeF`XX~tq1In^kmnJ~V@6iqSH`%L8 z>OfeLlHND6MQtIY*$>ZPnU_eV!by4-{I{EAqIhU?6j;ha!R#=!HTm4iGkbULJE$VY+pR{AjH~pn5QU zLP0xrL!|4czmGKAhuF{cQKOh9mCK$5W%|bAYRp-Q9^GgU`t&i{yYW|9lntrNAQ^l# z8DfR3@(^2r3rMf$_77frFT`4!Ty)?dO}ARZ5O_flVax+G;@tYWkW@0S((F>G=nNc8 z=;g3*nS+?MAY>T%^tpl^e-RWc)T~b+vLc>X$&;_lS*12&LmXW_ees52^y84CQ;~khuVDTujkuh4{ePIGXe^ z;QKnZ44YRnTVS&#!(YeMwt%iHk_sGEZ@f=@uQhyFWRLU8hyEn5IY0j=0ig$B5K(J< z2Fqh`9MAprN62+#b_V^4pu@QlcTdFjd<0`V=8M!)=K(!I^Eq%h(S5*x9Wb;r)kH_r z(LT^$YmqHBr<{}^!((8VDEGgk$YUc;wECLC7BFx>FTLki{cT!@?pLW&oNhD%r(WQh ziHEykhny}Qun4?sFZk)Su>^9!@?L-b-W$N=7&(OZI!-kHfL4e%)2+vHI6ee~=&uSk zYY$_=ct=HNB4X+o{G~=!P*DykXOi<~PFy9jLa)Y&H9%%xbmAuQjT9yDooBvnC0rI= zAO_CoEKT4Qghh_Tf36kZ@7i}<;Dy;-2&b{6H38Kc@r7Jd8hAg&Y<=M6k8wcOfDk_@ zbRskPpu0>3NBrEsE(R=J#IwweBZvP55?qz__m*`6cHx1{pyu$YVF?O$K@qH{451F8h@}Q0Q*1~=ld=9vZK`Pbk+MG7A>;I_37pZ_PrXpKrok^; zLNb<{>W@R00_uJZitqMZ0lOFu(DwhbZ-gREnn08dMYCRY4=?>zz!^z{H>Gm8Ova2g z{CQcz&LJiFo;64rJywx79pj`E)}_*{@VH@5EMVy>WtE z5Sae@Cvp-0f-ci zoM&_JZ?Yh$9P;89?IwDLh(qq}2O*GU&POQ66NjXsA|(h3wr#3xzKDH6MDkdvD#yDO z&$xPNDT#3$;XA{9E5+#EO<9xb+lO1tBZfYccH-EW812(5lui56$rq%CN z^Y`kqujStN^V-MY-vOtCyN;qVshkXM9J7jG-`>Mc{G4!Ry)REco*EIAc#K~(Q{Na$vYl42Rni?;3K$@YQuL7O z&>Wg{m*^*LTf61tFal))zcToKF521+Yr}A?;4v};8DowL#qZqG&z+ z-fzlNIIT|KJy^0AA8beb(ECcX$U*f_fv711KPiJ;>b*7~4Hwu3ri2N~e28|}L4Yq} zNBraWPD|F3i6EUPgg)T}?ldk;N^u;5*Nq6Sn2ui1!daM6KKO&C6`bSY>1>aC>$ls}wtRki zeei(?HM{=n_?T^WmdmUx=V_@)4URJI-4S|moit#%!cR$%fEha*tuSIqAbFfEMnW;z zYjkzwk4iR7ge3ffnfjl6W__ORa3ZwhIHX*3U5!W@4zAqNA0NMz@ee*)Xe+5mGP&S! zM&rNHVZ<{pi`|lq2?qw_y$o(@B}XxVFpnS2BD0#u-6H+4dM)Eu=pIW%d~M_Ow2!Z~ zfhf{@tyw4-ZfU@Lq%x4O851bRZ?qP(1T~W}#0W*tcvE`@=lKJrZ;eYEfAq>+3Sm_f z^GS*KT&iaj_8>fddjZWeA`tR;4%F~^Sg|tW8*fo=TyX;nqmxTn+e%z zFaqGW7`p!U88iy8gKZqn6!i+Bjq11G0`^+qpJ8O7@lp-?fI+P1Jkfh9YBLNt+JsM9xh5fDZqs8htB(&8jVQ5e9= zO8)f2lK?AHGK#$VP~>rqBbW(km%MV+bw&w!M0MR`j61*S2A^Z*8zEvdSB@EMttJF=Nk} z1YCOe+SZva+;kmZkNVla)w&~F2YrK|$5aFA05j0^x}w1jEtTguCx}uK;t3TRkH5>E z%X?pYp6M-{g{iqQQd8`5P^+T;i^GR-L3V2-UM}$-wAEDxqSU#Z`E!)SfMs7X9`o?OLX!g? zNi%x#zLz85&W4!#$Gb2f=&dB%sH2-D<|?k|s@K6n7i;MupD&;*;65>iN8OHv!H@N4 zD;x8>fsyr~Hh#_i8qUNj!g_g}I32b$)DenJ{aB!^2O){ad?QaBw?xC)LNtfm>=gBf zWcR0s@y3rOtf$H?tw+d0L2LwB!z(~V$(@^V+QQEnpwnU9CoH>T&5E|hern9b;~#z< zQy?rfV;n;;8g`x6r+Z8)&#Kb%c5ceUtN%2}OVZ}`qCN6zbkw+(ml^aelW!kGX|2Sk zB^T_l^5?oPD}e-%K)S8tlh~kP189aeX4V8?Y{WhNow8yb*fD9 zapBH}4-K*!!rAp7j6oRXn^CaiVvQ7NQr+^}hGAaDfz~ss?1L9QU!RfdIoNMx$KA#e zbXhTJ628s#XEpdV{ikK;9l`XMgdgKBgLKeUU^(pMXJ*CRTAD0OLdHXfR-qL^nbier za=PGQf`i0yRol@C^%kNmzhU_wq6VW7fj_O68oSn@GLuf> zgysp3rd(D?E8aAz-HO|^@AbZVDp~SpBEHs60CYI^Fq#I3#qmy&Nqckz!Drv+;ZCEL zD7YMV&T(}ajK-dv434JB{3EMgRPDLy&LFDbqz?J|kyTvI^^=?>+~i@sAH8Je_E*LC zPxqHhb^sk8sBbf!6BrTdZ{!7ZY~qEZhfFdV7r&Ai)8fYir3ou#D#cd&323@r$J|#q z>TE`#C55c>KQPfpZ2|AzGvcQd$bGqBiq$FhWUxnM$=LA9O_fsEf2x|0Q*`5kuOdP% zwRZ~Z!=L3w4V0J=ss{VjZ-Wd*vh|I3b-3u}h|t(tu|u#iA}HIkCovRXWXub7M5-#i ztu#IM(=${s{)M3JutThC+&LjAbVXR$ZV)TZ7xY{Y5HvQu-}#g`fOF7klVs-IhHr?u zmx`T@y7YE(^^Vi>TCQ6yw|}jZMb*5iFTH*uK4TMz=k14A5QsvHp~+X}*NGafhgqq0 z0b;M^Csix|#hUjOX&#pQ`%nK6vz&k2DS=JMJ}rLt(vw(m0;uXFHDDPB_E-#-)MTc< z=WbC!3j3Y2(lG0JNDLa;FCPkA#!VgOrwlb6VYF9QWXI|2k(M8Luca-J@PVEgP}uOq zzKLz%*{o*0J%kv6xHGY;R=YHRRv(0E_*SDJr&=9&LX-Br09<^z4F3So_?&F_CcXXw zdI)6QhyngQBhQuud`i(7?+vR?Fn#;_&LwAn9Yid4CK zCBz0%$R{VRd=AL|YI#L;Z|-7EKy;dU6h>;9GUUB-e`eX)_VQB%xCbAmvQW* zoM?$|#sNEn%orP+0nDttXvW?2Kja?3^$?t6sbqZf0z>Au72K3t{hHomP73NjSmQG_ zT}vFUsH}0^V5?4@$m4(Wc2fv$vA8(ZtG^Gpb(IElF5FDCE54y-j>h;E+hi<2F&yjA zPpvWhTcY`!dvuH4|FZ8699Sev0ADX)CxM})fMl_l{2DCaXjVKvdNHO^(}lT^GtwJ6 zP&i1rMBdXL7j%R%<)RBJ4gioL`UCqBGRG`pgq&gL?xPf<-~1!`?`oQz9At6Xq19vR zD01UADLnr5vSVVWf$<@q!M(u#i>bb+MvPtNVc2k!R{E4QCdCh)ep;&n#jx+W7intD z8>fc|Falav&6GD!ZvmY#MczGbv!cG#KdX({FFnejhL{dupDLyD<9gq@u5k7A1Sb4Y z_S%trwMD|Fc^^6J28A{gohOm=G6)NF!)K1-e8;WjhnVF_attp>ItyGK)K_Yq=|&1- zXl;<1woaNl;)dMKkm`5%fJKk-!07zvBN~ACoHYl+fd99b7-Qd?PDGhQ6~E1TnF=g; z5&C6gEbqHWdN+ed(;D~i3xcfC{uwS3ckwB+KR~z5O4yAN*`F+WqVt^5V^Zlcdh}bl zxU}4}-~t4}rbXXXKXd!_Yt&egpEc|&hQ zTGPey{itX!WH1QMG_t~CAVw8^%IVp>=Bbs{RR|K6iwVmw9A;+~G zMNy$9=IZ1yFot06hUP;|?Sb96T@ip@8}tdlac3xtU@Y-l6;lpp=B*dvGO|^Dc75D{ z0Jk95cjZd^UEQmy@zE>NdJ z7jU~s<}gR*-x%cH0D z4)Pc;vQ8&)P^^Hh^S#l73pKkvvG?}+i`k*}J#}zi%7@v>-Zmt6*4Kn`>Mm#DbW2MX zuy(vqgz>>+PnF{E`!tUG4ABR_^YA{13kR{LW=8D4$7k6>mGX8yIOU!>fbX3|n=}ux zv5`oA%{Y~6VdsW{IG*X-2KuzdIUG}{&-*c{)oBOi!RMn@0@OkufMXi7B z%>c?(V;c3>3k^yLVLR)f=IcZGv1njZ?Eb>Pzv$t^#19r3p;pX@l1(Be5xDOFcpYAw zPnH?+@yY&r7hY^l&lX}R;qYYXW5nG(>ORoOAH<=q8%$0Ey-8(qgGF{d(2PK0vYR6R zZiCni2>@2V&HOWMD$%W3nzD7E*9TC${lFr?)6KRpMgHld)*pux*fP_z4Y{|1g{T@H zT5+K&JHx;rP{Gq7oFCB7vI zf63bgybZRnz_QyUFenhCs7*+5;qUcRqzrpO`eaHIBfzRVn#=Lj1F8G!omWPAah1gm zfy-dK;5CYYwo9)MA>!^- zd_tyDwM>PE7CX9jEAxv!*k87FC8b6~Sbd9&M3>mC>mh^&tPk_;c>+ZHjtonqJ*zi< zoxEk92R>zW*GgEMKlJ7B93y0`6aQ6)(+g37I&hQsy_UA{?&!|y%J%|sbfpEGuqX9# zr_gQLM;RLG6|xUro`EsUtX2woF~igqo<&D6w-_1!IF0MW*~5e1bu|Hg9+QMt z7mhPB3k5<-CIFXi0q)(HdBS!Jd$_`*uaHd{$ir~ZVh=utAg0enwyyV`*f|U>11j6NmzQ1fZZDCGdEinxJ@-PpG`o$08?MS zM*u+zn~GyYcY>7xP^X14>=d!bR^mXjk25b#Ea#WBR)%YkqS2mmEQwFr&SVVd-F~Yd zfh@#&)7B&@k<+|jaQS#* zl6IG`UvCozISxPKL<5tH!!edvK9pJsx@i=nH$hf7AQEB)6u_-vsxHqMvM@HxmzSfM zR<&%C{6+l=_T^Sfl#g#Fcy-{0=ny5ym2_Kaz-BSKI$@?uX!KKIm^?2!ih*}4e)cZ# z4O$G4*bK!;(CA~JKcLYS7-Jkcx9KKZGn}CkfhC7sVUfDUv*aD*L{J($@IImErJBLP zC96mb>c^Og{6S;Z1ZEbv5LZXE8x9xXRX{M@(1{pJ|O#atIh#AtYo;&$8spsPQPe z37rH(Lq(`oOQ!1k>A$7`lNel4q{Lcw(~3`kRShJo-p^!{G+wwA@&Y*Y3Li8% zV1RuQO0?1km5(!ELx@0`lR&ctCCVG1k@Ao+8R#pNb5=HEj35;XjIGtcPsq* z%R2z|sl!Gw0F}XihGZ!r*6P2{gx~$yc)rLMw=rB*fystAT%W*Z}`9M2Mu? zM@EvA&1O@gAIR2l#^u#t8IY4tBdsK&P6Y9T-ZWnEmQX}dC+e_1w0VK)(wk5ottAI`rj~wU}@IcuGdZ?kC-Tn;u93A&= zY<)#d%y=1 z>Y$Eu@e4wNv>6F7s;Hthg9#RbV3?L5lp@Pd#&06JB!>M{FP+Zwy5@HJ$?D_aJ$>hh zjzm#j+$%Q^@VZ!9_CTw)S?2E`@P&oyO)$(IuEP9jZbZ?Q;?h7Bn;8y^^!r5j1j0};G7BZyp?8hGX>%az4rNc;sZ7`KWGJ|xU zzIx&9cY`GWojd7VbB)B3;fPVAxMD@AOM3)qJEUc5q|h>ux*L5K>Jb!Pzc{d!&OyVi zK;eShbDt{JD52f_bAz@~BkSAhVw;GT`e8P2m$RWJztsOS))f|~-zFbw?#d>u*09_x z4K_S3D9`N|+Rk<6+S)(oCRX4dd9Nt$0K|wB{eJre&f@F33`1}#8+YSR=B1C!n*i=v z)jUuDVcG0;)AJoT+=-GAgjwIQm|UN{*__W-k7vL$-6*u`Cr$(Ae0!VOlr%EldI06qg`z zP$Q<=kPTeYP``;Y%VX=%Bum}+t+|7!eJoHrc7bQ$BJSPaR^Y_mZm4?*QVH9p4plar zaqQS$WlV5iTx)*G_yiG^DPRfNQO0BB#!=;KV=B=v&>L1{X<{e!yEClQm$c_#0uGVa zqD|A|Mw0{7qsp92#@Ekgv)gNGo)l#{<9;S)b<;H8W~=R50+(v0nSYpBTH-%#GE%SL;zAIgl7WPLqrYP7qPCu zy@vtP+4BADyx)yPsSsRF>n}-e)WR&h{&|LjJ_LKgNi(q+VF}fyGe?PM+^Q;7BlYAi8bjnWkrk?%Dk02U{4Etq4%ntmaqHXrUALUJ` zWaVZ0;chW$yGbe{&jol+`Sy-kcg7}NET$n^y%J5jud2>WV+r_> zp@x4BHlyv2sjRHTN`!_RD#q7TL~&ZW{~Q3!mo+Z-?WM}|W8AVpjy-ZJW;UJR3N2<- z9I=@YVN?3-ZXJ??fi-2QaR6#e$ie45T&VFPSfyI}XO4{X56~)Yr35Q@eGSmX&)QR9 z-(ih^8NX(*lHO*N2(-4Xs@wwXpn!j~jo=#o(9s;_F!Y;ZZu%b`^{hTT7dc1YoQ{wTt`x7WCpcyp9K zXDv86MWn&HF-&CoDb}_JgHT0w4{G=1O^ah{Q(X!{OWl7`AP(hw=5@!DAb|9SM+TE2 z3*jwo+Cuyy2arMDDQ^S$l@b367iwhB&qK?!q2ho*0SmA%2$?LW(eJ-_MtFYB6mhNc zDTa6MCJ#Q?cF10RRMhEu-EsKI5bs*GX4Z5jX4Iom=f_ryI1EpPATkwUI=V;_I7lNx z_vbyboiUa&PV`xcJb@sNzUsKxyuk~y-=H0;&M4xNFS1r{P)!O{R!T+w8JNNC=RUW} zx0P*oX@xI4kqh%n_cQBFBZwR|pQ4>JE^8lJ_8V#%?8%1m{nM)i!s^bF)+lH!^d&~;Q^ zE2+(T#N@_Z{U+O%Y(%{}s^k$aKP_vCaX8-zLX;)3EZRmn>HbFtP6jmwB5iJ>5~Mw2 zyeFz8dBdv|mO8uv#CcS$^(ykJL9R1FD7^a<1r)m=oOx2>*GnY13w ziz?<1CLb0n2EC)9_nMLy6kG-?-s)y+KpSdUfm#%lvvv8cd~QxDc545*Bphc_xx`+T z>>M;c$2bD9UoZ5&i$u&#;93~wFH=Vs=CVfouoM|%kll=j@(SoZrra0@i$3D&np;oo zGiw{&FR3^9@?$Iw?0GAjlgBN8Q9aEe9X=f-&uMK$yAf<~c6KOK+8(RE0KFjLAZV_l zTHpdAvNEjQB(dN}hc8?DSk*qery<*DpnL3*%w?_X^Z?4f%%3TBe(g&v$X1Tx*RJQmFy zHanc=xUHDlwM(|rT>;Soo4|$zLHELH@8|dOn!A$pf6e=zY=xJw+=u3Rg=N)Q8mx z1AXxipoMr1uFAf$4#fa}LKPU9%K{2nY)&^cb|(B<3AZtTFItn%uLe7s*>5VpHMp*o zSvf2g2d*jw3QWn${65Az1i`o{k_-14I(e%o5s%W69@Fk%sSrLyQHWwDps6Z^_eKgF?xz z=;PxaM+n;ND6r-uG71ORKA1R%l7`<>+uI+YY!+%ah-RorSQ*Mfa1zs7paU=~n_ia3 zj>8+4kir=bF;gw1>SMRH8`u39E zRxR6`?zJFULrF5*2dyyS=ZSZ(ho}^MeuM=5PZ{myP{9O=8Z&(I1a=0a4bTJY~ zLMuso_f|3TacwV4<0t+?Dzb15?^6P?gPZy5bS6s!uSyEn*>vr)#fAHZ$+2kRLyp;{ zD9WUCrIKz*-?}1D#n*@`WvfMv+6;!(tJXQAB5HeeLyoIBt3lEV|GqSTU;6UI#h$Cg zbgaMBtSIKon1$=FYMn&bW&(Z;nGH|ewcgG@BiO#9_xo9kq_~wa1PmX7x*S>|*hY3= zeK>)o)XDplH9HxJUH6oOn8$$xy4+fPYOT|+p#DO>8}X0M%4(Qo+;tp(BJqFVgSs{` zc-XuQPLRia4c~An$Ib)+<7-ocpdu%tX_O{4F>K9AA-f+WX<-sw6r5Vwm6F_XFNZwV z;=t`apZ7nnvESYD8gTgGhSjNLbCg+c2+-diy7C4)zPdxB>cE8f(gEd-9+p1w+j?n# z&w0{xiWa`}ZtYlNTNRq}m%`-8vk<9%4Y^+CP;}1eTaV?Hw4{~(K2CNDy2_q6(N$*u zyZ4}5CCjZLRq6P9BU8#hcbZ&3V*bSrE63iNe^hZD=D5`?%_GLgi5=(d@W{fLd9*hBZYv~XS*(JAnei43~4wMm(S zR+9Mi3{O1NL4yGcuWeaN~9jc@6Yk)w7> zdxMc6pR9y!992tYMs1dErOwJ6fF8j$#ul{}O9>hb+@Vg{YS_nxafQ(l@Yd7>6QBZ> zYi5=}#SJ%lcz-BtZ+8sYPq0uB6W0PxEnSS1)B<)Lk-9?4cd~MLk)bBmb{v2UeJu{w z6V30xLAKJ-*{d8Si;Hn+D=M&pX1Q3s=1mm*wC>TSn~-p{cZE&mzi+^}_jWvGeVG)B zn59RuB~Dsn^zm*%GUzQf)j{6I3hD3(P1qZp5vgs^edaP2H!a5#(wW(OkVDn)rgz@> zbN6mbi?9Q&+@?%umOvy;t}mNnMt8x)G_;V}WT-l=he&*hhK}lj2kS^<99!jGC`{{H zjo3!H_Al(0@HCnOPBW!hNvj6IW^OrMK^?)w5;gYW?~=tlJ7X(5ujhW|R!o#?-(!z?C=V*C&N;fs)7C`>H9b#y;75dN>8T`C zyKBBp++pa8`}ywd=>7i2g8fb0d>K~CrDCw^fGkg)UAR=6G$s_J$qi#MRKBldCS0OI z(wrXxWT{bxFwd6}g=(4pv4E4(z(vZRZ5|lI2Gk%WI<0;bg!vku_`yMM00H2#%N&BYhNFQ& z#O1khnHj>#+G+;~+g=3*2Fc3T-HBIehY69Q>5x`VlA%9wjPO*#`xf85n|cy z-EyrDFVM$yysDehX^cyzC|$-#E{K zdluAzkRL`Gq=D2SOmZN<^@v2;<{w+jGH@rj6~%`3G&a?z_3up1m=9$W=li6>GhJo} z>AK9h|5tAe#h-NuD)qNh=)w8^;&@?i>0M#}0*r)cdFHL1ic0~Z?XaQv$gz9$B|Q*g z`Zk5QV)`_&N<~E|0QVdCpKw0RH$Y|fJ=^>0Tn-WFxbYkjW#j^RXz8??7%vpI_faI^ zETMI8Hv^PE&e@?_v11Phch@}bGuGMfd!{46kdxUqx6+bc8&f)l6$n6i$rEGZ2-l?qg zC~jl${6d*cH_*LFbc|vL>uf3*#;WQO*(8Lvw(hLmZePZpwoiAb#(f1k7VXtbVUCKf zp9+5i`rv>l6ZB?1zz|lwt*FE0exNf?10Ac_OJ)t!l@X{oG+D(ETQV8x2PnDw&LRl z;H9%P{KW1`y&Z%9z-A6|bZV56JwH}wgRIOF0eU~*zXx~t2K?=8iC%Itr;$W+^=Ih# zD^9Jg>FVw7C)b;CrI3EW{Yv32mV~%h%r|DRA`O_+>};{NOA9j&<+xv%?;`W0xy61n znBzz@@s7#HagI;B3SzU3crlv5lj8(|>9+m!nn%wsW|o1$&+FyM+ttOv%Y}Wcz2&@b z+nte|SzYM9AfJ7H!_j1wCw;rEX!edz$>X^Kio8IB)j(4=FsFHj8S5MRf5(p#FDLm! z+LeJsFaW?92>^ipFMc?hSeqC)nb0|z|4;t#{ZELm`Tt7sJ8D`=TO6pqZ*?5u(gAdh zFdJz%Y&#N3%{v>wxM9xt&`Ak48xQ(gwZ=k9tzTPj<8A9uK%O3(v+`RB&UAYpv*Z?T zV~&Hvj_I11qjb}#Fz-$bv=hX$sO&4iy3$`yY8_r9O=fl%?~}+P{1Ky6;{q*iE>K(WQ^MyN z;-|GPLR&=cS_V8AmSo9;6Wdaw8h79?yf+Oz6GSR`ESNzO4nQUDmj`HXeEJD%oN#1q z3{%=J-QyX`Goz*ZWAS?l0ti8}Bx>_1$F~g-O(Hp~5I$F85TBtcM zn)#C??g8xp0`bpwBUfp3B75=LY&Zc4mtK1?bS%+DDEwDx3KSA0OjVT23Y?Uu$_8X_ ztwSD8zm)sx$g6DQvhlF&z&pvTsYF@F>zWLHd`e5A=qH*~0e+YLU@b?vwqv`%*E|0a zqmp@-wW*gomuHn0N}8mvj1pPOXbflUPx5wHThhUMz68GK3HG|cVsdKdOzY&8vN5$= zSU(P-3uzmewg?Y$1pZ_@A}OWzY>y%TWo;Q{d;#&JU}H9qER40~yf>xc`A#j+v&!7f z&Z=J6KKeS*x(fzjO8J=r$s$}5UZWGM*4QuEG#iPu|kBLTG@QaC2^ByTM3VE z%Y6BYdb28e)JN$oasz+h4Smh=J_2x2)6x^#_&C8oGO+O}n|E2n-mnMnA}~CV)d8Y8 z?SmA!YU=GuinHpb*Nl10Kh8W##X@B$N0 zmDHLmt@sP%CXMaAj+D)tYJ3^#K`OnP?`o}1(O(~>BW0&)476nXDCOOAWx3E~+APmy z$$qXy+eblVAsl==OrbXQHq$tAXG_xJy*4;YkU_$76ro!FE1iNHK>}BXyUbzRX8HDB ze>pvVZQcK7%w4;IG>$haw~UM_`y`EAM~TnN2l%SRNK%%wDcabrXf8En@M<<~RUW3B zSknN*v}v41_*2m(#3QLit`@7lP^xO=wo5eYYrY*j?WKcT#Z}MZkm6>+<%9o4f})+R zv-N?r>=b^V$NXD;@&$xf^>cs_!CMokj^U%ox?SB^Rl}dIPk)F3t~vP6?=tpqe+`1& zhhctfU&7%~$od0nXCe76LFchakbLFCqgA5IlUZlyc~X|5%oZoM?}bGW&ja)}wfX8% z>z~Uwig-6}#i2X2u9#SCD1OTN(Ir`O2Gc ziww8#wm$VvpgC#KDA7U6{K{F!rYmJ2OspGtx6WlkLa%L)ZLL<`<9yE`G2G=3a6Ov1 zNs}!H#v3q_gVK=-ky`(1PqPLL(vuEMk#k*g%6%4JRwEg|`-u@Mg#NeBuFk^xg*?D=`Gt?R6RpD4qq zO;v?0fYY$?7lC?6`S6B5*7W|F_-cb34~jx&J^OAh#7!xrX-3I` zWh%(JMmIE2ulU)=f9;PC+_)vue5B)RhVdYV9@ATyT>6op+uQH&9*oJsti5iH&h;Cj z=^*Eibdfc7POql7&%92pxR}I@wTtN{=K*iO-bppAHy?D`REty)c!fkE3#KWc9$7Io zJ+Q#OFU3BcUp(5Er#j%wiZf+Y+(jOrD!syNGmp$sK@4MUzKHVytztKA_bQs$B5HKV z(%g0UtTL=`Zm(&(aE;#*sveccEvlK0zGyY@N^!F`9(FsT)i?)FA6(M4#8ZluW?%Q| z4$Pd_^{^H1S!TRg55Q81*U3Vl8`pf=3oGbd=9$?2ICEPRYp%ar_MbIde=zcaz^;mW zcSj5pi@=jwC--yw9N#;9I6klFni{p(W->`u^gJnBvI>&_r z1m@o9jqu1Yja1V{T%v)8l@hgoPVkE=IE^GTtfv_A_4ASWf9`;J{E! z{58$ioM6pG`VMH|*A(uc60!2<1|q|Z2)w0W4AVna@m)IURETQ&2UVC_QUp4meqlbr zvT6)WQ%cy-q{gX(?QT3!2kgNR9b}6g1zlsecKQ_(s1DGi)f7r>cKS^Kro9#}iT zYZo#;Bw#bxaSN8B0k94(e{l4?>h?Sl@LUKe7g=)yG;$gY=_)|vYy*u)h4{!ozrp63 z%*mYF3_tl5y>t0@WEL<@H|Bz@7MN>N1<2a*Nebvy)uJG~DBhv%gtDWBf#F{cF6>XUqXT1iCNQWxQ|etR6Rqd43$*QUl5l}WS)#Q@;o{QB3nnRo zJ7W2M$g49xIp1bHqNk`41e?oOU9?`!wNAL z7tjOFl_{^Q^e8(Pw1sxrDJ&45uSPC-J>jbN%w*x;IazP{CXjLiE|$TtVr-ZuD5*eZ z@2sEHY)59nkATEK?RgAtYBUW5m|t=2nbq^<;bujD;)e1zsM#hKMu$Mg-7yPjAQ-zT zN3VlC#W`McG%bNVo3!9#+W>F$L~1#uDX1M5ELkO0BxAirI4+^fnEN@*_m>~OVe$~G z-3&UjXw+YjEZk@dI8o z%g-!epW>#0X#+J;GOx9!pUH=U9_%j>M#*~=ACA}g0y2sQ@#Dxl3!9c2Oo2%*v-0HT z#(oa;xi~=MR8+d%GRtPMXL3iJH~xmKqncOD61BTzIQM|_4S?QbjaTZA2>}0CmbNo# z1hfEK&5r!A6>mJZ71VXwM)7dHh4TUhj${|~p9&=FuTe>4rcN+&GvAPln=$il71*|H zW}f(iIfJNhOyQ-q%Ttxtp4gdq*9;Qo1~fDyTZ;l(P=HQ5qSIo4!JFP-E}r!68KXf{ z4982ZaVw0^xt?ozqro^l%;LMz<0C>_21ntWKOi8>65+ zFXtTw13f5EkRwHiZiX9&w=xZO4uGtCu}oaN7Fdc4-Azy(gJ@0JpcT7%Ri$gRVAOJ0gc>Bn_Q zShn2~CD&zmfgxE|Y;W~bU~{GfHMm9Uq02Cp0pWm5ZV!$)WNHGn?Yd;Svt;0W>qJRb zAp;_6=D_mm4h8-6WozacQaJ?FBERtP&|=8Tf>m`;jYnFiOdpTVVYj~Wz{QB`E465F zg)R*QV6q-);!YP}|KW6G14Dy`Gei*-p3p&?t58a2%9kgt%v)Mvi!GtUSejFP$O-=m zTQA7&7jUx((w$b4+e!*Qe#F=++ALNnSbd!B*E-7-66Xhz%3r74b}kbRRyrAUzW)Xd zk0Vw$*eBvo5&^91^JZ!bY$MHFFQk6LW^1Gz8K%Duh!-*XOE9)&msDM4H5#O_Es+Q_ zfr0G?VvAxc#b9~N<+Bmo2CpNSD!PUvr$`ROAL1t(2wUYCs7DJb9l>=XpBZT%pJ_xt z0nno!yF^n{I)?oA!Hbj*4-VlfQdY#(TxrT(gq34>JwpEAdC1MhLfB&n+*VrLPFB~V z^->`+*{aaby-1Aluj1!OL-4`>Vo+2E&?iwHO^3S51S@e3fVg3M2Z4h-`?El@LXYX| zjs48f)J79E2_Zt94bX;`?+0^UlQPdErAkDVbF5V}CN~h7$XcydOF~Lqv)-a2qI#N@ z9aS{)so*qBiat)!?{WRNGV}|ye52Xdz4dy2%YrCxq!51hf6@Uk_$!3UFkcavFw7iz z!+ue@xpFk3JfpCdy#qVvztlQ6j@mmi{QbQ*aDMFJ z`^zVYe|LNo&TF$1C?MJn=vE^HT z@SqP$q>`)$TIY-3{dw1;ruVfwrItqDM-9Dy(`44;nMQw;i}3!2`z_~M-^R;CQAIkd zVQ)3swm2Vt1GPSR+xq?Q;r<%Z`RL)knB+6KcZs!KFOemOI;Z9p(X+k$1f-J0z?c@~ z5R{Z|K`YkqFL>F{z=&ZJX7WEVCt@cmgP`}c<$Mi5{w39=O<0WNw}AAAVFn*n)(%J> zK6`10Dp$#D<17V9o3g}{Ln8M6{^za;A;KFUc@P83NIz@`^=NMHA6_IQG0S-I{fW+NBh%H`FYhnJvMPHp`tG4hH5f1WU+b=An(`+jwlmZvR2U)Upmi*vN%3 zicqSX*fiyYM4IEOMB0neLpHDR`B$vfA^E1P+4JN5^MCf|2&F#{`kwbfE9%NuRWKz* zqR*rX-;Kj;E_LWYGDChfVuhM|LV^pMfx<#>mI=R5*SFP1VMCb0mBmAo#mw8q z82I%}_4>Y8!_O;gu-*HPwgCt@OWSM@$>8nb{0XQULg4>1OzRO*+g=h;eeqsqbe#DU zv%Vl#Wc}EF^$W@+!0i{QmkQjxAoLsAqX?y<+!#2u1VbJ__~-{_OE;KOgT?81+P3^C z!-2HVQaj1M8Om*2%2nFVnLgSgbp|Tm2n`%UcI`lD^B;7NS{hbbPGdSKIjNYxEx1Xd zKZ{eM9dZ(W-lv{ok|nTw8|ps~ef=|u1U+>0t(%PQb==b4J9&qI z8o9%S`I&Nj$81uKsE%Aor_NA5`TJ%P`uaC}p#}rX=cvwR-In<79bI1qVT5VgtBmB!%hq~(GW)K;(8bY*1#Bd} z97(JxW~V--Z+}vpLT^!i(LT62NkRx)0d|Ax@j+&P`3PRp^4l0fz5Ixhg?y=(|JiNH36!pBgoK@h@ zzFR*7U9ooAhWu(;;bvLT&dd$JQAdyXT}p+`Ve>6-+eVcx#wJfkGrHGWSq~o&Gn9*M zeQJuJ$)s-6DSUXNd_h)CZ~QB{s;@;Dj=WiU)>qX+{zH-c%UfVy)#e*ATpq&Yf~D|d zPN30Sao8r6zSfj+Tv**#_*3}A5pN&#zbYchxYH(!Sx}NS_ zZ%7Ax{(yx%44II)`fO*0J-;D&BF2k)mRx-zv=zh=vs?(s?;>gKa?T}t50i;e_q{^O zWQyn6q1NTMT(*I$9<)tkA|^43-3cSWwra3+1C&J^U+V#<9J`TT)Sm~3LQn91dRXWeFPQ#SphxCR@YQ*=QSL z@k5`^=R*g0P*V|OUzLPm#`RP_7)yl6a zyATG)6s#b3&&Cn2q%&VDgR>a10Q(w7+GuW4LMKwz%Te}^<46%u>g11lyMGsaW=h$n zbj@8kXz1>^CX+GqH4`V$aA@Qps;O%rDWR;aI}eXdk+2vl?&$bkru9ZLUyP2(j=t(5 zJ|1lvNkmR1MbJB@Uush2Dq36dpEEI@m;2^HNM9ZT!&%A>4d!OsbC0$AbXYlpa`5tS z63*ysl#xL7sHdo_!Y;yAnkqtG{BbM3z+E|NnTMA68617;k+ja9361ntN4Bgi_9#zY zAv2bD+pk{+JH=QvIS&A4p4l=jnqOt&mQZRN)5VOn905PqaMS@oF?ky5F{8QHWvns* zSnNTPSHu1Zn$0Pb-G*gNTtWmcai&=3%N4;fNvWOZ_0_kxWkF1QPpbJ!+-+7F?8nzo#n@5o@&?**#@efA;Zss1$~XiJz`{XUr$$=T%?l6>Rnq0`X}Xsner}J z#?4x5bk)-4I-P5?y7F3ZvVRa7$yfZ4YT75VXr4t0>N2jlvyDNw3*x9Dxgp)Hm`8MM zobu8dJon4DO$I3lITb(wo}g4{7&Hrs@Yiix#Zfh2W)RcY-=Pd^6y@V&DvMSzKG1rn_EA6@5%UA-(U; z;l0DjbcM-vSH*n2IGqMTr-ecF`K22`aCSD@zfeI{!jO0Ou&7EGfa=wWzgiw-&4=DH z-8EI!x3nEF*aiQbR6d`1!K~FD{s+vH2S`t)1u4+K71vQg@wQ>wGH#32V;wBX#e}5X zN2ekrfC%+`LRE8X6I`MGJ=aLcH##6i>InfN0W^(rUPFhgVFo>3dilF$SRTeG2p>jM zSCYUOc_$LlJOTp-Q4WKlupjpmUZ?gBUVoBdg~!09B7Y%H}I!&cl zuhkNs+^ntX$1F}(N3;ejH5_QzT8#qkg!}@t$amz~pYB5%A7%|xn*nk{_=r9}tiQGR z*5W-KdKx$BG`?UY6+o#!xHFu)!wc&P*3rV>*X<>DSr>dLnn=i1qYdlpmclhWf8qA* zPv;UGbd#!_`~X!ul}xT4WOU6w*k3F~SC>OSfW3~&iFstEK0)Eikho7BSoIyuQ}#*6 zcb1}vQ$ox5+|i}_9yIjvJ#kWU?oXKy-_a& z(r3m<^0^#{+7hda&CRoeqwb$l!8U(Ji4Gy#Pt^QFtUmzaVZMY>#Js(?hJcwX_@ntK z!?Kgaze6; z=Zgof)0(uXeVG67xO0*5>2EGz%hZ1Ca}D4}4`=jF_3l#K?{ys~vio`b z9w~T1BU>^zN5x{x;Qh!Jdi_iuyj#r~gt&Mquqs_SJx&n9+)7}&_rwUkY*oQgy)u(d zJ%?r!zf<*Udc?L)IG!&}{8GLLO6md5{9TS!t09@Nyc1-Vuee#{8HagKc{n(Rv@swo z^s4A2-LyOu=XB@QONnUb2;A&tqm#hVVQIE< z_>$ccmFqSJIj9jBPp4*Pr9TQis?N_9%30>yl!Sds678)?CXbLg&GX}b*1hG%@MO>r z$}f=Blp@2d)ttiRkvw>ZP+&J*`w4-u#D#xG#<&EJf4#lJEpp)*#b&)PsoNfd zU@<3PwzH`SOZIcpT&HPUjvIQtD2G3shmw#Wd&%q-TU&eg1_k?4<-C`T*r*|&&Bv_X zua$)5@mDfE2bxI;OOyGO=%mF;?pu_3N|W<`+V;h-^59WH0?wP*XK*v*ARoUyziDvriY5* z=^*GV5eyT(1`+JWz3AGt!o!vY%nD@(-722-dT?H9Joo^-uaF+v2dk|W%^Vz;iN`&~;RyFuq!|H>5Mr$$+qYR#L#;tgoJ+_+L9((Xsf zug7HzY6)CNM7BWRlP+TswmceZrsh55qPlD1cQ4OsH7(f+u@)N#hrhe(^x*>565>2G zh7zYYn$ztq&%@*mH_z0ep*0t5sjcP=#lE(2dI)iD;$Vq-F0=eq-Rg3bpwtAB7xOE- zSGruYsBtLx4_H2nj*#{@%{^6Ik6s>1=Le&edJWsV&N_R|;Y7D{D7+`Qxap;(pW?t8 zg_Xq~>9HkmxJM0RqV_|>y_&-~_3Z2CbLkhG#Mec5+WhPNG>F|tiMXjUE#BCx)=<9L zgP-KzhI3@?3 z@~47F#IA3V^e_z3T+_;|GcRmj8;p*em~_qrKEpSz-GZhS>&9_UTm;98viG76eOUiA z8>1LQqZhh-fUtWPT_6XNGch{x#^PqvuOvJ3reU2uuewOxabw{gg3=~qZlb|Gy@jGE z#%Z^n8TS~fu92qX6#|jAInPR)+kLhb_Tl+!(s+eo5Ml;;9OXk>oDmfltPD>PtD*9u zAa}f$c#h2L{TZ5cbq18O9o1iWcgC&h>gm*w(~}aXqzjwI;8zr zZL0S$vI=FJXh}R^>fNf+y-9XF=H>V=W|AJ!;Y+~HC&!wssxf3|;+?ksTV_wnuarwN zEP=`}X2S=4bCq#u*!!;X8r;|B6iO$9Krr)|T~mr7jc_Qga-M&ynoVN`y9UPY+X$1t zo11b#iy=hMMJy|EA$?HS2yy#E)?Ea)mQ^m=7&#z^Nq?%}>S^_9U*mi&fRAfPXJ%JL zXT2|A-S8t)$9JTj&#}j){J<9{h#U{S;fP^d#68iy7S4;(5vn1^;rWg|NYx*#Gl38+ zYj#rYrpUr!9vAFEc`bH`pLO3;-pK#@Vynz9Lq&Wedba~i(fa!g&DK}e;Q=uaACy^M zSbKJ>)2D`9N|ckLJGZbh#wa7(b%i1C5fwI@cz>{N6^DBDV5RVfP!9AwrTOk%&USSU z$W6-@T6FwW!o;-lU_2wk4DE_^bDfb z$GmrmNqi}4AyDpfxriC+henfVre@xfK-FW(*^m-g6N`wHO4PKR)G1usVrB9EoIYOB zz)1WvI9-~W+<*$3D+kf;VTAqYzQ#!G{x1c;B{dD75KK8ch8J2;0%+afWqJo_pL?G1 zYpJE%t2~F#-QMnM5+RRA=xc$KXv|vSyDk&z3kVw4hiYV7*iK@*X7J*`s`2mkj;z^n zas|UkERM2Y4tJjSqrVkgT#V?C$Q)!bvd_RY?Y3mdoO_Ea(LXc9$$j#-8Dui);nhd@ zFTzKh4|X1z68dQlS@bC%OnKw1Z%t^LJ2~jxL=gO54<*Dp5@Q9 z;+UUyhE~!41}TS__Kr6~7K~FyXXbKVaxbWj|I8&}J0bri+^%+g<;|PVM2a9uz=g52Ny(*InA}sh+P^uW2 zxYp&!RS$rI)fxx)INroMvf4VTYom>j%7%qwya{8rQV1)V{xTWO20c4w7MEU}np;^~ z$_VsLC`~|qn@i7w6O|@F+w833_{lxZMKPaR%IAS${>% zujcYubu`1Wn5F{QM?Kn{&e8(}&TaM7_=Kf1d2JCj0&`T9rf_5PcJhZu5SuQ|p2gA& z<)e9VMR>W~m2p0d6SY5|R3?IxI!E8jMh$q5bTPi{&UQ^WDVrmysFhE>pVS6`Jf%{f zMgj-iz9scZy$v=RF6dW=0vrloPmY3($dL0- z@Q_8|pKiiA1{WvkS$`j}I8&qu``nQx$e{$z24{JyN(iD|2v?ITbS z&n?|)2vc6TR{N&_8fGX^zM)ikOvk$X0ip9!nVKVW0+yYlmVAo~Z#;!;U-g{u<2C$P z8XB|GAYkRkP2R!mTFa=ekpL=+yA>W_QIl(ys245XLF)W9tD@TX++So(4!g6 zZ7$07vq?9mI07j_U|~^4=gi;1GXWHKo#isl=yHeljZ_OD(!pa~0ueO=#F`sny zlwC?uiSUP3Xjw5Hy2~rf3%9C)Sq};GSLD{{6)eNBZfmmDt0}VV@v>Q2qM@7&y0Bu_ zOt1rFh?TdJ>07d48>OphTpL9}{xf2FW$&%qKZEpD5=CpS#&vP_!!E09xYR1R|41`1 z0VT3P>W6mgl^G2P&nCxRS(DS5jeqB=9Rg?^FxQtReMyrDqjD*Mb*OWwSgh#`+AUg# z;1;btO5fx#iY1+akQU;$ElCDyF%4~mqF?)Ew}q6%!<4YO;lt~@anQ(<9BBXiwaGUj zg+xN__ZCYr%}bFke%7g*5gl3L4J1{XA{k#!XkBu1ife@mstLU$gCGH8@8)QSHHmqP z^Gc}EkIi5Qaz`>-5fSh42vXTGi+M6?6I>WmVa)smQgUOf5S;FT>M+&xKjx>f2|>5t z7ot>>zm}S7#LMk57XIo!B!onnZ}GQ^kt9))pDdgulAhLW{h%+4O-M0p0+Jf2(U3=H zguleNviC)sGw8|^{GF<4r5>fW)#hn?HpJOV4&}Xd{g#A6-X2d3NfqEj>gqL8JM6$W zx48!?xLUOc2fE&Y@ zhffPdb|R%EsaNykf*ieA$G5kK%q}2TeWQ2I3aK<@Kw{LKXF@MC99z;}!n7ai#Bb^} z|Eu)JUGf7@ z$sC-YPhHw|zTm4$4W>2j_-FjpMP+obYQ!T)jA^+rVJM*e>qCydogW5QF_A2wpx5p9 z=;c~pa)@?bLu(rTlQ6$vf^F;5nmpOFn((9<=0_!&g(}Yp>WhhfyKRC%EGtA|U%w?- zyW2?O!Dth0oR7~~N^-Dp?*M9-)MY#Ebd<1-Q9}2ySpt=+=~_|djIe4?dERs zkmMnG%uME|EC&aV2LJ$&0dYPzik5f=L&IbMfP*~%fCC@}*c#h`O^uDr-Rw^Ci*S1j!F z%AQ5ydYiWY=#Isbi>{x}{Hl@IQ0?{l`|1)b3QIu){n5?w==JSW^r;p=mhj^NBz;MCL2AiRHIF_2gA2N@u>wsJ5xw5@eU}`DOBG zL~h(eft&d7h)^*Oztp1_UC6rDO*(5{*xNJXqY0~quKGVE*>;((O2v7If>SHt_PEMSI*$%ApL#m?BRqD<=%RXYnq2>_6W1bggGW1ZU5hNo^|}RO2BnIiw%!)E+-jP8$FF#uM?%0E*So z9wBhTNt;Z;77mf26!@}JfU84~--~g_+a*n{y2n%bfk9r>mcIZWF$y2G$Z*h8;D7iy zyr)|V&X@+K4ruCzSgni60+9hKPha4228-eXfmua ze+ZA`vtK<7fhg2JO&fpk%hwg>O0t*n$-5S)dvof*ZhuWV{5t6{L>)NVAqPEaYZtVkHK4Bw+Qa?Iwpr=!-XCM;VWk!Nn~i$gRQx$`xtwxkuH3hb>`G@yBC;p5FlUAuMU*c`{`TK6 z!u7dj?${e+cT8Q!`tX&r>cdhx0AIp8d&CUe6)i*m0Z6Q9X4JQhRFE+sj-z&X;iRmMi7j?Gb6iO>0$gYH3x zT$WwbllG{}RXq7VYM*8$2!VW+rwWv1#@U2dNf4DF)_KC*i#WyK-2zXyY=k*vn89yF zEaaF%xd1E$_w$9!=ndT&ocvg{qojUj>GF6GGRn4Ef)7{ zmj@tQB`q(s?tRny{KI zP9iF5+BscozCR~`A8V>Pp)1kDfU))grcns_+q8ZD*M(}lh)$X|`fa4$1LR^1C(u0- zX3C;YWEj<&D#+oGQun7S8b(!PWfi>a>Pwlh_b~ z=(esEm$gtZ+N2lIi<(Jj2&HS6J<+Q%1fE53Au!F5OAPn(l&kW$&lP~zgASK#vybb) zZ>(!1NO_MVFT(2$Yc+0sZG-y=*2HjGIU60FK-QAo-j#x#6>jAnNUcagNvB?MIV#lt z>WSis^VO}j!LvZbCvj5I0O`7$yw7wp*byIz$NAsNdHvb@-+vOV&YW#~(YtAH^L3;9 zyWU*F|6PX8T$^F-IO%WgbyMZF9vn=4%uxr_Ml%z-jDJQsuKQ&nT0L?N(Sf%zuhCcA zB9n}XTh^ZVgcjRQI6>T@n5?Dpp7`hHAj(7e0!i|FB3t`8@l%|Z&z2OX8p*;ap2SaG zZHQOJfHWog{Zv5_iu<2(XV1e@5VrG&ixeC+Wy?LWpbf@#%_@E#0#u&R#h{XuP7NEK z4+iaH&asdtfM>8}{V{$xz_H%L37lI(9i!14Zayrv(we=QkIW?<0LC%uCz$HD{&ec& z1*0kM9%&B2{08719DRQCec7w;l8GTYy z8W`YHX^UAH2ovs9L;K*8J_PEQwy|^2paj8ZH3!@;=Q3raEUhlu3sh(Ne*_3;wL-Ht z)!xUV`B){W^#Xi@`?@hRE_nsK<%K(~%9Kfr%d4(5_IB@1cb|5ycP6pFFe*=whku~P zK920~p2#t=3k8G3V$=4pw?&YrlUTz8%`Jlp;)TrjZGMPQV9N3^>8&*BH><6}dAMKk zKOHMrm{rp1pcsfN=*rHWzdsG~d|di68M^z20MFh0Uq?Q4|4t3-ak$r5le@6x=G!t9 zlP|W%?G!6JNe8GzbfH_b;=s`YQ^ff4f|Wv1m-X;uxay8InNtYi@>!4EJ{CLCN*n&g$AL!C~uJsVD< zul*DxinKyPkMNmjAwUb#HsN^^JnumO>SE~ZBE7I}-?r;3$bo6~u>LtSH{@9r4!rP` zmY5>Uzicx|38Vy&;EKDpoNls?7@+$Shf7*EsfBfFOgR^|pG>Q;ww)0~CO0yhY?8u+ zG~`ZWDJ*arXD)t>igej?0=9HVPW3YV8ML-JWZ5C0?|G1kg5f`Y|Z#CJtjZ z2Lpoqd^dG*Bc*GJsx^RxBaq982yETLtB_yQXnC@g;TP`4!s%Q>#Vt0p24;` zDG~`B!(JS2dH>jLKp(FwC|!A6NwJ#@mH0%e6^J-O#_-^UU;p%7x%Agut&_1 z-w#6Y`CKEu2Hwt1AK!V2;=;W1yfFYkF_p(YfHBW47vfeF=UXOWSe)%NtpAh_wGQPL z-%y$)xwp~MVb9qHM?hc=br!s_z534_)m+i5?}_D<2Fxy>mm62-%fjtV8ohEqyz$GX zQC|h_`pSC!pE5PQw!vfrB~p~cf_MOgu(li?k#+X6)+lG*QC67_P8enHgA+*EK2}}N zqt{Pfpj!A3K9FcJg`OWia}r}rSDBxn3hLb{DAgr&+~!#Qq2n={O;2qghxls`mAwBNjSGQ|x88 z?;IE4`Lkj_^s^!F;Gq6(Joio8g14u{0Jo`4Ua~ni3ldZz8k|dO`mI3^iTO*LixYoV zT2>gFt*AY6wiE(G92UmYcxYeGYLZ#qV%4phlDdlcyQA7i=8P(uzmT{?%A=54?_+vX zlPFj5o^@bp2I7aJZhA=_CgFJM5Fy3Jjff)pxHdH5k2Ha)oMlt`@?PRiPCpK}8F(1} zP4Y9DBf2iI*2De7(eNg0)gn-?@$x=1p7TX#3dHl`Pl29Czv+srT-$~%l-ABhLdUwsJ(G3Bd-0>1xQvFZ!<@^TIVP zn##wddz(9IymFpA`GC=!FCs!CAHF~CFoy%C$bY~9KM)?Yl!yE$c3{;c)OF4UY(txD zL38l^iiHMi3sv*`H!C4Q>zSzNs$gUHihw26MUhM`KvB8!JZCPwGPMjjK^Ik!6s^8- zZn#`>_mT!N*o`nqS+kJHw}*8Evm_C9W>b1RIHLBkGfEar;S*G$rPT+eIaCU*5O4PC z>!wzT*Yt2%iD9q2wdGDy>Ez#$*=4(FH?Snosp>EzI6th4{*tt|9fHvUZZ@uEtF2u| z$%*r-ZUsl<8AeC36JzN9tDZ-2s-`BRXv znCB}rzZ!NAR~?k7Xti<|2B`RI;3Y8aPgsqI!BAR9`&(Py!jAAxhz|d-uioHC--=9?BhTkZmxEc6d2_lm%D-tDD}lQL-l7OoGt#~Rt51M z;vsYAoTVdg{)>ee@F8{F4MMQ;rFzeBFz*bLkGkOE!eC0m=71L;+a zxn#A|&&`L5nl)c}vSJY16I#|_WXuB~ViL*5Hvv(0udrc-S|xQw41?!OZop z0nBo4sQJRiMDLI=O#lXHW6JmKkH;Q4*xSaYie>8fPo`1-V)@JRu2qWL+G=~$kyXPd zpr*Vf9}0uhy5F1c4fy4Cow4>CRMbglm0oys^Vb!7deOz{04sl!hUn~sk8MQnTtskdO6`O zq0PNtVE}G}Srh=574S8cCR5Z{Ep+Um9!HBk#q)^pq@k7u#JFG^p~B7b-Kt|^oz7fG z79WpN`y7WjUZ0<-FQ`L$dhqsg`oZ(LWB*`9W1xO*-g~t`vNtnze@-BO5Ev*}k1C|~ z5gyFitrf$jQ+_l#t{-ys|kP z<UV{n!@9CT$I zZ#x>=HI+!M2P)BJ-ce}Z+%D0C`c7$k)fgIji*@0sbu5R-IJxWrk`)#iqgePR>d?dK zQ1S3#z@IoP^iTboN}QiOWVCpXk9wPn+N_>74GX`gkh9j% zCn3YNApgZes0|-hlN0Ziz)RvZ%(j0fZ`ULUhSe@!K~%@P7DjMfT9qHO4$yBPr5SsRK8^12bc+6vd9Cxg z0a=R(LWf$UIewDIs}QVTlcJTDBDiu;ca!WKy1TMUFlipA6ES@@WBOfKc!H8@%@&yz z$Ko97LpjdM$Eb98(N*VrsVh6$g$G2N*CSXd&a13Ho-DHdt=C-}9iSHUskPKB_Gpoa zg`-mfzXl?f%%!8hSb_3II8Y+9FDU2`__$3Rc`TBvknT=JqQ5;1tk}d_rEHv)f4ru~ zeRb%t*E$@ll7{2r?Mqf}mF>Se=L1ooht}q34*#{?K_??_fMc?QbE@TB78nzccRKSU zHVB~8zf}GHR>@U~dZ;wdR$oYCCQ!v-Z|a$>6MzVjVo6~xhvOvyGopc{V}9IeNVxF@ zDDr%Ao7ki6E^|qs$KLaF5*fKq&tOs_?m8uxf)DFw=SdRpMK7XAk zcPT%a=XbMs)ep+TIojFOUWMBf2i(=k4mgW6Gd)wO$(%c+BD&7`8RCr%XR`h%0ND zbUO}=hemMspr(p|u6`bUSE^9R-~gq+ktdZ*pF!-%u(d;6kzOwJF@{oBVYTa#zDecy zXe!tl3RO|3c=+os*j>=mYqL%YUYgK_pojZkQ~yCFVPj zY0aV;MEDz2VXV`WG;kaB;2DT$&%K;Dfi=ktG2s$!sAh8sH$Gt*+4Tq=p~+xzjn$IS ziO%yIjkG+vWJ*PMPX{`+M-;Yip=Q4p+Zqs~hsY+vp2p_UKj)US1T1I6hRui#nH5Ox zL@HlJ^C5^M+SxdV74>~oVhAPdSGspiKsAiUu8&X2@2Zpg(QR`P<%r|)!k*x_0(E?Q zWR9mNC$H{9)aeFLPAbF{%IQm{U3Hg##JI>7506CgllU~NTx+1Q<1Hf}?DF)NLy0hW zGbg-@Skfaz5~vYns8gRpv6G?N!RsE&Ki%I{6(V1Zm5ZuI^1t6hxU%;Nb__B4Z?)-T zZuZ9b$_y9trwBH%fFS|*csDxO#uCl>=v!a9mOhnFTndPR@^7A~(}JT`2y~i?snnMr zr@WC6rZ7k^`o5)8;JGmz2`U#C{xb?smpV3ji}b@coRqq2lUPOg2a+Y|SR$aCTRv(Z ze6px>cc1+pg6#YAnsV6XdY|=O>gft@d#@ksuumWHRUx$&XS5i|y<=zPl+ly(whao8a@;g{e~E23Cx4-y2Nb)%uABKdDQStzRT}7tUUITi3^%AXPr#FSwIP6y z9MzIRl&PN4!!Mw7Hw`DK3hR@Cn0@3Jl;#f0HPUD7ct z3{))3y!+f#o+q!?_FS4O+>W`wG&+u?9WmZ$;4$*^^zTM#-GcDO#nd)&2W~~44A{&| z!@CvuWx4C}q-10io={8SIV$@CB3L@`j(KkeaQVExUwQt|8qM1&dqo*o-mMU^8zT=%KhmahIn^>YQ=`Z zB9~uMMtn3!UtU)VXB@<3I^vgkuu67cHU9Yz8E&S%*6JI^AN9_kD0gnu$szJ)Ye{5?q{!0x?i9z?#eYdur`^y<&Cj^3zpLEJlY^-!6MTcYMRl?+%p&&f|lWA1YcQH)=V{N7IKohNx4Y2Ns0!PO-k@jCow#Q`rdCER0LK675>_c+Hmj zTx;lGia2x^lNi(@@UP4LOHUJ}8&*oxfV1S{gzz%!I10!9jr)!!sRGKBB@lCBA(4J^ zI;oPB9dRVzJZaZDBOUQlparU!&5SNfj8~D+qs#9uda>e@3pNO6z`Y;!kWxnkc}oto zHAu@7330)c&YGkwFU;aAqj z)eaxcWk$a39oM=`WOPd=XXgrB^G4@~p>ZR{x|HhXqjQVsv20|xpQ(Hw`LB8fm<)+= z{vgYO}yV8a&ap11#hb{7bH6_+g(T1=}ylK!=B-6LB6i9wjGlJ@+B; z3vX8PCI0QvVfh-#;BQ^?Fa+SM%%dQlc z0m{^-ijSa*u)wlb4Vx|ItEW+&?X`>AXWFnoTmuH5R`?4hn370OtmRj^NUoG(vao&# zxR_J3{%gou_lb3jgnZ0m)m0?-|9r@GFmaEi=7j$`+Joztb^6j217l4Q4>RH20~dKq zIR`t~uWFf<)>DL>UyKqzBT&$XlbRjQsKbcy{$(VaZb5n!S`<{2jzJ|C91l7TzQd@sT{7VAbn*?#5W5xP~!cq0FEojf-H5=U%T`Ua5({|R?4 zwdlqDOQUN%hi$yk(cvnst@GiFt8=!VJU`#W5Zd&~$2kHl>FW-fC3qp?l0_4)7Akte z(y`cu33&3@2)pRpk`818dSpVx4?LZPz`gm|g~O?O+n6qFl?AhU1UH|7l;aS50#HT9 za;2RZCqGPGQjTi$M&|8qVk-GXP;M|=OJ(V_IVSY`Ty-HTp`?*f;bR^Vci4qk-HOEM z8fYGcQqZE_s!@~fK&FxAm5ykCVOKdb;lN?H;Ko1}#{38}SbLtT#KL5+%W1G5hU}xd zE=VG2mzg{WX>@ih7F5Xj{Y{f-t_5OA#d`SO@C&y;tBIBT9ny=TTSF8I9ORzv`#?~Z zgMlT3`~Q(R;eS*P0Q?8ufa1Xass3L;j_`kg{QtsnWjO@I|CxaKFSP&5pUD5G`(LN& BlSTjl diff --git a/files/lambda_function.py b/files/lambda_function.py deleted file mode 100644 index 569e078..0000000 --- a/files/lambda_function.py +++ /dev/null @@ -1,865 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https://www.datadoghq.com/). -# Copyright 2018 Datadog, Inc. - -from __future__ import print_function - -import base64 -import gzip -import json -import os -import re -import socket -from botocore.vendored import requests -import time -import ssl -import urllib -import itertools -from io import BytesIO, BufferedReader - -import boto3 - -try: - # Datadog Lambda layer is required to forward metrics - from datadog_lambda.wrapper import datadog_lambda_wrapper - from datadog_lambda.metric import lambda_stats - DD_FORWARD_METRIC = True -except ImportError: - # For backward-compatibility - DD_FORWARD_METRIC = False - - -# Set this variable to `False` to disable log forwarding. -# E.g., when you only want to forward metrics from logs. -DD_FORWARD_LOG = os.getenv("DD_FORWARD_LOG", default="true").lower() == "true" - - -# Change this value to change the underlying network client (HTTP or TCP), -# by default, use the TCP client. -DD_USE_TCP = os.getenv("DD_USE_TCP", default="true").lower() == "true" - - -# Define the destination endpoint to send logs to -DD_SITE = os.getenv("DD_SITE", default="datadoghq.com") -if DD_USE_TCP: - DD_URL = os.getenv("DD_URL", default="lambda-intake.logs." + DD_SITE) - try: - if "DD_SITE" in os.environ and DD_SITE == "datadoghq.eu": - DD_PORT = int(os.environ.get("DD_PORT", 443)) - else: - DD_PORT = int(os.environ.get("DD_PORT", 10516)) - except Exception: - DD_PORT = 10516 -else: - DD_URL = os.getenv("DD_URL", default="lambda-http-intake.logs." + DD_SITE) - - -class ScrubbingRuleConfig(object): - def __init__(self, name, pattern, placeholder): - self.name = name - self.pattern = pattern - self.placeholder = placeholder - - -# Scrubbing sensitive data -# Option to redact all pattern that looks like an ip address / email address / custom pattern -SCRUBBING_RULE_CONFIGS = [ - ScrubbingRuleConfig( - "REDACT_IP", "\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", "xxx.xxx.xxx.xxx" - ), - ScrubbingRuleConfig( - "REDACT_EMAIL", - "[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+", - "xxxxx@xxxxx.com", - ), - ScrubbingRuleConfig( - "DD_SCRUBBING_RULE", - os.getenv("DD_SCRUBBING_RULE", default=None), - os.getenv("DD_SCRUBBING_RULE_REPLACEMENT", default="xxxxx") - ) -] - -# Use for include, exclude, and scrubbing rules -def compileRegex(rule, pattern): - if pattern is not None: - if pattern == "": - # If pattern is an empty string, raise exception - raise Exception("No pattern provided:\nAdd pattern or remove {} environment variable".format(rule)) - try: - return re.compile(pattern) - except Exception: - raise Exception("could not compile {} regex with pattern: {}".format(rule, pattern)) - - -# Filtering logs -# Option to include or exclude logs based on a pattern match -INCLUDE_AT_MATCH = os.getenv("INCLUDE_AT_MATCH", default=None) -include_regex = compileRegex("INCLUDE_AT_MATCH", INCLUDE_AT_MATCH) - -EXCLUDE_AT_MATCH = os.getenv("EXCLUDE_AT_MATCH", default=None) -exclude_regex = compileRegex("EXCLUDE_AT_MATCH", EXCLUDE_AT_MATCH) - - -# DD_API_KEY: Datadog API Key -DD_API_KEY = "" -if "DD_KMS_API_KEY" in os.environ: - ENCRYPTED = os.environ["DD_KMS_API_KEY"] - DD_API_KEY = boto3.client("kms").decrypt( - CiphertextBlob=base64.b64decode(ENCRYPTED) - )["Plaintext"] -elif "DD_API_KEY" in os.environ: - DD_API_KEY = os.environ["DD_API_KEY"] - -# Strip any trailing and leading whitespace from the API key -DD_API_KEY = DD_API_KEY.strip() - -# DD_API_KEY must be set -if DD_API_KEY == "" or DD_API_KEY == "": - raise Exception( - "You must configure your Datadog API key using " - "DD_KMS_API_KEY or DD_API_KEY" - ) -# Check if the API key is the correct number of characters -if len(DD_API_KEY) != 32: - raise Exception( - "The API key is not the expected length. " - "Please confirm that your API key is correct" - ) -# Validate the API key -validation_res = requests.get( - "https://api.{}/api/v1/validate?api_key={}".format(DD_SITE, DD_API_KEY) -) -if not validation_res.ok: - raise Exception("The API key is not valid.") - - -# DD_MULTILINE_LOG_REGEX_PATTERN: Datadog Multiline Log Regular Expression Pattern -DD_MULTILINE_LOG_REGEX_PATTERN = None -if "DD_MULTILINE_LOG_REGEX_PATTERN" in os.environ: - DD_MULTILINE_LOG_REGEX_PATTERN = os.environ["DD_MULTILINE_LOG_REGEX_PATTERN"] - try: - multiline_regex = re.compile( - "(?[^/]+)/(?P[^/]+)") - -DD_SOURCE = "ddsource" -DD_CUSTOM_TAGS = "ddtags" -DD_SERVICE = "service" -DD_HOST = "host" -DD_FORWARDER_VERSION = "1.4.1" - -# Pass custom tags as environment variable, ensure comma separated, no trailing comma in envvar! -DD_TAGS = os.environ.get("DD_TAGS", "") - - -class RetriableException(Exception): - pass - - -class ScrubbingException(Exception): - pass - - -class DatadogClient(object): - """ - Client that implements a exponential retrying logic to send a batch of logs. - """ - - def __init__(self, client, max_backoff=30): - self._client = client - self._max_backoff = max_backoff - - def send(self, logs): - backoff = 1 - while True: - try: - self._client.send(logs) - return - except RetriableException: - time.sleep(backoff) - if backoff < self._max_backoff: - backoff *= 2 - continue - - def __enter__(self): - self._client.__enter__() - return self - - def __exit__(self, ex_type, ex_value, traceback): - self._client.__exit__(ex_type, ex_value, traceback) - - -class DatadogTCPClient(object): - """ - Client that sends a batch of logs over TCP. - """ - - def __init__(self, host, port, api_key, scrubber): - self.host = host - self.port = port - self._api_key = api_key - self._scrubber = scrubber - self._sock = None - - def _connect(self): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock = ssl.wrap_socket(sock) - sock.connect((self.host, self.port)) - self._sock = sock - - def _close(self): - if self._sock: - self._sock.close() - - def _reset(self): - self._close() - self._connect() - - def send(self, logs): - try: - frame = self._scrubber.scrub( - "".join( - ["{} {}\n".format(self._api_key, log) for log in logs] - ) - ) - self._sock.sendall(frame.encode("UTF-8")) - except ScrubbingException: - raise Exception("could not scrub the payload") - except Exception: - # most likely a network error, reset the connection - self._reset() - raise RetriableException() - - def __enter__(self): - self._connect() - return self - - def __exit__(self, ex_type, ex_value, traceback): - self._close() - - -class DatadogHTTPClient(object): - """ - Client that sends a batch of logs over HTTP. - """ - - _POST = "POST" - _HEADERS = {"Content-type": "application/json"} - - def __init__(self, host, api_key, scrubber, timeout=10): - self._url = "https://{}/v1/input/{}".format(host, api_key) - self._scrubber = scrubber - self._timeout = timeout - self._session = None - - def _connect(self): - self._session = requests.Session() - self._session.headers.update(self._HEADERS) - - def _close(self): - self._session.close() - - def send(self, logs): - """ - Sends a batch of log, only retry on server and network errors. - """ - try: - resp = self._session.post( - self._url, - data=self._scrubber.scrub("[{}]".format(",".join(logs))), - timeout=self._timeout, - ) - except ScrubbingException: - raise Exception("could not scrub the payload") - except Exception: - # most likely a network error - raise RetriableException() - if resp.status_code >= 500: - # server error - raise RetriableException() - elif resp.status_code >= 400: - # client error - raise Exception( - "client error, status: {}, reason {}".format( - resp.status_code, resp.reason - ) - ) - else: - # success - return - - def __enter__(self): - self._connect() - return self - - def __exit__(self, ex_type, ex_value, traceback): - self._close() - - -class DatadogBatcher(object): - def __init__(self, max_log_size_bytes, max_size_bytes, max_size_count): - self._max_log_size_bytes = max_log_size_bytes - self._max_size_bytes = max_size_bytes - self._max_size_count = max_size_count - - def _sizeof_bytes(self, log): - return len(log.encode("UTF-8")) - - def batch(self, logs): - """ - Returns an array of batches. - Each batch contains at most max_size_count logs and - is not strictly greater than max_size_bytes. - All logs strictly greater than max_log_size_bytes are dropped. - """ - batches = [] - batch = [] - size_bytes = 0 - size_count = 0 - for log in logs: - log_size_bytes = self._sizeof_bytes(log) - if size_count > 0 and ( - size_count >= self._max_size_count - or size_bytes + log_size_bytes > self._max_size_bytes - ): - batches.append(batch) - batch = [] - size_bytes = 0 - size_count = 0 - # all logs exceeding max_log_size_bytes are dropped here - if log_size_bytes <= self._max_log_size_bytes: - batch.append(log) - size_bytes += log_size_bytes - size_count += 1 - if size_count > 0: - batches.append(batch) - return batches - - -class ScrubbingRule(object): - def __init__(self, regex, placeholder): - self.regex = regex - self.placeholder = placeholder - - -class DatadogScrubber(object): - def __init__(self, configs): - rules = [] - for config in configs: - if config.name in os.environ: - rules.append( - ScrubbingRule( - compileRegex(config.name, config.pattern), - config.placeholder - ) - ) - self._rules = rules - - def scrub(self, payload): - for rule in self._rules: - try: - payload = rule.regex.sub(rule.placeholder, payload) - except Exception: - raise ScrubbingException() - return payload - - -def datadog_forwarder(event, context): - """The actual lambda function entry point""" - events = parse(event, context) - metrics, logs = split(events) - if DD_FORWARD_LOG: - forward_logs(filter_logs(logs)) - if DD_FORWARD_METRIC: - forward_metrics(metrics) - - -if DD_FORWARD_METRIC: - # Datadog Lambda layer is required to forward metrics - lambda_handler = datadog_lambda_wrapper(datadog_forwarder) -else: - lambda_handler = datadog_forwarder - - -def forward_logs(logs): - """Forward logs to Datadog""" - scrubber = DatadogScrubber(SCRUBBING_RULE_CONFIGS) - if DD_USE_TCP: - batcher = DatadogBatcher(256 * 1000, 256 * 1000, 1) - cli = DatadogTCPClient(DD_URL, DD_PORT, DD_API_KEY, scrubber) - else: - batcher = DatadogBatcher(128 * 1000, 1 * 1000 * 1000, 25) - cli = DatadogHTTPClient(DD_URL, DD_API_KEY, scrubber) - - with DatadogClient(cli) as client: - for batch in batcher.batch(logs): - try: - client.send(batch) - except Exception as e: - print("Unexpected exception: {}, batch: {}".format(str(e), batch)) - - -def parse(event, context): - """Parse Lambda input to normalized events""" - metadata = generate_metadata(context) - try: - # Route to the corresponding parser - event_type = parse_event_type(event) - if event_type == "s3": - events = s3_handler(event, context, metadata) - elif event_type == "awslogs": - events = awslogs_handler(event, context, metadata) - elif event_type == "events": - events = cwevent_handler(event, metadata) - elif event_type == "sns": - events = sns_handler(event, metadata) - elif event_type == "kinesis": - events = kinesis_awslogs_handler(event, context, metadata) - except Exception as e: - # Logs through the socket the error - err_message = "Error parsing the object. Exception: {} for event {}".format( - str(e), event - ) - events = [err_message] - return normalize_events(events, metadata) - - -def generate_metadata(context): - metadata = { - "ddsourcecategory": "aws", - "aws": { - "function_version": context.function_version, - "invoked_function_arn": context.invoked_function_arn, - }, - } - # Add custom tags here by adding new value with the following format "key1:value1, key2:value2" - might be subject to modifications - dd_custom_tags_data = { - "forwardername": context.function_name.lower(), - "memorysize": context.memory_limit_in_mb, - "forwarder_version": DD_FORWARDER_VERSION, - } - metadata[DD_CUSTOM_TAGS] = ",".join( - filter( - None, - [ - DD_TAGS, - ",".join( - ["{}:{}".format(k, v) for k, v in dd_custom_tags_data.iteritems()] - ), - ], - ) - ) - - return metadata - - -def extract_metric(event): - """Extract metric from an event if possible""" - try: - metric = json.loads(event['message']) - required_attrs = {'m', 'v', 'e', 't'} - if all(attr in metric for attr in required_attrs): - return metric - else: - return None - except Exception: - return None - - -def split(events): - """Split events to metrics and logs""" - metrics, logs = [], [] - for event in events: - metric = extract_metric(event) - if metric: - metrics.append(metric) - else: - logs.append(json.dumps(event)) - return metrics, logs - - -# should only be called when INCLUDE_AT_MATCH and/or EXCLUDE_AT_MATCH exist -def filter_logs(logs): - """ - Applies log filtering rules. - If no filtering rules exist, return all the logs. - """ - if INCLUDE_AT_MATCH is None and EXCLUDE_AT_MATCH is None: - # convert to strings - return logs - # Add logs that should be sent to logs_to_send - logs_to_send = [] - # Test each log for exclusion and inclusion, if the criteria exist - for log in logs: - try: - if EXCLUDE_AT_MATCH is not None: - # if an exclude match is found, do not add log to logs_to_send - if re.search(exclude_regex, log): - continue - if INCLUDE_AT_MATCH is not None: - # if no include match is found, do not add log to logs_to_send - if not re.search(include_regex, log): - continue - logs_to_send.append(log) - except ScrubbingException: - raise Exception("could not filter the payload") - return logs_to_send - - -def forward_metrics(metrics): - """ - Forward custom metrics submitted via logs to Datadog in a background thread - using `lambda_stats` that is provided by the Datadog Python Lambda Layer. - """ - for metric in metrics: - try: - lambda_stats.distribution( - metric['m'], - metric['v'], - timestamp=metric['e'], - tags=metric['t'] - ) - except Exception as e: - print("Unexpected exception: {}, metric: {}".format(str(e), metric)) - - -# Utility functions - - -def normalize_events(events, metadata): - normalized = [] - for event in events: - if isinstance(event, dict): - normalized.append(merge_dicts(event, metadata)) - elif isinstance(event, str): - normalized.append(merge_dicts({"message": event}, metadata)) - else: - # drop this log - continue - return normalized - - -def parse_event_type(event): - if "Records" in event and len(event["Records"]) > 0: - if "s3" in event["Records"][0]: - return "s3" - elif "Sns" in event["Records"][0]: - return "sns" - elif "kinesis" in event["Records"][0]: - return "kinesis" - - elif "awslogs" in event: - return "awslogs" - - elif "detail" in event: - return "events" - raise Exception("Event type not supported (see #Event supported section)") - - -# Handle S3 events -def s3_handler(event, context, metadata): - s3 = boto3.client("s3") - - # Get the object from the event and show its content type - bucket = event["Records"][0]["s3"]["bucket"]["name"] - key = urllib.unquote_plus(event["Records"][0]["s3"]["object"]["key"]).decode("utf8") - - source = parse_event_source(event, key) - metadata[DD_SOURCE] = source - ##default service to source value - metadata[DD_SERVICE] = source - ##Get the ARN of the service and set it as the hostname - hostname = parse_service_arn(source, key, bucket, context) - if hostname: - metadata[DD_HOST] = hostname - - # Extract the S3 object - response = s3.get_object(Bucket=bucket, Key=key) - body = response["Body"] - data = body.read() - - # If the name has a .gz extension, then decompress the data - if key[-3:] == ".gz": - with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream: - # Reading line by line avoid a bug where gzip would take a very long time (>5min) for - # file around 60MB gzipped - data = "".join(BufferedReader(decompress_stream)) - - if is_cloudtrail(str(key)): - cloud_trail = json.loads(data) - for event in cloud_trail["Records"]: - # Create structured object and send it - structured_line = merge_dicts( - event, {"aws": {"s3": {"bucket": bucket, "key": key}}} - ) - yield structured_line - else: - # Check if using multiline log regex pattern - # and determine whether line or pattern separated logs - if DD_MULTILINE_LOG_REGEX_PATTERN and multiline_regex_start_pattern.match(data): - split_data = multiline_regex.split(data) - else: - split_data = data.splitlines() - - # Send lines to Datadog - for line in split_data: - # Create structured object and send it - structured_line = { - "aws": {"s3": {"bucket": bucket, "key": key}}, - "message": line, - } - yield structured_line - - -# Handle CloudWatch logs from Kinesis -def kinesis_awslogs_handler(event, context, metadata): - def reformat_record(record): - return { - "awslogs": { - "data": record["kinesis"]["data"] - } - } - - return itertools.chain.from_iterable(awslogs_handler(reformat_record(r), context, metadata) for r in event["Records"]) - - -# Handle CloudWatch logs -def awslogs_handler(event, context, metadata): - # Get logs - with gzip.GzipFile( - fileobj=BytesIO(base64.b64decode(event["awslogs"]["data"])) - ) as decompress_stream: - # Reading line by line avoid a bug where gzip would take a very long - # time (>5min) for file around 60MB gzipped - data = "".join(BufferedReader(decompress_stream)) - logs = json.loads(str(data)) - - # Set the source on the logs - source = logs.get("logGroup", "cloudwatch") - metadata[DD_SOURCE] = parse_event_source(event, source) - - # Default service to source value - metadata[DD_SERVICE] = metadata[DD_SOURCE] - - # Build aws attributes - aws_attributes = { - "aws": { - "awslogs": { - "logGroup": logs["logGroup"], - "logStream": logs["logStream"], - "owner": logs["owner"], - } - } - } - - # Set host as log group where cloudwatch is source - if metadata[DD_SOURCE] == "cloudwatch": - metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"] - - # When parsing rds logs, use the cloudwatch log group name to derive the - # rds instance name, and add the log name of the stream ingested - if metadata[DD_SOURCE] == "rds": - match = rds_regex.match(logs["logGroup"]) - if match is not None: - metadata[DD_HOST] = match.group("host") - metadata[DD_CUSTOM_TAGS] = ( - metadata[DD_CUSTOM_TAGS] + ",logname:" + match.group("name") - ) - # We can intuit the sourcecategory in some cases - if match.group("name") == "postgresql": - metadata[DD_CUSTOM_TAGS] + ",sourcecategory:" + match.group("name") - - # For Lambda logs we want to extract the function name, - # then rebuild the arn of the monitored lambda using that name. - # Start by splitting the log group to get the function name - if metadata[DD_SOURCE] == "lambda": - log_group_parts = logs["logGroup"].split("/lambda/") - if len(log_group_parts) > 1: - function_name = log_group_parts[1].lower() - # Split the arn of the forwarder to extract the prefix - arn_parts = context.invoked_function_arn.split("function:") - if len(arn_parts) > 0: - arn_prefix = arn_parts[0] - # Rebuild the arn by replacing the function name - arn = arn_prefix + "function:" + function_name - # Add the arn as a log attribute - arn_attributes = {"lambda": {"arn": arn}} - aws_attributes = merge_dicts(aws_attributes, arn_attributes) - # Add the function name as tag - metadata[DD_CUSTOM_TAGS] += ",functionname:" + function_name - # Set the arn as the hostname - metadata[DD_HOST] = arn - - # Create and send structured logs to Datadog - for log in logs["logEvents"]: - yield merge_dicts(log, aws_attributes) - - -# Handle Cloudwatch Events -def cwevent_handler(event, metadata): - - data = event - - # Set the source on the log - source = data.get("source", "cloudwatch") - service = source.split(".") - if len(service) > 1: - metadata[DD_SOURCE] = service[1] - else: - metadata[DD_SOURCE] = "cloudwatch" - ##default service to source value - metadata[DD_SERVICE] = metadata[DD_SOURCE] - - yield data - - -# Handle Sns events -def sns_handler(event, metadata): - - data = event - # Set the source on the log - metadata[DD_SOURCE] = parse_event_source(event, "sns") - - for ev in data["Records"]: - # Create structured object and send it - structured_line = ev - yield structured_line - - -def merge_dicts(a, b, path=None): - if path is None: - path = [] - for key in b: - if key in a: - if isinstance(a[key], dict) and isinstance(b[key], dict): - merge_dicts(a[key], b[key], path + [str(key)]) - elif a[key] == b[key]: - pass # same leaf value - else: - raise Exception( - "Conflict while merging metadatas and the log entry at %s" - % ".".join(path + [str(key)]) - ) - else: - a[key] = b[key] - return a - - -cloudtrail_regex = re.compile( - "\d+_CloudTrail_\w{2}-\w{4,9}-\d_\d{8}T\d{4}Z.+.json.gz$", re.I -) - - -def is_cloudtrail(key): - match = cloudtrail_regex.search(key) - return bool(match) - - -def parse_event_source(event, key): - if "elasticloadbalancing" in key: - return "elb" - for source in [ - "lambda", - "redshift", - "cloudfront", - "kinesis", - "mariadb", - "mysql", - "apigateway", - "route53", - "vpc", - "rds", - "sns", - "waf", - "docdb", - "fargate" - ]: - if source in key: - return source - if "API-Gateway" in key or "ApiGateway" in key: - return "apigateway" - if is_cloudtrail(str(key)) or ('logGroup' in event and event['logGroup'] == 'CloudTrail'): - return "cloudtrail" - if "awslogs" in event: - return "cloudwatch" - if "Records" in event and len(event["Records"]) > 0: - if "s3" in event["Records"][0]: - return "s3" - - return "aws" - - -def parse_service_arn(source, key, bucket, context): - if source == "elb": - # For ELB logs we parse the filename to extract parameters in order to rebuild the ARN - # 1. We extract the region from the filename - # 2. We extract the loadbalancer name and replace the "." by "/" to match the ARN format - # 3. We extract the id of the loadbalancer - # 4. We build the arn - idsplit = key.split("/") - # If there is a prefix on the S3 bucket, idsplit[1] will be "AWSLogs" - # Remove the prefix before splitting they key - if len(idsplit) > 1 and idsplit[1] == "AWSLogs": - idsplit = idsplit[1:] - keysplit = "/".join(idsplit).split("_") - # If no prefix, split the key - else: - keysplit = key.split("_") - if len(keysplit) > 3: - region = keysplit[2].lower() - name = keysplit[3] - elbname = name.replace(".", "/") - if len(idsplit) > 1: - idvalue = idsplit[1] - return "arn:aws:elasticloadbalancing:{}:{}:loadbalancer/{}".format( - region, idvalue, elbname - ) - if source == "s3": - # For S3 access logs we use the bucket name to rebuild the arn - if bucket: - return "arn:aws:s3:::{}".format(bucket) - if source == "cloudfront": - # For Cloudfront logs we need to get the account and distribution id from the lambda arn and the filename - # 1. We extract the cloudfront id from the filename - # 2. We extract the AWS account id from the lambda arn - # 3. We build the arn - namesplit = key.split("/") - if len(namesplit) > 0: - filename = namesplit[len(namesplit) - 1] - # (distribution-ID.YYYY-MM-DD-HH.unique-ID.gz) - filenamesplit = filename.split(".") - if len(filenamesplit) > 3: - distributionID = filenamesplit[len(filenamesplit) - 4].lower() - arn = context.invoked_function_arn - arnsplit = arn.split(":") - if len(arnsplit) == 7: - awsaccountID = arnsplit[4].lower() - return "arn:aws:cloudfront::{}:distribution/{}".format( - awsaccountID, distributionID - ) - if source == "redshift": - # For redshift logs we leverage the filename to extract the relevant information - # 1. We extract the region from the filename - # 2. We extract the account-id from the filename - # 3. We extract the name of the cluster - # 4. We build the arn: arn:aws:redshift:region:account-id:cluster:cluster-name - namesplit = key.split("/") - if len(namesplit) == 8: - region = namesplit[3].lower() - accountID = namesplit[1].lower() - filename = namesplit[7] - filesplit = filename.split("_") - if len(filesplit) == 6: - clustername = filesplit[3] - return "arn:aws:redshift:{}:{}:cluster:{}:".format( - region, accountID, clustername - ) - return \ No newline at end of file diff --git a/logshipping.tf b/logshipping.tf index 5fdbf5a..cffc7e9 100644 --- a/logshipping.tf +++ b/logshipping.tf @@ -75,15 +75,15 @@ resource "aws_iam_role_policy_attachment" "datadog-logshipping-lambda-attach3" { } resource "aws_lambda_function" "dd-log" { - filename = "${path.module}/files/dd_log_lambda.zip" + filename = "${path.module}/files/aws-dd-forwarder-3.5.0.zip" function_name = "${local.stack_prefix}DatadogLambdaFunction" role = aws_iam_role.dd-log-lambda.arn handler = "lambda_function.lambda_handler" description = "This lambda function will export logs to our orgs Datadog events" - source_code_hash = filebase64sha256("${path.module}/files/dd_log_lambda.zip") + source_code_hash = filebase64sha256("${path.module}/files/aws-dd-forwarder-3.5.0.zip") - runtime = "python2.7" + runtime = "python3.7" memory_size = "1024" timeout = "120"