From ef24e06b95be998bc26086598919eed7ab83c541 Mon Sep 17 00:00:00 2001 From: Dinesh Kumar Sellappan <40899231+selldinesh@users.noreply.github.com> Date: Thu, 17 Mar 2022 02:52:23 -0700 Subject: [PATCH] BGP Convergence Test for Benchmark Performance (#3715) BGP Convergence Test for Benchmark Performance * BGP Failover convergence for remote link failure * BGP RIB-IN Convergence * BGP RIB-IN Capacity * BGP Failover convergence for Local Link Failure Documentation https://github.com/Azure/sonic-mgmt/blob/master/docs/testplan/BGP%20Convergence%20Testplan%20for%20single%20DUT.md#test-case--1--convergence-performance-when-remote-link-fails-route-withdraw --- ...ence-Testplan-for-Benchmark-Performance.md | 190 ++++ docs/testplan/Img/Local_Link_Failure.png | Bin 0 -> 41025 bytes tests/common/snappi/common_helpers.py | 25 +- tests/common/snappi/snappi_fixtures.py | 99 +- tests/snappi/bgp/__init__.py | 0 tests/snappi/bgp/files/__init__.py | 0 .../bgp/files/bgp_convergence_helper.py | 945 ++++++++++++++++++ .../bgp/test_bgp_local_link_failover.py | 64 ++ .../bgp/test_bgp_remote_link_failover.py | 63 ++ tests/snappi/bgp/test_bgp_rib_in_capacity.py | 64 ++ .../snappi/bgp/test_bgp_rib_in_convergence.py | 65 ++ 11 files changed, 1511 insertions(+), 4 deletions(-) create mode 100644 docs/testplan/BGP-Convergence-Testplan-for-Benchmark-Performance.md create mode 100644 docs/testplan/Img/Local_Link_Failure.png create mode 100644 tests/snappi/bgp/__init__.py create mode 100644 tests/snappi/bgp/files/__init__.py create mode 100644 tests/snappi/bgp/files/bgp_convergence_helper.py create mode 100644 tests/snappi/bgp/test_bgp_local_link_failover.py create mode 100755 tests/snappi/bgp/test_bgp_remote_link_failover.py create mode 100644 tests/snappi/bgp/test_bgp_rib_in_capacity.py create mode 100644 tests/snappi/bgp/test_bgp_rib_in_convergence.py diff --git a/docs/testplan/BGP-Convergence-Testplan-for-Benchmark-Performance.md b/docs/testplan/BGP-Convergence-Testplan-for-Benchmark-Performance.md new file mode 100644 index 00000000000..3df9347587a --- /dev/null +++ b/docs/testplan/BGP-Convergence-Testplan-for-Benchmark-Performance.md @@ -0,0 +1,190 @@ +# BGP convergence test plan for benchmark performance + +- [BGP convergence test plan for benchmark performance](#bgp-convergence-test-plan-for-benchmark-performance) + - [Overview](#Overview) + - [Scope](#Scope) + - [Testbed](#Keysight-Testbed) + - [Topology](#Topology) + - [SONiC switch as ToR](#SONiC-switch-as-ToR) + - [SONiC switch as Leaf](#SONiC-switch-as-Leaf) + - [Setup configuration](#Setup-configuration) + - [Test methodology](#Test-methodology) + - [Test cases](#Test-cases) + - [Test case # 1 – BGP Remote Link Failover Convergence (route withdraw)](#test-case--1--convergence-performance-when-remote-link-fails-route-withdraw) + - [Test objective](#Test-objective) + - [Test steps](#Test-steps) + - [Test results](#Test-results) + - [Test case](#Test-case) + - [Test case # 2 – BGP RIB-IN Convergence](#Test-case--2--RIB-IN-Convergence) + - [Test objective](#Test-objective-1) + - [Test steps](#Test-steps-1) + - [Test results](#Test-results-1) + - [Test case](#Test-case-1) + - [Test case # 3 - BGP Local Link Failover Convergence](#Test-case--3--Failover-convergence-with-local-link-failure) + - [Test objective](#Test-objective-2) + - [Test steps](#Test-steps-2) + - [Test results](#Test-results-2) + - [Test case](#Test-case-2) + - [Call for action](#Call-for-action) + +## Overview +The purpose of these tests is to test the overall convergence of a data center network by simulating multiple network devices such as ToR/Leafs and using SONiC switch DUT as one of the ToR/Leaf, closely resembling production environment. + +### Scope +These tests are targeted on fully functioning SONiC system. The purpose of these tests are to measure convergence when some unexpected failures such as remote link failure, local link failure, node failure or link faults etc occur and some expected failures such as maintenance or upgrade of devices occur in the SONiC system. + +### Keysight Testbed +The tests will run on following testbeds: +* t0 + +![Single DUT Topology ](Img/Single_DUT_Topology.png) + +## Topology +### SONiC switch as ToR + +![SONiC DUT as ToR ](Img/Switch_as_ToR.png) + +### SONiC switch as Leaf + +![SONiC DUT as ToR ](Img/Switch_acting_as_leaf.png) + +## Setup configuration +IPv4 EBGP neighborship will be configured between SONiC DUT and directly connected test ports. Test ports inturn will simulate the ToR's and Leafs by advertising IPv4/IPv6, dual-stack routes. + +## Test Methodology +Following test methodologies will be used for measuring convergence. +* Traffic generator will be used to configure ebgp peering between chassis ports and SONiC DUT by advertising IPv4/IPv6, dual-stack routes. +* Receiving ports will be advertising the same VIP(virtual IP) addresses. +* Data traffic will be sent from server to these VIP addresses. +* Depending on the test case, the faults will be generated. Local link failures can be simulated on the port by "simulating link down" event. +* Remote link failures can be simulated by withdrawing the routes. +* Control to data plane convergence will be measured by noting down the precise time of the control plane event and the data plane event. Convergence will be measured by taking the difference between contol and data plane events. Traffic generator will create those events and provide us with the control to data plane convergence value under statistics. +* RIB-IN Convergence is the time it takes to install the routes in its RIB and then in its FIB to forward the traffic without any loss. In order to measure RIB-IN convergence, initially IPv4/IPv6 routes will not be advertised. Once traffic is sent, IPv4/IPv6 routes will be advertised and the timestamp will be noted. Once the traffic received rate goes above the configured threshold value, it will note down the data plane above threshold timestamp. The difference between these two event timestamps will provide us with the RIB-IN convergence value. +* Route capacity can be measured by advertising routes in a linear search fashion. By doing this we can figure out the maximum routes a switch can learn and install in its RIB and then in its FIB to forward traffic without any loss. + +## Test cases +### Test case # 1 – BGP Remote Link Failover Convergence (route withdraw) +#### Test objective +Measure the convergence time when remote link failure event happens with in the network. + +

+ + +

+ + +#### Test steps +* Configure IPv4 EBGP sessions between Keysight ports and the SONiC switch. +* Advertise IPv4 routes along with AS number via configured IPv4 BGP sessions. +* Configure and advertise same IPv4 routes from both the test ports. +* Configure another IPv4 session to send the traffic. This is the server port from which traffic will be sent to the VIP addresses. +* Start all protocols and verify that IPv4 BGP neighborship is established. +* Create a data traffic between the server port and receiver ports where the same VIP addresses are configured and enable tracking by "Destination Endpoint" and by "Destination session description". +* Set the desired threshold value for receiving traffic. By default we will be set to 90% of expected receiving rate. +* Apply and start the data traffic. +* Verify that traffic is equally distributed between the receiving ports without any loss. +* Simulate remote link failure by withdrawing the routes from one receiving port. +* Verify that the traffic is re-balanced and use the other available path to route the traffic. +* Drill down by "Destination Endpoint" under traffic statistics to get the control plane to data plane convergence value. +* In general the convergence value will fall in certain range. In order to achieve proper results, run the test multiple times and average out the test results. +* Set it back to default configuration. +#### Test results +| Event | Number Of IPv4 Routes | Convergence (ms) | +| :---: | :-: | :-: | +| Withdraw Routes | 1K | 388 | +| Withdraw Routes | 8K | 2870 | +| Withdraw Routes | 16K | 6188 | + +For above test case, below are the test results when multiple remote link fails. + +![Multiple link failure](Img/Multi_link_failure.png) + +| Event | Number Of IPv4 Routes | Convergence (ms) | +| :---: | :-: | :-: | +| Withdraw Routes | 1K | 438 | +| Withdraw Routes | 8K | 2800 | +| Withdraw Routes | 16K | 7176 | + +### Test Case +* sonic-mgmt/tests/ixia/bgp/test_bgp_remote_link_failover.py +### Test case # 2 – BGP RIB-IN Convergence +#### Test objective +Measure the convergence time to install the routes in its RIB and then in its FIB to forward the packets after the routes are advertised. + +

+ + +

+ +#### Test steps +* Configure IPv4 EBGP sessions between Keysight ports and the SONiC switch. +* Configure IPv4 routes via configured IPv4 BGP sessions. Initially disable the routes so that they don't get advertised after starting the protocols. +* Configure the same IPv4 routes from both the test receiving ports. +* Configure another IPv4 session to send the traffic. This is the server port from which traffic will be sent to the VIP addresses. +* Start all protocols and verify that IPv4 BGP neighborship is established. +* Create a data traffic between the server port and receiver ports where the same VIP addresses are configured and enable tracking by "Destination Endpoint" and by "Destination session description". +* Set the desired threshold value for receiving traffic. By default we will be set to 90% of expected receiving rate. +* Apply and start the data traffic. +* Verify that no traffic is being forwarded. +* Enable/advertise the routes which are already configured. +* Control plane event timestamp will be noted down and once the receiving traffic rate goes above the configured threshold value, it will note down the data plane threshold timestamp. +* The difference between these two event timestamp will provide us with the RIB-IN convergence time. +* In general the convergence value will fall in certain range. In order to achieve proper results, run the test multiple times and average out the test results. +* Set it back to default configuration. +#### Test results +| Event | Number Of IPv4 Routes | Convergence (ms) | +| :---: | :-: | :-: | +| Advertise Routes | 1K | 493 | +| Advertise Routes | 8K | 2953 | +| Advertise Routes | 64K | 28301 | +| Advertise Routes | 98K | 43109 | +| Advertise Routes | 196K | 90615 | + +In order to measure RIB-IN capacity of the switch, we can follow the same test methodology as RIB-IN convergence test. Below are the results for RIB-IN capacity test. + +| Event | Number Of IPv4 Routes | Convergence (ms) | Loss % | +| :---: | :-: | :-: | :-: | +| Advertise Routes | 256K | - | 25 | +| Advertise Routes | 198K | - | 0.50 | +| Advertise Routes | 196K | 85079 | 0 | +| Advertise Routes | 195K | 84487 | 0 | +| Advertise Routes | 194K | 83285 | 0 | + +### Test Case +* sonic-mgmt/tests/ixia/bgp/test_bgp_rib_in_convergence.py +* sonic-mgmt/tests/ixia/bgp/test_bgp_rib_in_capacity.py +### Test case # 3 - BGP Local Link Failover Convergence +#### Test objective +Measure the convergence time when local link failure event happens with in the network. +

+ +

+ +#### Test steps +* Configure IPv4 EBGP sessions between Keysight ports and the SONiC switch. +* Advertise IPv4 routes along with AS number via configured IPv4 BGP sessions. +* Configure and advertise same IPv4 routes from both the test ports. +* Configure another IPv4 session to send the traffic. This is the server port from which traffic will be sent to the VIP addresses. +* Start all protocols and verify that IPv4 BGP neighborship is established. +* Create a data traffic between the server port and receiver ports where the same VIP addresses are configured and enable tracking by "Destination Endpoint" and by "Destination session description". +* Set the desired threshold value for receiving traffic. By default it will be set to 95% of expected receiving rate. +* Apply and start the data traffic. +* Verify that traffic is equally distributed between the receiving ports without any loss. +* Simulate local link failure by making port down on test tool. +* Verify that the traffic is re-balanced and use the other available path to route the traffic. +* Compute the failover convergence by the below formula +* Data Convergence Time(seconds) = (Tx Frames - Rx Frames) / Tx Frame Rate + +### Test Results +Below table is the result of 3 way ECMP for 4 link flap iterations +| Event Name | No. of IPv4 Routes | Iterations | Avg Calculated Data Convergence Time(ms) | +| :---: | :-: | :-: | :-: | +| Test_Port_2 Link Failure | 1000 | 4 | 14.112 | +| Test_Port_3 Link Failure | 1000 | 4 | 14.336 | +| Test_Port_4 Link Failure | 1000 | 4 | 14.219 | + +### Test Case +* sonic-mgmt/tests/ixia/bgp/test_bgp_local_link_failover.py +### Call for action +* Solicit experience in multi-DUT system test scenarios. + diff --git a/docs/testplan/Img/Local_Link_Failure.png b/docs/testplan/Img/Local_Link_Failure.png new file mode 100644 index 0000000000000000000000000000000000000000..46195803622c7c250d8d74ae6eb7ee066ff95b1f GIT binary patch literal 41025 zcmZ^~by!qg_dh%|w+e`&SRkU@DvC}ADlpx_^o%*(os6g`Du`l>t$=|6Cn zATK-==!tPXo(=-_FL4sZPLo%yGpayIF#Nw)Nl>uX;&3Lx2uV<=%xWE{)X6kmG7 zLLZby64FdKgh@j|s*ozPm15RoU2<$DG0P3bS&0OVM!{4_EEG3~jg|0pcwlWLoSMNi zafntLS+AkO&{7zmrvS*|bpX#eoC59QE7=MZl>tUFB`zJ6!_mu-M5UZB$NFptgV0JN z{=*K=%{LlJ9<@>|Q5Z-BCWGv>;LSocPwmqi=`t=2WhAUSv zfmi6nIXPAZ3PuG+LLE}I0&kX?aXyq%PLiounOG>n$g~r!5-2!RLKpGS3@k*8hx3RQ zU_3Jm>qTqyDxOVB@`;otjhyNd;#GKr!$IevWe|bb=WrnrP#Fmbu4WiU;AC=%|pah(#D8|GrjG&~87=+@wUbatlOq=o}jaTO)<(4ax9YPqbluC6cU1E z5-mJ8jH&m@JZde;DB|+*L^vI(v5Or}sgvk}Lwr1moZtfs)O{yvSzY;q{= z482bWw%Zg6s)q*j!USj!f?#v#wQ|29df`^9O<_T}unN0KDl(Yt4yTjtVh|}{krGA+ zM(8O*j~tHGD>Wh|&IET-aYU_&j>BnX9;qCHP-2yyOe>1T*05+wv)O{g%Jo>849p^$ z?0UCE0D(x5JP9AKmgz|fw%2Cm2{TPnAB<&yJD5@jD$8c{YPoumj1G}|g>ZvNm?eeK zoHipun8`EIASf_LWH&iY04HpKHpEACavdTM*G(4FEntK}?<6U$E;&)3MW)~+0+*P{ zR5`H}Wv0bWr`X79F&|Fx;xR;qmtmCAj8MMAVNCJ6c%SQiVR)_QNuGyJP3*c&f{?CV7`$|q{=-!6$cmrEQn!HIQ3S3mIT40 zt3_Bhl>?Cym1aFtZeuuII2f74w`bX$LcG%L#V|=!iq1$=^W4fzmQcsCN%(T6Oi1FZ zAtHj-K{v|@5|3L7_&X?0=!O_{0LbEbXaY~7v$KtOIz-6FWYUosEikczB@%N@Zl&Fl zg=YykSeuh(&{?H)A5D(%aYRm<$R)ya)g%*%0#WgtDw&VxXPF{{J75kiUWV5A@9b9K z8%byCXzVNp$BDJ7h?x?(${+@#xFntf%iwwCE{V*igIMeuBodot$I7^5ky+=2LTPTb z&Ti#GoKB+*1pz!Tj*C`{Xf!g(LnSgLcq1iCs=#}+Sf1U>K{DJ_s?8uo`0#o@4I_cV z*kqYRi${V*DyV|P^YQI!28PPRgN0DN6($7CAE%??5iGL8N0XowF1E`c2i!5vDi+Fp z7_|h<#$v>NM=fLktDLX zvY1E>%L^eQ5ppTXpwO#zdWJ+MVkx*ds>furOSni3!LEX4+4w3j;BZ`4ginTa5Oo+m zu;aKaH;o2S^9g``Kp`}#7)20}h$fgur^C~rD2t3IQIVMhHi|1Xpv4xxUT!nu9Zsc# zmL(-hOcDqm%ce7FV5!(31k;Enp-kYyU}ac~$7b`Xd|o>h?82iUZl%*Jwab+{3r^^h zGYv8^UnV1XvfLi4L?N+3075#16U!FDEG zW_ksH$VnC(n1%v^G!h<*@v#&lm|ICCi?o1CAc>?Jf>~u`U}Qq2)X8S4l{#i7T8~gz zJu;lpWJQTh8am15b~_v-3I(AABUDlujbOI35MGScEW{Y40-BD+<7cUvGC4p{0w^TO zVF9+yrLvn~U>8j!Hq#7383M{vnZ-5}3$F7@QFbEKkVUX*B|v0HQaGJ@dOI$n~2dfaVg>EKGW`KK{7Q0nSO zao9WxL+zBC@Fs`V3q*W41Y2Mv^EgztL_+0R^t?>CluHmw&0ILv&CkTLPzn*%j*#nB zMzG(NI~^XIzyo#@5dt3t>ehk1Stz5JjB#4LN`YPsi~tsZ(4e`@ED8|tko>p@e2j8F z${@m1=sW|9Vgp-gKAsSp<%KKZI)_JYH5o*5A(vwTQ($Hr(?=t-9W1KFhC$oq5~x^9 zp|U72JVTddmzqr&Gm^_t8f+S|f@0?hOkSH1g2U=u5CI$l<$AKX5N0L{r_$01FbYCasR{Wl}}{JAaqhkL)m- z2`W;c#WJYK?KD9gUN-?rX4Bm~p;qLBploEaL#olqI9?W=ij>NUBAX5PsW8jJGpkH| zu7oaEz!*HTNvd^gI6jxc?<$=#IZ9}ea739|A|8(6wqaO)80WBUaHiIWWcWBxuQN-j z5Mr5G7P=D2;Q}g-h7-9mmqe^`;j|hOQUr&xyh4J7fYCa95{Ju$QAw~4gW2cElKKE* zR3}1AlFO7<868S;@(gq^%VwZCg?bs+B{F1YQO!^ooNnY1S!AYC3x<%|QLR>mj>#fX89u#2%40%oZl@l` zhVXfCV15mWDWNKCMjJ%z7pNUdMaW1V4S^%V0r;bW*yL(4!sON9O>h`RCzHdJQU#Lj z;lYe{i_`8kz+fV?9B<(mvIG!?Hj9|4B1z)snM=VEy1Wpw$6%pj z2r!C^%+jDm2pfZ`LU@=+DpZSu8PG_Hj_72dXga&VU{io`ZmPzLCgF@W7K_fX*$rM1 zodd@5ZRku3Q6%=qtp=}Mtp_XZHlI>&10r}CQ3y8s1Y9Kq3YMYu6rM^ib`b4cy3cIH zATe?tm9G-e83Mk*B6JA!YPF0Bv&qmT8kLJ;8Cc3pjFm}ac~#I%zq$I;6{dh`H!uZc z6V-`Eafx0c6l$^&7&Ma-;zG$O7%z_P=J2(d05hyiu^MinJGE#p+F@sT{a^}EKwCs8 zy^YL~vRGO*Lasuik!~29qQ;n^G`T-mg#mMs)EMC2-zqU~n7{qcBcR3ncYqtF7P{3& zs6}hW5WH3~+zvCLDMG6qiE%lycwh{K0p*G8Mk+}!DthQu1q zVqy#g85NI#a#1v!4Wct4rA#JUhPHq`2Ba3{(?S$psR{yw);Nb(Mz))bN-w~tLPMcR z2@Wj8fXZ|N$ts4Sx0>u=y@-xSP{Cdq3+C0KuqZymN`~cyJ~RqN^5}uE#GFOYDquJP5+Ts4lztdf;4uOy%OgcA99RU_&GiZ$ zP%gvaKois+3t&Jfm>SB}IPhF97=U^>4$kDUFfx@~#bUwW43v`KCYcl*3>n2y$Wdmr z$89rnaAKv;t6(uDKp@GJ`rv4XOaMfiW~R*sBnMKI$w@bXfw|FQy3}RhSXEg*n1f=% zSjj|=k0C=ibYeFF&N29uG9m}S8^6sd_&7L?uVlC(EDIL6cd5uQa+W{g17Hts#{g`Zz^ZX_uNJIF%g}5ogJeMS=`5@NvCYY{aP%yn)x|M`32G{y zkLIJO1P2p%#`h~35u*1qi8G5?L^+oxciJ&tiAuyVqZnwWi2%j%sWiX81hk37wfO8n zN{o|mQC5zL24#bJ{xH_fGDv_t3ukAuz(N(8$jQXu)JO(P#b@Cmz|UY7e~tU?mij`49wNQOwrLQ&9Q%ztF*=L~LAsDO0WOz{iO%Vz+& z7s4gByVWo#MI-YAixEMW`xR4eg+LWqYP*x|ma`y82owS%dZm0d8c+8}%|LdE^C4te zXb2Om&UEAHY#7IYVwm-;EIpnpmm0NhDv#!{xKMO37YgKh1TWKRH5kEgZKjc^z#yq= z49X%_WWjA_3(f}ka~#s-5aFCYEdqgr@tj0(mQ#iz%g8VkNAI_GF^MgPVBtESU;0?H zKX8LNB_^K9ibd&lZ~!r3I1?X5Cz;IpEF%@|V6)tCG1uia{UawCVD==NSpy?Mb#6RW zqJrtUNRv?vaiV-o2!Y}+P}pT0C0{_0(I8g5%5L@`{iXu+n`pKyh)Tg{VY$Gjuqj9) z3PT`~wO$d2s=|xlDm%rAaDXKQCC;ICv*|D*ho7b5=m>Cx1&@}vIDnFnyhIIxs@7-n z<(@1WPV6J8#a@@yV8FwjP_P-$Ih@(-q&QFn}}s6e1*2h^FwNRw2So)dO7?tcd9{ z%a~TRgUaN){Dnr6DoaJy$>DY)6RJ^W!iX3wm*^Hs{82ko$0dmCsNy=19KbV> z94I_c=)!Q-Duu_bCNlsiWMHLgrWFjNGFD&!L&$X7!CE6=gi5K^Cop9KPTJ{%5F92b z%$q4OQ_u*90pS6Lnur2`xSIt*u}PRrJye9yDYE?jfgsju&}x$p$y5?C91aB`_V=ni z{;H?{g9m>6--_u!^R?#c&7YN=lK@m6uT(K6BDf5 ze6;%&NK|KAc+~Uq;rinzeq?!}FE{--@#;s{`zNSZlX`~yObE;eCFX?x)65779yR9B zT;~7wfkve+5B{Ga{en48afsR&9Gwx|JAeQJ8jdp|>_3-4b2>S^xNU!rt4-nmC7T0u zWJD$ew)ECUf*u8)o&d65qzJ}?e#V2;cQ^|t?-dV<-i0gvyKBXb#()elLVON1BO)*_ zJpZqNm^gVsXlG(bFk?|!HZ6KfYIa>q#I;4@ww6 zvSVLKS#MB}UfGXBL5F+&j=v(Bo2bbiG=Q?N?MDp~bTu?*J;)l|G2D9zu^g&6kUsX{sf+31QKLzJX4|z~GyAo12?&-MxF4RUsUF zei$77wNbNmmYY8G)3UVA`UzvkT$$YcLtNV<8uN@N7Pl2`*^)K}3bntP7t%91Y}TKy z{{8zOn|{ox%f3>Sa_Zqx$E(@TqiV;`56WH=_3UDy;h3qp`sl-l57X~dO#ZQFT#fTx z-hu_pgsm#I`ueIbzw)2KK0JHJ1dR&Ikr$N43%@MR9)0iQj)^fzsY{2ZPn$4$^y@5( z<%9J7`}b0j=-%gJuJ)oMM~(~#KBQ~d8FlX5xq**|%4R=49kBMxs~dd-zMRi}vG?(E zSXvot^PXP~?DSb^`T6`C~f4S(-m-U)hKtGN+hqrSFb&CkxxK8U=i#M6HT zK-=0EN6rcwIB;P6qD6~Vg+|4nSQq{Gra@=U=I2BQ>z_VaGe$I|ZrRZ8xqJ7vzjL)W zrr>)=2c-3;&}jPX_br%fCHTDeTT)#GSAS36uY8IBUYfg8mx!dq#}6f(miu+S*RVnR zL320!B&*T+x|kvTK}!S?iTmB}9;DVSjhp%Rm@)0sd_JH1@W)ZGuyIEVB3E{PEY)pz zURpmdIlFhjwB8B1U7ufEc@+~I8wLs8ym|ADrSX$G`>KUOnpE%HL4^Y{P7mmo%jN11 zEk_;3*|TSlY}G$`@?-{60g6&bcvU#{S6vGrftVobYAWHlpq=DlDAz?A<%&?XjDN(usLht70!z zkFyVY)(l#H<^Gr}$-yKN3Hh$&=)va6mXmAVy?Ak^Uj*zOa3y*jR7Go;KD*<^ zg9SZfmj<5Z-CNjJlQ8lYwsCE+IC7Zu|G{PN0A%>p^{}!`y1%Yaf8E2vYbRoQgd5KGhn&-jZFe zJ)GvDuvp)h<>&9p)i}gG?Z&Ww@j2sl`0|+&VTCm0_3PIXkhL`?-X0wnkvM!fJ}+u$ zBz|G9qlP^VKaWDA*G-z~7E7fMMzqf44j56dlEjXi;$79zJ^N2*;g*S6P|Tc{yEQFG zY?)z^L)H<~!h7|aa(>GG9dpJn%*(5*l^j$v7v<;I->bRNvIzTLw?5US?{gG}?~aUZ zUHAz){&>Y#Dt8>KU=`EKumpPc6Rpl*VE<^yNYA`^}D+< zeNNZf!a;~+)pXs5tLy`;w25_j#SJJF+6i_Kn6Z9L z-FkbcJaO#Uw+XzhRm}N4cuSjF0W7mmox3f+_|mRb_2~mc`W}~u13sm`Cm|y?AaO*s zx%ENKQ8O;EOrY;k~gLskHeEYOOPF zC+$@CS!daM*l@45X#IMOB>kj(EOFab_oq*v?r%>Yj5EHzUJ8cAo!_~3ttc`Ywvezq zao)&WR${lZH{mUg>35#969SLDj#w!LY?wI^V}IW7bKN8u?BKTd<#XRZ=rIdYV#wRh ztMBf6Yx$~G&j^hQ=ZA;0?%yoIcPDLy!2Ukg@vHsbzL{^zluk{`y8A^NQI_wDM&EMe zI!oP`m)9o$zC|7yLB2JN$%4uIb&S0Bv*ccRZ_cK+kAo(9tsg%=ITu%2S{nc2tOUDY zfpS?+7bEEP{`ZOOS)e65ru3?~2ChEcur5LY8}$6gG1uSB4PWyrIcFDazrADhXZEow z{0+(QV)QNS`>DI5KoMg6{E&XRIVI^$j0f_j{j!0|Yv-R|TJ@@Z+(USETJG+;@@{Za zYOj=dMD^pO9GeJ?R@uj{mx_Zr?F^aO_Ij;+pz&-G!bfUn~U` zM801qix@b3hSv7?>&mWbvEr3>z&z5AO_6^P1FpWE-?e`5vF|R;n6LL%wmoPt4Q?27 zdjuc>zh2z;>%}CYxX=H^Ok*=Z#JrHlGQt1g*RmWgG3?>Vvs;G$-^j-y0XqsJPt05p z+gTj_-?2Hcz=CGz;?i6gIY#s(1f(rFGA;f7lXIYq*&nOY0_#sDzaQ3Jeq)m1P=4vT zL8BCdOUcFYpz}c)>7b8&PS%{Aa3mO1AC=aK1$F!walJmh{p0oi6+qAy@}_C0ws)H_ z1(_J1MQ8xkEe(k2UvQC%joZx&OF)ZH{WaukNXVZRw&z1X-$-TjWsez^dcJSZujgsu z%Nw-MK*g~j{4d)+!T4m7gKV=P=@nKqy;ohF$ zu!6{vrwe}LyZ*Y6GWxz?Q`>39%^d{xm02cn3fvc%v5Ne%a(WorR!TGS`QnEKO(_Eg zy=89>Y&tdc@x{Wrhuhj7Pgq%>K5x|LWtM5w+gtw8pXDQ_VkR)%pc7O6C`6Kt`HP>` z?L8mbA>22=?&gfG>;JNTOgR0nneE@j@xp}eC5s}S51l?_)uy|AGZznep4K*DLQqWX=(rBpAaVVx%!Zcq>ivvYH~XjtI$uR2iSOWh2PbARGu(1PW2Mx_Q9uPJ8$Bn0Rx z7(1$y#R64K7A-0~=1L*w|E^*)54S{q8lCz8Iw2_u^nCevd;LJn#m!kj$hL5A$Ke$K zfvcKjPWUee`tg0gtcK>FU#q1Ny$4BMw+0@)zcTCnqp+sh&2hfJ*ju<&VM#4FN8UUT-Pd^eW8B|>goa;kjQ1a}$X&y_jhE6hYbyA+=6d$!Zz$?NLGuT@^+@^g;R~YU ztNRQ=w@qdw_Ux6=y?jppo8*UrHOZ+T2RO67JV55w)_zP_oOY>1xlo;6J~w@*sC@2$ z5~N=*j29;DKD75p%E3{o+i}b5cA5O{QbR;~G%aZ46IbL>6Qj$PXhgiBN3esAA*=01K+k@i65VcUKkmY;JT!sS;0jq(eb7Spr1cMZy!ZjyK8mR zGXdP8tOIZ-Chjbl9x4P-Cn3LSL0NC`q_*b!0rTgB4jIC376;VqeRr+jkfI(@_QQX! zj1Ri$y!-9@h(brlflnW|0TS5zkLHYdzo>RL(;BR?tXdg=nZSzm=uwDn1GX%R!gFZ}K5s+nQJKK<3yEZoC#uiZF;4k+a)ua0E|w zMl(lDa<<)6OrAo_qlJ9@{Y8*2xUurh&?|pI=IJaCwhojAHLa@oeg8Sfsji$?4&qOS!(VFA~- zgxu5~Ssb}i|K@f`F>mD5<~acweLyXWXM>8e0t228jlDJZH}7ftwtp1$IrGZ~#~VEY zv|g3a6BZN*8dcD|A#$>;uKabvyH_s+i?%50Ks|a!%v9=k=5AkGUrzEKG5=`=?GFK= z%l91amr!bo?WiL;6S@iX8Dr23OV9O4Q1orYpZ-XmGyf$V({L8$Kfj%d4iYEsZW1MO@ieX^o{fWD7ZSH{NCoe z+}abrbjLi(xoxZ1p`fv0IgExiRrA+}Cv*>ZynK$DOZk4+54r#ZIF9LW){$f927&g6 zgY@XKxPWJW1w4P$kdfH!O%RQ{(T{9v2Uq*4_4paa_)!HwzP?l6 z@k?{Z{P1aT*+bj-CBL67U2il1_+ipkDdhOaU#}_B2EMx8cq`+0D(1-VOz!uONr!2( zrf$0npOO78=BM3wB{Xi|;%lC-WM8wj`)K>}+Nxrrfx^{=T_rbG5@uXW|MIf`+E|aL+S7GXvlk2F!9;(y-x9(< zf7kweRJ!t)Y{!Oz(0y*-&<5PfC6(Sy$Pvi5t@=MZM~^mDri{0=ac}&}+aHxS#rkzI z=0j1-r8x7eryFkBd`~AleI4LS)Bweddd#sV)UAZ&8;&Nwf0?_Nms1+=HV;C8o64?T zTdf!q+B*6Oe=p( z=ew6C#GP=oaZWCtjGDD0CB^XM<)e>ZgwGyL9eHW#wuY6sjDJlwhPmTixmkLl?xc+C zY_i&)j+{7Z_KZ^e{j$t-x5vZ!k?TR zR`o>OtZw(iGieR$hW!NN3@Y^C|!sm(1ncaX2lKUIivqW=WxTqrD$`$B#qs-(L#kF_K74`5`ld}u+&rzctOp)QC(Z^gZS6N8z|oLQ-lq(18GWqrh^GED zxgCAv$Ohl&pWAn9 zUTxOeaV)|8=IOct{r<3WuWxo7xl?x_A4CuS*Z7OBo|d;ngHaXSY11mF&9pw;V`maM zo26;fFC30LTtsib*niVgbL_ZL4=T%(o;w`v?D9)&)X@6KX+{pmi)N(`r4EG z2X_{|c=hA3$HQ*_IHzIf21QVNLg2;ToHs?1n7E&;@Oe2$b^R``pUzJ5T)94}`zUQk z?Za)vw|l#$o%wMzb5s?aQd^K^J=f$dG~NGtvid~%A~NU8rD5A1{k%J5f;cQ1JRe9QpNAx#?i=`Wr{L?x z?iY_e4L#^R9I1vj+tK>@#?o8k4qp((bQa zCj+KcNNb0}e;+zdNalefjKO7T}{+vHBk!C%!yLWw~ah0X2q%xJK)j`BQZ()esc%TxH1E9faWgHt|t= z(n-U;%D$i(pMlqP|4!Kb>gxK2QDeuBO-N2op8of&v*xDofZE&1jm|k&ZVg@7l5(O$ zTQaO5>dDQu8=87Y{&i?cIB2YRfAN_O2txr-16m5o*fN^&JTY}jU{FwV1**L`0;+!1rDtRaTD zCNSqB-|`|cm6G24`&D}PUt8e1+ey=>KeF@U>TW~_;KM&|7FLB^Uh;EPa>|!fd zQ0>j)l|O&}oM{X=ZO;eg<{WNF{x$A*;{o1`Wvg(+oEGikdFQg76~mANK14FMe$L&w zef#0f2iqT3Kf5?bDcTp`(KxEj;k)qp{erdQBlpEMzHHz5Cv<<;wqd(?+9?I$&$%(K z^%c+Ted5d|B_$?>LeZ`$SYY~DwP{>nBW>^7PS3N&sLz$X_HTRpJ|gS;W?kdbK#cTc zG5l0qJXAd(D-rpA$iwtQW5=g29sR$Ud&Xx~4Zip6OYIPW^UFo_ng{QX{pdNK(JTM$ ztRmA?`sFy*2E@Vggi|&63hD7?_U2C|N!9x+OQX@`%sWBV;q_GK^0ctR0g%!c#n{}> zHtguhJx1+qsb4X?dEbYeD^@G{2+Mg)Yf|+Z@y?f1fICf+a^MBL*>FBsq>e{tEfr*njZ$7gsi-4%L3Tk)FPM&dkt|IYTD; zzFHce;$D=G*n~XA@Kk2@<{H)ku=1_<+qi4;TCP0xVt;w=UkzQOw;Jyhtwq_t?LJlf zZGKepg*?R8RrJ^&S=O#8i91T;f0FAW>W)jw7u7nOF9T&~Hc-v|{^iTBnLufOQxUA5 z>Ex4E?0nq`;&d=0hZ1rUGJ@;oo}AfjVQ!t~Ue)z*vh?NCV;iI68PXjmC|_Q!7Nte4 zYb-fgQxI5oKWte`Cnia?y!Y8>v(H4Vt&r?3S}{}d=2%De!D@-&FIbo4L_EW^81)M=igf8#Zf5OUrCix~fw- zxnu4-TF~5K)z_ClP8d1w{Hw6Y zbIV?h-*UwD;UQ(%^7}}{9SYD&G(Gz(x3%--u&cH_8^q;=k&U#Pb+tSf-H?+8Ywud;L#+l-vjo{ z?OXcp<&xiNKb{F?l}jF@ZK&8z6aC5@#qXarX}{jW&h-CY(WQBas$3eMH+*uhY3*;{ zP8k+j?wpk{a&zNu!H0!ImWs$#!;tQhbS^GuJ6~8XezJDW8o`E5n;0zN-_M@nD`zb~ z+?Ft7%5nEK3vl$a8IQw#Uf64(eeIYG@wBw?0e#{j&D7ZHq4=L94!f-o5tequp1M|2UB4uTRlmR*CLjxDc&Md-!GR%x{NJT}a!~ zf;sf)*Uil{lhE-rL-s-Vv>J1(eGo)`K4Sd%@z7P+_uQj(FNQ?&xXiy`amNODSHAoB zTmw|DYk)HqVn$NnvuOVz%3mkubl2`pv!2U~I)zr&FUS3Pr07T{Y@ff(u(;qxWA=vI zy3Bi(%hYJfP@m<2)K^>l@x!l>km2jlitM3v_eNgb@%{DWypfHrA5&5L&utv|yzeDO z;nn}X3jlfI@O^r!p7W=xIgd}gzis{czzwY9Vyxx;MDNG=$+f3sk?y#Sy95Wr)isA7 zbTC6ByDk=W-GSG>(5Nt5r<}B}u0B+LW?u8r^vB*&6`!uH^y=V)j_umDs}wj(8`(ca z>TF(}1AgV$u_$utk~<_}>&m$COG8hOXvg+?u^-q8@ygR0ulxgh_VmshZx@{@`MSUO zCHv#nvEvFK9NB>`+dso$JF3C+jv{@pLoo%X29my2Y8<`SP40zuG@2C-1rT8-l1!iW>0vapVh02A<6hd#?j2gm`8@vl&@+iC!_1)} z8IfJbw%y(H;M>DPuQ%}J8;WlJaP-)v!@m;!?{S`}-<7-a6{_E55Q9UZZNl$e|rCIkE6~_JP?AU$h zK}>N;T*yb(zKF@S2ZhZ|OSApi?xQC8jXlx37<ZgVUQPnYP4GYD6uS#|+`rm|~ARb6>kQXi2Wl<*`dzu?4sM!r!{9SxI zV0S?Ty>o2QF#i87nZ7@3uaSdfi>SQjPqUZGk1OSL-BMjf6M;2t=B@B z{Oi~AEUn}2#SZBN<~x-V9H#S--fXPkA1cZoQ_JLmb-u4F-;F(^z9U#u~j#G}!)w8QKwxbJY zZ`{sbBKaE@{~Trj3SOP7)X{Vl0-VW=KxPS}fmUH>JKZ z6hA3BdvMD=+TQOFQQB?#w%a>-^eFpgW)q3*#~e*L;nR zzH{OzPHoR=Tw12RyLH82)T%>CY5Aa$08Q(>o;~RS0Y`TVR#u-Jn3klz6)^gI@!vxu zc4Wq~_swA@Z!)Li#{$Dyz2;wRq^H6@+PntyrE&a0Ax zST5u|mF_+h-g@cl{Mq}0v|~+9}mgSih6-`UdCUiPE?UL;>?SLgL4-pch=uq7kiTW_U*N&q%&VpRrv zEFq&(*Dvr&xk-^;GIt9z;I~D8b>+DJIMtdrMfrtggs?9?R}?I}Gi;WgysvfkqvX>C z#Va=d^{KC}IK2!sPG3O7`^t8#DL9?t#0L6{4;jmE65ZkTDHWy7?MFv%YoQ6;&zn^P zm<=Bo%cdqj7`_|GYmd#E^Tz8M^ttT9xKGNu9%GY&1J|H#-ByhT-FZLe#+1z{_oXNH zVGpnyKl=*~zvX-C<1axfN{7~;Hz}rX%jufi@>{o$n>VY^Ne=%}`QGyjzXyS&%li@r zekut$GI5gnFEjb@r;W9`fWEyt z+zz%}YDy~o=^lNkDdRfuPD##x-tbDU9%*OoZ3}aCeqWKcXqK?xY4eVz^B1q0EN??; zuKP`&-ejLl(EWK<6cBPxTX%b(4>(?Fg?!6?GT+@B$l-t!^=Xtr0ZH2h=ldsAEsB`O zelcsD`vHB~{yy3>ueZIcirG{+<%`{&h0)=!w9CP6Av3_&-nG!Z<1a`?w0i# z8B|?A$a3rwIX-r6S^rHd&)u2ZsrFrc`s3$m3mG?s-S0F@zL>jv+S@fJUXM?Ed-(!M z+Wj+XbU|0IQ8V||e{276X~*=YsM!E?oEg7Gd#xaR)aVLs-|_|?2|;=L>zu2kIQDqc z%?~w|KNCE~AFJBnrVOK_Kxhdrp3IH;_Gnr=YE=XI(=F@_4X(vLXhzJ<{mJ%cMbY7r zi8KCv$AOyoaa08^Gm8KHjgWR?a+2;>8)?vk_bH%bMgZ1y*-4<-rqGSF3(PSnllUOKWta#J@0+`(B)^# zuhHTT=I@k^Wu7_l`WF3R&4_ih-h+d;uAls9z+W+kOZ$w8?8)*h8}j;&>BZ$fbg&TD zdT4%aIxi!lbW~FSD=O@}^qXNtpU%Qr$~=Iq8!a2XA|Xe&W@HdV(Y&sIa?Y$6&Qf^A zJ9LWsHMO#YThKJa3OV$6_2Qq83-;ZDKPhKI;HcCi ziiHT{d_Z2kg3?ERt9a0_xFa|8&47)$RjbD#do7Au6ZdD!`^E={<{ufDIk@_9W;@b% zuS(L92tYz@)f&N#_@RNMNgT{5M-7b`u=%jb_yKT!o;&=|3!%qFiKdL-Ut*Zu5%BWD zmLsZB>!W4IAFrO;Q`{q(GY*)1_28M8zMgSJ4rktwAZe^ z^FGhYz$bth6TqeK?_MUB4e7t>fQ2bZ@_cRoG35huV^^kdUsX+~fBmfdcwqhOViRKq zx7g_ILu=NR&gjKFQ@!ikWO?73!u8ky#vJ8e7s50g&-b^l_qZI?ouD3xDD%Mc_d&wt z{Q!QH{a78Y(1zp|MS(nbf_gjYLC((~qc?Q*$*I+`G0wKy*|Q=$PL{Ne+t5RxU>x0l z4l^qwd&EsA6m%qjc{1*(?Mt>BQMG!A-}%<8kgT6 zSl9lZg~O_BBkKpg4@FmQ0IQQ8&OH-(@iO;_VL_01$l~{m{APU6iEw!Nl9nSc5KHe? z9bx!UY*ohPioxrIkhn|*Z1DZKmhL0>Z+B>VWZ-!Z%nhY$|Ck;OD%u}2_wu))F?aK? zRjcUDE9HNt|N3FsxdUDnkWuJ4NP*1z{c}la?$9B(dMpd)#|4It++#m|WHNE66se1` z*MxIE#&h_W54ByeOCZY5hMx*_I2sFSyk%>Utjnai8b} zsIQ}Cyc`yI*DsOe{2qWrCdNgM1A@HoS51nvz?`G-h`gP3<6e*2lhNF?erizZAH<>7 zWru)Q74Gb0hIKf}?Cjr*7PC4Cu_kI9zyx<$@si(wuUvHoE;53JX^$|s4j&oY-LtUA z%cSoTAMhW@RY%6W{>wHqAYtB4Szg}dv|;i;o$|$nZw4kj4H^R9vS?P(Q&vRn8RE>9 zeh60LN2b?7D!Bc4^a~4ruRCK|=Kj8{JAFgxs+@&aW{@usE1Nf6vh>_pajm8EBBhXY zGVDetsHyev-pwbEAOG@Ya!vnrq)0mDKddnT)_N2~7&uuP+@tV8qo7v;;GCdsNaHk1 zq1)A*087wMc(>#7qtV%CbiPxylQM`azD(#^+Ol7@;jm)vJIPRBg^hJXCg0h*Q25i_ zZ`1w#567(7()-7F5WGyQr|m1;*ljpjG_*!(JhQt`&4d32ED;}jgXF+09j*N#XOIr0=__0-ObSAza135DQ{sbN!N+RV}1A8w;^&o@14Cul+{z~QPR`jsjEyS_c>=8 z=gjM&Rt+^kfdc<|S5P23BKrp`;>zoJ(N|mozQR5e?6ZjF{1*|ND1N$tH~$H-AWKRK zn%uHJr1bauPq0QXG@zp?oT66`J-w{BZy6PlQb-AHt~thfPlH^^9E3KToS*)XP4an5 zCuN#A+LVW3i($dyW4`TQf&9P3<714xtQI~H41Hn z3%z*M94If}!H>nVCcVNI4noeN?&6^&EG0s#%*1|4q_Bns zN4;hQEi3vKb;l+M|C*~Gx?W(=;agcfoLh@Bd+nc9%P2sNc#5yQn@9VkKxVX5*DN1~ z`Y-|ySnSBAOP-5};I<|Nqo?;%m;TcHHOK+{-88F$K3aR9zKO7*8E~{xV->hy?bQQj zS|Fy}WnLDeT1frx+t@L)wLH7;zF0B@}_6fTfEm_olvj`GewPd9B= z`th6Iso~sKXGve{d?F&cFvJl-d!`TGnz`U0c;W4N)LjOlbT$P{rbC( z;UI|_)S-2aXKg`fRB)e6g^OiSMx5_QAXOYC`)+udV+S=uhXQoXx1C|doIB@N%={}J z{6(AV)x%vzIsaDk=s1u*g}jJdJ4RcD!}`zbi}O54#$q%*aj7>@DpYki<+rK5_Z59r ze|lwRCYa_EHc9j1(j)uA0?m)b3PY~+8%K+UEzbz7hZ60@q-QMBpS-3Y&LOf`ETsF+ z>%J_qY+$eb5i`y5hYoYwdf4yi;8V7?wiZ!VR_3f#vqTIUA0Jmw_qW zCAyWAEAs!66c6U{za_=f2g=B`QsFCvJ185w@2&*_p+#hfwCS^!nP=sf;>|PcLP3xWXYQo!d?v;-PyU568d+qu1 zo7n2FFb5h>ipM2pf`iQ*jh5!ekIR#Ba2#6kX5JCQiOi);bi8#!mam6lN}1XlF6P(m zB3q|=S`H31mp8{tAD&3y%+H_MLJ)D8q+%3BJwY~W)y5I()kY)Y9-f}TN!@+o&^bnB zALUm16T@Af9Ccc^&BK%uWH(Cc<#pXpBL zksRFDwA0N7DW6s#u*hmX%(-Y&YMl^41QZm7=K$cq6a6aBjV4EA}Q+Ai(LwSVC z!S*#I%U_7_@X;CyO>5)@T2WE=`RqqM#Gg$WQBID@E3f0lTE}o)&KpT3rPn%zS(Sz>LJk#jVi!V?ZCEfVo<}Zn?^$jfz&0cEX+tC6lNt`P?GwM#(pTypg z-b3V|;p=A6;ui~fXpA)@U)hQJ8PvnXnBa(ur)X-)j!&)5I9gxMiFhB)cC@ST$VE%b z02%5tFzB>bV}pZ~Qqs~q5Fq1`bFJ{VwUyxu0mXvLWFjl(;qk<6@6*ZgL_p9}6Cw}n z!yERj5%*7yJf3h;=G-3KSTez8+zCf$9U{HZPp(exj~ulJD+J~|zgI^jC@5%7@z~iW z%0(hfubUk_YmCH?yoM%tiMUv-S2f63SZ;pS*RzUM+jwn}bAS^Q6DK>JEH50F)~^oU zTwMIIoS`vxK-JL|U>QW4cq7sG&nt;(~___a3-&M$=|q3mxsXPoK!r*jt@u%+#_AcmHg* zl2~*FN!(crd!dzYYdSryEtKN#(XkmiHTReI%@c$D10L`OTobcSAaMb(3Yx90?e_#F z-4QVUkK%--etMe& zgHJyS7+_-0tQ}{`7h3;3;G+22tJzz29?=02k~M0}Kt(pOfrx~JG+U}(|5;2-Y`4+J z$M9a5@96Cu@7-+WHg{a($?TTDGlYDd{nCDSR2t-L6_Gbx^w_4^+&WWXjM8>#)HVDg zLQOg?C|f;Z-Y5^VSh=~vRKY~2-Ah%;8UuS;WF1FXPS~P;Vo=NhBsB#FB&1*4;09T) zeXPv;CTlSthwf49Ww+7Q02NOsNqEBro;(VTd1*wcT{jVaheX~#>hy9Ow4Gjbq$qrx zFPTp56|sG|F%UzR+IGK#y_+i$k32DUP(sug3~z^pztHLz5r&AUulBAdFsz?Iwt8Wy z{{0$8yy@`Px`IMM=(b7eP_ku=#t8yL_#kNnNVY+R5zZ0-L5~$(9FN6k}|> zhz5z^k03Q!Vwy$^ecchi7vB;B@5^jl0(qmQ9o6az2?$MpqhY)o22J84=oL)>5-KIVppBM^YjmEf2L?~(zwlwQmdy* zI|2fjYq3~9t6!cW0g1cgmF;7e?%LEa(!$VDDVlno926n&>{A^dJ*$P^<`F8m`pWM$ z)Ua||(*=hauFWQL`#8y8FvMq19p zrcW-58F1&AY@+R!)ZV6KU9Rv>nCD9$?Wt7J?m_dc8tb^$Qfsr^=xH<0TwFY}ZQ<^- z;5-yQZ3WH8c@A~RtDc}*^wc@mPiiv9xjDqwqKG;|P@C}S zfBuGOvE5$*J)~M`+UvtH1ou;aRHFSl>J{KxJya2rt>BRl4BHRlGL6;-WskA*F-D3H zjv5DPC&(yaoZ=1OKenlm=xdfpzc`4p0$xXqh-tFl+1;Er0$r2sv8H_4#!%FUhUO6z zy&doHtZt8^io6eycjCY>MY7Dy;->HxW{`OMRJUH93Z^I12S=_3rmNX*d+_T}JYBkih6YX!f`^OC-Sj&cv>b+Z@9kg8 zTDKP$gA>oGUa5Vis6_=5p?kjkj)K;QC` zZl3b4El-Zt+J{$1h+Dg33l(6m7g5s501l_`?tGKC!S&K;8%&RfhbQ^JNXwpJA~#Yd z@Wy;;SGVs0TQk$)6k1UXQ9mR8B46_p_?+KQ{85gU)Er>n`Ot#s!9a^(AS8k<=SFb< zAm{RF&{>1+W>!^o^%53VKq7ikUi`r1+xF+XU=4AZ98rpqq;8|{*P7a0sLUzFxY}L=7un_=D`WZ+F@LT}sp@Dca4^T%^RyQVS2dqOS zRW-~PWP=2n1%VRZ5x%Cr1G&EohX=SCIR|=(>f&Pa?NT4+5r>52x7kd8Xlc@xahKMe zr%#0zuN)gVYkPUmQ8rRAv(sR2AU(M?+0x?WR95?Pz-HO|Xkjr;l9l*iF<4;=Gc=C| zW|uDEHoLg;6WxQBZRQjcd$b}EwT?!Q#eBs~-N zg4@(%?iMvH_O?Q!5B^l9rd>-+*S3UK;~b~lw$5|eHN7%+-Zr?zUmvpy!q}Hd%NEWw zN%thrbJJ#D7izVuUW5&tKJ>+=lJ;;U0k0ZPj`y+miiKS zkoX66)M5HZofN<|tbB7ef)Iw9V-?7_$kY0)pXG1hNpf22hSBVsER)L&q5Ak09*G5@y5{-7TJXxoR8K5#mg!ajv<%!VQW39+*5 zVlt3`2Dg}p#i8|l5su4WfBSr0m{-!Z#+_j+KUCjqJ;AJNmO^4^izcPPh%4kUZ+81} zwN<{GP5;1f5*;`1?Qo3cEC|iq?RCP`5f_G?s7yuhPFq=hd2>~ML+Hauj#oMTY3xyPd*2Ed zLQ_F%iixYOb5`7o#0}J-B9)_v`gbk>;Kw;#-qO`gTMBy2a!dGDM%+6~M8zq20w)+; zaD<>(aP<((N4Lf1rW4DThrNguEJfSNdA7nKnyu-nl!~1=RXGM^1gK3ioWQrb?mCr5 zm1tLZo`+W@IP-tSa(lG38Fwdo3Iy@me%$(R8EtmG;EzUskGYqussH_Q(Q2k{ox8>n z0>9rkZp1wTly*_MHIMT(%x*NnVZVm^k(o?ogmXpo4 z#(KVX8B>N({xF%;FPM6vaVP!fy3P<(Ufzs9(710f=Swss`#wpu3f z?XKCiysfy=2wu~EW8E8bpG3t)dQWbo&(sa8#ryTD$0YClMFSIPruFck&Z|EYWK<%oHEjrv@Cqm+w9n0igdlEZJ8hq zxWW+*vN>n%2adFAaSI%FB8?%v%A-RVM7)=r`34Wn^@DFdw3?CLUA-Uf$t(d92Jz;3 zea7t#Dn!<=1(Hp0$*QP&zAC}zhjO|)4y*OT28aEGVbvAxZI5)NL0@&@!2l}gbP4g! zm;pC1{1zORg0rE97HTNy|NI>We|LPvtkU{Qozi6c8nsEz;S!O_|I&GV2qKeK z3=RwVxQdOM$`4dgr~rDrg;;Jlp9i=Zk0AJa5yW5UuRtPnj@|7?Ro`d3>j>NpP1Eo1 zoeVS!Qx*u;4t||sAY-dXXa_n5jdnMx%d80jF9)njTs`f8Xw;L-ulavIgLTBn2nLS_ z+2HwT3OrVx^?ck)XQ(9fe4&tb@ad1>KZOP9gJsz1g1d(AZV7z&Wi762vgwDhA$f#vY22Zk z1;|1jxL=ii-~3W&g zm3VblOF$#hsrAfm0{ZmX7vLm6ZDdXdNtzGYjK}k_MR}8me%S`TVcX`LA{>+-3bDma z{mIz6sI}<$5A-d5RXa6`aL`ECfakcg(|dBXdwvI6S`Y$$71Xype;#*`g%UD=lx-s2 zlP6C^t_AjZ-F~E+*f7Wsv>NdwtdkOXUhXfy zM*t!LX%HQ&IL-WPqwpERFg_LZAR>;qZ(7JytNHpOKE~;#i=KkB##_9tm-`=f$1Q@^4-lsv~;cw;~u(`?ybLGfbH0t1N97^nB|3)-3ls`rO7oh2DZv`ZS zg!llK3Jo`tr?46-*%As0s^-4>09Sy5y_JR}kv_ettg~|?7? zr-ouzb?Am@OIiiPZ&bII1=2H@3BENh76NTJ;$;s*cLxq5Mi$U3C?_+qiF&I1!{R)e!w?M$B%g9Ss(G_lYP`^dc zi+@%(Q(jjrAF#;**Q4QP!)eOm?>VX9t-7LpMrRiK)}!Vec_PmA>Mn?#-6bOgp#q;2 z7IIBRTW_1{16y~ky8^S(*V9AmDd(Ip+yPKF(J>-f|Nk07g{23uUL-`o0)kFif=ABf z709@fiv2(PU`7tQ2R)71;E;Eq5vsiWhwUPRLe~kllU?2U_{A`nhk?sKvM^M>&4p^h zO2$}?s1a8ZGsJv-L^1;K1^WYft(jtF=IrxzFiu`Mhl=jtSNVU<>S(t2sVSnAR~tGS z0YRjG-6^&ogoD@%&gFxaGOWd7G=w1ItK#l{PYX|ULWS(i^SunHIIM5z?6e?v&22DE z=@xQY1O(N-gWCTD>;c5)PM(Kw55I+3^!%hStDDVph(h4okPOWcfGz=YEc0zNT)(P| z_6T@ln6V5--nm|($1e)lWbCa6i|r&<0*tE%$$v7gk>?rKc%&t2F!yBoFB_6U-_}TH z!&EgL6uma4+3o%R0W7Bc<5uZHCu={^yVTTXU`ot5oHlQ)Ad)^>tyeKi+ zApe0sC#b7hw%mdbL0$_2HUQo#J7H)z=k$irR{UBAnmQH&R!0-?6vOZu%P~dR_(`mk z{rz5^qG+?R^=_wn@vEQTWIn2WJXrHx;6p!@NPqSR1*Y*Kf6K&6Fi89G-!a?nMEI2biYM+r2AxG& zg330-kH>6J9%su6ql*FnG`s#AlLtoK`K^7;PGC4(cMjSXgj~QGIY2>M_l6i7kZ~9| z-hPa)SFJBe&gS+=8{eO33QJr<_=X2-bW~aQ){tK11_1L{ze5lK|z|aA+Ca-KZ8Z^ z*$?w6j3jsWu0@F5^@C@Dn6@n7)hJ5#-G6ettN8zy94}@le6CgEua-TV8U_9_MDM+| zGKPR@@4%^@pwPbA_S zCXdCm?o`{_=+`a3!CzMAeQ#JmLeRLHmSan+%>obf5={{)2l|Z#A9aAfFvDaubC3#3 z&!-}(K`7y>T3#tUsFA%`rlR}~Cqi3cS8aazKE;^r3hr>s<%r#W#bBuvNY=^{yJlTF zn2fi>?u2@VI* zu_nw{!v#Q1Z;qh8fx~c4aAu&XEKWWe5%*BfK^jWmUzo35O@a~MKPgY>=I51A-{R#HiGN>{A)un8OAa`y2-k@h^O zdH5X4qG+Z@%n z7Q#U>tyo&#Jdj_x&~wZm=|G}HF?=N{-2zR}7%zEZv)TdQ!D`H|x2T~CRsW%zQ3n3s z=;kf_WVQra|EEnN4w7D8K$=8styQl)FT>nkdvB?U!2!AehBD+&O0W`r3h``(muNGV?RBAxFmo}lD_%Hi0$8owoQdHk$1sVl9E_b^Zr@@P$lamLLmR z^g{!upu-`^WaIe5C)ShA+nL=3HIL-p@!ocLi=h^cO>p@%jhJX=5{}I#zYL%{kLnRSWF!To^7Be-z!KJsaCXWNZV= zp-rg!KJBY0Z{}cl*E0kelI>H^jh^Ulz}31(AMEE|D7>|^u?tPsyDRg@-+Fvss-s{* z@1^#E?%Q#&h~s?Y=&s3~hxO$!Nd zOC;kJ)@C_*SmwboE^ zKa7^}p>+V{vki)*$hHb8rHX*^taLT9^zH|rAj98EKV#r~VZa!Q2llxD?+CV9+3U%T z?9dJ{kP0M-IVC2{=R5#(acW8s<;@ip`W^GFihQ;S!kcF0#h4@T(0Bz3;r@*e312I< zYJ|tyNhRWbVlo>2tR2bQ`@&;b40Vh=x zaVJYoh>sQ;AoMo?siL5?c^;zpHeKG|5qyB4+U?855Zwv{1cb!z)6xbm#lNiPpFf0d z{?Il8Isg=<9~~1@$~@IMz)DT+4GI0qbH5c+3rONId)ntdZLCiVqTr^icXj=Tc-C7E z+^{hGA>qHZ$==eBW2rfSx5M~PYRFVv?l(sjH8nM2*&-1y23!46$_^`kUuD~EHoB6B zPam!iS=}zS2Hw|ahGVnR9(4FZREsWjbaV(G0&%~uFRlOawyWW~AN;?_Vm(XN!&Bs& zkI6RYoB+5Hm(#=5zI`xFaA)|55-|bEYyXwg>1wO<#Q^W0(>$rP|q z-^Sh)##B^P_!Jdmsb32;8|`{*y8=Gwf2abUf_KrVRU&&cBq2rQ_P06kd@AK8h4ktrot82Smb+>-FzelyjJp_c`oML{CR903} z7%UTOcfGwoA4FvP`56gr!k|%`Q~;8IS-{(H6*0qDEES+~M(gYAYYE8WZNpRPaw?e$^ARw1l2JhU_7TJ}$AP_*t04rn zubKP`5T~6npQW0%0F5)+^0|rwQbJFnY8V3}L1i}jeYoeEAfETbtUnfO&9#Syhf^eZ z=1TPeDKTD%tjE%GK++C#eS7;?_izg$_Jf{CCS7I;IXTm-sN44}^3qI{%-kn^p_muy zloD`~>`~|uX7FDxD$3UaM+K0SR+Ok0xP)4+(R|23wtJtYMv zKPeA6@tPPQUZE#EgA>rtTW?pBYhBL&%q1$qTz<*kiv%y8eZNDuJeJooTF)MSP)3|vl*j;*EXEGT4)gE!)X8u~=}9g3@fJRn|$;agaE-+mmJ zoN53lOs*{rBMz5K-G3^orvIi$)&#SY(1ER<-Z@=Nfty9e@q^D4=zAKet27F(g|>)o zT=n}`lO&~mJor4S$dPS)#yi%(8e?7Nwpl;5FdvZ(!YZxq&Nd{Hna%5fusUrJ4d&rh zdj*A$GXR`&^nGu&`-GbY^vF?OUY@JLhhV3wSF!p8JReX6HEFOC&(7;qV&!dnlPaqc zjykg22Nw4B@87@0fGD4qq>|3z&X-`zGO8xW`m3xxM8TMw6 zoB3{bGA!VUyB=*0Hq4QNg5pxkW#jukV7P~^mCV^A9vM<_6nz%~fn3B6N~@!)^QBYmTu)nXjXa*O=NJK45pS^{&O)gZ z8jvLT2;T1-Rr?L5PBJ@6h_GA>1s;m`t^Pn-_p+R;vI+}t9y$@bp?Jv z$syxRXKd-^`LP{P;p84(Tl2o|4a06X*8U_x^v}^i4h9x2uUe`1U0X>>$rSkdHxL^P zD5X7TP|X36=)d8Z${)7YCq=#togWmJzkHplfdGD201!ddM@Ttd_alfCep89$dx73t zt`vZTVh^YuPSG0Da)_+OB_t&D#pQ4)n5xhf3<46i)?g42=pr}4^mY(4+~V}frKWBG zB>nO6vAlBu-ahg-M;w9AKB~lccT*e@-`(!V#*i1x>Lkk&lJsUE!A7Mi%r#} zO1D#LXlSsEBr+O@EI&v*kZ-#TEWGOXh3BpT(a1tvV~p}5R)xQ9T5fT@EN^XTxp@Z( zi6cLFXASKg_Eiwj68u(Uwes*gOE}aj85wcMql{@HGc;CODTi|9+M8XFS^`Da(t> z!5C~6BqE4~%HXNol#3)(a3Dez_$8bVNTi|g3iJKXmk1z&4s&Y_o2{KiGc3=@TjNXEnPC2_)uYNw z`VQ=GqIrx4D!w9+^sQaQ2__+o;HO|9_8+6;V2&way6-lj*CnB<{P{t`P@=D{n(7W3v3uT)W5C#B9{BUwLi2|fvv--vytAi01{Z-T}O{Wf7KhK@ga9W9gm>( z4E=D8FW0Cbk1m-{Chw36kbJRc;?<)CNI~n0Gy+|qby&T=|7x!FPQu>zuml;LOTN`O zqDbllHfW%vZ>$=ypj5E+0va4SfnO5K@04N-Y&xrdB2@xn0+>a`D zpFx2Is_1Bj8|QfDdB)7-yh3P6wxF(I(a~03d&XqXTVjYAyOwgqS|YDkd)*9TXb1u< zDbE>#PkFXA(=-Dg4T6mW+)%`c4$I?47G)-^R*Jg71QO8L)y|veMa9Y(8Un*P@|DNl z)*%v>+k({e7=Fu3$IMV|9=zHo!?7x+ZMQIn@>E1Ym71!F*XAY*^X3*|UE>H}KPsrI zllZzsMLQ!fx+34-rm1Fg>=c}oRiF{3)U27EDSv1R;Y9kmjwtlqxf}=_@Cx8$z3w1m zZ@gfp>9}E}hm66t4=kWbQE)%5!)t2-;v#!pL9#cO@C6CYI7eE?F!1%mVi&pUcopYe>R@1$1~deoZY+l(|n)J zVuQNJLf{R&AdA|{7>I(8Xa{9htZ~Z5x#Qz3)5vOI&zn7?e5f~L)_I44U(FuqW6 zx_sMk5v3^Sss7&nVLK@lfA$kTxXh36$uAi&AwG8H-`E{XUrnmY1^HvNWb-CK1&ZtQ zw2;e;v*3o-`Px8ic+|RuNXw(izk`{XtXh{kFPSP!=J+K%){2B|4c6 z-VB09+skj+q<6*~)IS=oUZYjVUDcI6?Ty5ZGrkF#ki`C)qVZ@s3dM5ebVZEXP(3W{ zoSi+nSoPA<70!ijv@0fgHL_|f_UBHapvazs$bGh)sA+5POL6{PYJL{AH*S(?@$F-* zcN_B2r#^o{NmfuF*>{nR9J-9HYJ5I*&+FP^XHA<=FR*NdLFeKcoZe;Rj zy>}+Fx&E6WBo#0So>}Qh5_1`QF|M*rv~c}Ih@2T6q&K8CDqkQJSGR;-$nXIBS3tgL zx+bbSK3uIh)qne*8=I7cd#p%_`VNS2qcr*pOvro>tb2UDX@eXrE0VnBWX>!4qATFX zpaaS8hae!RhK!dKn4vdRS>d9w(`_cLdgPEjsRyfMG{W7QfcB1}*)ZuCS z!(Lgr;1b!b;Nd6dT~f2?hcs8PBtM+Q1SEP1K=EF}IwOe{jNA1pp>Z3txD{$@2 z#Ad+}(TA?zRIx zB8R#P!o|X(BDd|lf#H|0bPua;2-l9+7?;znCR}Hme9t#h?ahv#Ssa~Ue~x0W1Tw^@ z_{sq}dapmVXr9~EzQ3gN*6K|=q91SoatQRXnjG(F6vp=S6raE1{?v_uyb|v-D%xMo#Ui6@TI(H|<{Z{opuql}Yw`Okd?g(?f?oXnJGnNxpsGM@D z|7>}h%3j)Ow)cJY-rey2-qBjy?X(O=L|t{epWNCVRLAAG^V-;W<=NM(r?KKt=U)6+ z2Nd>3agGp3TB%BsN&YXDb6W1c1|1|gTw79Iy1TBoE6(Q+hvf)cx~@A6!7zvP!J!g( z(I1crVr)tPRdGaH+ks8L2_Vgj!E};@iP^HG zT_u<%{n`D>`L-k>Cr95!>WcmZj$P!W)yd+i-*~(}Y_168I3i`jV}#brWuxzFZ?PIu z#);om=iOp7j#YBWRpa3buhwo)F)kZXhTRqsK4dKn{Hnqq1j>DKx7l9~C~&FnoothG z)Enf1lCZ2@6O)sE02@PwA}TJdT&k9v)9slE*PKFv>pAw~q%_wOI(|L42&VaG&vtYC zX}Gmu%^d6L#mnMlD}KE7z}f#{EUeJ*LWt>g@1aGy^{@^<%X}kj&%ZoJe}3fZ>^L%g z=)RTr`5nhDHJ>t{8HF5f>no4R!~Jo=k}36r*QL7!`v%w5%Xuw7MCNeg_7qh1ePHMX zp0cL07t{)`OMtn}xdADzuuM~Qcvhx89fmN6S)vWJ8|A)@w#ZCe^=;wIko&0?NAY2$ zHaHwyf9nm}gS*=WWn7&7ZuWktNas?+J5BphA(1lt>t;!zm4Ng9=Z!H}W_3+-n8h{+ zri0fVQHQk)h^vIb1RX#sj+OUCycB13$wH* zdB@3%YlR7yJ>Ro*jj&OeGGQ47h=b2ELYkhq58Cc9jT-P_%~Au+qk6lkQ(@g#C(EjV|%qXR4{G@=L`lX49xu8H>{qedz z*lGM4-Usy6A2c#00SNflaBv;c`j=qcNiX|MOgO*~c5bdVQ|)84P4Set&GMYz-#aA9;fa)*RMEO0O|un|cmeKg}L-`%J` z_@b^Es>7W}XPw08uCzoGDto&@;Tq*QM|e42c~Z)+?nMQ`VClj^_hOW#yEa>{@v|&< z=;9uWOp5RDmvQ#}>F$L&Deg=yY+z0OVXNnp+^Y@vqE$1>-dN^o;_ifNKbHd*Rb)>6 zbtJ_2MYXptYm?6K+U?clyweHuo@+UhcxBJm4Lt9fVF&6(X%GoI(rv)(Y@wV5y)sMLTHbgfbMDp9jECNTXq6$YE? zlC`CvG?BRM%il2j6h3kL?mE*D665rq2*8H=<+eLFIT_pdZUkX1ATRs1X&vivm$l-) zqj7Jg!89XOF~7#+guaH`<&y6xVcGpO`*GHN{JwOFKl>)`Q^eQHncXa}16pwo@e~c9 zso;C^T)_?~=CVb!)wFx&X_*2UhXI%ItkhhdO3$y0)rij%x0hDIxb5}r)uh^A+#r3)n;%OT+GSTJo?8f$dv(A{l1ZnMlLXGXE;ZRn zIdGRU;Tb~~916!oM@{~jM7nyfZaeGcJ=H6XsOihsw2nGg^vv65KTVMs<)AuX7nT!{ zg2KE+x8c5#a2ZMyUP;szc)nW6{RmjM5hpNbuJIb%!@BFvx9)k=%7O87pC8r5gqSOc z<&+nPwt9UK4Vj`wfg*n+wPbuQXLl$2o556S- zl&jr_CdMT}TH;|wz%BMh*s?RZ--7+=dXdH-Z%z7BK}@8UCV=qh6h6n_3Ael@q%MgpXkn8l!@msmo_^X80Febib%uVi`;p4!#l?z@fdb@qAt| zpEB)edY`KEaU}LYm#!^wkv36?V{rS6`P!3s*EtXxLTC#2YS4Vh3(To~w%FT41ryGw zLzJD2>hv()_@W*bAgF1?sN%60K$HPZ4&Dt7Mv99m`E{1IdWK*FasC_dIq>PvC}6^T zNEn}?$@lwph^ibCw|}hzz7}qdylo}AZRMrSfC6<2uu)OlsYxTrLZbK{@big@e6`O7 z3;{r$p*k56hsD6a_mYWU-f^CCB}iB-Hy~NJ-G4SgEY2bR zApETLKimG^{@3@R5|~7rflYzW8W#;6;$y!6zcbKir9v0tX*pux2GS>7SpUMihYU>-?Qp1^mFIm|o90jQl%C-b|SZXvXx(W|Ub{z76o( z#AN;f7#AzW0k`V!KY{{2Oy)|{pkD1ZBHyc#2?-GIsokbzGbgEhc{ zydjQ{I}~Jop;xxvfZs8)joffq3%g=ZE&`v*PyxD*5}FbRSjkriQedDrI|5vWCsAdV zUC^A0-kYc5_2oO!;Vc*pO-I0oCmFFI#>gZ&qyjFW50nv=bwGDWz}rsK!PiHJ->)rv z;JSc5yW8xz-wmA>6caJ}lmSNpJ^&1pXs@U}(0@K8=({#=*9>`zEC(cEG+OEc%LJf) zunaUgNXiJHi-YkJ^eAf3Cg^=c~pGs84qu!cTe_^bo-4<*-zB3tkSUXC2orbf2= z)=88IU}sdIF$gw7@NA}QhVtNe7$;iS$s7Jlginy~qu`6U;CXTGuJV0zC4A?!*H$Hp zgoj^8USe`2Q#o9ONKoinAw5pr-(6(o5L@)}zW0gFE1$5Kvla8e?8uyb`90ZQ%^EH4Y7nntl%0bd3pMXlH$FUe)Dp2p`Km?&l)@ z<-*EBC>YOQ3BL? z(GRa`OA6kJ4%M6Lf|rVv_4{reP()zMR$wVi9dW`C9yQ%u6&@Y*f%eDsBb^B5-PII{ zlkzV-FFcGYK|2!l3kcPKFluwRG^0Ezt_FE1*fPvI3CKC106~KU%5?R&=0S{b@0zKQ zwe2?B1}jwPLG60x`sd#dBRmTv)Z&mPUZY=>!BkMFy9~j6H+Q<6?%87!MMRT=QyVGd z3h5wsTtH9ZmKTJ70-Fg;x{o+7Jvir|j)TH3aah_haf%m(aKJNJaOK{Kk`#Qs-(iJe z&-x*%az7XiFD|}K&y29W^7C4Jf_d&F!hd&^r^WC%x#59SimNA>W9OIj_oIo(I`Te||EyN%o^?r4&;gO#V(j8b=S5}r3d8;KVVpQ@~S z6x(%5U&4Zc7m7kzvYHg1FPe8$8FrAy3l(b|RVv0gR4`lB=oZPTZjNC>k+FJ3JF;)N zCM1k`OkbK=uin_37yZc9WN+E#tX^T8M5@J-{NQJKh|224Q+)JETU<0?OV|hMmzY!6 z4N+hh>oNYYCxw~!`UYApYu|AYm!ct$REdxq)w2yDZr#=LQT=nhDVhGs*inR7N>?vR z?nT8hGdyGk6et==oWg)sD9a!kUVRd!cJ|L~C5@C=)flFVy*S4oQuu8W+?}N+%XzH4 z$%-|BEYavjx&GfFdlK`E|D&=q4~MdC|M*zSKK9*U%o2$#*(S0tV<|!s(IDB^A(E}3 zVeDgQETIgcvTvc1B?-|;h!C2d2_c!3?Y%v}<9+}C&3`k;ocCP!bzIkZe!l1TKIZCg zA)46mo1)00BAuH|f;kEm=1oktq=32mP{}MMdcBre5!T{Bg=)B2U$E$)VI1D|&f&z1vn)i))Wp22Xx>5`{CX48->d1ckkF3bUD-RlOU+z5RU*na`zR_lb z>S5sFRA|lB82x3;sqH*Vl*wDKcWKu!xta9`e6Sx)7%%GFuBl~UE!R7RnVe>k?h~L> zT9jS6KS-zw8vjIn=2tgTn}MoaN-Rhs6NLJHxc`$2K!PL{LwHZncte0eF!@wQQR-t# zud6)m;Sw`aJ)WuC2Xwo${FYbxgE@IKDlj)abJ?q=#i?KHsRfVb@v!VdY`sKiL`5Ou zZFzr?vI~`;VRnNs7AQ=p1}{UU%o!s&gOZ`Zac9v>V2(JMug?Wed_6P&b_veIIF)iT ze<@z8XJ?L-&q=o+Vug%R4|UCl_XqQ^WIy1fPTam`10fgN{tF+~(Y0Ji6sXzB1S9Xb z6~YODpmTnG0}5tTsK0Keu6baL9g&f^2b9w`RS^SZv{&>BHiT@Ae$uUAD&d2NVKG2|lB)Q>8g z0Tr)v)o8X1v_3<1;p;sv=m=WZburq@7`3eA3YKslaIJ$cG}HNFy8_2ClFkGk7G9rd zw?7}zgU3>t^|Ko3Dl-~^upNdBf-bwM%8}9qHJ`?&@T5t#OTo78>J9w7FI5=tKXqqH zT96tfQIy$Cf%)k#ldU7n1n7J&(yB=sJW4v=x!51Vd!>n9FKtz};ISpSl>gfc~ z=ru%@m8E$PXXR2+!N;)Lmz5FCzK#KL?;2DGjioe1>`4QAm6ItpWdXN`4^7jhZfwg` zoc-D>SrHayXUfwHYiD$j2-s||F*2#Pi4;;C?fvCI=%AF;2!n*x7naDuxx(6mJW|wp zJ(EoIR!2s^Mz`o$4}WuKRBNO0QUXT#JB_j)2in^N$Rp#w5)7%C-Ro1AZ6b)xL+6sC z>q3V29>5*B(d8ES=P#!&%Gn*8nv+oeE*#Da&ykXp>Z_y}uf8*>*nogao4Rs)x||-; z9A8T#5riwJ8>RmgMNP9JtgkDryt*_qO?iuKkbA|oRZCcz2VV$QvO*DC8|&9~sn{3rYz44<4?5R=0vjwYP#y# z_ar=^;Jl0}GDfH>0+cRKaR+Ieatv6wI-@H6%m6ml7%gF3-xozp|Axvo1+X`!AExvCgE_){<~CS_(>QG>yeJIa%@MOtTnBAt-C zm4?q+TBbHu?X;#k(scr*o7N+zo7sXc&R~`}d)2F@w{}nQM&E07{Yeoor8)j92tC-- zO{h?|=R<~4k@AB@+I9Yi#L$W6A$U>_xU-l~LI5I4TnMwszqK)?Jw=twv3R3IEMju? z`CRFnHBO==p>MAiDUZy^D~wD-x-Wsc`g8PY_qpx%xnQcA)!dyyEuSsjTcLe z{eklFkkbjdpjrNh9{jRPa~oGD6ls+9Z16o@@B3&9OevMa#Y7XlS9(#drm9NfZ=~#I z0egazSab#V4z#Qy8eu0y=QHwT(0M`D z3i+Si!&fnykVAgHf_j$xC{E`TqV#6CrN}vc*fI4L4j63mnAk?=y*u2bkZe|yYVzXW zT#-kR02AfAO-Mn87 zN3*SP4CR_71rak<22-V?;1DDxMGg`fKYZKQ0^ju&4@fiDVK`N^2s_2r=({B7&1$Uy zMl(%7Fbo5^L}+xhmaG9L+*d%_?gBsFo<(6c05r<1!t3r53v1UWkCl3aaITcD6k6XO z{k`vrQ(Yd|?dgcr-PYw_9;nK`%fMFdRg<-R@czb$COJHKu{xKKxaY{Sf)A^ zY`T}Ja8WA8Qh>mpBdSPG@oRG>W>)onXwFFKrRIB0Yl>;M))K*tq1N{)-|=WSca{*Q zQCCnoldY+7#E6(N&*gMVxll0yI{9v8^7pViuyg$>nBY@>B9S-p(no^(r8@eND$lg!gVtDyWRrCcnJptFmi^oq<6RHwRL zx_G4s937qoj7bxM!7F40k3^QI>*)M|7iBJir2M7=v-uk1EMHj9`Wiz|LR5b(AKLi| zJv70l->&c)*1!Q6XD#Ehpr#f?4{>_N3XxVf4*QS z+n2uOzj<@sJ+momO7vb_iMOI9>MiK;r3dUJVW87Nw=gHUT}2{>EzX>z9q0YAt@=9U z;tjx35X!m-5YXD|=(@2GVrq$A=-S=rx((Y5b-1vk@TOa;9Hh3UTm_ik`|?j^qswaU zekZPjc3cgNl0ex~Dh02b?DmN|xtb=&yk5f^DvTM0YKK~zd=plM2PR*HCSNuNsyNT7 z#T@KkSggD*Bnv&g4K!j}eLoO{5*y`gSAfk*hVwO}7lc23wS2@<_U?sPi^a3Q3s){k zVM3l7Z%wc>8=-1<=;6_MoPvR;Keh%dNHBHoHEtc;bju6tBUbG67is|(Va5{?2iC@; z-Xf|PIOk+e@asj3t&Ci`gPHLN&o^O!gQWJ)<2p<(c6M(kIP~L}-Ni;v@5K$td~ku< zDQ6IfLlJFa=pZrdzaE?lms77i-`AdBnYg#b`&SJ7=O5+v0DAnA`RG2sB}!n@l+fQ;!I#ung&a?ehaO8`JNi#L4@4i+IWM?Rp-T z!fM@gRCFYnk@T{h6Flb%;((T}95l7hBg^#nHP0Kr!KHPszxoaT=<8y*&+R3u`pkb! z9-ZBh&zFj?2PDO#axtKo8=lrUsAT=9nZ&!**A)18`LIn%bX#6Hg52;tjsN+K2luG( z_1x>UOH@0~9_aNvEJ@t<=~3^MlYQ!Y-pe_E&ir9LRyNm&L)nyrh=YJ(GKz`KZM(bs z8(bm2>pR@_&p&UOTzKNpXtvy_mtJuCg5jS{Sp&D(&*oub;|Ao!1#d9{W@$IK*f(`vP!u=NES?)WW-!D3_Qvb2oEM)(qtZ8jFEKF$)~5PCr#~ES=rF;2 zG@oYz7BN`XSLCSYmSsS;F~6(?7lZREe(Xx!`f+-eyZcZp^4CIpeC5+%?AvUtMUtCz z3Qba**C{n$@%>HlG!r@c9QVSiY)!4W&g2=?BSnp+o{f-vF;#)$(XPkx)mMQBsQ76& z(jDcFFGR}6#RYe64kb!g%%u=7kv)l#^t^ud_p2-uR15{C47QnCo(Y=kOHRFOoQ~_& z)HxmSd!9v7chK{-gxZ8+|(Uj_!FS>yB11 zreZqLrI_uJbT5DD5+q&4QK0n!^0A->)1l_oX9aDr?7QvFfcEhz<*ziVuYVix>EE;j+V? zGw1qpz~PNOVB2RZ<1ggLb#ISPVc#UD$f=gyGOiV^_oSwr>4O^@$t?ygYYx1M^oci~ z5&zU&%ze#eLm%O2H>Vg%%=Dnyv?a3x$oLQSsIyFv1xE1X${EqelTQZ@>`yFe)zuAD zAHEiy*g3;W!9h6Fz*Yise$2E7fydT3t&70Bb8>&q2S9wZdT=?B5Dymq>*G+nu#w1Q zc2l4viuAp}UF2;DuMb46w#Ci3^iRa}2c(fx_yaDNJ(ph>9#1n9!L8+ZX5!}G=i?&P zn%86vQC!_hhRt69vB(xFGzhLhTB3;3eM0k!bPJ|@?;||B$w~zP+0lDV15tNLeEw5X zUK)*-VVV2GM_i7IkuKYF>5}Q`vVcnjP8DIvY{v%;K72#3-OT}iuS#;*2&s(i=e^^c zo`%Cbj`vEZF{|kL_AFwjD7{FL*234B<2hIuc1krU@ zmM~TffHSLY8j!;bw3#s!Tu_sMj)P^Y(m<__woR>)p)tF*wx85s2o zsy|e@q&C9moar?an5`lO$6Xk=CFJtS%pQ|Jh8)sb9|>{h#!OaN>AY~>Kw$~?<8zw+ zZA!u749UAzGUGIKn#{^+27~%!!&4@WqnISpSvV3*1Pk!Y;f<&b+U%*y-NoU3FSNIJ z6*mM|YGURd{7e=f-`MagWD*jnfIeBGT|d1*Fn3yfRVUqI98!Pwe2LPgZsUvVm(b=T zwULl^MdlU*l!-oCW+#^WYV|cQnas*Gp@_1|xd~yXLD5Ji*H5UPpG7Dq!lAWOSJ;Q( zk2xaxP${@$Co5APfk)C^VJ|5}<;5L{YQyZ#L_jQ(9@P?Sbs(o{CfEWas-45nj(Ffq zec5=d5oi;txqFoGVbpxxPb%NDnB?PCTElwU7cB4m(kl6;=h`G##^S_!7kV@x_Rckb zC40;Z$F#nLn;vP7(bbbwJfyHvTK&7GlfppU-%DbcIPZ_pTecEq{mxPM8uzFR1CdeW z@su>J_XI;%)iozh{Y-pfAtlQL<8Cd}V$I6jdJ7@Lxt|gX{CrfyI|jaH2!S<2 zYczX8kp)(7bmnp@qxt;5hDqWTSqPfF0AHR26VVULIk};Q;PADdeK5H5j`kUZEGX-b zCS4iG4{#{N!ZW53sB1bCigI5SMD!65wEC6G^<*rmx`X29(22Pe z8I4+0_6m!&zh}H9?++Qn-@nOj@;vc|65ZlTWq)?&gT4zu4oU}}F_`~7U_Zi?@KUir zexlu~LNYJu^2#OFw>W@?A!s}uf3-#kil$=RUXW-i!=EZ)K`p|@C|9T8ij7}Yw=8=8mGkop2lBp=qujFrl_hywF+C`4SQsqP$pW8NkWZUU1e=7TTphxPthcwCY literal 0 HcmV?d00001 diff --git a/tests/common/snappi/common_helpers.py b/tests/common/snappi/common_helpers.py index 4a8404db1b4..f5de796dbd6 100644 --- a/tests/common/snappi/common_helpers.py +++ b/tests/common/snappi/common_helpers.py @@ -14,7 +14,8 @@ import ipaddr from netaddr import IPNetwork from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice - +from ipaddress import IPv6Network, IPv6Address +from random import getrandbits def increment_ip_address(ip, incr=1): """ @@ -654,3 +655,25 @@ def enable_packet_aging(duthost): duthost.command("docker cp /tmp/packets_aging.py syncd:/") duthost.command("docker exec syncd python /packets_aging.py enable") duthost.command("docker exec syncd rm -rf /packets_aging.py") + +def get_ipv6_addrs_in_subnet(subnet, number_of_ip): + """ + Get N IPv6 addresses in a subnet. + Args: + subnet (str): IPv6 subnet, e.g., '2001::1/64' + number_of_ip (int): Number of IP addresses to get + Return: + Return n IPv6 addresses in this subnet in a list. + """ + + subnet = str(IPNetwork(subnet).network) + "/" + str(subnet.split("/")[1]) + subnet = unicode(subnet, "utf-8") + ipv6_list = [] + for i in range(number_of_ip): + network = IPv6Network(subnet) + address = IPv6Address( + network.network_address + getrandbits( + network.max_prefixlen - network.prefixlen)) + ipv6_list.append(str(address)) + + return ipv6_list \ No newline at end of file diff --git a/tests/common/snappi/snappi_fixtures.py b/tests/common/snappi/snappi_fixtures.py index 0f884c39034..73fbf73edb3 100644 --- a/tests/common/snappi/snappi_fixtures.py +++ b/tests/common/snappi/snappi_fixtures.py @@ -3,16 +3,16 @@ """ import pytest import snappi +import snappi_convergence from ipaddress import ip_address, IPv4Address from tests.common.fixtures.conn_graph_facts import conn_graph_facts,\ fanout_graph_facts from tests.common.snappi.common_helpers import get_addrs_in_subnet,\ - get_peer_snappi_chassis + get_ipv6_addrs_in_subnet, get_peer_snappi_chassis from tests.common.snappi.snappi_helpers import SnappiFanoutManager, get_snappi_port_location from tests.common.snappi.port import SnappiPortConfig, SnappiPortType from tests.common.helpers.assertions import pytest_assert - @pytest.fixture(scope="module") def snappi_api_serv_ip(tbinfo): """ @@ -428,4 +428,97 @@ def snappi_testbed_config(conn_graph_facts, snappi_ports=snappi_ports) pytest_assert(config_result is True, 'Fail to configure L3 interfaces') - return config, port_config_list \ No newline at end of file + return config, port_config_list + +@pytest.fixture(scope="module") +def tgen_ports(duthost, + conn_graph_facts, + fanout_graph_facts): + + """ + Populate tgen ports info of T0 testbed and returns as a list + Args: + duthost (pytest fixture): duthost fixture + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts (pytest fixture): fanout graph + Return: + [{'card_id': '1', + 'ip': '22.1.1.2', + 'ipv6': '3001::2', + 'ipv6_prefix': u'64', + 'location': '10.36.78.238;1;2', + 'peer_device': 'sonic-s6100-dut', + 'peer_ip': u'22.1.1.1', + 'peer_ipv6': u'3001::1', + 'peer_port': 'Ethernet8', + 'port_id': '2', + 'prefix': u'24', + 'speed': 'speed_400_gbps'}, + {'card_id': '1', + 'ip': '21.1.1.2', + 'ipv6': '2001::2', + 'ipv6_prefix': u'64', + 'location': '10.36.78.238;1;1', + 'peer_device': 'sonic-s6100-dut', + 'peer_ip': u'21.1.1.1', + 'peer_ipv6': u'2001::1', + 'peer_port': 'Ethernet0', + 'port_id': '1', + 'prefix': u'24', + 'speed': 'speed_400_gbps'}] + """ + + speed_type = {'50000': 'speed_50_gbps', + '100000': 'speed_100_gbps', + '200000': 'speed_200_gbps', + '400000': 'speed_400_gbps'} + + snappi_fanout = get_peer_snappi_chassis(conn_data=conn_graph_facts, + dut_hostname=duthost.hostname) + snappi_fanout_id = list(fanout_graph_facts.keys()).index(snappi_fanout) + snappi_fanout_list = SnappiFanoutManager(fanout_graph_facts) + snappi_fanout_list.get_fanout_device_details(device_number = snappi_fanout_id) + snappi_ports = snappi_fanout_list.get_ports(peer_device = duthost.hostname) + port_speed = None + + for i in range(len(snappi_ports)): + if port_speed is None: + port_speed = int(snappi_ports[i]['speed']) + + elif port_speed != int(snappi_ports[i]['speed']): + """ All the ports should have the same bandwidth """ + return None + + config_facts = duthost.config_facts(host=duthost.hostname, + source="running")['ansible_facts'] + for port in snappi_ports: + port['location'] = get_snappi_port_location(port) + port['speed'] = speed_type[port['speed']] + try: + for port in snappi_ports: + peer_port = port['peer_port'] + int_addrs = config_facts['INTERFACE'][peer_port].keys() + ipv4_subnet = [ele for ele in int_addrs if "." in ele][0] + if not ipv4_subnet: + raise Exception("IPv4 is not configured on the interface {}".format(peer_port)) + port['peer_ip'], port['prefix'] = ipv4_subnet.split("/") + port['ip'] = get_addrs_in_subnet(ipv4_subnet, 1)[0] + ipv6_subnet = [ele for ele in int_addrs if ":" in ele][0] + if not ipv6_subnet: + raise Exception("IPv6 is not configured on the interface {}".format(peer_port)) + port['peer_ipv6'], port['ipv6_prefix'] = ipv6_subnet.split("/") + port['ipv6'] = get_ipv6_addrs_in_subnet(ipv6_subnet, 1)[0] + except: + raise Exception('Configure IPv4 and IPv6 on DUT interfaces') + + return snappi_ports + + +@pytest.fixture(scope='module') +def cvg_api(snappi_api_serv_ip, + snappi_api_serv_port): + api = snappi_convergence.api(location=snappi_api_serv_ip + ':' + str(snappi_api_serv_port),ext='ixnetwork') + yield api + if getattr(api, 'assistant', None) is not None: + api.assistant.Session.remove() + \ No newline at end of file diff --git a/tests/snappi/bgp/__init__.py b/tests/snappi/bgp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/snappi/bgp/files/__init__.py b/tests/snappi/bgp/files/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/snappi/bgp/files/bgp_convergence_helper.py b/tests/snappi/bgp/files/bgp_convergence_helper.py new file mode 100644 index 00000000000..e5152d8b1ab --- /dev/null +++ b/tests/snappi/bgp/files/bgp_convergence_helper.py @@ -0,0 +1,945 @@ +from tabulate import tabulate +from statistics import mean +from tests.common.utilities import (wait, wait_until) +from tests.common.helpers.assertions import pytest_assert +logger = logging.getLogger(__name__) + +TGEN_AS_NUM = 65200 +DUT_AS_NUM = 65100 +TIMEOUT = 30 +BGP_TYPE = 'ebgp' +temp_tg_port=dict() +NG_LIST = [] +aspaths = [65002, 65003] + +def run_bgp_local_link_failover_test(cvg_api, + duthost, + tgen_ports, + iteration, + multipath, + number_of_routes, + route_type, + port_speed,): + """ + Run Local link failover test + + Args: + cvg_api (pytest fixture): snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of T0 testbed + iteration: number of iterations for running convergence test on a port + multipath: ecmp value for BGP config + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + port_count = multipath+1 + + """ Create bgp config on dut """ + duthost_bgp_config(duthost, + tgen_ports, + port_count, + route_type,) + + """ Create bgp config on TGEN """ + tgen_bgp_config = __tgen_bgp_config(cvg_api, + port_count, + number_of_routes, + route_type, + port_speed,) + + """ + Run the convergence test by flapping all the rx + links one by one and calculate the convergence values + """ + get_convergence_for_local_link_failover(cvg_api, + tgen_bgp_config, + iteration, + multipath, + number_of_routes, + route_type,) + + """ Cleanup the dut configs after getting the convergence numbers """ + cleanup_config(duthost) + + +def run_bgp_remote_link_failover_test(cvg_api, + duthost, + tgen_ports, + iteration, + multipath, + number_of_routes, + route_type, + port_speed,): + """ + Run Remote link failover test + + Args: + cvg_api (pytest fixture): snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of T0 testbed + iteration: number of iterations for running convergence test on a port + multipath: ecmp value for BGP config + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + port_count = multipath+1 + """ Create bgp config on dut """ + duthost_bgp_config(duthost, + tgen_ports, + port_count, + route_type,) + + """ Create bgp config on TGEN """ + tgen_bgp_config = __tgen_bgp_config(cvg_api, + port_count, + number_of_routes, + route_type, + port_speed,) + + """ + Run the convergence test by withdrawing all the route ranges + one by one and calculate the convergence values + """ + get_convergence_for_remote_link_failover(cvg_api, + tgen_bgp_config, + iteration, + multipath, + number_of_routes, + route_type,) + + """ Cleanup the dut configs after getting the convergence numbers """ + cleanup_config(duthost) + + + +def run_rib_in_convergence_test(cvg_api, + duthost, + tgen_ports, + iteration, + multipath, + number_of_routes, + route_type, + port_speed,): + """ + Run RIB-IN Convergence test + + Args: + cvg_api (pytest fixture): snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of T0 testbed + iteration: number of iterations for running convergence test on a port + multipath: ecmp value for BGP config + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + port_count = multipath+1 + + """ Create bgp config on dut """ + duthost_bgp_config(duthost, + tgen_ports, + port_count, + route_type,) + + """ Create bgp config on TGEN """ + tgen_bgp_config = __tgen_bgp_config(cvg_api, + port_count, + number_of_routes, + route_type, + port_speed,) + + """ + Run the convergence test by withdrawing all routes at once and + calculate the convergence values + """ + get_rib_in_convergence(cvg_api, + tgen_bgp_config, + iteration, + multipath, + number_of_routes, + route_type,) + + """ Cleanup the dut configs after getting the convergence numbers """ + cleanup_config(duthost) + + +def run_RIB_IN_capacity_test(cvg_api, + duthost, + tgen_ports, + multipath, + start_value, + step_value, + route_type, + port_speed,): + + """ + Run RIB-IN Capacity test + + Args: + cvg_api (pytest fixture): snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of T0 testbed + multipath: ecmp value for BGP config + start_value: start value of number of routes + step_value: step value of routes to be incremented at every iteration + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + port_count = multipath+1 + """ Create bgp config on dut """ + duthost_bgp_config(duthost, + tgen_ports, + port_count, + route_type,) + + + """ Run the RIB-IN capacity test by increasig the route count step by step """ + get_RIB_IN_capacity(cvg_api, + multipath, + start_value, + step_value, + route_type, + port_speed,) + + """ Cleanup the dut configs after getting the convergence numbers """ + cleanup_config(duthost) + + +def duthost_bgp_config(duthost, + tgen_ports, + port_count, + route_type,): + """ + Configures BGP on the DUT with N-1 ecmp + + Args: + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of T0 testbed + port_count:multipath + 1 + multipath: ECMP value for BGP config + route_type: IPv4 or IPv6 routes + """ + duthost.command("sudo config save -y") + duthost.command("sudo cp {} {}".format("/etc/sonic/config_db.json", "/etc/sonic/config_db_backup.json")) + global temp_tg_port + temp_tg_port = tgen_ports + for i in range(0, port_count): + intf_config = ( + "sudo config interface ip remove %s %s/%s \n" + "sudo config interface ip remove %s %s/%s \n" + ) + intf_config %= (tgen_ports[i]['peer_port'], tgen_ports[i]['peer_ip'], tgen_ports[i]['prefix'], tgen_ports[i]['peer_port'], tgen_ports[i]['peer_ipv6'], tgen_ports[i]['ipv6_prefix']) + logger.info('Removing configured IP and IPv6 Address from %s' % (tgen_ports[i]['peer_port'])) + duthost.shell(intf_config) + + for i in range(0, port_count): + portchannel_config = ( + "sudo config portchannel add PortChannel%s \n" + "sudo config portchannel member add PortChannel%s %s\n" + "sudo config interface ip add PortChannel%s %s/%s\n" + "sudo config interface ip add PortChannel%s %s/%s\n" + ) + portchannel_config %= (i+1, i+1, tgen_ports[i]['peer_port'], i+1, tgen_ports[i]['peer_ip'], tgen_ports[i]['prefix'], i+1, tgen_ports[i]['peer_ipv6'], 64) + logger.info('Configuring %s to PortChannel%s with IPs %s,%s' % (tgen_ports[i]['peer_port'], i+1, tgen_ports[i]['peer_ip'], tgen_ports[i]['peer_ipv6'])) + duthost.shell(portchannel_config) + bgp_config = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'no bgp ebgp-requires-policy' " + "-c 'bgp bestpath as-path multipath-relax' " + "-c 'maximum-paths %s' " + "-c 'exit' " + ) + bgp_config %= (DUT_AS_NUM, port_count-1) + duthost.shell(bgp_config) + if route_type == 'IPv4': + for i in range(1, port_count): + bgp_config_neighbor = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'neighbor %s remote-as %s' " + "-c 'address-family ipv4 unicast' " + "-c 'neighbor %s activate' " + "-c 'exit' " + ) + bgp_config_neighbor %= (DUT_AS_NUM, tgen_ports[i]['ip'], TGEN_AS_NUM, tgen_ports[i]['ip']) + logger.info('Configuring BGP v4 Neighbor %s' % tgen_ports[i]['ip']) + duthost.shell(bgp_config_neighbor) + else: + for i in range(1, port_count): + bgp_config_neighbor = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'neighbor %s remote-as %s' " + "-c 'address-family ipv6 unicast' " + "-c 'neighbor %s activate' " + "-c 'exit' " + ) + bgp_config_neighbor %= (DUT_AS_NUM, tgen_ports[i]['ipv6'], TGEN_AS_NUM, tgen_ports[i]['ipv6']) + logger.info('Configuring BGP v6 Neighbor %s' % tgen_ports[i]['ipv6']) + duthost.shell(bgp_config_neighbor) + + +def __tgen_bgp_config(cvg_api, + port_count, + number_of_routes, + route_type, + port_speed,): + """ + Creating BGP config on TGEN + + Args: + cvg_api (pytest fixture): snappi API + port_count: multipath + 1 + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + global NG_LIST + conv_config = cvg_api.convergence_config() + config = conv_config.config + for i in range(1, port_count+1): + config.ports.port(name='Test_Port_%d' % i, location=temp_tg_port[i-1]['location']) + c_lag = config.lags.lag(name="lag%d" % i)[-1] + lp = c_lag.ports.port(port_name='Test_Port_%d' % i)[-1] + lp.ethernet.name = 'lag_eth_%d' % i + if len(str(hex(i).split('0x')[1])) == 1: + m = '0'+hex(i).split('0x')[1] + else: + m = hex(i).split('0x')[1] + lp.protocol.lacp.actor_system_id = "00:10:00:00:00:%s" % m + lp.ethernet.name = "lag_Ethernet %s" % i + lp.ethernet.mac = "00:10:01:00:00:%s" % m + config.devices.device(name='Topology %d' % i) + + config.options.port_options.location_preemption = True + layer1 = config.layer1.layer1()[-1] + layer1.name = 'port settings' + layer1.port_names = [port.name for port in config.ports] + layer1.ieee_media_defaults = False + layer1.auto_negotiation.rs_fec = True + layer1.auto_negotiation.link_training = False + layer1.speed = port_speed + layer1.auto_negotiate = False + + def create_v4_topo(): + eth = config.devices[0].ethernets.add() + eth.port_name = config.lags[0].name + eth.name = 'Ethernet 1' + eth.mac = "00:00:00:00:00:01" + ipv4 = eth.ipv4_addresses.add() + ipv4.name = 'IPv4 1' + ipv4.address = temp_tg_port[0]['ip'] + ipv4.gateway = temp_tg_port[0]['peer_ip'] + ipv4.prefix = int(temp_tg_port[0]['prefix']) + rx_flow_name = [] + for i in range(2, port_count+1): + NG_LIST.append('Network_Group%s'%i) + if len(str(hex(i).split('0x')[1])) == 1: + m = '0'+hex(i).split('0x')[1] + else: + m = hex(i).split('0x')[1] + + ethernet_stack = config.devices[i-1].ethernets.add() + ethernet_stack.port_name = config.lags[i-1].name + ethernet_stack.name = 'Ethernet %d' % i + ethernet_stack.mac = "00:00:00:00:00:%s" % m + ipv4_stack = ethernet_stack.ipv4_addresses.add() + ipv4_stack.name = 'IPv4 %d' % i + ipv4_stack.address = temp_tg_port[i-1]['ip'] + ipv4_stack.gateway = temp_tg_port[i-1]['peer_ip'] + ipv4_stack.prefix = int(temp_tg_port[i-1]['prefix']) + bgpv4 = config.devices[i-1].bgp + bgpv4.router_id = temp_tg_port[i-1]['peer_ip'] + bgpv4_int = bgpv4.ipv4_interfaces.add() + bgpv4_int.ipv4_name = ipv4_stack.name + bgpv4_peer = bgpv4_int.peers.add() + bgpv4_peer.name = 'BGP %d' % i + bgpv4_peer.as_type = BGP_TYPE + bgpv4_peer.peer_address = temp_tg_port[i-1]['peer_ip'] + bgpv4_peer.as_number = int(TGEN_AS_NUM) + route_range = bgpv4_peer.v4_routes.add(name=NG_LIST[-1]) #snappi object named Network Group 2 not found in internal db + route_range.addresses.add(address='200.1.0.1', prefix=32, count=number_of_routes) + as_path = route_range.as_path + as_path_segment = as_path.segments.add() + as_path_segment.type = as_path_segment.AS_SEQ + as_path_segment.as_numbers = aspaths + rx_flow_name.append(route_range.name) + return rx_flow_name + + def create_v6_topo(): + eth = config.devices[0].ethernets.add() + eth.port_name = config.lags[0].name + eth.name = 'Ethernet 1' + eth.mac = "00:00:00:00:00:01" + ipv6 = eth.ipv6_addresses.add() + ipv6.name = 'IPv6 1' + ipv6.address = temp_tg_port[0]['ipv6'] + ipv6.gateway = temp_tg_port[0]['peer_ipv6'] + ipv6.prefix = int(temp_tg_port[0]['ipv6_prefix']) + rx_flow_name = [] + for i in range(2, port_count+1): + NG_LIST.append('Network_Group%s'%i) + if len(str(hex(i).split('0x')[1])) == 1: + m = '0'+hex(i).split('0x')[1] + else: + m = hex(i).split('0x')[1] + ethernet_stack = config.devices[i-1].ethernets.add() + ethernet_stack.port_name = config.lags[i-1].name + ethernet_stack.name = 'Ethernet %d' % i + ethernet_stack.mac = "00:00:00:00:00:%s" % m + ipv6_stack = ethernet_stack.ipv6_addresses.add() + ipv6_stack.name = 'IPv6 %d' % i + ipv6_stack.address = temp_tg_port[i-1]['ipv6'] + ipv6_stack.gateway = temp_tg_port[i-1]['peer_ipv6'] + ipv6_stack.prefix = int(temp_tg_port[i-1]['ipv6_prefix']) + + bgpv6 = config.devices[i-1].bgp + bgpv6.router_id = temp_tg_port[i-1]['peer_ip'] + bgpv6_int = bgpv6.ipv6_interfaces.add() + bgpv6_int.ipv6_name = ipv6_stack.name + bgpv6_peer = bgpv6_int.peers.add() + bgpv6_peer.name = 'BGP+_%d' % i + bgpv6_peer.as_type = BGP_TYPE + bgpv6_peer.peer_address = temp_tg_port[i-1]['peer_ipv6'] + bgpv6_peer.as_number = int(TGEN_AS_NUM) + route_range = bgpv6_peer.v6_routes.add(name=NG_LIST[-1]) + route_range.addresses.add(address='3000::1', prefix=64, count=number_of_routes) + as_path = route_range.as_path + as_path_segment = as_path.segments.add() + as_path_segment.type = as_path_segment.AS_SEQ + as_path_segment.as_numbers = aspaths + rx_flow_name.append(route_range.name) + return rx_flow_name + + if route_type == 'IPv4': + rx_flows = create_v4_topo() + flow = config.flows.flow(name='IPv4 Traffic')[-1] + elif route_type == 'IPv6': + rx_flows = create_v6_topo() + flow = config.flows.flow(name='IPv6 Traffic')[-1] + else: + raise Exception('Invalid route type given') + flow.tx_rx.device.tx_names = [config.devices[0].name] + flow.tx_rx.device.rx_names = rx_flows + flow.size.fixed = 1024 + flow.rate.percentage = 100 + flow.metrics.enable = True + return conv_config + +def get_flow_stats(cvg_api): + """ + Args: + cvg_api (pytest fixture): Snappi API + """ + request = cvg_api.convergence_request() + request.metrics.flow_names = [] + return cvg_api.get_results(request).flow_metric + +def get_convergence_for_local_link_failover(cvg_api, + bgp_config, + iteration, + multipath, + number_of_routes, + route_type,): + """ + Args: + cvg_api (pytest fixture): snappi API + bgp_config: __tgen_bgp_config + config: TGEN config + iteration: number of iterations for running convergence test on a port + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + """ + rx_port_names = [] + for i in range(1, len(bgp_config.config.ports)): + rx_port_names.append(bgp_config.config.ports[i].name) + bgp_config.rx_rate_threshold = 90/(multipath-1) + cvg_api.set_config(bgp_config) + + """ Starting Protocols """ + logger.info("Starting all protocols ...") + cs = cvg_api.convergence_state() + cs.protocol.state = cs.protocol.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Protocols To start") + + def get_avg_dpdp_convergence_time(port_name): + """ + Args: + port_name: Name of the port + """ + + table, avg, tx_frate, rx_frate, avg_delta = [], [], [], [], [] + for i in range(0, iteration): + logger.info('|---- {} Link Flap Iteration : {} ----|'.format(port_name, i+1)) + + """ Starting Traffic """ + logger.info('Starting Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Traffic To start") + flow_stats = get_flow_stats(cvg_api) + tx_frame_rate = flow_stats[0].frames_tx_rate + assert tx_frame_rate != 0, "Traffic has not started" + """ Flapping Link """ + logger.info('Simulating Link Failure on {} link'.format(port_name)) + cs = cvg_api.convergence_state() + cs.link.port_names = [port_name] + cs.link.state = cs.link.DOWN + cvg_api.set_state(cs) + wait(TIMEOUT, "For Link to go down") + flows = get_flow_stats(cvg_api) + for flow in flows: + tx_frate.append(flow.frames_tx_rate) + rx_frate.append(flow.frames_tx_rate) + assert sum(tx_frate) == sum(rx_frate), "Traffic has not converged after link flap: TxFrameRate:{},RxFrameRate:{}".format(sum(tx_frate), sum(rx_frate)) + logger.info("Traffic has converged after link flap") + """ Get control plane to data plane convergence value """ + request = cvg_api.convergence_request() + request.convergence.flow_names = [] + convergence_metrics = cvg_api.get_results(request).flow_convergence + for metrics in convergence_metrics: + logger.info('CP/DP Convergence Time (ms): {}'.format(metrics.control_plane_data_plane_convergence_us/1000)) + avg.append(int(metrics.control_plane_data_plane_convergence_us/1000)) + avg_delta.append(int(flows[0].frames_tx)-int(flows[0].frames_rx)) + """ Performing link up at the end of iteration """ + logger.info('Simulating Link Up on {} at the end of iteration {}'.format(port_name, i+1)) + cs = cvg_api.convergence_state() + cs.link.port_names = [port_name] + cs.link.state = cs.link.UP + cvg_api.set_state(cs) + table.append('%s Link Failure' % port_name) + table.append(route_type) + table.append(number_of_routes) + table.append(iteration) + table.append(mean(avg_delta)) + table.append(mean(avg)) + return table + table = [] + """ Iterating link flap test on all the rx ports """ + for i, port_name in enumerate(rx_port_names): + table.append(get_avg_dpdp_convergence_time(port_name)) + columns = ['Event Name', 'Route Type', 'No. of Routes', 'Iterations', 'Delta Frames', 'Avg Calculated Data Convergence Time (ms)'] + logger.info("\n%s" % tabulate(table, headers=columns, tablefmt="psql")) + + +def get_convergence_for_remote_link_failover(cvg_api, + bgp_config, + iteration, + multipath, + number_of_routes, + route_type,): + """ + Args: + cvg_api (pytest fixture): snappi API + bgp_config: __tgen_bgp_config + config: TGEN config + iteration: number of iterations for running convergence test on a port + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + """ + route_names = NG_LIST + bgp_config.rx_rate_threshold = 90/(multipath-1) + cvg_api.set_config(bgp_config) + def get_avg_cpdp_convergence_time(route_name): + """ + Args: + route_name: name of the route + + """ + table, avg, tx_frate, rx_frate, avg_delta = [], [], [], [], [] + """ Starting Protocols """ + logger.info("Starting all protocols ...") + cs = cvg_api.convergence_state() + cs.protocol.state = cs.protocol.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Protocols To start") + for i in range(0, iteration): + logger.info('|---- {} Route Withdraw Iteration : {} ----|'.format(route_name, i+1)) + """ Starting Traffic """ + logger.info('Starting Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Traffic To start") + flow_stats = get_flow_stats(cvg_api) + tx_frame_rate = flow_stats[0].frames_tx_rate + assert tx_frame_rate != 0, "Traffic has not started" + + """ Withdrawing routes from a BGP peer """ + logger.info('Withdrawing Routes from {}'.format(route_name)) + cs = cvg_api.convergence_state() + cs.route.names = [route_name] + cs.route.state = cs.route.WITHDRAW + cvg_api.set_state(cs) + wait(TIMEOUT, "For routes to be withdrawn") + flows = get_flow_stats(cvg_api) + for flow in flows: + tx_frate.append(flow.frames_tx_rate) + rx_frate.append(flow.frames_tx_rate) + assert sum(tx_frate) == sum(rx_frate), "Traffic has not converged after lroute withdraw TxFrameRate:{},RxFrameRate:{}".format(sum(tx_frate), sum(rx_frate)) + logger.info("Traffic has converged after route withdraw") + + """ Get control plane to data plane convergence value """ + request = cvg_api.convergence_request() + request.convergence.flow_names = [] + convergence_metrics = cvg_api.get_results(request).flow_convergence + for metrics in convergence_metrics: + logger.info('CP/DP Convergence Time (ms): {}'.format(metrics.control_plane_data_plane_convergence_us/1000)) + avg.append(int(metrics.control_plane_data_plane_convergence_us/1000)) + avg_delta.append(int(flows[0].frames_tx)-int(flows[0].frames_rx)) + """ Advertise the routes back at the end of iteration """ + cs = cvg_api.convergence_state() + cs.route.names = [route_name] + cs.route.state = cs.route.ADVERTISE + cvg_api.set_state(cs) + logger.info('Readvertise {} routes back at the end of iteration {}'.format(route_name, i+1)) + + table.append('%s route withdraw' % route_name) + table.append(route_type) + table.append(number_of_routes) + table.append(iteration) + table.append(mean(avg_delta)) + table.append(mean(avg)) + return table + table = [] + """ Iterating route withdrawal on all BGP peers """ + for route in route_names: + table.append(get_avg_cpdp_convergence_time(route)) + + columns = ['Event Name', 'Route Type', 'No. of Routes', 'Iterations', 'Frames Delta', 'Avg Control to Data Plane Convergence Time (ms)'] + logger.info("\n%s" % tabulate(table, headers=columns, tablefmt="psql")) + + +def get_rib_in_convergence(cvg_api, + bgp_config, + iteration, + multipath, + number_of_routes, + route_type,): + """ + Args: + cvg_api (pytest fixture): snappi API + bgp_config: __tgen_bgp_config + config: TGEN config + iteration: number of iterations for running convergence test on a port + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + """ + route_names = NG_LIST + bgp_config.rx_rate_threshold = 90/(multipath) + cvg_api.set_config(bgp_config) + table, avg, tx_frate, rx_frate, avg_delta = [], [], [], [], [] + for i in range(0, iteration): + logger.info('|---- RIB-IN Convergence test, Iteration : {} ----|'.format(i+1)) + """ withdraw all routes before starting traffic """ + logger.info('Withdraw All Routes before starting traffic') + cs = cvg_api.convergence_state() + cs.route.names = route_names + cs.route.state = cs.route.WITHDRAW + cvg_api.set_state(cs) + wait(TIMEOUT-25, "For Routes to be withdrawn") + """ Starting Protocols """ + logger.info("Starting all protocols ...") + cs = cvg_api.convergence_state() + cs.protocol.state = cs.protocol.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Protocols To start") + """ Start Traffic """ + logger.info('Starting Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Traffic To start") + flow_stats = get_flow_stats(cvg_api) + tx_frame_rate = flow_stats[0].frames_tx_rate + rx_frame_rate = flow_stats[0].frames_rx_rate + assert tx_frame_rate != 0, "Traffic has not started" + assert rx_frame_rate == 0 + + """ Advertise All Routes """ + logger.info('Advertising all Routes from {}'.format(route_names)) + cs = cvg_api.convergence_state() + cs.route.names = route_names + cs.route.state = cs.route.ADVERTISE + cvg_api.set_state(cs) + wait(TIMEOUT-25, "For all routes to be ADVERTISED") + flows = get_flow_stats(cvg_api) + for flow in flows: + tx_frate.append(flow.frames_tx_rate) + rx_frate.append(flow.frames_tx_rate) + assert sum(tx_frate) == sum(rx_frate), "Traffic has not convergedv, TxFrameRate:{},RxFrameRate:{}".format(sum(tx_frate), sum(rx_frate)) + logger.info("Traffic has converged after route advertisement") + + """ Get RIB-IN convergence """ + request = cvg_api.convergence_request() + request.convergence.flow_names = [] + convergence_metrics = cvg_api.get_results(request).flow_convergence + for metrics in convergence_metrics: + logger.info('RIB-IN Convergence time (ms): {}'.format(metrics.control_plane_data_plane_convergence_us/1000)) + avg.append(int(metrics.control_plane_data_plane_convergence_us/1000)) + avg_delta.append(int(flows[0].frames_tx)-int(flows[0].frames_rx)) + """ Stop traffic at the end of iteration """ + logger.info('Stopping Traffic at the end of iteration{}'.format(i+1)) + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.STOP + cvg_api.set_state(cs) + wait(TIMEOUT-20, "For Traffic To stop") + """ Stopping Protocols """ + logger.info("Stopping all protocols ...") + cs = cvg_api.convergence_state() + cs.protocol.state = cs.protocol.STOP + cvg_api.set_state(cs) + wait(TIMEOUT-20, "For Protocols To STOP") + table.append('Advertise All BGP Routes') + table.append(route_type) + table.append(number_of_routes) + table.append(iteration) + table.append(mean(avg_delta)) + table.append(mean(avg)) + columns = ['Event Name', 'Route Type', 'No. of Routes','Iterations', 'Frames Delta', 'Avg RIB-IN Convergence Time(ms)'] + logger.info("\n%s" % tabulate([table], headers=columns, tablefmt="psql")) + + +def get_RIB_IN_capacity(cvg_api, + multipath, + start_value, + step_value, + route_type, + port_speed,): + """ + Args: + cvg_api (pytest fixture): snappi API + temp_tg_port (pytest fixture): Ports mapping info of T0 testbed + multipath: ecmp value for BGP config + start_value: Start value of the number of BGP routes + step_value: Step value of the number of BGP routes to be incremented + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used in test + """ + def tgen_capacity(routes): + conv_config = cvg_api.convergence_config() + config = conv_config.config + for i in range(1, 3): + config.ports.port(name='Test_Port_%d' % i, location=temp_tg_port[i-1]['location']) + c_lag = config.lags.lag(name="lag%d" % i)[-1] + lp = c_lag.ports.port(port_name='Test_Port_%d' % i)[-1] + lp.ethernet.name = 'lag_eth_%d' % i + if len(str(hex(i).split('0x')[1])) == 1: + m = '0'+hex(i).split('0x')[1] + else: + m = hex(i).split('0x')[1] + lp.protocol.lacp.actor_system_id = "00:10:00:00:00:%s" % m + lp.ethernet.name = "lag_Ethernet %s" % i + lp.ethernet.mac = "00:10:01:00:00:%s" % m + config.devices.device(name='Topology %d' % i) + + config.options.port_options.location_preemption = True + layer1 = config.layer1.layer1()[-1] + layer1.name = 'port settings' + layer1.port_names = [port.name for port in config.ports] + layer1.ieee_media_defaults = False + layer1.auto_negotiation.rs_fec = True + layer1.auto_negotiation.link_training = False + layer1.speed = port_speed + layer1.auto_negotiate = False + + def create_v4_topo(): + eth = config.devices[0].ethernets.add() + eth.port_name = config.lags[0].name + eth.name = 'Ethernet 1' + eth.mac = "00:00:00:00:00:01" + ipv4 = eth.ipv4_addresses.add() + ipv4.name = 'IPv4 1' + ipv4.address = temp_tg_port[0]['ip'] + ipv4.gateway = temp_tg_port[0]['peer_ip'] + ipv4.prefix = int(temp_tg_port[0]['prefix']) + rx_flow_name = [] + for i in range(2, 3): + if len(str(hex(i).split('0x')[1])) == 1: + m = '0'+hex(i).split('0x')[1] + else: + m = hex(i).split('0x')[1] + ethernet_stack = config.devices[i-1].ethernets.add() + ethernet_stack.port_name = config.lags[i-1].name + ethernet_stack.name = 'Ethernet %d' % i + ethernet_stack.mac = "00:00:00:00:00:%s" % m + ipv4_stack = ethernet_stack.ipv4_addresses.add() + ipv4_stack.name = 'IPv4 %d' % i + ipv4_stack.address = temp_tg_port[i-1]['ip'] + ipv4_stack.gateway = temp_tg_port[i-1]['peer_ip'] + ipv4_stack.prefix = int(temp_tg_port[i-1]['prefix']) + bgpv4 = config.devices[i-1].bgp + bgpv4.router_id = temp_tg_port[i-1]['peer_ip'] + bgpv4_int = bgpv4.ipv4_interfaces.add() + bgpv4_int.ipv4_name = ipv4_stack.name + bgpv4_peer = bgpv4_int.peers.add() + bgpv4_peer.name = 'BGP %d' % i + bgpv4_peer.as_type = BGP_TYPE + bgpv4_peer.peer_address = temp_tg_port[i-1]['peer_ip'] + bgpv4_peer.as_number = int(TGEN_AS_NUM) + route_range = bgpv4_peer.v4_routes.add(name="Network_Group%d" % i) #snappi object named Network Group 2 not found in internal db + route_range.addresses.add(address='200.1.0.1', prefix=32, count=number_of_routes) + as_path = route_range.as_path + as_path_segment = as_path.segments.add() + as_path_segment.type = as_path_segment.AS_SEQ + as_path_segment.as_numbers = aspaths + rx_flow_name.append(route_range.name) + return rx_flow_name + + def create_v6_topo(): + eth = config.devices[0].ethernets.add() + eth.port_name = config.lags[0].name + eth.name = 'Ethernet 1' + eth.mac = "00:00:00:00:00:01" + ipv6 = eth.ipv6_addresses.add() + ipv6.name = 'IPv6 1' + ipv6.address = temp_tg_port[0]['ipv6'] + ipv6.gateway = temp_tg_port[0]['peer_ipv6'] + ipv6.prefix = int(temp_tg_port[0]['ipv6_prefix']) + rx_flow_name = [] + for i in range(2, 3): + if len(str(hex(i).split('0x')[1])) == 1: + m = '0'+hex(i).split('0x')[1] + else: + m = hex(i).split('0x')[1] + ethernet_stack = config.devices[i-1].ethernets.add() + ethernet_stack.port_name = config.lags[i-1].name + ethernet_stack.name = 'Ethernet %d' % i + ethernet_stack.mac = "00:00:00:00:00:%s" % m + ipv6_stack = ethernet_stack.ipv6_addresses.add() + ipv6_stack.name = 'IPv6 %d' % i + ipv6_stack.address = temp_tg_port[i-1]['ipv6'] + ipv6_stack.gateway = temp_tg_port[i-1]['peer_ipv6'] + ipv6_stack.prefix = int(temp_tg_port[i-1]['ipv6_prefix']) + + bgpv6 = config.devices[i-1].bgp + bgpv6.router_id = temp_tg_port[i-1]['peer_ip'] + bgpv6_int = bgpv6.ipv6_interfaces.add() + bgpv6_int.ipv6_name = ipv6_stack.name + bgpv6_peer = bgpv6_int.peers.add() + bgpv6_peer.name = 'BGP+_%d' % i + bgpv6_peer.as_type = BGP_TYPE + bgpv6_peer.peer_address = temp_tg_port[i-1]['peer_ipv6'] + bgpv6_peer.as_number = int(TGEN_AS_NUM) + route_range = bgpv6_peer.v6_routes.add(name="Network Group %d" % i) + route_range.addresses.add(address='3000::1', prefix=64, count=number_of_routes) + as_path = route_range.as_path + as_path_segment = as_path.segments.add() + as_path_segment.type = as_path_segment.AS_SEQ + as_path_segment.as_numbers = aspaths + rx_flow_name.append(route_range.name) + return rx_flow_name + conv_config.rx_rate_threshold = 90/(multipath) + if route_type == 'IPv4': + rx_flows = create_v4_topo() + flow = config.flows.flow(name='IPv4_Traffic_%d' % routes)[-1] + elif route_type == 'IPv6': + rx_flows = create_v6_topo() + flow = config.flows.flow(name='IPv6_Traffic_%d' % routes)[-1] + else: + raise Exception('Invalid route type given') + flow.tx_rx.device.tx_names = [config.devices[0].name] + flow.tx_rx.device.rx_names = rx_flows + flow.size.fixed = 1024 + flow.rate.percentage = 100 + flow.metrics.enable = True + flow.metrics.loss = True + return conv_config + + def run_traffic(routes): + logger.info('|-------------------- RIB-IN Capacity test, No.of Routes : {} ----|'.format(routes)) + conv_config = tgen_capacity(routes) + cvg_api.set_config(conv_config) + """ Starting Protocols """ + logger.info("Starting all protocols ...") + cs = cvg_api.convergence_state() + cs.protocol.state = cs.protocol.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Protocols To start") + """ Starting Traffic """ + logger.info('Starting Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.START + cvg_api.set_state(cs) + wait(TIMEOUT, "For Traffic To start") + + try: + for j in range(start_value, 100000000000, step_value): + tx_frate, rx_frate = [], [] + run_traffic(j) + flow_stats = get_flow_stats(cvg_api) + logger.info('Loss% : {}'.format(flow_stats[0].loss)) + for flow in flow_stats: + tx_frate.append(flow.frames_tx_rate) + rx_frate.append(flow.frames_rx_rate) + logger.info("Tx Frame Rate : {}".format(tx_frate)) + logger.info("Rx Frame Rate : {}".format(rx_frate)) + if float(flow_stats[0].loss) > 0.001: + if j == start_value: + raise Exception('Traffic Loss Encountered in first iteration, reduce the start value and run the test') + logger.info('Loss greater than 0.001 occured') + logger.info('Reducing the routes and running test') + b = j-step_value + logger.info('Stopping Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.STOP + cvg_api.set_state(cs) + wait(TIMEOUT-20, "For Traffic To stop") + break + logger.info('Stopping Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.STOP + cvg_api.set_state(cs) + wait(TIMEOUT-20, "For Traffic To stop") + l = [] + l.append(b+int(step_value/8)) + l.append(b+int(step_value/4)) + l.append(b+int(step_value/2)) + l.append(b+step_value-int(step_value/4)) + l.append(b+step_value-int(step_value/8)) + for i in range(0,len(l)): + run_traffic(l[i]) + flow_stats = get_flow_stats(cvg_api) + logger.info('Loss% : {}'.format(flow_stats[0].loss)) + if float(flow_stats[0].loss) <= 0.001: + max_routes = start_value + pass + else: + max_routes = l[i]-int(step_value/8) + break + logger.info('Stopping Traffic') + cs = cvg_api.convergence_state() + cs.transmit.state = cs.transmit.STOP + cvg_api.set_state(cs) + wait(TIMEOUT-20, "For Traffic To stop") + """ Stopping Protocols """ + logger.info("Stopping all protocols ...") + cs = cvg_api.convergence_state() + cs.protocol.state = cs.protocol.STOP + cvg_api.set_state(cs) + wait(TIMEOUT-20, "For Protocols To STOP") + except Exception as e: + logger.info(e) + finally: + columns = ['Test Name', 'Maximum no. of Routes'] + logger.info("\n%s" % tabulate([['RIB-IN Capacity Test',max_routes]], headers=columns, tablefmt="psql")) + +def cleanup_config(duthost): + """ + Cleaning up dut config at the end of the test + + Args: + duthost (pytest fixture): duthost fixture + """ + duthost.command("sudo cp {} {}".format("/etc/sonic/config_db_backup.json","/etc/sonic/config_db.json")) + duthost.shell("sudo config reload -y \n") + logger.info("Wait until all critical services are fully started") + pytest_assert(wait_until(360, 10, 1, duthost.critical_services_fully_started), "Not all critical services are fully started") + logger.info('Convergence Test Completed') \ No newline at end of file diff --git a/tests/snappi/bgp/test_bgp_local_link_failover.py b/tests/snappi/bgp/test_bgp_local_link_failover.py new file mode 100644 index 00000000000..e1e9b31c419 --- /dev/null +++ b/tests/snappi/bgp/test_bgp_local_link_failover.py @@ -0,0 +1,64 @@ +from tests.common.snappi.snappi_fixtures import cvg_api +from tests.common.snappi.snappi_fixtures import ( + snappi_api_serv_ip, snappi_api_serv_port, tgen_ports) +from files.bgp_convergence_helper import run_bgp_local_link_failover_test +from tests.common.fixtures.conn_graph_facts import ( + conn_graph_facts, fanout_graph_facts) +import pytest + + +@pytest.mark.parametrize('multipath', [2]) +@pytest.mark.parametrize('convergence_test_iterations', [1]) +@pytest.mark.parametrize('number_of_routes', [1000]) +@pytest.mark.parametrize('route_type', ['IPv4']) +@pytest.mark.parametrize('port_speed',['speed_100_gbps']) +def test_bgp_convergence_for_local_link_failover(cvg_api, + duthost, + tgen_ports, + conn_graph_facts, + fanout_graph_facts, + multipath, + convergence_test_iterations, + number_of_routes, + route_type, + port_speed,): + + """ + Topo: + TGEN1 --- DUT --- TGEN(2..N) + + Steps: + 1) Create BGP config on DUT and TGEN respectively + 2) Create a flow from TGEN1 to (N-1) TGEN ports + 3) Send Traffic from TGEN1 to (N-1) TGEN ports having the same route range + 4) Simulate link failure by bringing down one of the (N-1) TGEN Ports + 5) Calculate the packet loss duration for convergence time + 6) Clean up the BGP config on the dut + + Verification: + 1) Send traffic without flapping any link + Result: Should not observe traffic loss + 2) Flap one of the N TGEN link + Result: The traffic must be routed via rest of the ECMP paths + + Args: + cvg_api (pytest fixture): Snappi Convergence API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts (pytest fixture): fanout graph + multipath: ECMP value + convergence_test_iterations: number of iterations the link failure test has to be run for a port + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + #convergence_test_iterations, multipath, number_of_routes and route_type parameters can be modified as per user preference + run_bgp_local_link_failover_test(cvg_api, + duthost, + tgen_ports, + convergence_test_iterations, + multipath, + number_of_routes, + route_type, + port_speed,) diff --git a/tests/snappi/bgp/test_bgp_remote_link_failover.py b/tests/snappi/bgp/test_bgp_remote_link_failover.py new file mode 100755 index 00000000000..c992b061d12 --- /dev/null +++ b/tests/snappi/bgp/test_bgp_remote_link_failover.py @@ -0,0 +1,63 @@ +from tests.common.snappi.snappi_fixtures import cvg_api +from tests.common.snappi.snappi_fixtures import ( + snappi_api_serv_ip, snappi_api_serv_port, tgen_ports) +from files.bgp_convergence_helper import run_bgp_remote_link_failover_test +from tests.common.fixtures.conn_graph_facts import ( + conn_graph_facts, fanout_graph_facts) +import pytest + +@pytest.mark.parametrize('multipath',[2]) +@pytest.mark.parametrize('convergence_test_iterations',[1]) +@pytest.mark.parametrize('number_of_routes',[1000]) +@pytest.mark.parametrize('route_type',['IPv4']) +@pytest.mark.parametrize('port_speed',['speed_100_gbps']) +def test_bgp_convergence_for_remote_link_failover(cvg_api, + duthost, + tgen_ports, + conn_graph_facts, + fanout_graph_facts, + multipath, + convergence_test_iterations, + number_of_routes, + route_type, + port_speed,): + + """ + Topo: + TGEN1 --- DUT --- TGEN(2..N) + + Steps: + 1) Create BGP config on DUT and TGEN respectively + 2) Create a flow from TGEN1 to (N-1) TGEN ports + 3) Send Traffic from TGEN1 to (N-1) TGEN ports having the same route range + 4) Simulate route withdraw from one of the (N-1) BGP peers which is the equivalent of remote link failure + 5) Calculate the cp/dp for convergence time + 6) Clean up the BGP config on the dut + + Verification: + 1) Send traffic with all routes advertised by BGP peers + Result: Should not observe traffic loss + 2) Withdraw all routes from one of the BGP peer + Result: The traffic must be routed via rest of the ECMP paths and should not observe traffic loss + + Args: + snappi_api (pytest fixture): Snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts (pytest fixture): fanout graph + multipath: ECMP value + convergence_test_iterations: number of iterations the cp/dp convergence test has to be run for a port + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + #convergence_test_iterations, multipath, number_of_routes, port_speed and route_type parameters can be modified as per user preference + run_bgp_remote_link_failover_test(cvg_api, + duthost, + tgen_ports, + convergence_test_iterations, + multipath, + number_of_routes, + route_type, + port_speed,) diff --git a/tests/snappi/bgp/test_bgp_rib_in_capacity.py b/tests/snappi/bgp/test_bgp_rib_in_capacity.py new file mode 100644 index 00000000000..76d79104066 --- /dev/null +++ b/tests/snappi/bgp/test_bgp_rib_in_capacity.py @@ -0,0 +1,64 @@ +from tests.common.snappi.snappi_fixtures import cvg_api +from tests.common.snappi.snappi_fixtures import ( + snappi_api_serv_ip, snappi_api_serv_port, tgen_ports) +from files.bgp_convergence_helper import run_RIB_IN_capacity_test +from tests.common.fixtures.conn_graph_facts import ( + conn_graph_facts, fanout_graph_facts) +import pytest + + +@pytest.mark.parametrize('multipath', [2]) +@pytest.mark.parametrize('start_value', [1000]) +@pytest.mark.parametrize('step_value', [1000]) +@pytest.mark.parametrize('route_type', ['IPv4']) +@pytest.mark.parametrize('port_speed',['speed_100_gbps']) +def test_RIB_IN_capacity(cvg_api, + duthost, + tgen_ports, + conn_graph_facts, + fanout_graph_facts, + multipath, + start_value, + step_value, + route_type, + port_speed,): + + """ + Topo: + TGEN1 --- DUT --- TGEN(2..N) + + Steps: + 1) Create a BGP config on DUT and TGEN respectively + 2) Create a flow from TGEN1 to TGEN2 port + 3) Send Traffic from TGEN1 to TGEN2 port route range + 4) Check if there is any loss observed + 5) Increment the routes in terms of step_value and repeat test untill loss is observed + 6) Note down the number of routes upto which no loss was observed which is the RIB-IN capacity value + 7) Clean up the BGP config on the dut + Note: + confihgure DUT interfaces prior to running test + Verification: + 1) Send traffic and make sure there is no loss observed + 2) If loss is observed quit the test and note down the maximum routes upto which there was no loss + + Args: + snappi_api (pytest fixture): Snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts (pytest fixture): fanout graph + multipath: ECMP value + start_value: Start value of the number of BGP routes + step_value: Step value of the number of BGP routes to be incremented + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + #multipath, start_value, step_value and route_type, port_speed parameters can be modified as per user preference + run_RIB_IN_capacity_test(cvg_api, + duthost, + tgen_ports, + multipath, + start_value, + step_value, + route_type, + port_speed,) diff --git a/tests/snappi/bgp/test_bgp_rib_in_convergence.py b/tests/snappi/bgp/test_bgp_rib_in_convergence.py new file mode 100644 index 00000000000..c6f71c114ae --- /dev/null +++ b/tests/snappi/bgp/test_bgp_rib_in_convergence.py @@ -0,0 +1,65 @@ +from tests.common.snappi.snappi_fixtures import cvg_api +from tests.common.snappi.snappi_fixtures import ( + snappi_api_serv_ip, snappi_api_serv_port, tgen_ports) +from files.bgp_convergence_helper import run_rib_in_convergence_test +from tests.common.fixtures.conn_graph_facts import ( + conn_graph_facts, fanout_graph_facts) +import pytest + + +@pytest.mark.parametrize('multipath', [2]) +@pytest.mark.parametrize('convergence_test_iterations', [1]) +@pytest.mark.parametrize('number_of_routes', [1000]) +@pytest.mark.parametrize('route_type', ['IPv4']) +@pytest.mark.parametrize('port_speed',['speed_100_gbps']) +def test_rib_in_convergence(cvg_api, + duthost, + tgen_ports, + conn_graph_facts, + fanout_graph_facts, + multipath, + convergence_test_iterations, + number_of_routes, + route_type, + port_speed,): + + """ + Topo: + TGEN1 --- DUT --- TGEN(2..N) + + Steps: + 1) Create BGP config on DUT and TGEN respectively + 2) Create a flow from TGEN1 to (N-1) TGEN ports + 3) Withdraw the routes from all the BGP peers + 4) Send Traffic from TGEN1 to (N-1) TGEN ports having the same route range + 4) Advertise the routes when traffic is running + 5) Calculate the RIB-IN convergence time + 6) Clean up the BGP config on the dut + + Verification: + 1) Send traffic after withdrawing routes from all BGP peers + Result: Should not observe any traffic in the receiving side + 2) Advertise the routes when the traffic is running + Result: The traffic must be routed via the ECMP paths + + Args: + snappi_api (pytest fixture): Snappi API + duthost (pytest fixture): duthost fixture + tgen_ports (pytest fixture): Ports mapping info of testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts (pytest fixture): fanout graph + multipath: ECMP value + convergence_test_iterations: number of iterations the link failure test has to be run for a port + number_of_routes: Number of IPv4/IPv6 Routes + route_type: IPv4 or IPv6 routes + port_speed: speed of the port used for test + """ + #convergence_test_iterations, multipath, number_of_routes port_speed and route_type parameters can be modified as per user preference + run_rib_in_convergence_test(cvg_api, + duthost, + tgen_ports, + convergence_test_iterations, + multipath, + number_of_routes, + route_type, + port_speed,)