From cd855698c345a5c2f95e19ac6a7701259085cb27 Mon Sep 17 00:00:00 2001 From: JunhongMao <134556118+JunhongMao@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:02:55 -0500 Subject: [PATCH] [VOQ][saidump] Modify generate_dump: replace save_saidump with save_saidump_by_route_size (#2972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * * [saidump] • Saidump for DNX-SAI https://github.com/sonic-net/sonic-buildimage/issues/13561 Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, #1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null #2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" #3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null #4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" #5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh * * [saidump] • Saidump for DNX-SAI https://github.com/sonic-net/sonic-buildimage/issues/13561 --- scripts/generate_dump | 115 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 102 insertions(+), 13 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index ac9528ff3..2179c17ca 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -50,6 +50,7 @@ SKIP_BCMCMD=0 SAVE_STDERR=true RETURN_CODE=$EXT_SUCCESS DEBUG_DUMP=false +ROUTE_TAB_LIMIT_DIRECT_ITERATION=24000 # lock dirs/files LOCKDIR="/tmp/techsupport-lock" @@ -863,24 +864,114 @@ save_redis() { } ############################################################################### -# SAI DUMP from syncd +# GET ROUTE table size by ASIC id and ip version +# Globals: +# TIMEOUT_MIN +# TIMEOUT_EXIT_CODE +# Arguments: +# asic id +# IP version +# Returns: +# Status: 0 success, otherwise failure +############################################################################### +get_route_table_size_by_asic_id_and_ipver() { + local asic_id="$1" + local ip_ver="$2" + local filepath="/tmp/route_summary.txt" + local ns="" + RC=0 + + if [[ $NUM_ASICS -gt 1 ]] ; then + ns="-n ${asic_id}" + fi + + if [ $ip_ver = "ipv4" ]; then + cmd="vtysh ${ns} -c 'show ip route summary json'" + elif [ $ip_ver = "ipv6" ]; then + cmd="vtysh ${ns} -c 'show ipv6 route summary json'" + else + echo "Wrong argument $ip_ver." + return 255 + fi + + local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m" + local cmds="$cmd > '$filepath'" + + eval "${timeout_cmd} bash -c \"${cmds}\"" || RC=$? + + if [ $RC -eq $TIMEOUT_EXIT_CODE ]; then + echo "Command: $cmds timedout after ${TIMEOUT_MIN} minutes." + return $RC + elif [ $RC -ne 0 ]; then + echo "Command: $cmds failed with RC $RC" + return $RC + fi + + local route_tab_size=$(python3 -c "\ +import json +with open('$filepath') as json_file: + data = json.load(json_file) + print(data['routesTotal'])") + rm $filepath + echo "$route_tab_size" +} + +############################################################################### +# SAI DUMP based on the route table size +# if the route table has more than ROUTE_TAB_LIMIT_DIRECT_ITERATION +# then dump by Redis Save command, +# otherwize, dump it by directly iteration the Redis +# # Globals: # NUM_ASICS +# ROUTE_TAB_LIMIT_DIRECT_ITERATION # Arguments: # None # Returns: # None ############################################################################### -save_saidump() { +save_saidump_by_route_size() { trap 'handle_error $? $LINENO' ERR - if [[ ( "$NUM_ASICS" == 1 ) ]] ; then - save_cmd "docker exec syncd saidump" "saidump" - else - for (( i=0; i<$NUM_ASICS; i++ )) - do - save_cmd "docker exec syncd$i saidump" "saidump$i" - done - fi + + for (( i=0; i<$NUM_ASICS; i++ )) + do + route_size_ipv4=`get_route_table_size_by_asic_id_and_ipver $i ipv4` + ret=$? + + if [ $ret -ne 0 ]; then + echo "Get route table's size by asicid $i and ipv4 failed." + return $ret + fi + + route_size_ipv6=`get_route_table_size_by_asic_id_and_ipver $i ipv6` + ret=$? + + if [ $ret -ne 0 ]; then + echo "Get route table's size by asicid $i and ipv6 failed." + return $ret + fi + + route_size=`expr $route_size_ipv4 + $route_size_ipv6` + echo "The route table's size is $route_size(ipv4 $route_size_ipv4, ipv6 $route_size_ipv6)" + + if [[ $route_size -gt $ROUTE_TAB_LIMIT_DIRECT_ITERATION ]]; then + echo "Dump by using Redis SAVE." + + if [[ ( "$NUM_ASICS" == 1 ) ]] ; then + save_cmd "docker exec syncd saidump.sh" "saidump" + else + save_cmd "docker exec syncd$i saidump.sh" "saidump$i" + fi + else + echo "Dump by using direct iteration of Redis DB." + + if [[ ( "$NUM_ASICS" == 1 ) ]] ; then + save_cmd "docker exec syncd saidump" "saidump" + else + save_cmd "docker exec syncd$i saidump" "saidump$i" + fi + fi + done } ############################################################################### @@ -1807,9 +1898,7 @@ main() { save_cmd "ps -AwwL -o user,pid,lwp,ppid,nlwp,pcpu,pri,nice,vsize,rss,tty,stat,wchan:12,start,bsdtime,command" "ps.extended" & wait - if [[ "$device_type" != "SpineRouter" ]]; then - save_saidump - fi + save_saidump_by_route_size if [ "$asic" = "barefoot" ]; then collect_barefoot