diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..72d60fe --- /dev/null +++ b/install.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Install Mellanox tools and scripts not yet part of an +# Oracle or upstream RPM +# +# /usr/bin/tc_wrap.py +# /usr/bin/mlnx_qos +# /usr/sbin/cma_roce_mode +# /usr/sbin/show_gids + +# This script does not make backups of any existing versions of +# the tools. +# +# This script doesn't do any error checking. +# +# This script does not have an uninstaller. + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi + +parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) + +echo "Installing cma_roce_mode..." +install -o root -g root -m 0755 ${parent_path}/ofed_scripts/cma_roce_mode /usr/sbin/cma_roce_mode + +echo "Installing show_gids..." +install -o root -g root -m 0755 ${parent_path}/ofed_scripts/show_gids /usr/sbin/show_gids + +echo "Installing Mellanox Python utils..." +cd ${parent_path}/ofed_scripts/utils/ +/usr/bin/env python setup.py install +cd ${parent_path} + +exit 0 diff --git a/ofed_scripts/cma_roce_mode b/ofed_scripts/cma_roce_mode new file mode 100755 index 0000000..484be80 --- /dev/null +++ b/ofed_scripts/cma_roce_mode @@ -0,0 +1,117 @@ +#!/bin/bash +# +# Copyright (c) 2016 Mellanox Technologies. All rights reserved. +# +# This Software is licensed under one of the following licenses: +# +# 1) under the terms of the "Common Public License 1.0" a copy of which is +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/cpl.php. +# +# 2) under the terms of the "The BSD License" a copy of which is +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/bsd-license.php. +# +# 3) under the terms of the "GNU General Public License (GPL) Version 2" a +# copy of which is available from the Open Source Initiative, see +# http://www.opensource.org/licenses/gpl-license.php. +# +# Licensee has the right to choose one of the above licenses. +# +# Redistributions of source code must retain the above copyright +# notice and one of the license notices. +# +# Redistributions in binary form must reproduce both the above copyright +# notice, one of the license notices in the documentation +# and/or other materials provided with the distribution. +# +# Author: Moni Shoua +# + +DEVICE=mlx4_0 +PORT=1 +MODE=0 + +usage="Set/Show RoCE mode of RDMA_CM applications + +Usage: + cma_roce_mode +Options: + -d use IB device (default mlx4_0) + -p use port of IB device (default 1) + -m set RoCE mode of RDMA_CM applications (default 1)" + + +while getopts "d:m:hp:" arg; do +case $arg in + h) + echo "$usage" + exit + ;; + d) + DEVICE=$OPTARG + ;; + p) + PORT=$OPTARG + ;; + m) + MODE=$OPTARG + ;; + esac +done + +ID=$(id -u) +if [ $ID -ne 0 ] ; then + echo you must be root to run this + exit 1 +fi + +if (! cat /proc/mounts |grep /sys/kernel/config > /dev/null) ; then + if (! mount -t configfs none /sys/kernel/config) ; then + echo "Fail to mount configfs" + exit 1 + fi +fi + +if (modinfo configfs &> /dev/null) ; then + if (! cat /proc/modules|grep configfs > /dev/null) ; then + modprobe configfs + fi +fi + +cd /sys/kernel/config/ + +if [ ! -d rdma_cm ] ; then + echo module rdma_cm is not loaded or does not support configfs + exit 1 +fi + +cd rdma_cm + +if [ ! -d $DEVICE ] ; then + sudo mkdir $DEVICE + if [ $? -ne 0 ] ; then + echo failed to creat configuration for $DEVICE + exit 1 + fi +fi + +if [ ! -d $DEVICE/ports/$PORT ] ; then + echo device $DEVICE port $PORT does not exist + exit 1 +fi + +cd $DEVICE/ports/$PORT + +if [ $MODE -eq 1 ] ; then + echo "IB/RoCE v1" |sudo tee default_roce_mode +elif [ $MODE -eq 2 ] ; then + echo "RoCE v2" |sudo tee default_roce_mode +else + cat default_roce_mode +fi + +cd ../../.. + +sudo rmdir $DEVICE + diff --git a/ofed_scripts/show_gids b/ofed_scripts/show_gids new file mode 100755 index 0000000..bb3eba2 --- /dev/null +++ b/ofed_scripts/show_gids @@ -0,0 +1,105 @@ +#!/bin/bash +# +# Copyright (c) 2016 Mellanox Technologies. All rights reserved. +# +# This Software is licensed under one of the following licenses: +# +# 1) under the terms of the "Common Public License 1.0" a copy of which is +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/cpl.php. +# +# 2) under the terms of the "The BSD License" a copy of which is +# available from the Open Source Initiative, see +# http://www.opensource.org/licenses/bsd-license.php. +# +# 3) under the terms of the "GNU General Public License (GPL) Version 2" a +# copy of which is available from the Open Source Initiative, see +# http://www.opensource.org/licenses/gpl-license.php. +# +# Licensee has the right to choose one of the above licenses. +# +# Redistributions of source code must retain the above copyright +# notice and one of the license notices. +# +# Redistributions in binary form must reproduce both the above copyright +# notice, one of the license notices in the documentation +# and/or other materials provided with the distribution. +# +# Author: Moni Shoua +# + +black='\E[30;50m' +red='\E[31;50m' +green='\E[32;50m' +yellow='\E[33;50m' +blue='\E[34;50m' +magenta='\E[35;50m' +cyan='\E[36;50m' +white='\E[37;50m' + +bold='\033[1m' + +gid_count=0 + +# cecho (color echo) prints text in color. +# first parameter should be the desired color followed by text +function cecho () +{ + echo -en $1 + shift + echo -n $* + tput sgr0 +} + +# becho (color echo) prints text in bold. +becho () +{ + echo -en $bold + echo -n $* + tput sgr0 +} + +function print_gids() +{ + dev=$1 + port=$2 + for gf in /sys/class/infiniband/$dev/ports/$port/gids/* ; do + gid=$(cat $gf); + if [ $gid = 0000:0000:0000:0000:0000:0000:0000:0000 ] ; then + continue + fi + echo -e $(basename $gf) "\t" $gid + done +} + +echo -e "DEV\tPORT\tINDEX\tGID\t\t\t\t\tIPv4 \t\tVER\tDEV" +echo -e "---\t----\t-----\t---\t\t\t\t\t------------ \t---\t---" +DEVS=$1 +if [ -z "$DEVS" ] ; then + DEVS=$(ls /sys/class/infiniband/) +fi +for d in $DEVS ; do + for p in $(ls /sys/class/infiniband/$d/ports/) ; do + for g in $(ls /sys/class/infiniband/$d/ports/$p/gids/) ; do + gid=$(cat /sys/class/infiniband/$d/ports/$p/gids/$g); + if [ $gid = 0000:0000:0000:0000:0000:0000:0000:0000 ] ; then + continue + fi + if [ $gid = fe80:0000:0000:0000:0000:0000:0000:0000 ] ; then + continue + fi + _ndev=$(cat /sys/class/infiniband/$d/ports/$p/gid_attrs/ndevs/$g 2>/dev/null) + __type=$(cat /sys/class/infiniband/$d/ports/$p/gid_attrs/types/$g 2>/dev/null) + _type=$(echo $__type| grep -o "[Vv].*") + if [ $(echo $gid | cut -d ":" -f -1) = "0000" ] ; then + ipv4=$(printf "%d.%d.%d.%d" 0x${gid:30:2} 0x${gid:32:2} 0x${gid:35:2} 0x${gid:37:2}) + echo -e "$d\t$p\t$g\t$gid\t$ipv4 \t$_type\t$_ndev" + else + echo -e "$d\t$p\t$g\t$gid\t\t\t$_type\t$_ndev" + fi + gid_count=$(expr 1 + $gid_count) + done #g (gid) + done #p (port) +done #d (dev) + +echo n_gids_found=$gid_count diff --git a/ofed_scripts/utils/dcbnetlink.py b/ofed_scripts/utils/dcbnetlink.py new file mode 100644 index 0000000..c656f7e --- /dev/null +++ b/ofed_scripts/utils/dcbnetlink.py @@ -0,0 +1,347 @@ +#!/usr/bin/python + +import sys +import os +if os.path.exists('/usr/share/pyshared'): + sys.path.append('/usr/share/pyshared') +import socket +import struct + + +import array + +from netlink import hexdump, parse_attributes, Message, Nested, U8Attr, StrAttr, NulStrAttr, Connection, NETLINK_GENERIC, U32Attr, NLM_F_REQUEST +#from genetlink import Controller, GeNlMessage + +NETLINK_ROUTE = 0 +RTM_GETDCB = 78 +AF_UNSPEC = 0 + +DCB_CMD_UNDEFINED = 0 +DCB_CMD_GSTATE = 1 +DCB_CMD_SSTATE = 2 +DCB_CMD_PGTX_GCFG = 3 +DCB_CMD_PGTX_SCFG = 4 +DCB_CMD_PGRX_GCFG = 5 +DCB_CMD_PGRX_SCFG = 6 +DCB_CMD_PFC_GCFG = 7 +DCB_CMD_PFC_SCFG = 8 +DCB_CMD_SET_ALL = 9 +DCB_CMD_GPERM_HWADDR = 10 +DCB_CMD_GCAP = 11 +DCB_CMD_GNUMTCS = 12 +DCB_CMD_SNUMTCS = 13 +DCB_CMD_PFC_GSTATE = 14 +DCB_CMD_PFC_SSTATE = 15 +DCB_CMD_BCN_GCFG = 16 +DCB_CMD_BCN_SCFG = 17 +DCB_CMD_GAPP = 18 +DCB_CMD_SAPP = 19 +DCB_CMD_IEEE_SET = 20 +DCB_CMD_IEEE_GET = 21 +DCB_CMD_GDCBX = 22 +DCB_CMD_SDCBX = 23 +DCB_CMD_GFEATCFG = 24 +DCB_CMD_SFEATCFG = 25 +DCB_CMD_CEE_GET = 26 +DCB_CMD_IEEE_DEL = 27 + +DCB_ATTR_UNDEFINED = 0 +DCB_ATTR_IFNAME = 1 +DCB_ATTR_STATE = 2 +DCB_ATTR_PFC_STATE = 3 +DCB_ATTR_PFC_CFG = 4 +DCB_ATTR_NUM_TC = 5 +DCB_ATTR_PG_CFG = 6 +DCB_ATTR_SET_ALL = 7 +DCB_ATTR_PERM_HWADDR = 8 +DCB_ATTR_CAP = 9 +DCB_ATTR_NUMTCS = 10 +DCB_ATTR_BCN = 11 +DCB_ATTR_APP = 12 +DCB_ATTR_IEEE = 13 +DCB_ATTR_DCBX = 14 +DCB_ATTR_FEATCFG = 15 +DCB_ATTR_CEE = 16 + +DCB_ATTR_IEEE_UNSPEC = 0 +DCB_ATTR_IEEE_ETS = 1 +DCB_ATTR_IEEE_PFC = 2 +DCB_ATTR_IEEE_APP_TABLE = 3 +DCB_ATTR_IEEE_PEER_ETS = 4 +DCB_ATTR_IEEE_PEER_PFC = 5 +DCB_ATTR_IEEE_PEER_APP = 6 +DCB_ATTR_IEEE_MAXRATE = 7 +DCB_ATTR_IEEE_QCN = 8 +DCB_ATTR_IEEE_QCN_STATS = 9 + +class DcbnlHdr: + def __init__(self, len, type): + self.len = len + self.type = type + def _dump(self): + return struct.pack("BBxx", self.len, self.type) + +class DcbNlMessage(Message): + def __init__(self, type, cmd, attrs=[], flags=0): + self.type = type + self.cmd = cmd + self.attrs = attrs + Message.__init__(self, type, flags=flags, + payload=[DcbnlHdr(len=0, type=self.cmd)]+attrs) + + @staticmethod + def recv(conn): + msgs = conn.recv() + packet = msgs[0].payload + + dcb_family, cmd = struct.unpack("BBxx", packet[:4]) + + dcbnlmsg = DcbNlMessage(dcb_family, cmd) + dcbnlmsg.attrs = parse_attributes(packet[4:]) + + return dcbnlmsg + +class DcbController: + def __init__(self, intf): + self.conn = Connection(NETLINK_ROUTE) + self.intf = intf + + def check_err(self, m, attr_type): + if m.attrs[attr_type].u8(): + err = OSError("Netlink error: Bad value. see dmesg.") + raise err + + def __parse_array(self,arr, n): + lst = [] + for i in range (0, len(arr), n): + lst.append(arr[i:i+8]) + return lst + + def get_dcb_state(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_GSTATE, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + return m.attrs[0].u8() + + def set_dcb_state(self, state): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + state_attr = U8Attr(DCB_ATTR_STATE, state) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_SSTATE, + flags=NLM_F_REQUEST, attrs=[a, state_attr]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + self.check_err(m, DCB_ATTR_STATE) + + def get_dcbx(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_GDCBX, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + return m.attrs[DCB_ATTR_DCBX].u8() + + def set_dcbx(self, mode): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + mode_attr = U8Attr(DCB_ATTR_DCBX , mode) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_SDCBX, + flags=NLM_F_REQUEST, attrs=[a, mode_attr]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + self.check_err(m, DCB_ATTR_DCBX) + + def get_ieee_pfc(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_GET, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + + ieee = m.attrs[DCB_ATTR_IEEE].nested() + + a = array.array('B') + a.fromstring(ieee[DCB_ATTR_IEEE_PFC].str()[0:]) + + return a[1] + + def get_ieee_ets(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_GET, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + + ieee = m.attrs[DCB_ATTR_IEEE].nested() + + willing, ets_cap, cbs = struct.unpack_from("BBB", ieee[DCB_ATTR_IEEE_ETS].str(), 0) + + a = array.array('B') + a.fromstring(ieee[DCB_ATTR_IEEE_ETS].str()[3:]) + + f = lambda A, n=8: [A[i:i+n] for i in range(0, len(A), n)] + + tc_tc_bw, tc_rx_bw, tc_tsa, prio_tc, tc_reco_bw, tc_reco_tsa, reco_prio_tc = f(a,8) + + return prio_tc, tc_tsa, tc_tc_bw + + def set_ieee_pfc(self, _pfc_en): + pfc_cap = 8 + mbc = 0 + delay = 0 + + requests = array.array('B', '\0' * 64) + indications = array.array('B', '\0' * 64) + + #netlink packet is 64bit alignment + pads = array.array('B', '\0' * 3) + + pfc = struct.pack("BBBBB", pfc_cap, _pfc_en, mbc, delay, delay) + (requests + indications + pads).tostring() + + intf = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + ieee_pfc = StrAttr(DCB_ATTR_IEEE_PFC, pfc) + ieee = Nested(DCB_ATTR_IEEE, [ieee_pfc]); + + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_SET, + flags=NLM_F_REQUEST, attrs=[intf, ieee]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + self.check_err(m, DCB_ATTR_IEEE) + + def set_ieee_ets(self, _prio_tc, _tsa, _tc_bw): + willing = 0 + ets_cap = 0 + cbs = 0 + tc_rx_bw = array.array('B', '\0' * 8) + tc_reco_bw = array.array('B', '\0' * 8) + tc_reco_tsa = array.array('B', '\0' * 8) + reco_prio_tc = array.array('B', '\0' * 8) + + tc_tc_bw = array.array('B', '\0' * 8) + tc_tsa = array.array('B', '\0' * 8) + prio_tc = array.array('B', '\0' * 8) + + for up in range(len(_prio_tc)): prio_tc[up] = _prio_tc[up] + for tc in range(len(_tsa)): tc_tsa[tc] = _tsa[tc] + for tc in range(len(_tc_bw)): tc_tc_bw[tc] = _tc_bw[tc] + + ets = struct.pack("BBB", willing, ets_cap, cbs) + (tc_tc_bw + tc_rx_bw + + tc_tsa + prio_tc + tc_reco_bw + tc_reco_tsa + + reco_prio_tc).tostring() + + intf = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + ieee_ets = StrAttr(DCB_ATTR_IEEE_ETS, ets) + ieee = Nested(DCB_ATTR_IEEE, [ieee_ets]); + + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_SET, + flags=NLM_F_REQUEST, attrs=[intf, ieee]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + self.check_err(m, DCB_ATTR_IEEE) + + def get_ieee_maxrate(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_GET, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + + ieee_nested = m.attrs[DCB_ATTR_IEEE] + + ieee = m.attrs[DCB_ATTR_IEEE].nested() + + tc_maxrate = struct.unpack_from("QQQQQQQQ",ieee[DCB_ATTR_IEEE_MAXRATE].str(), 0); + + return tc_maxrate + + def set_ieee_maxrate(self, _tc_maxrate): + tc_maxrate = struct.pack("QQQQQQQQ", + _tc_maxrate[0], + _tc_maxrate[1], + _tc_maxrate[2], + _tc_maxrate[3], + _tc_maxrate[4], + _tc_maxrate[5], + _tc_maxrate[6], + _tc_maxrate[7], + ) + + intf = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + ieee_maxrate = StrAttr(DCB_ATTR_IEEE_MAXRATE, tc_maxrate) + ieee = Nested(DCB_ATTR_IEEE, [ieee_maxrate]); + + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_SET, + flags=NLM_F_REQUEST, attrs=[intf, ieee]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + self.check_err(m, DCB_ATTR_IEEE) + + def get_ieee_qcn(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_GET, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + + ieee = m.attrs[DCB_ATTR_IEEE].nested() + + rpg_enable = array.array('B') + rpg_enable.fromstring(ieee[DCB_ATTR_IEEE_QCN].str()[:8]) + a = array.array('I') + a.fromstring(ieee[DCB_ATTR_IEEE_QCN].str()[8:]) + + lst_params = self.__parse_array(a,8) + + rppp_max_rps = lst_params[0] + rpg_time_reset = lst_params[1] + rpg_byte_reset = lst_params[2] + rpg_threshold = lst_params[3] + rpg_max_rate = lst_params[4] + rpg_ai_rate = lst_params[5] + rpg_hai_rate = lst_params[6] + rpg_gd = lst_params[7] + rpg_min_dec_fac = lst_params[8] + rpg_min_rate = lst_params[9] + cndd_state_machine = lst_params[10] + + return rpg_enable, rppp_max_rps, rpg_time_reset, rpg_byte_reset, rpg_threshold, rpg_max_rate, rpg_ai_rate, rpg_hai_rate, rpg_gd, rpg_min_dec_fac, rpg_min_rate, cndd_state_machine + + def get_ieee_qcnstats(self): + a = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_GET, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + + ieee = m.attrs[DCB_ATTR_IEEE].nested() + + rppp_rp_centiseconds = struct.unpack_from("QQQQQQQQ",ieee[DCB_ATTR_IEEE_QCN_STATS].str(), 0); + a = array.array('I') + a.fromstring(ieee[DCB_ATTR_IEEE_QCN_STATS].str()[64:]) + + lst_statistics = self.__parse_array(a,8) + + rppp_created_rps = lst_statistics[0] + ignored_cnm = lst_statistics[1] + estimated_total_rate = lst_statistics[2] + cnms_handled_successfully = lst_statistics[3] + min_total_limiters_rate = lst_statistics[4] + max_total_limiters_rate = lst_statistics[5] + + return rppp_rp_centiseconds, rppp_created_rps, ignored_cnm, estimated_total_rate, cnms_handled_successfully, min_total_limiters_rate, max_total_limiters_rate + + # @_qcn: struct of arrays, each array (_qcn[0], _qcn[1].. etc.) holds the values of a certain qcn parameter for all priorities. + def set_ieee_qcn(self, _qcn): + + qcn = _qcn[0].tostring() + (_qcn[1] + _qcn[2] + _qcn[3] + _qcn[4] + _qcn[5] + _qcn[6] + _qcn[7] + _qcn[8] + _qcn[9] + _qcn[10] + _qcn[11]).tostring() + + intf = NulStrAttr(DCB_ATTR_IFNAME, self.intf) + ieee_qcn = StrAttr(DCB_ATTR_IEEE_QCN, qcn) + ieee = Nested(DCB_ATTR_IEEE, [ieee_qcn]); + + m = DcbNlMessage(type = RTM_GETDCB, cmd = DCB_CMD_IEEE_SET, + flags=NLM_F_REQUEST, attrs=[intf, ieee]) + m.send(self.conn) + m = DcbNlMessage.recv(self.conn) + self.check_err(m, DCB_ATTR_IEEE) diff --git a/ofed_scripts/utils/genetlink.py b/ofed_scripts/utils/genetlink.py new file mode 100644 index 0000000..aab7bd5 --- /dev/null +++ b/ofed_scripts/utils/genetlink.py @@ -0,0 +1,80 @@ +''' +Netlink message generation/parsing + +Copyright 2007 Johannes Berg + +GPLv2+; See copying for details. +''' + +import sys +import os +if os.path.exists('/usr/share/pyshared'): + sys.path.append('/usr/share/pyshared') +import struct +from netlink import NLM_F_REQUEST, NLMSG_MIN_TYPE, Message, parse_attributes +from netlink import NulStrAttr, Connection, NETLINK_GENERIC + +CTRL_CMD_UNSPEC = 0 +CTRL_CMD_NEWFAMILY = 1 +CTRL_CMD_DELFAMILY = 2 +CTRL_CMD_GETFAMILY = 3 +CTRL_CMD_NEWOPS = 4 +CTRL_CMD_DELOPS = 5 +CTRL_CMD_GETOPS = 6 + +CTRL_ATTR_UNSPEC = 0 +CTRL_ATTR_FAMILY_ID = 1 +CTRL_ATTR_FAMILY_NAME = 2 +CTRL_ATTR_VERSION = 3 +CTRL_ATTR_HDRSIZE = 4 +CTRL_ATTR_MAXATTR = 5 +CTRL_ATTR_OPS = 6 + +class GenlHdr: + def __init__(self, cmd, version = 0): + self.cmd = cmd + self.version = version + def _dump(self): + return struct.pack("BBxx", self.cmd, self.version) + +def _genl_hdr_parse(data): + return GenlHdr(*struct.unpack("BBxx", data)) + +GENL_ID_CTRL = NLMSG_MIN_TYPE + +class GeNlMessage(Message): + def __init__(self, family, cmd, attrs=[], flags=0, seq=0): + self.cmd = cmd + self.attrs = attrs + self.family = family + Message.__init__(self, family, flags=flags, + payload=[GenlHdr(self.cmd)]+attrs, seq=0) + + @staticmethod + def recv(conn): + msgs = conn.recv() + genlmsgs = []; + for msg in msgs: + packet = msg.payload + try: + hdr = _genl_hdr_parse(packet[:4]) + except: + return genlmsgs + genlmsgs.append(GeNlMessage(msg.type, hdr.cmd, [], msg.flags)) + genlmsgs[-1].attrs = parse_attributes(packet[4:]) + genlmsgs[-1].version = hdr.version + return genlmsgs + +class Controller: + def __init__(self, conn): + self.conn = conn + def get_family_id(self, family): + a = NulStrAttr(CTRL_ATTR_FAMILY_NAME, family) + m = GeNlMessage(GENL_ID_CTRL, CTRL_CMD_GETFAMILY, + flags=NLM_F_REQUEST, attrs=[a]) + m.send(self.conn) + m = GeNlMessage.recv(self.conn) + return m[0].attrs[CTRL_ATTR_FAMILY_ID].u16() + +#connection = Connection(NETLINK_GENERIC) +#controller = Controller(connection) diff --git a/ofed_scripts/utils/mlnx_dump_parser b/ofed_scripts/utils/mlnx_dump_parser new file mode 100644 index 0000000..7babe58 --- /dev/null +++ b/ofed_scripts/utils/mlnx_dump_parser @@ -0,0 +1,219 @@ +#!/usr/bin/python +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# See the COPYING file for license information. +# +# Copyright (c) 2011 Mellanox Technologies. All rights reserved +# Add support for parsing driver diagnostic dump + +# Author: Parvi Kaustubhi + +import struct +from optparse import OptionParser +import os, sys + +MLX5_DIAG_DRV_VERSION = 0 +MLX5_DIAG_DEVICE_NAME = 1 +MLX5_DIAG_MST = 2 +MLX5_DIAG_SQ = 3 +MLX5_DIAG_RQ = 4 +MLX5_DIAG_CQ = 5 +MLX5_DIAG_EQ = 6 + +MLX5_MCION_STATUS_PRESENT = 1 +MLX5_MCION_STATUS_RX_LOS = 2 +MLX5_MCION_STATUS_TX_FAULT = 4 + +class mlx5_diag_wq: + def __init__(self): + wq_type = 0 #unsigned int + wqn = 0 # unsigned int + pi = 0 # short + ci = 0 # short + wqe_stride = 0 # char + rsvd = 0 #char + size = 0 # short + wqe_num = 0 # unsigned int + group_id = 0 # unsigned int + +class mlx5_diag_eq: + def __init__(self): + eq_type = 0# unsigned int + ci = 0 # unsigned int + size = 0 # int + irqn = 0 # unsigned int + eqn = 0 # char + nent = 0 # int + mask = 0 #long unsigned int + index = 0 # int + group_id = 0# unsigned int + +class mlx5_diag_blk: + def __init__(self): + blk_type = 0 # unsigned int + length = 0 # unsigned int + data = 0 #char [1024] + +class mlx5_diag_dump: + def __init__(self): + version = 0 # unsigned int + flag = 0 # unsigned int + num_blocks = 0 # unsigned int + total_length = 0 #unsigned int + dump = 0 #char [0] + +class header: + def __init__(self): + version = 0 # unsigned int + flag = 0 # unsigned int + num_blocks = 0 #unsigned int + total_length = 0 #unsigned int + +def bstr(n): + return ''.join([str(n >> x & 1) for x in (7, 6, 5, 4, 3, 2, 1, 0)]) + +def read_binary(dump, mst_dump_file, ring_dump_file): + diag_blk = mlx5_diag_blk() + f = open(dump, 'rb') + ring = open(ring_dump_file, 'w') + version = struct.unpack('I', f.read(4))[0] + + flag = struct.unpack('I', f.read(4))[0] + + num_blocks = struct.unpack('I', f.read(4))[0] + + length = struct.unpack('I', f.read(4))[0] + print 'Version: %s Flag: %s Number of blocks: %s Length %s' % (version, flag, num_blocks, length) + + module_no = struct.unpack('I', f.read(4))[0] + + module_status = struct.unpack('I', f.read(4))[0] + + print "MCION module number:", + print module_no, + print "status:", + + if (module_status & MLX5_MCION_STATUS_PRESENT): + print "| present |", + else: + print "| non-present |", + + if (module_status & MLX5_MCION_STATUS_RX_LOS): + print "| rx los |", + + if (module_status & MLX5_MCION_STATUS_TX_FAULT): + print "| tx fault |", + + print '' + + blk_type = struct.unpack('I', f.read(4))[0] + blk_len = struct.unpack('I', f.read(4))[0] + if (blk_type == 0): + drv_version = struct.unpack('64s', f.read(64))[0] + print 'DRIVER VERSION: %s' % drv_version + blk_type = struct.unpack('I', f.read(4))[0] + blk_len = struct.unpack('I', f.read(4))[0] + if (blk_type == 1): + dev_name = struct.unpack('64s', f.read(64))[0] + print 'DEVICE NAME %s' % dev_name + + for x in xrange(0, num_blocks - 2): + diag_blk.blk_type = struct.unpack('I', f.read(4))[0] + diag_blk.length = struct.unpack('I', f.read(4))[0] + if (diag_blk.blk_type == MLX5_DIAG_MST): + mst = open(mst_dump_file, 'w') + for i in xrange(0, diag_blk.length, 8): + off = struct.unpack('I', f.read(4))[0] + data = struct.unpack('I', f.read(4))[0] + mst.write("0x%.8lx 0x%.8lx\n" %(off, data)) + mst.close() + elif (diag_blk.blk_type == MLX5_DIAG_SQ): + diag_wq = mlx5_diag_wq() + diag_wq.wq_type = struct.unpack('I', f.read(4))[0] + diag_wq.wqn = struct.unpack('I', f.read(4))[0] + diag_wq.pi = struct.unpack('H', f.read(2))[0] + diag_wq.ci = struct.unpack('H', f.read(2))[0] + diag_wq.wqe_stride = struct.unpack('B', f.read(1))[0] + diag_wq.rsvd = struct.unpack('B', f.read(1))[0] + diag_wq.size = struct.unpack('H', f.read(2))[0] + diag_wq.wqe_num = struct.unpack('I', f.read(4))[0] + diag_wq.group_id = struct.unpack('I', f.read(4))[0] + ring.write('SQ TYPE: %d, WQN: %d, PI: %d, CI: %d, STRIDE: %d, SIZE: %d, WQE_NUM: %d, GROUP_IP: %d\n' %(diag_wq.wq_type, diag_wq.wqn, diag_wq.pi, diag_wq.ci, diag_wq.wqe_stride, diag_wq.size, diag_wq.wqe_num, diag_wq.group_id)) + + elif (diag_blk.blk_type == MLX5_DIAG_RQ): + diag_wq = mlx5_diag_wq() + diag_wq.wq_type = struct.unpack('I', f.read(4))[0] + diag_wq.wqn = struct.unpack('I', f.read(4))[0] + diag_wq.pi = struct.unpack('H', f.read(2))[0] + diag_wq.ci = struct.unpack('H', f.read(2))[0] + diag_wq.wqe_stride = struct.unpack('B', f.read(1))[0] + diag_wq.rsvd = struct.unpack('B', f.read(1))[0] + diag_wq.size = struct.unpack('H', f.read(2))[0] + diag_wq.wqe_num = struct.unpack('I', f.read(4))[0] + diag_wq.group_id = struct.unpack('I', f.read(4))[0] + ring.write('RQ TYPE: %d, WQN: %d, PI: %d, CI: %d, STRIDE: %d, SIZE: %d, WQE_NUM: %d, GROUP_IP: %d\n' %(diag_wq.wq_type, diag_wq.wqn, diag_wq.pi, diag_wq.ci, diag_wq.wqe_stride, diag_wq.size, diag_wq.wqe_num, diag_wq.group_id)) + + elif (diag_blk.blk_type == MLX5_DIAG_CQ): + diag_wq = mlx5_diag_wq() + diag_wq.wq_type = struct.unpack('I', f.read(4))[0] + diag_wq.wqn = struct.unpack('I', f.read(4))[0] + diag_wq.pi = struct.unpack('H', f.read(2))[0] + diag_wq.ci = struct.unpack('H', f.read(2))[0] + diag_wq.wqe_stride = struct.unpack('B', f.read(1))[0] + diag_wq.rsvd = struct.unpack('B', f.read(1))[0] + diag_wq.size = struct.unpack('H', f.read(2))[0] + diag_wq.wqe_num = struct.unpack('I', f.read(4))[0] + diag_wq.group_id = struct.unpack('I', f.read(4))[0] + ring.write('CQ TYPE: %d, WQN: %d, PI: %d, CI: %d, STRIDE: %d, SIZE: %d, WQE_NUM: %d, GROUP_IP: %d\n' %(diag_wq.wq_type, diag_wq.wqn, diag_wq.pi, diag_wq.ci, diag_wq.wqe_stride, diag_wq.size, diag_wq.wqe_num, diag_wq.group_id)) + + elif (diag_blk.blk_type == MLX5_DIAG_EQ): + diag_eq = mlx5_diag_eq() + diag_eq.eq_type = struct.unpack('I', f.read(4))[0] + diag_eq.ci = struct.unpack('I', f.read(4))[0] + diag_eq.size = struct.unpack('I', f.read(4))[0] + diag_eq.irqn = struct.unpack('I', f.read(4))[0] + diag_eq.eqn = struct.unpack('B', f.read(1))[0] + diag_eq.nent = struct.unpack('I', f.read(4))[0] + diag_eq.mask = struct.unpack('Q', f.read(8))[0] + diag_eq.index = struct.unpack('I', f.read(4))[0] + diag_eq.group_id = struct.unpack('I', f.read(4))[0] + ring.write('EQ TYPE: %d, CI: %d, SIZE: %d, IRQN: %d, EQN: %d, NENT: %d, MASK: %d, INDEX: %d, GROUP_ID: %d\n' %(diag_eq.eq_type, diag_eq.ci, diag_eq.size, diag_eq.irqn, diag_eq.eqn, diag_eq.nent, diag_eq.mask, diag_eq.index, diag_eq.group_id)) + else: + print 'Unknown block type' + ring.close() + f.close() + if os.stat(ring_dump_file).st_size == 0: + os.remove(ring_dump_file) + + return 'Parsing Complete!' + + +parser = OptionParser(usage="%prog -f -m -r ", version="%prog 1.0") + +parser.add_option("-f", "--dump_file", dest="dump_file", help="Dump file name") + +parser.add_option("-m", "--mst_dump_file", dest="mst_dump_file", default='mst_dump.txt', help="File name for parsed MST data") + +parser.add_option("-r", "--ring_dump_file", dest="ring_dump_file", default='ring_dump.txt', help="File name for parsed ring dump") + +(options, args) = parser.parse_args() + +if (options.dump_file == None): + print "Name of dump file is required" + parser.print_usage() + sys.exit(1) + +str = read_binary(options.dump_file, options.mst_dump_file, options.ring_dump_file) +print "%s" % str diff --git a/ofed_scripts/utils/mlnx_get_vfs.pl b/ofed_scripts/utils/mlnx_get_vfs.pl new file mode 100755 index 0000000..91cb327 --- /dev/null +++ b/ofed_scripts/utils/mlnx_get_vfs.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl + +use File::Basename; + +my $numvfs_file = "/sys/module/mlx4_core/parameters/num_vfs"; +open (my $NUMVFS_FILE, "<", $numvfs_file) || die "Can't find num_vfs parameter in sysfs"; + + +my $probevf_file = "/sys/module/mlx4_core/parameters/probe_vf"; +open (my $PROBEVF_FILE, "<", $probevf_file) || die "Can't find probe_vf parameter in sysfs"; + +my $numvfs = <$NUMVFS_FILE>; +my $probevf = <$PROBEVF_FILE>; +my $iters = 0; + +while ($numvfs =~ m/((([[:xdigit:]]{4}:)?[[:xdigit:]]{2}:)[[:xdigit:]]{2}.[[:xdigit:]])-([[:digit:]]+)(;([[:digit:]]+))?(;([[:digit:]]+))?,?/g) +{ + $iters++; + my $bdf = $1; + my $first = $4; + my $second = $6; + my $third = $8; + my $p1, $p2, $both; + if (!defined($second) && !defined($third)) { + $both = $first; + $p1 = $p2 = 0; + } else { + $p1 = $first; + $p2 = $second; + $both = $third; + } + my @ports = (["\tPort 1: " . int($p1) . " \n", int($p1), 0], + ["\tPort 2: " . int($p2) . " \n", int($p2), int($p1)], + ["\tBoth: " . int($both) . " \n", int($both), int($p1) + int($p2)]); + + open(LSPCI, "lspci -D -s $bdf 2>/dev/null |") || next; + if (!eof(LSPCI)) { + print "BDF $bdf\n"; + parse_bdf(\@ports, $2, $bdf); + } +} + +if (!$iters) { + if ($numvfs =~ m/([[:digit:]]+)(,([[:digit:]]+))?(,([[:digit:]]+))?/g) { + my $first = $1; + my $second = $3; + my $third = $5; + my $p1, $p2, $both; + if (!defined($second) && !defined($third)) { + $both = $first; + $p1 = $p2 = 0; + } else { + $p1 = $first; + $p2 = $second; + $both = $third; + } + my @ports = (["\tPort 1: " . int($p1) . " \n", int($p1), 0], + ["\tPort 2: " . int($p2) . " \n", int($p2), int($p1)], + ["\tBoth: " . int($both) . " \n", int($both), int($p1) + int($p2)]); + open(my $LSPCI, "lspci -D | grep Mellanox | grep -v \"Virtual Function\" |") || die "Failed $!\n"; + while (<$LSPCI>) { + my ($full_bdf) = $_ =~ /(^[^ ]*)/; + my ($bdf) = $full_bdf =~ /(^(([[:xdigit:]]{4}:)?[[:xdigit:]]{2}:))/; + print "BDF $full_bdf\n"; + parse_bdf(\@ports, $bdf, $full_bdf); + } + + } else { + print "No devices found\n"; + } +} + +sub map_bdf_vf { + my $pf_bdf = $_[0]; + my %hash = (); + my $sysfs_dir = "/sys/bus/pci/devices/*$pf_bdf"; + my @virt_fns = <$sysfs_dir/virtfn*>; + foreach (@virt_fns) { + my ($fn) = fileparse($_) =~ /([0-9]+)$/; + $fn = "vf$fn"; + my $link = readlink($_); + my ($bdf) = $link =~ /(([:.]*[[:xdigit:]])+)/; + $link = "$sysfs_dir/" . $link; + $hash{$bdf} = $fn; + } + return %hash; +} + +sub parse_bdf { + my ($ports_ref, $bdf, $full_bdf) = @_; + my @ports = @$ports_ref; + my $counter = 0; + my $ports_index = 0; + my %hash = map_bdf_vf($full_bdf); + open(my $LSPCI, "lspci -D -s $bdf | tail -n +2 |") || die "Failed $!\n"; + while (<$LSPCI>) { + while ($counter == $ports[$ports_index]->[1] + $ports[$ports_index]->[2]) { + print $ports[$ports_index]->[0]; + $ports_index++; + } + + if ($counter == $ports[$ports_index]->[2]) { + print $ports[$ports_index]->[0]; + } + my ($vf_bdf) = $_ =~ /(^[^ ]*)/; + print "\t\t$hash{\"$vf_bdf\"}\t$vf_bdf\n"; + $counter++; + } + while ($ports_index++ < $#ports) { + print $ports[$ports_index]->[0]; + } +} + + diff --git a/ofed_scripts/utils/mlnx_perf b/ofed_scripts/utils/mlnx_perf new file mode 100755 index 0000000..d583f53 --- /dev/null +++ b/ofed_scripts/utils/mlnx_perf @@ -0,0 +1,133 @@ +#!/usr/bin/python + +from optparse import OptionParser +from subprocess import Popen, PIPE +import sys +import time +import re +import math +from collections import defaultdict +import glob +import tempfile + +def thous(x, sep=',', dot='.'): + frac, num = math.modf(x) + num = str(int(num)) + frac = int(frac * 100) + num = re.sub(r'(\d{3})(?=\d)', r'\1'+sep, num[::-1])[::-1] + if frac > 0: + num += dot + str(frac) + return num + +def get_stats(intf, keys = None): + with tempfile.NamedTemporaryFile(delete=True) as stdout_pipe: + process = Popen('ethtool -S %s'%intf, + shell=True, bufsize=0, + stdout=stdout_pipe) + rc = process.wait() + stdout_pipe.seek(0) + if (rc): + print "error running ethtool(%d):" % (process.returncode) + sys.exit(0) + + map = {} + output = stdout_pipe.readlines()[1:] + for line in output: + key, val = line.strip().split(":") + + if not keys == None: + keys += [key] + + if val == "": + val = "0" + + map[key] = int(val) + return map + +parser = OptionParser(usage="%prog -i [options]", version="%prog 1.0") + +parser.add_option("-i", "--interface", dest="intf", help="Interface name") +parser.add_option("-t", "--interval", dest="interval", default=1, + help="Interval between measurements in seconds") +parser.add_option("-c", "--count", dest="count", default=-1, type="int", + help="Exit counter - exit after counting number of intervals ( default is -1: do not exit) ") + +(options, args) = parser.parse_args() + +if (options.intf == None): + print "Interface name is required" + parser.print_usage() + sys.exit(1) + +print "Initializing mlnx_perf..." + +# keys must be ordered, so can't use 'for key in map' +keys = [] +prev = get_stats(options.intf, keys) + +for key in keys: + m = re.match("tx(\d+)_bytes", key) + if m: + ring = int(m.groups()[0]) + +count = int(options.count) + +if count < -1 or count == 0: + print "Error, please use positive value for \"count\" or \"-1\" for no exit " + sys.exit(1) + +print "Sampling started." + +while count != 0: + time.sleep(float(options.interval)) + count -= 1 + + curr = get_stats(options.intf) + + if (curr.has_key('timestamp') and prev.has_key('timestamp')): + secs = float(curr['timestamp'] - prev['timestamp']) / 1000 + else: + secs = float(options.interval) + + up_bw = defaultdict(int) + up_packets = defaultdict(int) + total_bw = 0 + total_packets = 0 + something_printed = False + for key in keys: + if key in ["timestamp"]: + continue + + bw = (curr[key]-prev[key]) / secs + if (bw > 0): + if "bytes" in key: + # Calculate throughput rate in Mbps from the Bytes counter + print "%30s: %-20s = %-20s" % (key, thous(bw) + " Bps", + thous(bw * 8 / 1000000) + " Mbps") + else: + print "%30s: %s" % (key, thous(bw)) + something_printed = True + + m = re.match("tx_prio_?(\d+)_bytes", key) + if m: + up = int(m.groups()[0]) + up_bw[up] += bw + total_bw += bw + + m = re.match("tx_prio_?(\d+)_packets", key) + if m: + up = int(m.groups()[0]) + up_packets[up] += bw + total_packets += bw + + prev = curr + if something_printed: + for up in up_bw: + # Calculate throughput rate in Mbps from the Bytes counter + print "%30s: %-20s Mbps = %-2.2f%%" % ("UP " + str(up), thous(up_bw[up] * + 8 / 1000000), 100.0 * up_bw[up] / total_bw) + for up in up_packets: + print "%30s: %-20s Tran/sec = %-2.2f%%" % ("UP " + str(up), thous(up_packets[up]), 100.0 * up_packets[up] / total_packets) + + print "--------" + sys.stdout.flush() diff --git a/ofed_scripts/utils/mlnx_qcn b/ofed_scripts/utils/mlnx_qcn new file mode 100755 index 0000000..f918092 --- /dev/null +++ b/ofed_scripts/utils/mlnx_qcn @@ -0,0 +1,248 @@ +#!/usr/bin/python + +import sys +import os +if os.path.exists('/usr/share/pyshared'): + sys.path.append('/usr/share/pyshared') +import ast +from optparse import OptionParser +from dcbnetlink import DcbController +from collections import defaultdict +from subprocess import Popen, PIPE +from tc_wrap import * + +DCB_CAP_DCBX_VER_IEEE = 0x8 +NUMBER_OF_QCN_PARAMS = 12 +NUMBER_OF_QCN_STATS = 10 +NUMBER_OF_TCS = 8 + +list_params = ["rpg_enable", "rppp_max_rps", "rpg_time_reset", "rpg_byte_reset", "rpg_threshold", "rpg_max_rate", "rpg_ai_rate", "rpg_hai_rate", "rpg_gd", "rpg_min_dec_fac", "rpg_min_rate", "cndd_state_machine"] +list_statistics = ["rppp_rp_centiseconds", "rppp_created_rps", "ignored_cnm", "estimated_total_rate", "cnms_handled_successfully", "min_total_limiters_rate", "max_total_limiters_rate"] + + +class QCN: + def get(self): + raise "Not implemented" + def set(self, qcn): + raise "Not implemented" + def getstats(self): + raise "Not implemented" + + +class QCNNL(QCN): + def __init__(self,ctrl): + self.ctrl = ctrl + def get(self): + return ctrl.get_ieee_qcn() + + def set(self, qcn): + ctrl.set_ieee_qcn(qcn) + + def get_statistics(self): + return ctrl.get_ieee_qcnstats() + +class QCNSysfs(QCN): + def __init__(self, path): + self.path = path + + def get(self): + qcn_params = [] + f = open(self.path, "r") + for item in f.read().split(): + if (item == "priority" or item == "|priority"): + lastItem = item + continue + elif (item == ":" or item == "|"): + continue + elif (lastItem == "priority" or lastItem == "|priority"): + lastItem = "" + continue + qcn_params.append(int(item)) + f.close() + return qcn_params + + def set(self, qcn): + f = open(self.path, "w") + f.write(" ".join(str(r) for r in qcn)) + f.close() + +class QCNStatsSysfs(QCN): + def __init__(self, path): + self.path = path + + def get_statistics(self): + qcn_stats = [] + f = open(self.path, "r") + for item in f.read().split(): + if (item == "priority" or item == "|priority"): + lastItem = item + continue + elif (item == ":" or item == "|"): + continue + elif (lastItem == "priority" or lastItem == "|priority"): + lastItem = "" + continue + qcn_stats.append(int(item)) + f.close() + return qcn_stats + +def pretty_print_qcn(qcn, values): + # Deal with an additions of qcn parameters which were not added to the tool # + additions = (len(qcn) - (NUMBER_OF_TCS * NUMBER_OF_QCN_PARAMS)) / NUMBER_OF_TCS + + for tc in range(NUMBER_OF_TCS): + print "priority %d:" % tc + for val in values: + if isinstance(qcn, list): + print "\t%s: %d" % (val,qcn[(tc * (len(values) + additions)) + values.index(val)]) + else: + print "\t%s: %d" % (val,qcn[values.index(val)][tc]) + print + + +def update_paremeter(qcn, listPerTc, param): + # Deal with an additions of qcn parameters which were not added to the tool # + additions = (len(qcn) - (NUMBER_OF_TCS * NUMBER_OF_QCN_PARAMS)) / NUMBER_OF_TCS + + for tc in range(NUMBER_OF_TCS): + if (listPerTc[tc] != -1): + if isinstance(qcn, list): + qcn[(tc * (NUMBER_OF_QCN_PARAMS + additions)) + list_params.index(param)] = listPerTc[tc] + else: + qcn[list_params.index(param)][tc] = listPerTc[tc] + return qcn + + + +parser = OptionParser(usage="%prog -i [options]", version="%prog 1.0") + +parser.add_option("-i", "--interface", dest="intf", + help="Interface name") +parser.add_option("-g", "--get_type", dest="type", choices=["parameters", "statistics"], + help="Type of information to get: \"statistics\" or \"parameters\"") + +parser.add_option("--rpg_enable", nargs=8, type="int", dest="rpg_enable_list", + help="Set value of rpg_enable according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rppp_max_rps", nargs=8, type="int", dest="rppp_max_rps_list", + help="Set value of rppp_max_rps according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_time_reset", nargs=8, type="int", dest="rpg_time_reset_list", + help="Set value of rpg_time_reset according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_byte_reset", nargs=8, type="int", dest="rpg_byte_reset_list", + help="Set value of rpg_byte_reset according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_threshold", nargs=8, type="int", dest="rpg_threshold_list", + help="Set value of rpg_threshold according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_max_rate", nargs=8, type="int", dest="rpg_max_rate_list", + help="Set value of rpg_max_rate according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_ai_rate", nargs=8, type="int", dest="rpg_ai_rate_list", + help="Set value of rpg_ai_rate according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_hai_rate", nargs=8, type="int", dest="rpg_hai_rate_list", + help="Set value of rpg_hai_rate according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_gd", nargs=8, type="int", dest="rpg_gd_list", + help="Set value of rpg_gd according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_min_dec_fac", nargs=8, type="int", dest="rpg_min_dec_fac_list", + help="Set value of rpg_min_dec_fac according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--rpg_min_rate", nargs=8, type="int", dest="rpg_min_rate_list", + help="Set value of rpg_min_rate according to priority, use spaces between values and -1 for unknown values.") + +parser.add_option("--cndd_state_machine", nargs=8, type="int", dest="cndd_state_machine_list", + help="Set value of cndd_state_machine according to priority, use spaces between values and -1 for unknown values.") + +(options, args) = parser.parse_args() + +if len(args) > 0: + print "Bad arguments" + parser.print_usage() + sys.exit(1) + +if (options.intf == None): + print "Interface name is required" + parser.print_usage() + sys.exit(1) + +tmp_list = ast.literal_eval(options.__str__()).values() + +if (tmp_list.count(None) == len(tmp_list) - 1): + print "No action was asked for, choose -h to see options" + parser.print_usage() + sys.exit(1) + +qcn_path = "/sys/class/net/" + options.intf + "/qos/qcn" +qcn_stats_path = "/sys/class/net/" + options.intf + "/qos/qcn_stats" + +ctrl = DcbController(options.intf) + +try: + qcn_main = None + if (not os.path.exists(qcn_path)): + qcn_main = QCNNL(ctrl) + else: + if (options.type == "statistics"): + qcn_main = QCNStatsSysfs(qcn_stats_path) + else: + qcn_main = QCNSysfs(qcn_path) + + if (options.type == "parameters"): + qcn = qcn_main.get() + pretty_print_qcn(qcn, list_params) + elif (options.type == "statistics"): + qcn = qcn_main.get_statistics() + pretty_print_qcn(qcn, list_statistics) + + else: + + qcn = qcn_main.get() + + if (options.rpg_enable_list != None): + qcn = update_paremeter(qcn, options.rpg_enable_list, "rpg_enable") + + if (options.rppp_max_rps_list != None): + qcn = update_paremeter(qcn, options.rppp_max_rps_list, "rppp_max_rps") + + if (options.rpg_time_reset_list != None): + qcn = update_paremeter(qcn, options.rpg_time_reset_list, "rpg_time_reset") + + if (options.rpg_byte_reset_list != None): + qcn = update_paremeter(qcn, options.rpg_byte_reset_list, "rpg_byte_reset") + + if (options.rpg_threshold_list != None): + qcn = update_paremeter(qcn, options.rpg_threshold_list, "rpg_threshold") + + if (options.rpg_max_rate_list != None): + qcn = update_paremeter(qcn, options.rpg_max_rate_list, "rpg_max_rate") + + if (options.rpg_ai_rate_list != None): + qcn = update_paremeter(qcn, options.rpg_ai_rate_list, "rpg_ai_rate") + + if (options.rpg_hai_rate_list != None): + qcn = update_paremeter(qcn, options.rpg_hai_rate_list, "rpg_hai_rate") + + if (options.rpg_gd_list != None): + qcn = update_paremeter(qcn, options.rpg_gd_list, "rpg_gd") + + if (options.rpg_min_dec_fac_list != None): + qcn = update_paremeter(qcn, options.rpg_min_dec_fac_list, "rpg_min_dec_fac") + + if (options.rpg_min_rate_list != None): + qcn = update_paremeter(qcn, options.rpg_min_rate_list, "rpg_min_rate") + + if (options.cndd_state_machine_list != None): + qcn = update_paremeter(qcn, options.cndd_state_machine_list, "cndd_state_machine") + + qcn_main.set(qcn) + + +except: + print "QCN is not supported on your system!" + qcn = [] + + diff --git a/ofed_scripts/utils/mlnx_qos b/ofed_scripts/utils/mlnx_qos new file mode 100755 index 0000000..a820afd --- /dev/null +++ b/ofed_scripts/utils/mlnx_qos @@ -0,0 +1,309 @@ +#!/usr/bin/python + +import sys +import os +if os.path.exists('/usr/share/pyshared'): + sys.path.append('/usr/share/pyshared') +from optparse import OptionParser +from dcbnetlink import DcbController +from collections import defaultdict +from subprocess import Popen, PIPE + +DCB_CAP_DCBX_HOST = 0x1 +DCB_CAP_DCBX_LLD_MANAGED = 0x2 +DCB_CAP_DCBX_VER_CEE = 0x4 +DCB_CAP_DCBX_VER_IEEE = 0x8 +DCB_CAP_DCBX_STATIC = 0x10 + +IEEE_8021QAZ_TSA_STRICT = 0 +IEEE_8021QAZ_TSA_CB_SHAPER = 1 +IEEE_8021QAZ_TSA_ETS = 2 +IEEE_8021QAZ_TSA_VENDOR = 255 + +class Maxrate: + def get(self): + pass + def set(self, ratelimit): + pass + + def prepare(self, ratelimit): + old_ratelimit = self.get() + ratelimit += old_ratelimit[len(ratelimit):8] + return ratelimit + +class MaxrateNL(Maxrate): + def __init__(self, ctrl): + self.ctrl = ctrl + def get(self): + return ctrl.get_ieee_maxrate() + + def set(self, ratelimit): + ratelimit = self.prepare(ratelimit) + ctrl.set_ieee_maxrate(ratelimit) + +class MaxrateSysfs(Maxrate): + def __init__(self, path): + self.path = path + + def get(self): + ratelimit = [] + f = open(self.path, "r") + for item in f.read().split(): + ratelimit.append(float(item)) + f.close() + + return ratelimit + + def set(self, ratelimit): + ratelimit = self.prepare(ratelimit) + f = open(self.path, "w") + f.write(" ".join(str(r) for r in ratelimit)) + f.close() + +def pretty_print(prio_tc, tsa, tcbw, ratelimit, pfc_en): + tc2up = defaultdict(list) + + if (printall == True): + for i in range(8): + tc2up.setdefault(i,[]) + + print "PFC configuration:" + print "\tpriority 0 1 2 3 4 5 6 7" + msg = "\tenabled " + for up in range(8): + msg += "%1d " % ((pfc_en >> up) & 0x01) + print msg + print "" + + for up in range(len(prio_tc)): + tc = prio_tc[up] + tc2up[int(tc)].append(up) + + for tc in tc2up: + r = "unlimited" + msg = "" + try: + if ratelimit[tc] > 0: + r = "%.1f Gbps" % (float(ratelimit[tc] / 1000)/1000) + msg = "tc: %d ratelimit: %s, tsa: " % (tc, r) + except Exception, err: + pass + try: + if (tsa[tc] == IEEE_8021QAZ_TSA_ETS): + msg +="ets, bw: %s%%" % (tcbw[tc]) + elif (tsa[tc] == IEEE_8021QAZ_TSA_STRICT): + msg += "strict" + elif (tsa[tc] == IEEE_8021QAZ_TSA_VENDOR): + msg += "vendor" + else: + msg += "unknown" + except Exception, err: + pass + + if msg: + print msg + + try: + for up in tc2up[tc]: + print "\t priority: ", up + except Exception, err: + pass + +def parse_int(str, min, max, description): + try: + v = int(str) + + if (v < min or v > max): + raise ValueError("%d is not in the range %d..%d" % (v, min, max)) + + return v + except ValueError, e: + print "Bad value for %s: %s" % (description, e) + parser.print_usage() + sys.exit(1) + +parser = OptionParser(usage="%prog -i [options]", version="%prog 1.0") + +parser.add_option("-f", "--pfc", dest="pfc", + help="Set priority flow control for each priority. LIST is " + + "comma separated value for each priority starting from 0 to 7. " + + "Example: 0,0,0,0,1,1,1,1 enable PFC on TC4-7", metavar="LIST") +parser.add_option("-p", "--prio_tc", dest="prio_tc", + help="maps UPs to TCs. LIST is 8 comma seperated TC numbers. " + + "Example: 0,0,0,0,1,1,1,1 maps UPs 0-3 to TC0, and UPs 4-7 to " + + "TC1", metavar="LIST") +parser.add_option("-s", "--tsa", dest="tsa", help="Transmission algorithm for " + + "each TC. LIST is comma seperated algorithm names for each TC. " + + "Possible algorithms: strict, etc. Example: ets,strict,ets sets " + + "TC0,TC2 to ETS and TC1 to strict. The rest are unchanged.", + metavar="LIST") +parser.add_option("-t", "--tcbw", dest="tc_bw", + help="Set minimal guaranteed %BW for ETS TCs. LIST is comma " + + "seperated percents for each TC. Values set to TCs that are " + + "not configured to ETS algorithm are ignored, but must be " + + "present. Example: if TC0,TC2 are set to ETS, then 10,0,90 " + + "will set TC0 to 10% and TC2 to 90%. Percents must sum to " + + "100.", metavar="LIST") +parser.add_option("-r", "--ratelimit", dest="ratelimit", + help="Rate limit for TCs (in Gbps). LIST is a comma seperated " + + "Gbps limit for each TC. Example: 1,8,8 will limit TC0 to " + + "1Gbps, and TC1,TC2 to 8 Gbps each.", metavar="LIST") +parser.add_option("-d", "--dcbx", dest="dcbx", + help="get the dcbx mode(get) or set dcbx mode to firmware controlled(fw) or " + + "os controlled(os)") +parser.add_option("-i", "--interface", dest="intf", + help="Interface name") + +parser.add_option("-a", action="store_true", dest="printall", default=False, + help="Show all interface's TCs") + +(options, args) = parser.parse_args() + +if len(args) > 0: + print "Bad arguments" + parser.print_usage() + sys.exit(1) + +if (options.intf == None): + print "Interface name is required" + parser.print_usage() + + sys.exit(1) + +ratelimit_path = "/sys/class/net/" + options.intf + "/qos/maxrate" + +pfc_en = 0 +tsa = [IEEE_8021QAZ_TSA_STRICT, IEEE_8021QAZ_TSA_STRICT,IEEE_8021QAZ_TSA_STRICT,IEEE_8021QAZ_TSA_STRICT,IEEE_8021QAZ_TSA_STRICT,IEEE_8021QAZ_TSA_STRICT,IEEE_8021QAZ_TSA_STRICT,IEEE_8021QAZ_TSA_STRICT] +tc_bw = [0, 0, 0, 0, 0, 0, 0, 0] +prio_tc = [0, 0, 0, 0, 0, 0, 0, 0] +printall = False + +ctrl = DcbController(options.intf) + +if (options.dcbx != None): + if (options.dcbx == "os"): + ctrl.set_dcbx(ctrl.get_dcbx() | DCB_CAP_DCBX_HOST); + elif (options.dcbx == "fw"): + ctrl.set_dcbx(0); + + if (ctrl.get_dcbx() & DCB_CAP_DCBX_HOST): + print ("DCBX mode: OS controlled") + else: + print ("DCBX mode: Firmware controlled") + + sys.exit(1) + +try: + ratelimit = [] + maxrate = None + if (not os.path.exists(ratelimit_path)): + maxrate = MaxrateNL(ctrl) + else: + maxrate = MaxrateSysfs(ratelimit_path) + + if options.ratelimit: + for r in options.ratelimit.split(","): + r = parse_int(r, 0, 1000000, "ratelimit") + + ratelimit += [r * 1000 * 1000] + try: + maxrate.set(ratelimit) + except: + print "Rate limit is not supported on your system!" + + try: + ratelimit = maxrate.get() + except: + print "Rate limit is not supported on your system!" +except: + if options.ratelimit: + sys.exit(1) + else: + ratelimit = [] + +try: + if (not (ctrl.get_dcbx() & DCB_CAP_DCBX_VER_IEEE)): + ctrl.set_dcbx(DCB_CAP_DCBX_VER_IEEE | DCB_CAP_DCBX_HOST) + + prio_tc, tsa, tc_bw = ctrl.get_ieee_ets() + + pfc_en = ctrl.get_ieee_pfc() + +except: + print "ETS features are not supported on your system" + +if (options.tsa): + i = 0 + for t in options.tsa.split(","): + if i >= 8: + print "Too many items for TSA" + sys.exit(1) + + if (t == "strict"): + tsa[i] = IEEE_8021QAZ_TSA_STRICT + elif (t == 'ets'): + tsa[i] = IEEE_8021QAZ_TSA_ETS + else: + print "Bad TSA value: ", t + parser.print_usage() + sys.exit(1) + i += 1 + +if options.printall: + printall = True + +if options.pfc: + i = 0 + pfc_en = 0 + + for t in options.pfc.split(","): + if i >= 8: + print "Too many items for PFC" + sys.exit(1) + + temp = parse_int(t, 0, 1, "PFC") + pfc_en |= (temp << i) + + i += 1 + + try: + ctrl.set_ieee_pfc(_pfc_en = pfc_en) + except OSError, e: + print e + sys.exit(1) + +if options.tc_bw: + i = 0 + for t in options.tc_bw.split(","): + if i >= 8: + print "Too many items for ETS BW" + sys.exit(1) + + bw = parse_int(t, 0, 100, "ETS BW") + + if tsa[i] == IEEE_8021QAZ_TSA_STRICT and bw != 0: + print "ETS BW for a strict TC must be 0" + parser.print_usage() + sys.exit(1) + + tc_bw[i] = bw + i += 1 + +if options.prio_tc: + i = 0 + for t in options.prio_tc.split(","): + if i >= 8: + print "Too many items in UP => TC mapping" + sys.exit(1) + + prio_tc[i] = parse_int(t, 0, 7, "UP => TC mapping") + i += 1 + +if options.tsa or options.tc_bw or options.prio_tc: + try: + ctrl.set_ieee_ets(_prio_tc = prio_tc, _tsa = tsa, _tc_bw = tc_bw) + except OSError, e: + print e + sys.exit(1) + +pretty_print(prio_tc, tsa, tc_bw, ratelimit, pfc_en) diff --git a/ofed_scripts/utils/mlx_fs_dump b/ofed_scripts/utils/mlx_fs_dump new file mode 100755 index 0000000..6b44eaf --- /dev/null +++ b/ofed_scripts/utils/mlx_fs_dump @@ -0,0 +1,394 @@ +#!/usr/bin/python +""" +This is a tool to parse the flow steering information. + +It can either parse the output from command: + + sudo mlxdump -d /dev/mst/mt4115_pciconf0 fsdump --type FT --no_zero=true > fs_log + +by doing: mlx_fs_dump.py -f fs_log + +Or just run this script and it will run the command(make sure mst is running). + +See ./mlx_fs_dump -h for help. +""" + +__author__ = 'Bodong Wang - bodong@mellanox.com' +__author__ = 'Huy Nguyen - huyn@mellanox.com' +__author__ = 'Sunil Sudhakar Rani - sunils@mellanox.com' +__version__= '1.0' + +import sys +import os +import re +import glob +import subprocess +import argparse +import traceback +from enum import Enum +from argparse import RawTextHelpFormatter +try: + from anytree import Node, RenderTree, NodeMixin, AsciiStyle +except ImportError: + sys.exit("- ERR - Please install anytree (e.g, pip install anytree)") + +try: + from termcolor import colored +except ImportError: + sys.exit("- ERR - Please install termcolor (e.g, pip install termcolor)") + +# Global +tableList = [] +groupList = [] +entryList = [] +color = 1 + +ftb_type = ["NIC_RX", "NIC_TX", "ESW_EGRESS_ACL","ESW_INGRESS_ACL", "ESW_FDB", "SNIFFER_RX", "SNIFFER_TX", "TX_RDMA", "TX_RDMA"] +fte_action = ["", "ALLOW", "DROP", "", "FWD"] +fte_match = ["OUT_HDR", "MISC", "IN_HDR", "", ""] +dest_type = ["VPORT", "FLOW_TABLE", "TIR", "QP"] + +# Dict to rename the long name to short one +match_criteria = { + "outer_headers.smac_47_16" : "out.smac_47_16", + "outer_headers.ethertype" : "out.ethtype", + "outer_headers.smac_15_0" : "out.smac_15_0", + "outer_headers.dmac_47_16" : "out.dmac_47_16", + "outer_headers.first_vid" : "out.1st_vid", + "outer_headers.first_cfi" : "out.1st_cfi", + "outer_headers.first_prio" : "out.1st_prio", + "outer_headers.dmac_15_0" : "out.dmac_15_0", + "outer_headers.tcp_flags" : "out.tcp_flags", + "outer_headers.frag" : "out.frag", + "outer_headers.svlan_tag" : "out.svlan_tag", + "outer_headers.cvlan_tag" : "out.cvlan_tag", + "outer_headers.ip_ecn" : "out.ip_ecn", + "outer_headers.ip_dscp" : "out.ip_dscp", + "outer_headers.ip_protocol" : "out.ip_prot", + "outer_headers.tcp_dport" : "out.tcp_dport", + "outer_headers.tcp_sport" : "out.tcp_sport", + "outer_headers.udp_dport" : "out.udp_dport", + "outer_headers.udp_sport" : "out.udp_sport", + "outer_headers.src_ip_127_96" : "out.src_ip_127_96", + "outer_headers.src_ip_95_64" : "out.src_ip_95_64", + "outer_headers.src_ip_63_32" : "out.src_ip_63_32", + "outer_headers.src_ip_31_0" : "out.src_ip_31_0", + "outer_headers.dst_ip_127_96" : "out.dst_ip_127_96", + "outer_headers.dst_ip_95_64" : "out.dst_ip_95_64", + "outer_headers.dst_ip_63_32" : "out.dst_ip_63_32", + "outer_headers.dst_ip_31_0" : "out.dst_ip_31_0", + "misc_parameters.source_sqn" : "src_sqn", + "misc_parameters.source_port" : "src_port", + "misc_parameters.inner_second_vid" : "in_2nd_vid", + "misc_parameters.inner_second_cfi" : "in_2nd_cfi", + "misc_parameters.inner_second_prio" : "in_2nd_prio", + "misc_parameters.outer_second_vid" : "out_2nd_vid", + "misc_parameters.outer_second_cfi" : "out_2nd_cfi", + "misc_parameters.outer_second_prio" : "out_2nd_prio", + "misc_parameters.gre_protocol" : "gre_protocol", + "misc_parameters.inner_second_svlan_ta" : "in_2nd_svlan_ta", + "misc_parameters.outer_second_svlan_ta" : "out_2nd_svlan_ta", + "misc_parameters.inner_second_cvlan_ta" : "in_2nd_cvlan_ta", + "misc_parameters.outer_second_cvlan_ta" : "out_2nd_cvlan_ta", + "misc_parameters.gre_key_l" : "gre_key_l", + "misc_parameters.gre_key_h" : "gre_key_h", + "misc_parameters.vxlan_vni" : "vxlan_vni", + "misc_parameters.geneve_oam" : "geneve_oam", + "misc_parameters.geneve_vni" : "geneve_vni", + "misc_parameters.outer_ipv6_flow_label" : "out_ipv6_flow_label", + "misc_parameters.inner_ipv6_flow_label" : "in_ipv6_flow_label", + "misc_parameters.geneve_protocol_type" : "geneve_prot_type", + "misc_parameters.bth_dst_qp" : "bth_dst_qp", + "inner_headers.smac_47_16" : "in.smac_47_16", + "inner_headers.ethertype" : "in.ethtype", + "inner_headers.smac_15_0" : "in.smac_15_0", + "inner_headers.dmac_47_16" : "in.dmac_47_16", + "inner_headers.first_vid" : "in.1st_vid", + "inner_headers.first_cfi" : "in.1st_cfi", + "inner_headers.first_prio" : "in.1st_prio", + "inner_headers.dmac_15_0" : "in.dmac_15_0", + "inner_headers.tcp_flags" : "in.tcp_flags", + "inner_headers.frag" : "in.frag", + "inner_headers.svlan_tag" : "in.svlan_tag", + "inner_headers.cvlan_tag" : "in.cvlan_tag", + "inner_headers.ip_ecn" : "in.ip_ecn", + "inner_headers.ip_dscp" : "in.ip_dscp", + "inner_headers.ip_protocol" : "in.ip_prot", + "inner_headers.tcp_dport" : "in.tcp_dport", + "inner_headers.tcp_sport" : "in.tcp_sport", + "inner_headers.udp_dport" : "in.udp_dport", + "inner_headers.udp_sport" : "in.udp_sport", + "inner_headers.src_ip_127_96" : "in.src_ip_127_96", + "inner_headers.src_ip_95_64" : "in.src_ip_95_64", + "inner_headers.src_ip_63_32" : "in.src_ip_63_32", + "inner_headers.src_ip_31_0" : "in.src_ip_31_0", + "inner_headers.dst_ip_127_96" : "in.dst_ip_127_96", + "inner_headers.dst_ip_95_64" : "in.dst_ip_95_64", + "inner_headers.dst_ip_63_32" : "in.dst_ip_63_32", + "inner_headers.dst_ip_31_0" : "in.dst_ip_31_0" +} + +# Dict to find the ethType name +eth_type = { + "0x800" : "IPv4", + "0x806" : "APR", + "33024" : "VLAN_TAGGED", + "0x8914": "FCOE", + "0x86dd": "IPv6", + "0x8915": "RCOE" +} + +# Dict to find the IP_Protocal name +ip_proto = { + 1 : "ICMP", + 2 : "IGMP", + 4 : "IP-in-IP", + 6 : "TCP", + 17 : "UDP", + 41 : "IPv6", + 50 : "ESP", + 51 : "AH", +} + +def test_bit(node=[], bit=None): + try: + return node[node.index(bit) + 1] + except ValueError: + return "0" + +def green(string): + if color: + return colored(string, 'green') + else: + return string + +def blue(string): + if color: + return colored(string, 'blue') + else: + return string + +def red(string): + if color: + return colored(string, 'red') + else: + return string + +class flow_table(NodeMixin): + def __init__(self, parent=None, table_id=None, level=None, type=None, table_miss_id=None, rootFlag=1): + self.parent = parent + self.table_id = table_id + self.vport = 0 + self.level = level + self.type = type # Use flow_table_type + self.table_miss_id = table_miss_id + self.rootFlag = rootFlag + +class flow_group(NodeMixin): + def __init__(self, parent=None, table_id=None, group_id=None, match_enable=None, match_cr=[]): + self.parent = parent + self.table_id = table_id + self.group_id = group_id + self.match_enable = match_enable + self.match_cr = match_cr + +class flow_entry(NodeMixin): + def __init__(self, parent=None, table_id=None,\ + group_id=None, action=None, flow_index=None,\ + dst_type=None, dst_value=None, match_cr=[]): + self.parent = parent + self.table_id = table_id + self.group_id = group_id + self.action = action + self.flow_index = flow_index + self.dst_type = dst_type + self.dst_value = dst_value + self.cr_found = [] + + for i, cr in enumerate(match_cr): + # Only check the cr name + if not i % 2: + try: + cr_rename = match_criteria[cr] + except KeyError: + continue + # Save cr name + self.cr_found.append(cr_rename) + # Save cr value + self.cr_found.append(match_cr[i+1]) + +def printFt(node): + return ("level: %s, type: %s" % (node.level, node.type)) + +def printFg(node): + return ("%s" % (node.match_enable)) + +def printFte(node): + ret = " " + + if node.dst_type is not "0": + ret += "to (" + node.dst_type + ":" + node.dst_value + ") " + + for i, cr in enumerate(node.cr_found): + # Only print cr with none 0 value + if (i % 2) and (cr is not "0"): + cr_name = node.cr_found[i-1] + if "ethtype" in cr_name: + try: + cr = eth_type[cr] + except KeyError: + pass + elif "ip_prot" in cr_name: + try: + cr = ip_proto[int(cr, 0)] + except KeyError: + pass + elif "out.dmac_47_16" in cr_name: + cr_name = "out.dmac" + try: + index = node.cr_found.index("out.dmac_15_0") + except ValueError: + index = -1 + + if index is not -1: + cr = cr.zfill(8) + node.cr_found[index+1].lstrip("0x").zfill(4) + node.cr_found[index] = "0" + node.cr_found[index+1] = "0" + elif "out.smac_47_16" in cr_name: + cr_name = "out.smac" + try: + index = node.cr_found.index("out.smac_15_0") + except ValueError: + index = -1 + + if index is not -1: + cr = cr.zfill(8) + node.cr_found[index+1].lstrip("0x").zfill(4) + node.cr_found[index] = "0" + node.cr_found[index+1] = "0" + + + ret += cr_name + ":" + cr + " " + + return ret + +def printNode(pre, node, substring): + if isinstance(node, flow_table): + print(blue("%s%s%s (%s)") % (pre, "FT: ", node.table_id, printFt(node))) + elif isinstance(node, flow_group): + print(red("%s%s%s (%s)") % (pre, "FG: ", node.group_id, printFg(node))) + elif isinstance(node, flow_entry): + print(green("%s%s%s (%s)%s") % \ + (pre, "FTE: ", node.flow_index, node.action, printFte(node))) + +class fancy_dump(): + + def __init__(self, args=[]): + global color + parser = argparse.ArgumentParser(\ + description='-'*65 + "\n" + + 'Dump flow table in a fancy way.\n\n' + \ + 'Package/Tool required:\n' +\ + '\t - mst\n' +\ + '\t - python packages (anytree, termcolor)\n' + \ + '-'*65, + formatter_class=RawTextHelpFormatter) + # Add arguments + parser.add_argument('-g', type=int, help='Gvmi number', dest='gvmi', required=False, default=0) + parser.add_argument('-c', type=int, help='Print with color', dest='color', required=False, default=1) + parser.add_argument('-f', help='Parse a file from mlxdump', dest='in_file', required=False, default=None) + # Array for all arguments passed to script + arg = parser.parse_args() + self.gvmi = arg.gvmi + color = arg.color + self.in_file = arg.in_file + + def run(self): + if self.in_file is None: + #run fsdump command + self.cmd = "sudo mlxdump -d /dev/mst/mt4115_pciconf0 fsdump --type FT --no_zero=true" + " --gvmi " + str(self.gvmi) + + p = subprocess.Popen(self.cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + out, err = p.communicate() + + if err: + return ("[%s] Failed\n- ERROR - %s" % (self.cmd, err)) + + if b"-E-" in out: + return ("[%s] Failed\n- %s" % (self.cmd, out)) + else: + with open(self.in_file, 'rb') as f: + out = f.read() + + # split the output with empty lines + out_1 = out.decode('utf8').split(os.linesep + os.linesep) + + # split with , + for i in out_1: + node = i.replace("\n", ",").replace("-", "").replace(" ", "").replace("=", ",").replace(":", ",").lstrip(',').split(',') + #print node + + table_id = test_bit(node, "table_id") + group_id = test_bit(node, "group_id") + + if node[0] == "FT": + vport = test_bit(node, "gvmi") + type = ftb_type[int(test_bit(node, "table_type"), 0)] + level = test_bit(node, "level") + table_miss_id = test_bit(node, "table_miss_id") + + tableList.append(flow_table(None, table_id, level, type, table_miss_id, 1)) + + if node[0] == "FG": + match_enable = test_bit(node, "match_criteria_enable") + + if int(match_enable, 0): + match_cr = node[node.index("match_criteria_enable") + 2:] + else: + match_cr = [] + + match_enable = fte_match[int(match_enable, 0)] + + groupList.append(flow_group(None, table_id, group_id, match_enable, match_cr)) + + if node[0] == "FTE": + action = fte_action[int(test_bit(node, "action"), 0)] + valid = test_bit(node,"valid") + flow_index = test_bit(node,"flow_index") + dst_value = test_bit(node,"destination[0].destination_id") + dst_type = test_bit(node, "destination[0].destination_type").\ + split("(")[0].rstrip('_') + + if int(valid, 0): + match_cr = node[node.index("valid") + 2:] + else: + match_cr = [] + entryList.append(flow_entry(None, table_id,\ + group_id, action, flow_index,\ + dst_type, dst_value, match_cr)) + + for ftb_idx, ftb in enumerate(tableList): + for fg_idx, fg in enumerate(groupList): + for fte_idx, fte in enumerate(entryList): + if int(fte.group_id, 0) == int(fg.group_id, 0): + entryList[fte_idx].parent = groupList[fg_idx] + if int(ftb.table_id, 0) == int(fg.table_id, 0): + groupList[fg_idx].parent = tableList[ftb_idx] + + for fte_idx, fte in enumerate(entryList): + if fte.dst_type == "FLOW_TABLE": + for ftb_idx, ftb in enumerate(tableList): + if int(ftb.table_id, 0) == int(fte.dst_value, 0): + tableList[ftb_idx].parent = entryList[fte_idx] + tableList[ftb_idx].rootFlag = 0 + + for i in tableList: + if not i.rootFlag: + continue + + for pre, _, node in RenderTree(i, style=AsciiStyle()): + printNode(pre, node, 0) + +if __name__ == "__main__": + test = fancy_dump(sys.argv[1:]) + rc = test.run() + sys.exit(rc) diff --git a/ofed_scripts/utils/netlink.py b/ofed_scripts/utils/netlink.py new file mode 100644 index 0000000..f746921 --- /dev/null +++ b/ofed_scripts/utils/netlink.py @@ -0,0 +1,266 @@ +''' +Netlink message generation/parsing + +Copyright 2007 Johannes Berg + +GPLv2+; See copying for details. +''' + +import os +import socket +import struct + +def hexdump(msg, data): + arr="" + for i in range(len(data)): + if (i and i % 16 == 0): + arr +="\n" + arr += '%02x ' % ord(data[i]) +# hex = lambda data: ' '.join('{:02X}'.format(data[i]) for i in range(len(data))) + print msg, ":" + print arr + + +try: + # try to use python 2.5's netlink support + _dummysock = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, 0) + _dummysock.bind((0, 0)) + del _dummysock + def _nl_bind(descriptor, addr): + descriptor.bind(addr) + def _nl_getsockname(descriptor): + return descriptor.getsockname() + def _nl_send(descriptor, msg): + descriptor.send(msg) + def _nl_recv(descriptor, bufs=16384): + return descriptor.recvfrom(bufs) +except socket.error: + # or fall back to the _netlink C module + try: + import _netlink + def _nl_bind(descriptor, addr): + _netlink.bind(descriptor.fileno(), addr[1]) + def _nl_getsockname(descriptor): + return _netlink.getsockname(descriptor.fileno()) + def _nl_send(descriptor, msg): + _netlink.send(descriptor.fileno(), msg) + def _nl_recv(descriptor, bufs=16384): + return _netlink.recvfrom(descriptor.fileno(), bufs) + except ImportError: + # or fall back to the ctypes module + import ctypes + + libc = ctypes.CDLL(None) + + class SOCKADDR_NL(ctypes.Structure): + _fields_ = [("nl_family", ctypes.c_ushort), + ("nl_pad", ctypes.c_ushort), + ("nl_pid", ctypes.c_int), + ("nl_groups", ctypes.c_int)] + + def _nl_bind(descriptor, addr): + addr = SOCKADDR_NL(socket.AF_NETLINK, 0, os.getpid(), 0) + return libc.bind(descriptor.fileno(), + ctypes.pointer(addr), + ctypes.sizeof(addr)) + + def _nl_getsockname(descriptor): + addr = SOCKADDR_NL(0, 0, 0, 0) + len = ctypes.c_int(ctypes.sizeof(addr)); + libc.getsockname(descriptor.fileno(), + ctypes.pointer(addr), + ctypes.pointer(len)) + return addr.nl_pid, addr.nl_groups; + + def _nl_send(descriptor, msg): + return libc.send(descriptor.fileno(), msg, len(msg), 0); + + def _nl_recv(descriptor, bufs=16384): + addr = SOCKADDR_NL(0, 0, 0, 0) + len = ctypes.c_int(ctypes.sizeof(addr)) + buf = ctypes.create_string_buffer(bufs) + + r = libc.recvfrom(descriptor.fileno(), + buf, bufs, 0, + ctypes.pointer(addr), ctypes.pointer(len)) + + ret = ctypes.string_at(ctypes.pointer(buf), r) + return ret, (addr.nl_pid, addr.nl_groups) + + +# flags +NLM_F_REQUEST = 1 +NLM_F_MULTI = 2 +NLM_F_ACK = 4 +NLM_F_ECHO = 8 + +NLM_F_ROOT = 0x100 +NLM_F_MATCH = 0x200 +NLM_F_ATOMIC = 0x400 +NLM_F_DUMP = (NLM_F_ROOT | NLM_F_MATCH) + 5 + +# types +NLMSG_NOOP = 1 +NLMSG_ERROR = 2 +NLMSG_DONE = 3 +NLMSG_OVERRUN = 4 +NLMSG_MIN_TYPE = 0x10 + +class Attr: + def __init__(self, attr_type, data, *values): + self.type = attr_type + if len(values): + self.data = struct.pack(data, *values) + else: + self.data = data + + def _dump(self): + hdr = struct.pack("HH", len(self.data)+4, self.type) + length = len(self.data) + pad = ((length + 4 - 1) & ~3 ) - length + return hdr + self.data + '\0' * pad + + def __repr__(self): + return '' % (self.type, repr(self.data)) + + def u8(self): + return struct.unpack('B', self.data)[0] + def u16(self): + return struct.unpack('H', self.data)[0] + def s16(self): + return struct.unpack('h', self.data)[0] + def u32(self): + return struct.unpack('I', self.data)[0] + def s32(self): + return struct.unpack('i', self.data)[0] + def str(self): + return self.data + def nulstr(self): + return self.data.split('\0')[0] + def nested(self): + return parse_attributes(self.data) + +class StrAttr(Attr): + def __init__(self, attr_type, data): + Attr.__init__(self, attr_type, "%ds" % len(data), data) + +class NulStrAttr(Attr): + def __init__(self, attr_type, data): + Attr.__init__(self, attr_type, "%dsB" % len(data), data, 0) + +class U32Attr(Attr): + def __init__(self, attr_type, val): + Attr.__init__(self, attr_type, "I", val) + +class U8Attr(Attr): + def __init__(self, attr_type, val): + Attr.__init__(self, attr_type, "B", val) + +class Nested(Attr): + def __init__(self, attr_type, attrs): + self.attrs = attrs + self.type = attr_type + + def _dump(self): + contents = [] + for attr in self.attrs: + contents.append(attr._dump()) + contents = ''.join(contents) + length = len(contents) + hdr = struct.pack("HH", length+4, self.type) + return hdr + contents + +NETLINK_ROUTE = 0 +NETLINK_UNUSED = 1 +NETLINK_USERSOCK = 2 +NETLINK_FIREWALL = 3 +NETLINK_INET_DIAG = 4 +NETLINK_NFLOG = 5 +NETLINK_XFRM = 6 +NETLINK_SELINUX = 7 +NETLINK_ISCSI = 8 +NETLINK_AUDIT = 9 +NETLINK_FIB_LOOKUP = 10 +NETLINK_CONNECTOR = 11 +NETLINK_NETFILTER = 12 +NETLINK_IP6_FW = 13 +NETLINK_DNRTMSG = 14 +NETLINK_KOBJECT_UEVENT = 15 +NETLINK_GENERIC = 16 + +class Message: + def __init__(self, msg_type, flags=0, seq=-1, payload=None): + self.type = msg_type + self.flags = flags + self.seq = seq + self.pid = -1 + payload = payload or [] + if isinstance(payload, list): + contents = [] + for attr in payload: + contents.append(attr._dump()) + self.payload = ''.join(contents) + else: + self.payload = payload + + def send(self, conn): + if self.seq == -1: + self.seq = conn.seq() + + self.pid = conn.pid + length = len(self.payload) + + hdr = struct.pack("IHHII", length + 4*4, self.type, + self.flags, self.seq, self.pid) + + conn.send(hdr + self.payload) + + def __repr__(self): + return '' % ( + self.type, self.pid, self.seq, self.flags, repr(self.payload)) + +class Connection: + def __init__(self, nltype, groups=0, unexpected_msg_handler=None): + self.descriptor = socket.socket(socket.AF_NETLINK, + socket.SOCK_RAW, nltype) + self.descriptor.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + self.descriptor.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + _nl_bind(self.descriptor, (os.getpid(), groups)) + self.pid, self.groups = _nl_getsockname(self.descriptor) + self._seq = 0 + self.unexpected = unexpected_msg_handler + def send(self, msg): + _nl_send(self.descriptor, msg) + def recv(self): + msgs = [] + offset = 0 + contents, (nlpid, nlgrps) = _nl_recv(self.descriptor) + # XXX: python doesn't give us message flags, check + # len(contents) vs. msglen for TRUNC + while (offset < len(contents)) : + msglen, msg_type, flags, seq, pid = struct.unpack("IHHII", contents[offset:offset + 16]) + + msg = Message(msg_type, flags, seq, contents[offset + 16:offset + msglen]) + msg.pid = pid + if msg.type == NLMSG_ERROR: + errno = -struct.unpack("i", msg.payload[:4])[0] + if errno != 0: + err = OSError("Netlink error: %s (%d)" % (os.strerror(errno), errno)) + err.errno = errno + raise err + msgs.append(msg) + offset += msglen + return msgs + + def seq(self): + self._seq += 1 + return self._seq + +def parse_attributes(data): + attrs = {} + while len(data): + attr_len, attr_type = struct.unpack("HH", data[:4]) + attrs[attr_type] = Attr(attr_type, data[4:attr_len]) + attr_len = ((attr_len + 4 - 1) & ~3 ) + data = data[attr_len:] + return attrs diff --git a/ofed_scripts/utils/setup.py b/ofed_scripts/utils/setup.py new file mode 100755 index 0000000..370181d --- /dev/null +++ b/ofed_scripts/utils/setup.py @@ -0,0 +1,23 @@ +from distutils.core import setup +from subprocess import Popen, PIPE +from sys import argv + +# I would absolutely *LOVE* to be informed of a sexier way to do this, +# preferably without hard-coding Ubuntu as a special case... +try: + if 'Ubuntu\n' in Popen(('lsb_release', '-si'), + stdout=PIPE).communicate(): + argv.append('--install-layout=deb') +except OSError: + pass + + + +setup(name='ofed-le-utils', + version='1.0.3', + author='Amir Vadai', + author_email='amirv@mellanox.co.il', + url='www.mellanox.co.il', + scripts=['mlnx_qos', 'tc_wrap.py', 'mlnx_perf', 'mlnx_get_vfs.pl', 'mlnx_qcn', 'mlnx_dump_parser', 'mlx_fs_dump'], + py_modules=['netlink', 'dcbnetlink', 'genetlink'], + ) diff --git a/ofed_scripts/utils/tc_wrap.py b/ofed_scripts/utils/tc_wrap.py new file mode 100755 index 0000000..3045196 --- /dev/null +++ b/ofed_scripts/utils/tc_wrap.py @@ -0,0 +1,225 @@ +#!/usr/bin/python + +import sys +import os +import re +from subprocess import Popen, PIPE +from collections import defaultdict +from optparse import OptionParser + +tctool = 'tc' +port_num = 1 +max_tc_num = "8" +skprio2tos = { 0 : 0, 2 : 8, 4 : 24, 6 : 16 } + +class skprio2up: + def __init__(self, path, intf): + self.path = path + self.map = [] + self.intf = intf + self.up2skprio = defaultdict(list) + if (options.skprio_up is not None): + self.parse_args(options.skprio_up.split(",")) + + + def get_tagged(self): + output = Popen('grep -H "EGRESS" /proc/net/vlan/' + self.intf + + "* 2> /dev/null", shell=True, bufsize=4096, + stdout=PIPE).stdout + for line in output: + param, val=line.strip().split(":", 1) + vlan = param.split('.')[-1] + for item in val.split(":", 1)[1].split(): + skprio, up = item.split(':') + skprio = int(skprio) + str = "%d (vlan %s" % (skprio, vlan) + if skprio2tos.get(skprio): + str += " tos: %d" % (skprio2tos[skprio]) + str += ")" + self.up2skprio[int(up)].append(str) + + def refresh(self): + skprio = 0 + for up in self.map: + s = str(skprio) + if skprio2tos.get(skprio): + s += " (tos: %s)" % (str(skprio2tos[skprio])) + self.up2skprio[int(up)].append(s) + skprio += 1 + self.get_tagged() + + def parse_args(self, new): + for i, up in enumerate(new): + _up = int(up) + if (_up > 8 or _up < 0): + print "Bad user prio: %s - should be in the range: 0-7" % up + sys.exit(1) + + self.map.append(up) + + def set(self, new): + self.parse_args(new) + self.refresh() + f = open(self.path, "w") + f.write(" ".join(self.map).strip()) + f.close() + + def get(self): + f = open(self.path, "r") + self.map = f.read().split() + f.close() + +class tcnum: + def __init__(self, intf): + self.map = [] + self.intf = intf + self.tc_num = str(8) + + def set(self, dummy): + raise NotImplementedError("Setting skprio<=>up mapping is not implemented yet") + +class tcnum_sysfs(tcnum): + def __init__(self, path, intf): + tcnum.__init__(self, intf) + self.path = path + + def set(self, new): + self.get() + if self.tc_num == int(new): + return + f = open(self.path, "w") + f.write(new) + f.close() + + def get(self): + f = open(self.path, "r") + self.tc_num = int(f.read().strip()) + f.close() + + +class tcnum_mqprio(tcnum): + def __init__(self, intf): + tcnum.__init__(self, intf) + + + def set(self, new): + try: + output = Popen("%s qdisc del dev %s root" % (tctool, self.intf), + shell=True, + bufsize=4096, stdout=PIPE, stderr=PIPE).stdout + + output = Popen("%s qdisc add dev %s root mqprio num_tc %s" % (tctool, + self.intf, new), + shell=True, + bufsize=4096, stdout=PIPE).stdout + + except: + print "QoS is not supported via mqprio" + sys.exit(1) + + def get(self): + empty = True + output = Popen(tctool + " qdisc show dev " + self.intf, shell=True, + bufsize=4096, stdout=PIPE).stdout + + for line in output: + empty=False + m = re.search(r'tc (\d)', line) + if m: + self.tc_num = m.group(1) + + if (empty): + raise IOError("tc tool returned empty output") + + + +if __name__ == "__main__": + parser = OptionParser(usage="%prog -i [options]", version="%prog 1.0") + + parser.add_option("-i", "--interface", dest="intf", + help="Interface name") + + parser.add_option("-u", "--skprio_up", dest="skprio_up", + help="maps sk_prio to priority for RoCE. LIST is <=16 comma separated priority. " + + "index of element is sk_prio.") + + parser.add_option("-n", "--show_tc_num", action="store_true", default=False, + dest="show_tc_num", help="Show number of TCs for the interface and exists.") + + (options, args) = parser.parse_args() + + if (options.intf == None): + print "Interface name is required" + parser.print_usage() + + sys.exit(1) + + output = Popen("ls /sys/class/net/%s/device/infiniband/ 2> /dev/null"%options.intf, shell=True, + bufsize=4096, stdout=PIPE).stdout + + mlx_dev = None + for line in output: + m = re.search(r'mlx\d_\d', line) + if m: + mlx_dev = m.group(0) + + if not mlx_dev and options.skprio_up: + print "Couldn't find RDMA device for %s. Can't set skprio."%options.intf + sys.exit(1) + + empty = True + output = Popen("ibdev2netdev", shell=True, + bufsize=4096, stdout=PIPE).stdout + + for line in output: + m = re.search(r'port (\d+) ==> (\w+)', line) + if m: + if (m.group(2) == options.intf): + empty = False + port_num = m.group(1) + if (empty): + print "Could not find interface %s in ibdev2netdev output"%options.intf + sys.exit(1) + + # try using sysfs - if not exist fallback to tc tool + tc_num_path = "/sys/class/net/%s/qos/tc_num"%options.intf + skprio2up_path = "/sys/class/infiniband/%s/ports/%s/skprio2up"%(mlx_dev, port_num) + + try: + if (os.path.exists(tc_num_path)): + tcnum = tcnum_sysfs(tc_num_path, options.intf) + else: + tcnum = tcnum_mqprio(options.intf) + + except Exception, e: + print e + sys.exit(1) + + tcnum.set(max_tc_num) + + try: + skprio2up = skprio2up(skprio2up_path, options.intf) + + if (os.path.exists(skprio2up_path) and options.skprio_up is not None): + skprio2up.set(options.skprio_up.split(",")) + else: + if (options.skprio_up is not None): + print "skprio2up is available only for RoCE in kernels that don't support set_egress_map" + + except Exception, e: + print e + sys.exit(1) + + if options.show_tc_num: + print tcnum.tc_num + sys.exit(0) + + tcnum.get() + print "Traffic classes are set to %s"%tcnum.tc_num + + skprio2up.refresh() + + for up in range(8): + print "UP ", up + for skprio in skprio2up.up2skprio[int(up)]: + print "\tskprio: " + skprio