[pytest/hash] Add hash keys: 'src-mac', 'dst-mac', 'ip-proto', 'vlan-…

…id'. (#1512)

* Revise according to the following review comments:
1. Not enable src-mac, dst-mac and vlan-id hash keys by default.
2. Get all untag vlan ports move to a separate function.
3. setup vlans for hash_key vlan-id test when vlan-id in hash_keys.

ansible/roles/test/files/ptftests/hash_test.py

@@ -5,11 +5,8 @@
 #---------------------------------------------------------------------
 # Global imports
 #---------------------------------------------------------------------
-import ipaddress
 import logging
 import random
-import socket
-import sys
 
 from ipaddress import ip_address, ip_network
 
@@ -48,14 +45,17 @@ def setUp(self):
         '''
         self.dataplane = ptf.dataplane_instance
         self.fib = fib.Fib(self.test_params['fib_info'])
+        self.testbed_type = self.test_params['testbed_type']
         self.router_mac = self.test_params['router_mac']
         self.in_ports = self.test_params['in_ports']
 
         self.src_ip_range = [unicode(x) for x in self.test_params['src_ip_range'].split(',')]
         self.dst_ip_range = [unicode(x) for x in self.test_params['dst_ip_range'].split(',')]
         self.src_ip_interval = lpm.LpmDict.IpInterval(ip_address(self.src_ip_range[0]), ip_address(self.src_ip_range[1]))
         self.dst_ip_interval = lpm.LpmDict.IpInterval(ip_address(self.dst_ip_range[0]), ip_address(self.dst_ip_range[1]))
+        self.vlan_ids = self.test_params.get('vlan_ids', [])
         self.hash_keys = self.test_params.get('hash_keys', ['src-ip', 'dst-ip', 'src-port', 'dst-port'])
+        self.dst_macs = self.test_params.get('dst_macs', [])    # TODO
 
         self.balancing_range = self.test_params.get('balancing_range', self.DEFAULT_BALANCING_RANGE)
 
@@ -108,27 +108,38 @@ def check_ipv4_route(self, hash_key, in_port, dst_port_list):
         @param in_port: index of port to use for sending packet to switch
         @param dst_port_list: list of ports on which to expect packet to come back from the switch
         '''
-        src_mac = self.dataplane.get_mac(0, 0)
+        base_mac = self.dataplane.get_mac(0, 0)
         ip_src = self.src_ip_interval.get_random_ip() if hash_key == 'src-ip' else self.src_ip_interval.get_first_ip()
         ip_dst = self.dst_ip_interval.get_random_ip() if hash_key == 'dst-ip' else self.dst_ip_interval.get_first_ip()
         sport = random.randint(0, 65535) if hash_key == 'src-port' else 1234
         dport = random.randint(0, 65535) if hash_key == 'dst-port' else 80
+        src_mac = (base_mac[:-5] + "%02x" % random.randint(0, 255) + ":" + "%02x" % random.randint(0, 255)) if hash_key == 'src-mac' else base_mac
+        dst_mac = random.choice(self.dst_macs) if hash_key == 'dst-mac' else self.router_mac
+        vlan_id = random.choice(self.vlan_ids) if hash_key == 'vlan-id' else 0
+        ip_proto = random.randint(100, 200) if hash_key == 'ip-proto' else None
 
-        pkt = simple_tcp_packet(
-                            eth_dst=self.router_mac,
+        pkt = simple_tcp_packet(pktlen=100 if vlan_id == 0 else 104,
+                            eth_dst=dst_mac,
                             eth_src=src_mac,
+                            dl_vlan_enable=False if vlan_id == 0 else True,
+                            vlan_vid=vlan_id,
+                            vlan_pcp=0,
                             ip_src=ip_src,
                             ip_dst=ip_dst,
                             tcp_sport=sport,
                             tcp_dport=dport,
                             ip_ttl=64)
         exp_pkt = simple_tcp_packet(
-                            eth_src=self.router_mac,
+                            eth_src=dst_mac,
                             ip_src=ip_src,
                             ip_dst=ip_dst,
                             tcp_sport=sport,
                             tcp_dport=dport,
                             ip_ttl=63)
+
+        if hash_key == 'ip-proto':
+            pkt['IP'].proto = ip_proto
+            exp_pkt['IP'].proto = ip_proto
         masked_exp_pkt = Mask(exp_pkt)
         masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst")
 
@@ -146,27 +157,39 @@ def check_ipv6_route(self, hash_key, in_port, dst_port_list):
         @param dst_port_list: list of ports on which to expect packet to come back from the switch
         @return Boolean
         '''
-        src_mac = self.dataplane.get_mac(0, 0)
+        base_mac = self.dataplane.get_mac(0, 0)
         ip_src = self.src_ip_interval.get_random_ip() if hash_key == 'src-ip' else self.src_ip_interval.get_first_ip()
         ip_dst = self.dst_ip_interval.get_random_ip() if hash_key == 'dst-ip' else self.dst_ip_interval.get_first_ip()
         sport = random.randint(0, 65535) if hash_key == 'src-port' else 1234
         dport = random.randint(0, 65535) if hash_key == 'dst-port' else 80
+        src_mac = (base_mac[:-5] + "%02x" % random.randint(0, 255) + ":" + "%02x" % random.randint(0, 255)) if hash_key == 'src-mac' else base_mac
+        dst_mac = random.choice(self.dst_macs) if hash_key == 'dst-mac' else self.router_mac
+        vlan_id = random.choice(self.vlan_ids) if hash_key == 'vlan-id' else 0
+        ip_proto = random.randint(100, 200) if hash_key == "ip-proto" else None
 
-        pkt = simple_tcpv6_packet(
-                                eth_dst=self.router_mac,
+        pkt = simple_tcpv6_packet(pktlen=100 if vlan_id == 0 else 104,
+                                eth_dst=dst_mac,
                                 eth_src=src_mac,
+                                dl_vlan_enable=False if vlan_id == 0 else True,
+                                vlan_vid=vlan_id,
+                                vlan_pcp=0,
                                 ipv6_dst=ip_dst,
                                 ipv6_src=ip_src,
                                 tcp_sport=sport,
                                 tcp_dport=dport,
                                 ipv6_hlim=64)
         exp_pkt = simple_tcpv6_packet(
-                                eth_src=self.router_mac,
+                                eth_src=dst_mac,
                                 ipv6_dst=ip_dst,
                                 ipv6_src=ip_src,
                                 tcp_sport=sport,
                                 tcp_dport=dport,
                                 ipv6_hlim=63)
+
+        if hash_key == 'ip-proto':
+            pkt['IPv6'].nh = ip_proto
+            exp_pkt['IPv6'].nh = ip_proto
+
         masked_exp_pkt = Mask(exp_pkt)
         masked_exp_pkt.set_do_not_care_scapy(scapy.Ether,"dst")
 

tests/fib/test_fib.py

@@ -1,19 +1,21 @@
 import pytest
-from netaddr import *
 import time
 import json
 import logging
-import requests
 from ptf_runner import ptf_runner
 from datetime import datetime 
 
 logger = logging.getLogger(__name__)
 
-HASH_KEYS = ['src-ip', 'dst-ip', 'src-port', 'dst-port', 'ingress-port']
+# Usually src-mac, dst-mac, vlan-id are optional hash keys. Not all the platform supports these optional hash keys. Not enable these three by default. 
+# HASH_KEYS = ['src-ip', 'dst-ip', 'src-port', 'dst-port', 'ingress-port', 'src-mac', 'dst-mac', 'ip-proto', 'vlan-id']
+HASH_KEYS = ['src-ip', 'dst-ip', 'src-port', 'dst-port', 'ingress-port', 'ip-proto']
 SRC_IP_RANGE = ['8.0.0.0', '8.255.255.255']
 DST_IP_RANGE = ['9.0.0.0', '9.255.255.255']
 SRC_IPV6_RANGE = ['20D0:A800:0:00::', '20D0:A800:0:00::FFFF']
 DST_IPV6_RANGE = ['20D0:A800:0:01::', '20D0:A800:0:01::FFFF']
+VLANIDS = range(1032, 1279)
+VLANIP = '192.168.{}.1/24'
 
 g_vars = {}
 
@@ -59,6 +61,21 @@ def build_fib(duthost, config_facts, fibfile, t):
             else:
                 ofp.write("{} []\n".format(prefix))
 
+def get_vlan_untag_ports(config_facts):
+    """
+    get all untag vlan ports
+    """
+    vlan_untag_ports = []
+    vlans = config_facts.get('VLAN_INTERFACE', {}).keys()
+    for vlan in vlans:
+        vlan_member_info = config_facts.get('VLAN_MEMBER', {}).get(vlan, {})
+        if vlan_member_info:
+            for port_name, tag_mode in vlan_member_info.items():
+                if tag_mode['tagging_mode'] == 'untagged':
+                    vlan_untag_ports.append(port_name)
+
+    return vlan_untag_ports
+
 def get_router_interface_ports(config_facts, testbed):
     """
     get all physical ports associated with router interface (physical router interface, port channel router interface and vlan router interface)
@@ -73,13 +90,7 @@ def get_router_interface_ports(config_facts, testbed):
             for port_name in config_facts.get('PORTCHANNEL', {})[po_name]['members']:
                 portchannels_member_ports.append(port_name)
     if 't0' in testbed['topo']['name']:
-        vlans = config_facts.get('VLAN_INTERFACE', {}).keys()
-        for vlan in vlans:
-            vlan_member_info = config_facts.get('VLAN_MEMBER', {}).get(vlan, {})
-            if vlan_member_info:
-                for port_name, tag_mode in vlan_member_info.items():
-                    if tag_mode['tagging_mode'] == 'untagged':
-                        vlan_untag_ports.append(port_name)
+        vlan_untag_ports = get_vlan_untag_ports(config_facts)
 
     router_interface_ports = ports + portchannels_member_ports + vlan_untag_ports
 
@@ -145,6 +156,10 @@ def setup_hash(self, testbed, duthost, ptfhost):
         ptfhost.copy(src="ptftests", dest="/root")
         logging.info("run ptf test")
 
+        # TODO
+        if 'dst-mac' in self.hash_keys:
+            self.hash_keys.remove('dst-mac')
+
         # do not test load balancing on L4 port on vs platform as kernel 4.9
         # can only do load balance base on L3
         meta = config_facts.get('DEVICE_METADATA')
@@ -155,9 +170,30 @@ def setup_hash(self, testbed, duthost, ptfhost):
         g_vars['testbed_type'] = testbed['topo']['name']
         g_vars['router_mac'] = duthost.shell('sonic-cfggen -d -v \'DEVICE_METADATA.localhost.mac\'')["stdout_lines"][0].decode("utf-8")
 
+        vlan_untag_ports = get_vlan_untag_ports(config_facts)
         in_ports_name = get_router_interface_ports(config_facts, testbed)
         g_vars['in_ports'] = [config_facts.get('port_index_map', {})[p] for p in in_ports_name]
 
+        # add some vlan for hash_key vlan-id test
+        if 't0' in g_vars['testbed_type'] and 'vlan-id' in self.hash_keys:
+            for vlan in VLANIDS:
+                duthost.shell('config vlan add {}'.format(vlan))
+                for port in vlan_untag_ports:
+                    duthost.shell('config vlan member add {} {}'.format(vlan, port))
+                duthost.shell('config interface ip add Vlan{} '.format(vlan) + VLANIP.format(vlan%256))
+            time.sleep(5)
+
+        yield
+
+        # remove added vlan
+        if 't0' in g_vars['testbed_type'] and 'vlan-id' in self.hash_keys:
+            for vlan in VLANIDS:
+                duthost.shell('config interface ip remove Vlan{} '.format(vlan) + VLANIP.format(vlan%256))
+                for port in vlan_untag_ports:
+                    duthost.shell('config vlan member del {} {}'.format(vlan, port))
+                duthost.shell('config vlan del {}'.format(vlan))
+            time.sleep(5)
+
     def test_hash_ipv4(self, ptfhost):
         log_file = "/tmp/hash_test.HashTest.ipv4.{}.log".format(self.t)
         logging.info("PTF log file: %s" % log_file)
@@ -174,6 +210,7 @@ def test_hash_ipv4(self, ptfhost):
                         "src_ip_range": ",".join(src_ip_range),
                         "dst_ip_range": ",".join(dst_ip_range),
                         "in_ports": g_vars['in_ports'],
+                        "vlan_ids": VLANIDS,
                         "hash_keys": self.hash_keys },
                 log_file=log_file,
                 socket_recv_size=16384)
@@ -194,6 +231,7 @@ def test_hash_ipv6(self, ptfhost):
                         "src_ip_range": ",".join(src_ip_range),
                         "dst_ip_range": ",".join(dst_ip_range),
                         "in_ports": g_vars['in_ports'],
+                        "vlan_ids": VLANIDS,
                         "hash_keys": self.hash_keys },
                 log_file=log_file,
                 socket_recv_size=16384)