[Mellanox] Adjust test cases for fan led support

tests/platform/mellanox/mellanox_thermal_control_test_helper.py

@@ -255,6 +255,12 @@ def __init__(self, mock_helper, naming_rule, index):
         """
         self.index = index
         self.helper = mock_helper
+        dut_hwsku = self.helper.dut.facts["hwsku"]
+        if SWITCH_MODELS[dut_hwsku]['fans']['hot_swappable']:
+            self.name = 'drawer{}'.format(index)
+        else:
+            self.name = 'N/A'
+        self.fan_data_list = []
         self.mocked_presence = None
         self.mocked_direction = None
         if 'presence' in naming_rule:
@@ -336,7 +342,15 @@ def get_status_led(self):
         else:
             assert 0, 'Invalid FAN led color for FAN: {}, green={}, red={}'.format(self.name, green_led_value,
                                                                                    red_led_value)
+    def get_expect_led_color(self):
+        if self.mocked_presence == 'Not Present':
+            return 'red'
+
+        for fan_data in self.fan_data_list:
+            if fan_data.get_expect_led_color() == 'red':
+                return 'red'
 
+        return 'green'
 
 class FanData:
     """
@@ -346,6 +360,9 @@ class FanData:
     # MAX PWM value.
     PWM_MAX = 255
 
+    # Speed tolerance
+    SPEED_TOLERANCE = 0.2
+
     def __init__(self, mock_helper, naming_rule, index):
         """
         Constructor of FAN data.
@@ -436,7 +453,25 @@ def get_target_speed(self):
         target_speed = int(round(pwm * 100.0 / FanData.PWM_MAX))
         return target_speed
 
+    def get_expect_led_color(self):
+        """
+        Get expect LED color.
+        :return: Return the LED color that this FAN expect to have.
+        """
+        if self.mocked_status == 'Not OK':
+            return 'red'
+
+        target_speed = self.get_target_speed()
+        mocked_speed = int(self.mocked_speed)
+        if mocked_speed > target_speed * (1 + FanData.SPEED_TOLERANCE):
+            return 'red'
+
+        if mocked_speed < target_speed * (1 - FanData.SPEED_TOLERANCE):
+            return 'red'
 
+        return 'green'
+
+
 class TemperatureData:
     """
     Data mocker of a thermal.
@@ -531,6 +566,7 @@ def __init__(self, dut):
         """
         FanStatusMocker.__init__(self, dut)
         self.mock_helper = MockerHelper(dut)
+        self.drawer_list = []
         self.expected_data = {}
 
     def deinit(self):
@@ -555,6 +591,7 @@ def mock_data(self):
             try:
                 if (fan_index - 1) % MockerHelper.FAN_NUM_PER_DRAWER == 0:
                     drawer_data = FanDrawerData(self.mock_helper, naming_rule, drawer_index)
+                    self.drawer_list.append(drawer_data)
                     drawer_index += 1
                     presence = random.randint(0, 1)
                     drawer_data.mock_presence(presence)
@@ -563,11 +600,14 @@ def mock_data(self):
                         presence = 1
 
                 fan_data = FanData(self.mock_helper, naming_rule, fan_index)
+                drawer_data.fan_data_list.append(fan_data)
                 fan_index += 1
                 if presence == 1:
                     fan_data.mock_status(random.randint(0, 1))
                     fan_data.mock_speed(random.randint(0, 100))
                     self.expected_data[fan_data.name] = [
+                        drawer_data.name,
+                        'N/A', # update this value later 
                         fan_data.name,
                         '{}%'.format(fan_data.mocked_speed),
                         drawer_data.mocked_direction,
@@ -576,6 +616,8 @@ def mock_data(self):
                     ]
                 else:
                     self.expected_data[fan_data.name] = [
+                        drawer_data.name,
+                        'red',
                         fan_data.name,
                         'N/A',
                         'N/A',
@@ -586,6 +628,13 @@ def mock_data(self):
                 logging.info('Failed to mock fan data: {}'.format(e))
                 continue
 
+        # update led color here
+        for drawer_data in self.drawer_list:
+            for fan_data in drawer_data.fan_data_list:
+                if drawer_data.mocked_presence == 'Present':
+                    expected_data = self.expected_data[fan_data.name]
+                    expected_data[1] = drawer_data.get_expect_led_color()
+
         dut_hwsku = self.mock_helper.dut.facts["hwsku"]
         psu_count = SWITCH_MODELS[dut_hwsku]["psus"]["number"]
         naming_rule = FAN_NAMING_RULE['psu_fan']
@@ -597,6 +646,8 @@ def mock_data(self):
                 fan_data.mock_speed(speed)
 
                 self.expected_data[fan_data.name] = [
+                    'N/A',
+                    '',
                     fan_data.name,
                     '{}RPM'.format(fan_data.mocked_speed),
                     NOT_AVAILABLE,
@@ -610,14 +661,16 @@ def mock_data(self):
     def check_result(self, actual_data):
         """
         Check actual data with mocked data.
-        :param actual_data: A dictionary contains actual command line data. Key of the dictionary  is FAN name. Value
+        :param actual_data: A dictionary contains actual command line data. Key of the dictionary is FAN name. Value
                             of the dictionary is a list of field values for a line of FAN data.
         :return: True if match else False.
         """
         for name, fields in self.expected_data.items():
             if name in actual_data:
                 actual_fields = actual_data[name]
                 for i, expected_field in enumerate(fields):
+                    if name.find('psu') != -1 and i ==1: 
+                        continue # skip led status check for PSU because we don't mock it
                     if expected_field != actual_fields[i]:
                         logging.error('Check fan status for {} failed, ' \
                                      'expected: {}, actual: {}'.format(name, expected_field, actual_fields[i]))
@@ -844,6 +897,7 @@ def mock_all_normal(self):
 
         for fan_data in self.fan_data_list:
             try:
+                fan_data.mock_status(0)
                 fan_data.mock_speed(AbnormalFanMocker.TARGET_SPEED_VALUE)
                 fan_data.mock_target_speed(AbnormalFanMocker.TARGET_SPEED_VALUE)
             except SysfsNotExistError as e:
@@ -854,6 +908,7 @@ def mock_normal(self):
         Change the mocked FAN status to 'Present' and normal speed.
         :return:
         """
+        self.mock_status(0)
         self.mock_presence()
         self.mock_normal_speed()
 
@@ -873,6 +928,15 @@ def mock_presence(self):
         self.fan_drawer_data.mock_presence(1)
         self.expect_led_color = 'green'
 
+    def mock_status(self, status):
+        """
+        Change the mocked FAN status to good or bad
+        :param status: bool value indicate the target status of the FAN.
+        :return:
+        """
+        self.fan_data.mock_status(0 if status else 1)
+        self.expect_led_color = 'green' if status else 'red'
+
     def mock_over_speed(self):
         """
         Change the mocked FAN speed to faster than target speed and exceed speed tolerance.

tests/platform/test_platform_info.py

@@ -32,10 +32,14 @@
 LOG_EXPECT_POLICY_FILE_INVALID = '.*Caught exception while initializing thermal manager.*'
 LOG_EXPECT_FAN_REMOVE_RE = '.*Fan removed warning:.*'
 LOG_EXPECT_FAN_REMOVE_CLEAR_RE = '.*Fan removed warning cleared:.*'
-LOG_EXPECT_FAN_UNDER_SPEED_RE = '.*Fan under speed warning:.*'
-LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE = '.*Fan under speed warning cleared:.*'
-LOG_EXPECT_FAN_OVER_SPEED_RE = '.*Fan over speed warning:*'
-LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE = '.*Fan over speed warning cleared:.*'
+LOG_EXPECT_FAN_FAULT_RE = '.*Fan fault warning:.*'
+LOG_EXPECT_FAN_FAULT_CLEAR_RE = '.*Fan fault warning cleared:.*'
+LOG_EXPECT_FAN_UNDER_SPEED_RE = '.*Fan low speed warning:.*'
+LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE = '.*Fan low speed warning cleared:.*'
+LOG_EXPECT_FAN_OVER_SPEED_RE = '.*Fan high speed warning:*'
+LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE = '.*Fan high speed warning cleared:.*'
+LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE = '.*Insufficient number of working fans warning:.*'
+LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE = '.*Insufficient number of working fans warning cleared:.*'
 
 
 def check_sensord_status(ans_host):
@@ -305,7 +309,7 @@ def test_show_platform_syseeprom(testbed_devices):
 def check_show_platform_fanstatus_output(lines):
     """
     @summary: Check basic output of 'show platform fan'. Expect output are:
-              "Fan Not detected" or a table of fan status data with 6 columns.
+              "Fan Not detected" or a table of fan status data with 8 columns.
     """
     assert len(lines) > 0, 'There must be at least one line output for show platform fans'
     if len(lines) == 1:
@@ -314,7 +318,7 @@ def check_show_platform_fanstatus_output(lines):
         assert len(lines) > 2, 'There must be at least two lines output for show platform fans if any FAN is detected'
         second_line = lines[1]
         field_ranges = get_field_range(second_line)
-        assert len(field_ranges) == 6, 'There must be 6 columns in output of show platform fans'
+        assert len(field_ranges) == 8, 'There must be 8 columns in output of show platform fans'
 
 
 def test_show_platform_fanstatus(testbed_devices, mocker_factory):
@@ -336,7 +340,7 @@ def test_show_platform_fanstatus(testbed_devices, mocker_factory):
     logging.info('Mock FAN status data...')
     mocker.mock_data()
     logging.info('Wait and check actual data with mocked FAN status data...')
-    result = check_cli_output_with_mocker(dut, mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+    result = check_cli_output_with_mocker(dut, mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
 
     assert result, 'FAN mock data mismatch'
 
@@ -525,38 +529,50 @@ def test_thermal_control_fan_status(testbed_devices, mocker_factory):
         time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
 
         if single_fan_mocker.is_fan_removable():
-            loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_RE]
+            loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE]
             with loganalyzer:
                 logging.info('Mocking an absence FAN...')
                 single_fan_mocker.mock_absence()
-                check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+                check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
 
-            loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_CLEAR_RE]
+            loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE]
             with loganalyzer:
                 logging.info('Make the absence FAN back to presence...')
                 single_fan_mocker.mock_presence()
-                check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+                check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
 
+        loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE]
+        with loganalyzer:
+            logging.info('Mocking a fault FAN...')
+            single_fan_mocker.mock_status(False)
+            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
+
+        loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE]
+        with loganalyzer:
+            logging.info('Mocking the fault FAN back to normal...')
+            single_fan_mocker.mock_status(True)
+            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
+
         loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_RE]
         with loganalyzer:
             logging.info('Mocking an over speed FAN...')
             single_fan_mocker.mock_over_speed()
-            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
 
         loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE]
         with loganalyzer:
             logging.info('Make the over speed FAN back to normal...')
             single_fan_mocker.mock_normal_speed()
-            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
 
         loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_RE]
         with loganalyzer:
             logging.info('Mocking an under speed FAN...')
             single_fan_mocker.mock_under_speed()
-            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
 
         loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE]
         with loganalyzer:
             logging.info('Make the under speed FAN back to normal...')
             single_fan_mocker.mock_normal_speed()
-            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME)
+            check_cli_output_with_mocker(dut, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)

tests/platform/thermal_control_test_helper.py

@@ -174,6 +174,14 @@ def mock_presence(self):
         """
         pass
 
+    def mock_status(self, status):
+        """
+        Change the mocked FAN status to good or bad
+        :param status: bool value indicate the target status of the FAN.
+        :return:
+        """
+        pass
+
     def mock_normal_speed(self):
         """
         Change the mocked FAN speed to a normal value.
@@ -254,7 +262,7 @@ def get_fields(line, field_ranges):
     return fields
 
 
-def check_cli_output_with_mocker(dut, mocker_object, command, max_wait_time):
+def check_cli_output_with_mocker(dut, mocker_object, command, max_wait_time, key_index=0):
     """
     Check the command line output matches the mocked data.
     :param dut: DUT object representing a SONiC switch under test. 
@@ -273,7 +281,7 @@ def check_cli_output_with_mocker(dut, mocker_object, command, max_wait_time):
     actual_data = {}
     for line in output["stdout_lines"][2:]:
         fields = get_fields(line, field_ranges)
-        actual_data[fields[0]] = fields
+        actual_data[fields[key_index]] = fields
 
     return mocker_object.check_result(actual_data)