[rqd] Rqd tests were not being executed (AcademySoftwareFoundation#1560)

The unit tests were not reporting test failures and interruptions as
expected, which caused us to be running with failed unit tests for a
long time.

This PR replaces `unittest` with `pytest` for RQD and fixes some of the
relevant unit tests.

---------

Signed-off-by: Diego Tavares <[email protected]>

ci/run_python_tests.sh

@@ -26,7 +26,7 @@ python -m unittest discover -s pycue/tests -t pycue -p "*.py"
 PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
 PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
 PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
-python -m unittest discover -s rqd/tests -t rqd -p "*.py"
+python -m pytest rqd/tests
 
 
 # Xvfb no longer supports Python 2.

requirements.txt

@@ -15,6 +15,7 @@ pylint==2.15.10;python_version>="3.7"
 pynput==1.7.6
 PyYAML==5.1
 six==1.16.0
+pytest==8.3.3
 
 # Optional requirements
 # Sentry support for rqd

rqd/rqd/rqcore.py

@@ -131,6 +131,7 @@ def _createCommandFile(self, command):
         @rtype:  string
         @return: Command file location"""
         # TODO: this should use tempfile to create the files and clean them up afterwards
+        commandFile = None
         try:
             if platform.system() == "Windows":
                 rqd_tmp_dir = os.path.join(tempfile.gettempdir(), 'rqd')

rqd/rqd/rqmachine.py

@@ -215,10 +215,23 @@ def __updateGpuAndLlu(self, frame):
             frame.lluTime = int(stat)
 
     def _getStatFields(self, pidFilePath):
+        """ Read stats file and return list of values
+        Stats file can star with these formats:
+         - 105 name ...
+         - 105 (name) ...
+         - 105 (name with space) ...
+         - 105 (name with) (space and parenthesis) ...
+        """
         with open(pidFilePath, "r", encoding='utf-8') as statFile:
-            stats = statFile.read().split()
-            stats[1] = stats[1].strip('()')
-            return stats
+            txt = statFile.read()
+            try:
+                open_par_index = txt.index('(')
+                close_par_index = txt.rindex(')')
+                name = txt[open_par_index:close_par_index].strip("()")
+                reminder = (txt[0:open_par_index] + txt[close_par_index + 1:]).split()
+                return reminder[0:1] + [name] + reminder[1:]
+            except ValueError:
+                return txt.split()
 
     def rssUpdate(self, frames):
         """Updates the rss and maxrss for all running frames"""
@@ -269,6 +282,9 @@ def rssUpdate(self, frames):
                         # Fetch swap usage
                         "swap": self._getProcSwap(pid),
                     }
+
+                    # TODO: Improve this logic to avoid collecting data from all running procs.
+                    # instead, focus on the monitored procs hierarchy
                     # cmdline:
                     p = psutil.Process(int(pid))
                     pids[pid]["cmd_line"] = p.cmdline()

rqd/rqd/rqnimby.py

@@ -58,7 +58,7 @@ def getNimby(rqCore):
                 # Ideally ImportError could be used here, but pynput
                 # can throw other kinds of exception while trying to
                 # access runpy components
-                log.exception("Failed to import pynput, falling back to Select module")
+                log.debug("Failed to import pynput, falling back to Select module")
                 # Still enabling the application start as hosts can be manually locked
                 # using the API/GUI
                 return NimbyNop(rqCore)
@@ -242,8 +242,7 @@ def openEvents(self):
                 if device.startswith("event") or device.startswith("mice"):
                     try:
                         # pylint: disable=consider-using-with
-                        self.fileObjList.append(open("/dev/input/%s" % device, "rb",
-                                                     encoding='utf-8'))
+                        self.fileObjList.append(open("/dev/input/%s" % device, "rb"))
                     except IOError:
                         # Bad device found
                         log.exception("IOError: Failed to open /dev/input/%s", device)

rqd/tests/rqconstants_tests.py → rqd/tests/rqconstants_test.py

@@ -39,7 +39,7 @@
 import rqd.rqutil
 import rqd.compiled_proto.report_pb2
 
-from .rqmachine_tests import (
+from .rqmachine_test import (
     CPUINFO,
     LOADAVG_LOW_USAGE,
     MEMINFO_MODERATE_USAGE,
@@ -121,26 +121,14 @@ def makeRqMachine(self):
         """
 [Override]
 DEFAULT_FACILITY =  test_facility
+RQD_TAGS =  test_tag1 test_tag2  test_tag3
 """,
     )
     def test_facility(self):
         self.assertEqual(rqd.rqconstants.DEFAULT_FACILITY, "test_facility")
 
         machine = self.makeRqMachine()
         self.assertEqual(machine.renderHost.facility, "test_facility")
-
-    @MockConfig(
-        tempdir,
-        """
-[Override]
-RQD_TAGS =  test_tag1 test_tag2  test_tag3
-""",
-    )
-    def test_tags(self):
-        self.assertEqual(rqd.rqconstants.RQD_TAGS, "test_tag1 test_tag2  test_tag3")
-
-        machine = self.makeRqMachine()
-        self.assertEqual(machine.renderHost.facility, "cloud")
         self.assertTrue(
             set(["test_tag1", "test_tag2", "test_tag3"]).issubset(
                 machine.renderHost.tags