[rqd] Fix permission issues when becoming a user (#1496)

Changes implemented by #1416 impacted the locking mechanism for handling
permissions on rqd, causing multiple threads to compete for permission
settings and access to passwords.

Besides fixing the bug, this PR also introduces a fix for a potential
security issue that would allow frames to run as root if the frame user
didn't exist and the process to create this user fails.

rqd/rqd/rqcore.py

@@ -295,10 +295,6 @@ def runLinux(self):
 
         self.__createEnvVariables()
         self.__writeHeader()
-        if rqd.rqconstants.RQD_CREATE_USER_IF_NOT_EXISTS:
-            rqd.rqutil.permissionsHigh()
-            rqd.rqutil.checkAndCreateUser(runFrame.user_name, runFrame.uid, runFrame.gid)
-            rqd.rqutil.permissionsLow()
 
         tempStatFile = "%srqd-stat-%s-%s" % (self.rqCore.machine.getTempPath(),
                                              frameInfo.frameId,
@@ -508,17 +504,17 @@ def run(self):
             runFrame.log_dir_file = os.path.join(runFrame.log_dir, runFrame.log_file)
 
             try:  # Exception block for all exceptions
-
-                # Change to frame user if needed:
-                if runFrame.HasField("uid"):
-                    # Do everything as launching user:
-                    runFrame.gid = rqd.rqconstants.LAUNCH_FRAME_USER_GID
-                    rqd.rqutil.permissionsUser(runFrame.uid, runFrame.gid)
-
+                # Ensure permissions return to Low after this block
                 try:
-                    #
-                    # Setup proc to allow launching of frame
-                    #
+                    if rqd.rqconstants.RQD_CREATE_USER_IF_NOT_EXISTS and runFrame.HasField("uid"):
+                        rqd.rqutil.checkAndCreateUser(runFrame.user_name,
+                                                      runFrame.uid,
+                                                      runFrame.gid)
+                        # Do everything as launching user:
+                        runFrame.gid = rqd.rqconstants.LAUNCH_FRAME_USER_GID
+                        rqd.rqutil.permissionsUser(runFrame.uid, runFrame.gid)
+
+                    # Setup frame logging
                     try:
                         self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file)
                         self.rqlog.waitForFile()

rqd/rqd/rqutil.py

@@ -79,14 +79,17 @@ def permissionsHigh():
     """Sets the effective gid/uid to processes original values (root)"""
     if platform.system() == "Windows" or not rqd.rqconstants.RQD_BECOME_JOB_USER:
         return
-    with PERMISSIONS:
-        os.setegid(os.getgid())
-        os.seteuid(os.getuid())
-        try:
-            os.setgroups(HIGH_PERMISSION_GROUPS)
-        # pylint: disable=broad-except
-        except Exception:
-            pass
+    # PERMISSIONS gets locked here and unlocked at permissionsLow()
+    # therefore 'with' should not be used here
+    # pylint: disable=consider-using-with
+    PERMISSIONS.acquire()
+    os.setegid(os.getgid())
+    os.seteuid(os.getuid())
+    try:
+        os.setgroups(HIGH_PERMISSION_GROUPS)
+    # pylint: disable=broad-except
+    except Exception:
+        pass
 
 
 def permissionsLow():
@@ -114,10 +117,9 @@ def permissionsUser(uid, gid):
             groups = [20] + [g.gr_gid for g in grp.getgrall() if username in g.gr_mem]
             os.setgroups(groups)
         # pylint: disable=broad-except
-        except Exception:
-            pass
-        os.setegid(gid)
-        os.seteuid(uid)
+        finally:
+            os.setegid(gid)
+            os.seteuid(uid)
 
 
 def __becomeRoot():
@@ -134,24 +136,29 @@ def __becomeRoot():
 
 def checkAndCreateUser(username, uid=None, gid=None):
     """Check to see if the provided user exists, if not attempt to create it."""
-    # TODO(gregdenton): Add Windows and Mac support here. (Issue #61)
-    if not rqd.rqconstants.RQD_BECOME_JOB_USER:
+    if platform.system() == "Windows" or not rqd.rqconstants.RQD_BECOME_JOB_USER:
         return
     try:
         pwd.getpwnam(username)
         return
     except KeyError:
-        cmd = [
-            'useradd',
-            '-p', str(uuid.uuid4()),  # generate a random password
-        ]
-        if uid:
-            cmd += ['-u', str(uid)]
-        if gid:
-            cmd += ['-g', str(gid)]
-        cmd.append(username)
-        log.info("Frame's username not found on host. Adding user with: %s", cmd)
-        subprocess.check_call(cmd)
+        # Multiple processes can be trying to access passwd, permissionHigh and
+        # permissionLow handle locking
+        permissionsHigh()
+        try:
+            cmd = [
+                'useradd',
+                '-p', str(uuid.uuid4()),  # generate a random password
+            ]
+            if uid:
+                cmd += ['-u', str(uid)]
+            if gid:
+                cmd += ['-g', str(gid)]
+            cmd.append(username)
+            log.info("Frame's username not found on host. Adding user with: %s", cmd)
+            subprocess.check_call(cmd)
+        finally:
+            permissionsLow()
 
 
 def getHostIp():