Changes to be able to use the new rendered views

ThreeD_Model.py

@@ -8,9 +8,11 @@
 
 
 class FaceModel:
-    def __init__(self, path, name):
+    def __init__(self, path, name, getMask):
         self.load_model(path, name)
-        self.eyemask = self.getEyeMask(width=8,plot=False)
+        self.eyemask = None
+        if getMask:
+            self.eyemask = self.getEyeMask(width=8,plot=False)
 
     def load_model(self, path, name):
         model = scio.loadmat(path)[name]

camera_calibration.py

@@ -5,19 +5,27 @@
 import math
 
 
-def estimate_camera(model3D, fidu_XY):
-    rmat, tvec = calib_camera(model3D, fidu_XY)
+def estimate_camera(model3D, fidu_XY, pose_db_on=False):
+    if pose_db_on:
+        rmat, tvec = calib_camera(model3D, fidu_XY, pose_db_on=True)
+        tvec = tvec.reshape(3,1)
+    else:
+        rmat, tvec = calib_camera(model3D, fidu_XY)
     RT = np.hstack((rmat, tvec))
     projection_matrix = model3D.out_A * RT
     return projection_matrix, model3D.out_A, rmat, tvec
 
-def calib_camera(model3D, fidu_XY):
+def calib_camera(model3D, fidu_XY, pose_db_on=False):
     #compute pose using refrence 3D points + query 2D point
     ## np.arange(68)+1 since matlab starts from 1
-    goodind = np.setdiff1d(np.arange(68)+1, model3D.indbad)
-    goodind=goodind-1
-    fidu_XY = fidu_XY[goodind,:]
-    ret, rvecs, tvec = cv2.solvePnP(model3D.model_TD, fidu_XY, model3D.out_A, None, None, None, False)
+    if pose_db_on:
+        rvecs = fidu_XY[0:3]
+        tvec = fidu_XY[3:6]
+    else:
+        goodind = np.setdiff1d(np.arange(68)+1, model3D.indbad)
+        goodind=goodind-1
+        fidu_XY = fidu_XY[goodind,:]
+        ret, rvecs, tvec = cv2.solvePnP(model3D.model_TD, fidu_XY, model3D.out_A, None, None, None, False)
     rmat, jacobian = cv2.Rodrigues(rvecs, None)
 
     inside = calc_inside(model3D.out_A, rmat, tvec, model3D.size_U[1], model3D.size_U[0], model3D.model_TD)

config.ini

@@ -28,13 +28,17 @@ background = yes
 symmetry = yes
 
 ## Parameters to get a bit far from the face when sampling the background
-scaleFaceX = 0.5
+scaleFaceX = 0.25
 
 ## Activate Near View Rendering
 nearView = yes
 
+## If we want to use the new Rendered Views with the head better visible and more poses
+newRenderedViews = no
+
 ## (used if symmetry is applied)
 [symmetry]
 
 ## If we want to flip the background in symmetry or no
-flipBackground = no
+flipBackground = no
+

demo.py

@@ -16,13 +16,19 @@
 this_path = os.path.dirname(os.path.abspath(__file__))
 opts = config.parse()
 ## 3D Models we are gonna use to to the rendering {0, -40, -75}
-pose_models = ['model3D_aug_-00','model3D_aug_-40','model3D_aug_-75',]
+newModels = opts.getboolean('renderer', 'newRenderedViews')
+if opts.getboolean('renderer', 'newRenderedViews'):
+    pose_models_folder = '/models3d_new/'
+    pose_models = ['model3D_aug_-00_00','model3D_aug_-22_00','model3D_aug_-40_00','model3D_aug_-55_00','model3D_aug_-75_00']
+else:
+    pose_models_folder = '/models3d/'
+    pose_models = ['model3D_aug_-00','model3D_aug_-40','model3D_aug_-75',]
 ## In case we want to crop the final image for each pose specified above/
 ## Each bbox should be [tlx,tly,brx,bry]
 resizeCNN = opts.getboolean('general', 'resizeCNN')
 cnnSize = opts.getint('general', 'cnnSize')
 if not opts.getboolean('general', 'resnetON'):
-    crop_models = [None,None,None]  # <-- with this no crop is done.     
+    crop_models = [None,None,None,None,None]  # <-- with this no crop is done.     
 else:
     #In case we want to produce images for ResNet
     resizeCNN=False #We can decide to resize it later using the CNN software or now here.
@@ -39,7 +45,7 @@ def demo():
     # http://sourceforge.net/projects/dclib/files/d.10/shape_predictor_68_face_landmarks.dat.bz2
     check.check_dlib_landmark_weights()
     ## Preloading all the models for speed
-    allModels = myutil.preload(this_path,pose_models,nSub)
+    allModels = myutil.preload(this_path,pose_models_folder,pose_models,nSub)
 
     for f in fileList:
         if '#' in f: #skipping comments
@@ -63,7 +69,7 @@ def demo():
             ## To refine the estimation of yaw. Yaw can change from model to model...
             img_display = img.copy()
             img, lmarks, yaw = myutil.flipInCase(img,lmarks,allModels)
-            listPose = myutil.decidePose(yaw,opts)
+            listPose = myutil.decidePose(yaw,opts, newModels)
             ## Looping over the poses
             for poseId in listPose:
             	posee = pose_models[poseId]

myutil.py

@@ -52,7 +52,7 @@ def isFrontal(pose):
 		return True
 	return False
 
-def preload(this_path, pose_models,nSub):
+def preload(this_path, pose_models_folder, pose_models,nSub):
     print '> Preloading all the models for efficiency'
     allModels= dict()
     for posee in pose_models:
@@ -61,7 +61,12 @@ def preload(this_path, pose_models,nSub):
             pose =   posee + '_' + str(subj).zfill(2) +'.mat'
             # load detections performed by dlib library on 3D model and Reference Image
             print "> Loading pose model in " + pose
-            model3D = ThreeD_Model.FaceModel(this_path + "/models3d/" + pose, 'model3D')
+            #model3D = ThreeD_Model.FaceModel(this_path + "/models3d_new/" + pose, 'model3D')
+            if '-00' in posee:
+                    model3D = ThreeD_Model.FaceModel(this_path + pose_models_folder + pose, 'model3D', True)
+            else:
+                    model3D = ThreeD_Model.FaceModel(this_path + pose_models_folder + pose, 'model3D', False)
+
             allModels[pose] = model3D
     return allModels
 
@@ -143,14 +148,49 @@ def show(img_display, img, lmarks, frontal_raw, \
     enter = raw_input("Press [enter] to continue.")
     plt.clf()
 
-def decidePose(yaw,opts):
-	if opts.getboolean('renderer', 'nearView'):
-		yaw = abs(yaw)
-		# If yaw is near-frontal we render everything
-		if yaw < 15:
-			return [0,1,2]
-		# otherwise we render only 2 profiles (from profile to frontal is noisy)
-		else:
-			return [1,2]
+# def decidePose(yaw,opts):
+# 	if opts.getboolean('renderer', 'nearView'):
+# 		yaw = abs(yaw)
+# 		# If yaw is near-frontal we render everything
+# 		if yaw < 15:
+# 			return [0,1,2]
+# 		# otherwise we render only 2 profiles (from profile to frontal is noisy)
+# 		else:
+# 			return [1,2]
+# 	else:
+# 		return [0,1,2]
+
+def decidePose(yaw,opts, newModels=True):
+	if newModels == True:
+	    if opts.getboolean('renderer', 'nearView'):
+	        yaw = abs(yaw)
+	        # If yaw is near-frontal we render everything                                                                                                                                                                            
+	        if yaw < 15:
+	                return [0,1,2,3,4]
+	        # otherwise we render only 2 profiles (from profile to frontal is noisy)                                                                                                                                                 
+	        elif yaw > 40:
+	                return [3,4]
+	        else:
+	                return [2,3,4]
+	    else:
+	        return [0,1,2,3,4]
 	else:
-		return [0,1,2]
+		if opts.getboolean('renderer', 'nearView'):
+			yaw = abs(yaw)
+			# If yaw is near-frontal we render everything
+			if yaw < 15:
+				return [0,1,2]
+			# otherwise we render only 2 profiles (from profile to frontal is noisy)
+			else:
+				return [1,2]
+		else:
+			return [0,1,2]
+def decideSide_from_db(img, pose_Rt, allModels):
+	## Check if we need to flip the image                                                                                                                                                                                            
+	#model3D = ThreeD_Model.FaceModel(this_path + "/models3d/" + pose_models[0] +'_01.mat', 'model3D')                                                                                                                               
+	## Getting yaw estimate over poses and subjects                                                                                                                                                                                  
+	mm = allModels.values()[0]
+	proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(mm, pose_Rt, pose_db_on=True)
+	yaw =  calib.get_yaw(rmat)
+	print '> Yaw value mean: ',  yaw
+	return yaw