diff --git a/README.md b/README.md new file mode 100644 index 0000000..4932fca --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +Exploiting temporal context for 3D human pose estimation in the wild +== + + +[Exploiting temporal context for 3D human pose estimation in the wild](http://arxiv.org/abs/1905.04266) uses temporal information from videos to correct errors in single-image 3D pose estimation. In this repository, we provide results from applying this algorithm on the [Kinetics-400](https://deepmind.com/research/open-source/open-source-datasets/kinetics/) dataset. Note that this is not an exhaustive labeling: at most one person is labeled per frame, and frames which the algorithm has identified as outliers are not labeled. + +The archive contains a single `.pkl` file for each video where bundle adjustment succeeded. Let `N` be the number of frames that the algorithm considers inliers. Then the `.pkl` file contains a map with the following keys: + +* `time`: Array of size `N`, where each element is the time in seconds since the start of the 10-second kinetics clip (not the start of the whole video) +* `smpl_shape`: Array of size `Nx10`, where each row is the SMPL shape for one example. +* `smpl_pose`: Array of size `Nx72`, where each row is the SMPL pose for one example. +* `3d_keypoints`: Array of size `Nx24x3` where each slice is the 19 cocoplus joints obtained from the SMPL model using the custom keypoint regressor described below. +* `2d_keypoints`: Array of size `Nx19x2`, where each slice is the 19 cocoplus joints reprojected from the SMPL model, using the custom keypoint regressor described below, in `(x,y)` coordinates. These coordinates are normalized to the image frame: therefore, (0, 0) and (1,1) are the top-left and bottom-right corners respectively. +* `cameras`: Array of size `Nx3`, containing the translation and scale that maps the SMPL 3D joint locations to `2d_keypoints`. `cameras[:,0]` is scale and `cameras[:,1:3]` is translation. Thus, if `x` is a `19x3` array of 3D keypoints in the format `(x,y,z)` produced byt the SMPL model, then `2d_keypoints` can be computed as `cameras[:,0:1]*(x[:,0:2]+cameras[:,1:3])`. +* `vertices`: Array of size `Nx6890x3`. These are the vertices of the SMPL mesh computed from `smpl_shape` and `smpl_pose` computing with the neutral body model from [HMR](https://github.com/akanazawa/hmr). + +The dataset can be downloaded [here](https://storage.cloud.google.com/temporal-3d-pose-kinetics/temporal_3d_pose_kinetics.tar.gz) (325 GB), as well as an significantly smaller archive which does not contain `vertices`, but is otherwise identical, [here](https://storage.cloud.google.com/temporal-3d-pose-kinetics/temporal_3d_pose_kinetics_noverts.tar.gz) (2.7 GB). + +## Joint regressor + +We also have a custom [joint regressor](https://storage.cloud.google.com/temporal-3d-pose-kinetics/custom_joint_regressor.pkl) that is specific to our pose estimator (since there are slight differences between the 2D joints we used for bundle adjustment and those used for SMPL). This is a `6890x19` array that can be used as a drop-in replacement for the `cocoplus_regressor` that is distributed in the public [HMR repository](https://github.com/akanazawa/hmr), and is required to extract the `3d_keypoints` above from the estimated poses. It was learned using ground-truth from the [Human3.6m dataset](http://vision.imar.ro/human3.6m/). + +## Pretrained Model +This [Tensorflow checkpoint](https://storage.cloud.google.com/temporal-3d-pose-kinetics/model-894621.tar.gz) was trained using the procedure outlined in our paper. That is, it uses the above dataset as well as standard HMR 3D data. The checkpoint is compatible with [HMR](https://github.com/akanazawa/hmr). + +## Visualising data + +- You need to install [`youtube-dl`](https://github.com/ytdl-org/youtube-dl) and [`ffmpeg`](http://ffmpeg.org) to download the Kinetics videos to visualise. +- Download the faces of the SMPL mesh for visualisation: `wget https://github.com/akanazawa/hmr/raw/master/src/tf_smpl/smpl_faces.npy` +- The python packages needed are in `requirements.txt`. We recommend create a new virtual environment, and running `pip install -r requirements.txt`. + +To run the demo: + +`python run_visualise --filename ` + +## Credits +- The Kinetics download scripts are from [ActivityNet](https://github.com/activitynet/ActivityNet/tree/master/Crawler/Kinetics) +- The renderer to visualise the SMPL model is from [HMR]( https://github.com/akanazawa/hmr) + +## Reference + +If you use this data, please cite + +```tex +@InProceedings{Arnab_CVPR_2019, + author = {Arnab, Anurag* and + Doersch, Carl* and + Zisserman, Andrew}, + title = {Exploiting temporal context for 3D human pose estimation in the wild}, + booktitle = {Computer Vision and Pattern Recognition (CVPR)}, + month = {June}, + year = {2019} +} +``` diff --git a/plot_utils.py b/plot_utils.py new file mode 100644 index 0000000..f0fe9e2 --- /dev/null +++ b/plot_utils.py @@ -0,0 +1,180 @@ +# Copyright 2018 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Create plots with Matplotlib to visualise the result.""" + +import StringIO +import matplotlib.pyplot as plt +import numpy as np + +HMR_JOINT_NAMES = [ + 'right_ankle', + 'right_knee', + 'right_hip', + 'left_hip', + 'left_knee', + 'left_ankle', + 'right_wrist', + 'right_elbow', + 'right_shoulder', + 'left_shoulder', + 'left_elbow', + 'left_wrist', + 'neck', + 'head_top', + 'nose', + 'left_eye', + 'right_eye', + 'left_ear', + 'right_ear', +] + +MSCOCO_JOINT_NAMES = [ + 'nose', 'left_eye', 'right_eye', 'left_ear', 'right_ear', 'left_shoulder', + 'right_shoulder', 'left_elbow', 'right_elbow', 'left_wrist', 'right_wrist', + 'left_hip', 'right_hip', 'left_knee', 'right_knee', 'left_ankle', + 'right_ankle' +] + +coco_to_hmr = [] +for name in MSCOCO_JOINT_NAMES: + index = HMR_JOINT_NAMES.index(name) + coco_to_hmr.append(index) + +PARENTS_COCO_PLUS = [ + 1, 2, 8, 9, 3, 4, 7, 8, 12, 12, 9, 10, 14, -1, 13, -1, -1, 15, 16 +] +COLOURS = [] +for name in HMR_JOINT_NAMES: + if name.startswith('left'): + c = 'r' + elif name.startswith('right'): + c = 'g' + else: + c = 'm' + COLOURS.append(c) + + +def plot_keypoints_2d(image, + joints_2d, + ax=None, + show_plot=False, + title='', + is_coco_format=False): + """Plot 2d keypoints overlaid on image.""" + + if ax is None: + fig = plt.figure() + ax = fig.add_subplot(111) + + if hasattr(ax, 'set_axis_off'): + ax.set_axis_off() + + if is_coco_format: + kp = np.zeros((len(HMR_JOINT_NAMES), 2)) + kp[coco_to_hmr, :] = joints_2d + joints_2d = kp + + if image is not None: + ax.imshow(image) + + joint_colour = 'c' if not is_coco_format else 'b' + s = 30 * np.ones(joints_2d.shape[0]) + for i in range(joints_2d.shape[0]): + x, y = joints_2d[i, :] + if x == 0 and y == 0: + s[i] = 0 + + ax.scatter( + joints_2d[:, 0].squeeze(), + joints_2d[:, 1].squeeze(), + s=30, + c=joint_colour) + + for idx_i, idx_j in enumerate(PARENTS_COCO_PLUS): + if idx_j >= 0: + pair = [idx_i, idx_j] + x, y = joints_2d[pair, 0], joints_2d[pair, 1] + if x[0] > 0 and y[0] > 0 and x[1] > 0 and y[1] > 0: + ax.plot(x.squeeze(), y.squeeze(), c=COLOURS[idx_i], linewidth=1.5) + + ax.set_xlim([0, image.shape[1]]) + ax.set_ylim([image.shape[0], 0]) + + if title: + ax.set_title(title) + + if show_plot: + plt.show() + + return ax + + +def plot_summary_figure(img, + joints_2d, + rend_img_overlay, + rend_img, + rend_img_vp1, + rend_img_vp2, + save_name=None): + """Create plot to visulise results.""" + + fig = plt.figure(1, figsize=(20, 12)) + plt.clf() + + plt.subplot(231) + plt.imshow(img) + plt.title('Input') + plt.axis('off') + + ax_skel = plt.subplot(232) + ax_skel = plot_keypoints_2d(img, joints_2d, ax_skel) + plt.title('Joint Projection') + plt.axis('off') + + plt.subplot(233) + plt.imshow(rend_img_overlay) + plt.title('3D Mesh overlay') + plt.axis('off') + + plt.subplot(234) + plt.imshow(rend_img) + plt.title('3D mesh') + plt.axis('off') + + plt.subplot(235) + plt.imshow(rend_img_vp1) + plt.title('Other viewpoint (+60 degrees)') + + plt.axis('off') + plt.subplot(236) + plt.imshow(rend_img_vp2) + plt.title('Other viewpoint (-60 degrees)') + plt.axis('off') + + plt.draw() + + if save_name is not None: + buf = StringIO.StringIO() + plt.savefig(buf, format='jpg') + buf.seek(0) + + with open(save_name, 'w') as fp: + fp.write(buf.read(-1)) + else: + plt.show() + + return fig + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..517c4ff --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +matplotlib==2.2.3 +numpy==1.11.3 +absl-py +scipy==1.2.1 +scikit-video==1.1.11 +opencv-python==4.0.0.21 +opendr==0.78 diff --git a/run_visualise.py b/run_visualise.py new file mode 100644 index 0000000..648b18e --- /dev/null +++ b/run_visualise.py @@ -0,0 +1,244 @@ +# Copyright 2018 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Visualise bundle adjustment results. + +Example to run: +python run_visualise.py --filename KV4jIAq3WJo_155_165.pkl +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import cPickle as pickle +import errno +import os +import subprocess +import sys + +from absl import flags +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import plot_utils +import skvideo.io +from third_party.activity_net.download import download_clip +import third_party.hmr.renderer as vis_util + +# Input +flags.DEFINE_string('filename', '', 'The annoation pickle file') +flags.DEFINE_string('smpl_face_path', 'smpl_faces.npy', + 'Path to smpl model face file.') + +# Output +flags.DEFINE_string( + 'output_dir', 'results', 'Where to write results to.' + 'Directory automatically created.') + + +def mkdir(dirname): + """Create directory if it does not exist.""" + try: + os.makedirs(dirname) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def im_save_cv(image, filename): + """Write image with OpenCV, converting from BGR to RGB format.""" + cv2.imwrite(filename, image[:, :, (2, 1, 0)]) + + +def visualize(img, + joints, + vertices, + camera, + image_name, + output_dir, + renderer=None, + color_id=0): + """Renders the result in original image coordinate frame. + + Args: + img: The image + joints: 2D keypoints, in the image coordinate frame. + vertices: Vertices of the SMPL mesh. + camera: Camera predicted. + image_name: Name of image for saving. + output_dir: Directory to save results to + renderer: Renderer object to use. + color_id: 0 is blue, and 1 is light pink. For the visualisation. The + colours are defined in the renderer. + """ + + cam_for_render = camera * img.shape[0] + vert_shifted = np.copy(vertices) + + # Approximate an orthographic camera: + # move points away and adjust the focal length to zoom in. + vert_shifted[:, -1] = vert_shifted[:, -1] + 100. + cam_for_render[0] *= 100. + + rend_img_overlay = renderer( + vert_shifted, + cam=cam_for_render, + img=img, + do_alpha=True, + color_id=color_id) + rend_img = renderer( + vert_shifted, + cam=cam_for_render, + img_size=img.shape[:2], + color_id=color_id) + rend_img_vp1 = renderer.rotated( + vert_shifted, + 60, + cam=cam_for_render, + img_size=img.shape[:2], + color_id=color_id) + rend_img_vp2 = renderer.rotated( + vert_shifted, + -60, + cam=cam_for_render, + img_size=img.shape[:2], + color_id=color_id) + + save_name = os.path.join(output_dir, image_name + '.jpg') + fig = plot_utils.plot_summary_figure(img, joints, rend_img_overlay, rend_img, + rend_img_vp1, rend_img_vp2, save_name) + plt.close(fig) + + +def transform_keypoints_to_image(keypoints, img): + """Transform keypoints from range [0, 1] to image coordinates.""" + + keypoints[:, :, 0] *= img.shape[0] + keypoints[:, :, 1] *= img.shape[ + 0] # The saved keypoints are scaled by image height. + + return keypoints + + +def parse_filename(filename): + """Parse filename of the pickle file.""" + + name = os.path.basename(filename) + name = name.replace('.pkl', '') + tokens = name.split('_') + end_time = int(tokens[-1]) + start_time = int(tokens[-2]) + video_id = '_'.join(tokens[0:-2]) + + return video_id, start_time, end_time + + +def get_frame_rate(video_path): + """Get frame rate of the video from its metadata.""" + + meta_data = skvideo.io.ffprobe(video_path) + if 'video' in meta_data.keys(): + meta_data = meta_data['video'] + + if '@avg_frame_rate' in meta_data: + frame_rate = eval(meta_data['@avg_frame_rate']) + else: + frame_rate = None + + return frame_rate + + +def video_from_images(directory, save_name): + """Create video from images saved in directory using ffmpeg.""" + + command = [ + 'ffmpeg', '-framerate', '25', '-pattern_type', + 'glob -i \'{}/*.jpg\''.format(directory), '-c:v', 'libx264', '-pix_fmt', + 'yuv420p', '-loglevel', 'panic', save_name + ] + command = ' '.join(command) + try: + _ = subprocess.check_output( + command, shell=True, stderr=subprocess.STDOUT) + except: + pass + + +def load_pickle(filename): + """Read pickle file.""" + + with open(filename) as fp: + data = pickle.load(fp) + return data + + +def main(config): + + data = load_pickle(config.filename) + + video_id, start_time, end_time = parse_filename(config.filename) + video_path = '/tmp/' + video_id + '.mp4' + + status, message = download_clip(video_id, video_path, start_time, end_time) + + if not status: + print('Video not downloaded') + print(message) + sys.exit() + + video = skvideo.io.vread(video_path) + frame_rate = get_frame_rate(video_path) + + if not frame_rate: + print('Error. Could not determine frame rate of video') + sys.exit() + + output_dir = os.path.join(config.output_dir, video_id) + mkdir(output_dir) + + keypoints = transform_keypoints_to_image(data['2d_keypoints'], + video[0].squeeze()) + renderer = vis_util.SMPLRenderer(face_path=config.smpl_face_path) + + for i in range(data['time'].size): + + idx = int(round(data['time'][i] * frame_rate)) + if idx >= video.shape[0]: + break + + img = video[idx].squeeze() + image_name = '{:>04}'.format(i) + + visualize( + img, + joints=keypoints[i].squeeze(), + vertices=data['vertices'][i].squeeze(), + camera=data['camera'][i].squeeze(), + image_name=image_name, + output_dir=output_dir, + renderer=renderer) + + if i % 20 == 0: + print('Processed {:3d} / {:3d}'.format(i + 1, data['time'].size)) + + video_from_images(output_dir, os.path.join(output_dir, video_id + '.mp4')) + + +if __name__ == '__main__': + config_ = flags.FLAGS + config_(sys.argv) + + main(config_) diff --git a/third_party/__init__.py b/third_party/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/third_party/activity_net/LICENSE b/third_party/activity_net/LICENSE new file mode 100644 index 0000000..5ba1240 --- /dev/null +++ b/third_party/activity_net/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 ActivityNet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/activity_net/__init__.py b/third_party/activity_net/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/third_party/activity_net/download.py b/third_party/activity_net/download.py new file mode 100644 index 0000000..be50615 --- /dev/null +++ b/third_party/activity_net/download.py @@ -0,0 +1,88 @@ +""" +https://github.com/activitynet/ActivityNet/blob/master/Crawler/Kinetics/download.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import glob +import os +import subprocess +import uuid + + +def download_clip(video_identifier, + output_filename, + start_time, + end_time, + tmp_dir='/tmp/kinetics', + num_attempts=5, + url_base='https://www.youtube.com/watch?v='): + """Download a video from youtube if exists and is not blocked. + + + Args: + video_identifier: str. Unique YouTube video identifier (11 characters) + output_filename: str File path where the video will be stored. + start_time: float Indicates the beginning time in seconds from where the + video will be trimmed. + end_time: float Indicates the ending time in seconds of the trimmed + video. + + Returns: + status: boolean. Whether the downloaded succeeded + message: str. Error message if download did not succeed + + """ + # Defensive argument checking. + assert isinstance(video_identifier, str), 'video_identifier must be string' + assert isinstance(output_filename, str), 'output_filename must be string' + assert len(video_identifier) == 11, 'video_identifier must have length 11' + + if os.path.exists(output_filename): + return True, 'Downloaded' + + status = False + # Construct command line for getting the direct video link. + tmp_filename = os.path.join(tmp_dir, '%s.%%(ext)s' % uuid.uuid4()) + command = [ + 'youtube-dl', '--quiet', '--no-warnings', '-f', 'mp4', '-o', + '"%s"' % tmp_filename, + '"%s"' % (url_base + video_identifier) + ] + command = ' '.join(command) + attempts = 0 + while True: + try: + _ = subprocess.check_output( + command, shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + attempts += 1 + if attempts == num_attempts: + return status, err.output + else: + break + + tmp_filename = glob.glob('%s*' % tmp_filename.split('.')[0])[0] + # Construct command to trim the videos (ffmpeg required). + command = [ + 'ffmpeg', '-i', + '"%s"' % tmp_filename, '-ss', + str(start_time), '-t', + str(end_time - start_time), '-c:v', 'libx264', '-c:a', 'copy', '-threads', + '1', '-loglevel', 'panic', + '"%s"' % output_filename + ] + command = ' '.join(command) + try: + output = subprocess.check_output( + command, shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + return status, err.output + + # Check if the video was successfully saved. + status = os.path.exists(output_filename) + os.remove(tmp_filename) + return status, 'Downloaded' + diff --git a/third_party/hmr/LICENSE b/third_party/hmr/LICENSE new file mode 100644 index 0000000..ee95691 --- /dev/null +++ b/third_party/hmr/LICENSE @@ -0,0 +1,25 @@ +MIT License + +This code base itself is MIT, but please follow the license for SMPL, MoSh data, +and the respective dataset. + +Copyright (c) 2018 akanazawa +Copyright 2019 DeepMind Technologies Limited. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/hmr/__init__.py b/third_party/hmr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/third_party/hmr/renderer.py b/third_party/hmr/renderer.py new file mode 100644 index 0000000..5421b7a --- /dev/null +++ b/third_party/hmr/renderer.py @@ -0,0 +1,233 @@ +"""Render meshes using OpenDR. + + Code is from: + https://github.com/akanazawa/hmr/blob/master/src/util/renderer.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import cv2 +import numpy as np + +from opendr.camera import ProjectPoints +from opendr.renderer import ColoredRenderer +from opendr.lighting import LambertianPointLight + +colors = { + # colorbline/print/copy safe: + 'light_blue': [0.65098039, 0.74117647, 0.85882353], + 'light_pink': [.9, .7, .7], # This is used to do no-3d +} + + +class SMPLRenderer(object): + """Utility class to render SMPL models.""" + def __init__(self, img_size=224, flength=500., face_path='smpl_faces.npy'): + self.faces = np.load(face_path) + self.w = img_size + self.h = img_size + self.flength = flength + + def __call__(self, + verts, + cam=None, + img=None, + do_alpha=False, + far=None, + near=None, + color_id=0, + img_size=None): + + # cam is 3D [f, px, py] + if img is not None: + h, w = img.shape[:2] + elif img_size is not None: + h = img_size[0] + w = img_size[1] + else: + h = self.h + w = self.w + + if cam is None: + cam = [self.flength, w / 2., h / 2.] + + use_cam = ProjectPoints( + f=cam[0] * np.ones(2), + rt=np.zeros(3), + t=np.zeros(3), + k=np.zeros(5), + c=cam[1:3]) + + if near is None: + near = np.maximum(np.min(verts[:, 2]) - 25, 0.1) + if far is None: + far = np.maximum(np.max(verts[:, 2]) + 25, 25) + + imtmp = render_model( + verts, + self.faces, + w, + h, + use_cam, + do_alpha=do_alpha, + img=img, + far=far, + near=near, + color_id=color_id) + + return (imtmp * 255).astype('uint8') + + def rotated(self, + verts, + deg, + cam=None, + axis='y', + img=None, + do_alpha=True, + far=None, + near=None, + color_id=0, + img_size=None): + + if axis == 'y': + around = cv2.Rodrigues(np.array([0, math.radians(deg), 0]))[0] + elif axis == 'x': + around = cv2.Rodrigues(np.array([math.radians(deg), 0, 0]))[0] + else: + around = cv2.Rodrigues(np.array([0, 0, math.radians(deg)]))[0] + center = verts.mean(axis=0) + new_v = np.dot((verts - center), around) + center + + return self.__call__( + new_v, + cam, + img=img, + do_alpha=do_alpha, + far=far, + near=near, + img_size=img_size, + color_id=color_id) + + +def _create_renderer(w=640, + h=480, + rt=np.zeros(3), + t=np.zeros(3), + f=None, + c=None, + k=None, + near=.5, + far=10.): + + f = np.array([w, w]) / 2. if f is None else f + c = np.array([w, h]) / 2. if c is None else c + k = np.zeros(5) if k is None else k + + rn = ColoredRenderer() + + rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k) + rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w} + return rn + + +def _rotateY(points, angle): + """Rotate the points by a specified angle.""" + ry = np.array([[np.cos(angle), 0., np.sin(angle)], [0., 1., 0.], + [-np.sin(angle), 0., np.cos(angle)]]) + return np.dot(points, ry) + + +def simple_renderer(rn, + verts, + faces, + yrot=np.radians(120), + color=colors['light_pink']): + # Rendered model color + rn.set(v=verts, f=faces, vc=color, bgcolor=np.ones(3)) + albedo = rn.vc + + # Construct Back Light (on back right corner) + rn.vc = LambertianPointLight( + f=rn.f, + v=rn.v, + num_verts=len(rn.v), + light_pos=_rotateY(np.array([-200, -100, -100]), yrot), + vc=albedo, + light_color=np.array([1, 1, 1])) + + # Construct Left Light + rn.vc += LambertianPointLight( + f=rn.f, + v=rn.v, + num_verts=len(rn.v), + light_pos=_rotateY(np.array([800, 10, 300]), yrot), + vc=albedo, + light_color=np.array([1, 1, 1])) + + # Construct Right Light + rn.vc += LambertianPointLight( + f=rn.f, + v=rn.v, + num_verts=len(rn.v), + light_pos=_rotateY(np.array([-500, 500, 1000]), yrot), + vc=albedo, + light_color=np.array([.7, .7, .7])) + + return rn.r + + +def get_alpha(imtmp, bgval=1.): + h, w = imtmp.shape[:2] + alpha = (~np.all(imtmp == bgval, axis=2)).astype(imtmp.dtype) + + b_channel, g_channel, r_channel = cv2.split(imtmp) + + im_RGBA = cv2.merge( + (b_channel, g_channel, r_channel, alpha.astype(imtmp.dtype))) + return im_RGBA + + +def append_alpha(imtmp): + alpha = np.ones_like(imtmp[:, :, 0]).astype(imtmp.dtype) + if np.issubdtype(imtmp.dtype, np.uint8): + alpha = alpha * 255 + b_channel, g_channel, r_channel = cv2.split(imtmp) + im_RGBA = cv2.merge((b_channel, g_channel, r_channel, alpha)) + return im_RGBA + + +def render_model(verts, + faces, + w, + h, + cam, + near=0.5, + far=25, + img=None, + do_alpha=False, + color_id=None): + rn = _create_renderer( + w=w, h=h, near=near, far=far, rt=cam.rt, t=cam.t, f=cam.f, c=cam.c) + + # Uses img as background, otherwise white background. + if img is not None: + rn.background_image = img / 255. if img.max() > 1 else img + + if color_id is None: + color = colors['light_blue'] + else: + color_list = colors.values() + color = color_list[color_id % len(color_list)] + + imtmp = simple_renderer(rn, verts, faces, color=color) + + # If white bg, make transparent. + if img is None and do_alpha: + imtmp = get_alpha(imtmp) + elif img is not None and do_alpha: + imtmp = append_alpha(imtmp) + + return imtmp