[Version] Bump version to v0.6.1

README.md

@@ -40,7 +40,7 @@ pip install xrfeitoria
 - `Python >= 3.8`
 - (optional) `Unreal Engine >= 5.1`
     - [x] Windows
-    - [ ] Linux
+    - [x] Linux
     - [ ] MacOS
 - (optional) `Blender >= 3.0`
     - [x] Windows

docs/en/README.md

@@ -30,6 +30,12 @@ docs/en/make.bat html
 docs/en/make html
 ```
 
+### Build `.pyi` files
+
+```bash
+# for instance
+stubgen xrfeitoria/sequence/sequence_base.py --include-private
+```
 
 ### ~~Draw UML~~
 

samples/blender/07_amass.py

@@ -10,6 +10,7 @@
 
 ** It is recommended to run this script with Blender >= 3.6 **
 """
+
 from pathlib import Path
 
 import xrfeitoria as xf
@@ -20,26 +21,26 @@
 
 # prepare the assets
 ####################
-root = Path('.cache/sample-amass').resolve()  # modify this to your own path
+asset_root = Path('.cache/sample-amass').resolve()  # modify this to your own path
 
 # 1. Download Amass from https://amass.is.tue.mpg.de/download.php
 # For example, download ACCAD (SMPL-X N) from https://download.is.tue.mpg.de/download.php?domain=amass&sfile=amass_per_dataset/smplx/neutral/mosh_results/ACCAD.tar.bz2
 # and use `ACCAD/s001/EricCamper04_stageii.npz` from the uncompressed folder
-amass_file = root / 'EricCamper04_stageii.npz'
+amass_file = asset_root / 'EricCamper04_stageii.npz'
 
 # 2.1 Download SMPL-XL model from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.fbx
 # or from https://openxlab.org.cn/datasets/OpenXDLab/SynBody/tree/main/Assets
 # With downloading this, you are agreeing to CC BY-NC-SA 4.0 License (https://creativecommons.org/licenses/by-nc-sa/4.0/).
-smpl_xl_file = root / 'SMPL-XL-001.fbx'
+smpl_xl_file = asset_root / 'SMPL-XL-001.fbx'
 # 2.2 Download the meta information from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.npz
-smpl_xl_meta_file = root / 'SMPL-XL-001.npz'
+smpl_xl_meta_file = asset_root / 'SMPL-XL-001.npz'
 
 # 3. Define the output file path
-seq_name = 'seq_amass'
-output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender' / Path(__file__).stem
-output_path.mkdir(parents=True, exist_ok=True)
-saved_humandata_file = output_path / 'output.npz'
-saved_blend_file = output_path / 'output.blend'
+seq_name = Path(__file__).stem
+output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender'
+seq_dir = output_path / seq_name
+saved_humandata_file = seq_dir / 'smplx' / 'output.npz'
+saved_blend_file = seq_dir / 'output.blend'
 
 
 @remote_blender()
@@ -56,14 +57,15 @@ def main(background: bool = False):
 
     motion = load_amass_motion(amass_file)
     motion.convert_fps(30)  # convert the motion from 120fps (amass) to 30fps
+    motion.cut_motion(end_frame=10)  # cut the motion to 10 frames, for demonstration purpose
     motion_data = motion.get_motion_data()
 
     # modify this to your blender executable path
     xf_runner = xf.init_blender(
         exec_path='C:/Program Files/Blender Foundation/Blender 3.6/blender.exe', background=background
     )
 
-    with xf_runner.Sequence.new(seq_name=seq_name, seq_length=motion.n_frames) as seq:
+    with xf_runner.sequence(seq_name=seq_name, seq_length=motion.n_frames) as seq:
         # Import SMPL-XL model
         actor = xf_runner.Actor.import_from_file(smpl_xl_file)
         apply_scale(actor.name)  # SMPL-XL model is imported with scale, we need to apply scale to it

samples/unreal/07_amass.py

@@ -8,82 +8,96 @@
 SMPL-XL: a parametric human model based on SMPL-X in a layered representation, introduced in https://synbody.github.io/
 Amass: a large database of human motion, introduced in https://amass.is.tue.mpg.de/
 """
+
 from pathlib import Path
 
 import xrfeitoria as xf
 from xrfeitoria.data_structure.models import RenderPass
 from xrfeitoria.utils import setup_logger
-from xrfeitoria.utils.anim import dump_humandata, load_amass_motion
+from xrfeitoria.utils.anim import dump_humandata, load_amass_motion, refine_smpl_x_from_actor_info
 
 from ..config import unreal_exec, unreal_project
 
 # prepare the assets
 ####################
-root = Path('.cache/sample-amass').resolve()  # modify this to your own path
+assets_root = Path('.cache/sample-amass').resolve()  # modify this to your own path
 
 # 1. Download Amass from https://amass.is.tue.mpg.de/download.php
 # For example, download ACCAD (SMPL-X N) from https://download.is.tue.mpg.de/download.php?domain=amass&sfile=amass_per_dataset/smplx/neutral/mosh_results/ACCAD.tar.bz2
 # and use `ACCAD/s001/EricCamper04_stageii.npz` from the uncompressed folder
-amass_file = root / 'EricCamper04_stageii.npz'
+amass_file = assets_root / 'EricCamper04_stageii.npz'
 
 # 2.1 Download SMPL-XL model from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.fbx
 # or from https://openxlab.org.cn/datasets/OpenXDLab/SynBody/tree/main/Assets
 # With downloading this, you are agreeing to CC BY-NC-SA 4.0 License (https://creativecommons.org/licenses/by-nc-sa/4.0/).
-smpl_xl_file = root / 'SMPL-XL-001.fbx'
+smpl_xl_file = assets_root / 'SMPL-XL-001.fbx'
 # 2.2 Download the meta information from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.npz
-smpl_xl_meta_file = root / 'SMPL-XL-001.npz'
+smpl_xl_meta_file = assets_root / 'SMPL-XL-001.npz'
 
 # 3. Define the output file path
-seq_name = 'seq_amass'
-output_path = Path(__file__).resolve().parents[2] / 'output/samples/unreal' / Path(__file__).stem
-output_path.mkdir(parents=True, exist_ok=True)
-saved_humandata_file = output_path / 'output.npz'
+seq_name = Path(__file__).stem
+output_path = Path(__file__).resolve().parents[2] / 'output/samples/unreal'
+seq_dir = output_path / seq_name
+saved_humandata_file = seq_dir / 'smplx' / 'output.npz'
 
 
 def main(background: bool = False):
     logger = setup_logger()
 
     motion = load_amass_motion(amass_file)
     motion.convert_fps(30)  # convert the motion from 120fps (amass) to 30fps
+    motion.cut_motion(end_frame=10)  # cut the motion to 10 frames, for demonstration purpose
     motion_data = motion.get_motion_data()
 
     xf_runner = xf.init_unreal(exec_path=unreal_exec, project_path=unreal_project, background=background)
 
     # Import SMPL-XL model
-    actor_path = xf_runner.utils.import_asset(smpl_xl_file)
+    actor_path = xf_runner.utils.import_asset(smpl_xl_file, replace=False)
 
-    with xf_runner.Sequence.new(seq_name=seq_name, level='/Game/Levels/Playground', seq_length=motion.n_frames) as seq:
+    with xf_runner.sequence(
+        seq_name=seq_name, level='/Game/Levels/Playground', seq_length=motion.n_frames, replace=True
+    ) as seq:
         seq.show()
 
         # Spawn the actor, and add motion data as FK animation
         actor = seq.spawn_actor(
             actor_asset_path=actor_path,
-            location=(0, 0, 0),
+            location=(3, 0, 0),
             rotation=(0, 0, 0),
             stencil_value=1,
             motion_data=motion_data,
         )
+        actor_name = actor.name
 
         camera = seq.spawn_camera(
-            location=(0, 2.5, 0.6),
+            location=(3, 2.5, 0.6),
             rotation=(0, 0, -90),
         )
 
         # Add render job to renderer
         seq.add_to_renderer(
             output_path=output_path,
             resolution=(1920, 1080),
-            render_passes=[RenderPass('img', 'png')],
+            render_passes=[RenderPass('img', 'jpg')],
+            export_skeleton=True,
+            export_vertices=True,
         )
 
     # Save the motion data as annotation in humandata format defined in https://github.com/open-mmlab/mmhuman3d/blob/main/docs/human_data.md
-    dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file)
+    dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file, actor_name=actor_name)
 
     # render
     xf_runner.render()
 
+    # refine smplx parameters
+    refine_smpl_x_from_actor_info(
+        smpl_x_file=saved_humandata_file,
+        actor_info_file=seq_dir / 'actor_infos' / f'{actor_name}.npz',
+        replace_smpl_x_file=True,
+    )
+
     logger.info('🎉 [bold green]Success!')
-    output_img = output_path / seq_name / 'img' / camera.name / '0000.png'
+    output_img = seq_dir / 'img' / camera.name / '0000.png'
     if output_img.exists():
         logger.info(f'Check the output in "{output_img.as_posix()}"')
     if not background:

samples/utils.py

@@ -38,7 +38,10 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat
     logger.info('Visualizing vertices')
     # fixed file structure
     img_path = seq_output_path / 'img' / camera_name / f'{frame_idx:04d}.png'
-    camera_param_json = seq_output_path / 'camera_params' / f'{camera_name}.json'
+    if (seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json').exists():
+        camera_param_json = seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json'
+    else:
+        camera_param_json = seq_output_path / 'camera_params' / f'{camera_name}.json'
 
     # load img and camera parameters
     img = np.array(Image.open(img_path.as_posix()))
@@ -58,7 +61,7 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat
         # draw vertices on image
         img = projector.draw_points3d(verts[frame_idx], camera_param, image=img, color=colors[idx])
 
-    save_path = vis_dir / f'{frame_idx:04d}-overlap.png'
+    save_path = vis_dir / f'{camera_name}-{frame_idx:04d}-overlap.png'
     Image.fromarray(img).save(save_path.as_posix())
     logger.info(f'Original image: "{img_path.as_posix()}"')
     logger.info(f'Overlap image saved to: "{save_path.as_posix()}"')

src/XRFeitoriaBpy/core/factory.py

@@ -1113,6 +1113,7 @@ def apply_motion_data_to_action(
         motion_data: 'List[MotionFrame]',
         action: 'bpy.types.Action',
         scale: float = 1.0,
+        is_first_frame_as_origin: bool = True,
     ) -> None:
         """Apply motion data in dict to object.
 
@@ -1121,9 +1122,8 @@ def apply_motion_data_to_action(
                 containing rotation (quaternion) and location.
             action (bpy.types.Action): Action.
             scale (float, optional): Scale of movement in location of animation. Defaults to 1.0.
+            is_first_frame_as_origin (bool, optional): Whether to set the first frame as the origin. Defaults to True.
         """
-        import numpy as np
-
         num_frames = len(motion_data)
         fcurves_map = {(fc.data_path, fc.array_index): fc for fc in action.fcurves}
 
@@ -1137,9 +1137,8 @@ def _get_fcurve(data_path: str, index: int):
             return fcurve
 
         # Set keyframes
-        frames_iter = range(num_frames)
         loc0 = [0, 0, 0]
-        for f in frames_iter:
+        for f in range(num_frames):
             for bone_name in motion_data[0].keys():
                 # rotation_quaternion
                 data_path = f'pose.bones["{bone_name}"].rotation_quaternion'
@@ -1154,25 +1153,33 @@ def _get_fcurve(data_path: str, index: int):
                 if 'location' in motion:
                     data_path = f'pose.bones["{bone_name}"].location'
                     location_ = motion['location']
-                    if f < 1:
-                        loc0 = location_
-                        location_ = np.zeros(3)
-                    else:
-                        location_ = np.subtract(location_, loc0) * scale
+                    if is_first_frame_as_origin:
+                        if f < 1:
+                            loc0 = location_
+                            location_ = np.zeros(3)
+                        else:
+                            location_ = np.subtract(location_, loc0) * scale
                     for idx, val in enumerate(location_):
                         fcurve = _get_fcurve(data_path=data_path, index=idx)
                         # fcurve.keyframe_points[f].co = (f, val)
                         fcurve.keyframe_points.insert(frame=f, value=val, options={'FAST'})
 
-    def apply_motion_data_to_actor(motion_data: 'List[MotionFrame]', actor_name: str) -> None:
+    def apply_motion_data_to_actor(
+        motion_data: 'List[MotionFrame]',
+        actor_name: str,
+        is_first_frame_as_origin: bool = True,
+    ) -> None:
         """Applies motion data to a given actor.
 
         Args:
             motion_data: A list of dictionaries containing motion data (quaternion) for the actor.
             actor_name: The name of the actor to apply the motion data to.
+            is_first_frame_as_origin: Whether to set the first frame as the origin.
         """
         action = bpy.data.actions.new('Action')
-        XRFeitoriaBlenderFactory.apply_motion_data_to_action(motion_data=motion_data, action=action)
+        XRFeitoriaBlenderFactory.apply_motion_data_to_action(
+            motion_data=motion_data, action=action, is_first_frame_as_origin=is_first_frame_as_origin
+        )
         XRFeitoriaBlenderFactory.apply_action_to_actor(action, actor=bpy.data.objects[actor_name])
 
     def apply_shape_keys_to_mesh(shape_keys: 'List[Dict[str, float]]', mesh_name: str) -> None:

src/XRFeitoriaUnreal/Content/Python/constants.py

@@ -1,7 +1,8 @@
+import json
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, TypedDict, Union
 
 import unreal
 
@@ -19,7 +20,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]:
 
 PLUGIN_NAME = 'XRFeitoriaUnreal'
 MATERIAL_PATHS = {
-    'depth': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_depth_EXR',
+    'depth': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_depth',
     'mask': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_mask_MRQ',
     'flow': f'/{PLUGIN_NAME}/Materials/PPM_velocity',
     'diffuse': f'/{PLUGIN_NAME}/Materials/PPM_diffusecolor',
@@ -41,7 +42,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]:
 ENGINE_MAJOR_VERSION = int(unreal.SystemLibrary.get_engine_version().split('.')[0])
 ENGINE_MINOR_VERSION = int(unreal.SystemLibrary.get_engine_version().split('.')[1])
 DEFAULT_PATH = f'/Game/{PLUGIN_NAME}'
-DEFAULT_SEQUENCE_PATH = f'{DEFAULT_PATH}/Sequences'
+DEFAULT_SEQUENCE_DIR = f'{DEFAULT_PATH}/Sequences'
 DEFAULT_ASSET_PATH = f'{DEFAULT_PATH}/Assets'
 DEFAULT_SEQUENCE_DATA_ASSET = f'/{PLUGIN_NAME}/DefaultSequenceData'
 MRQ_JOB_UPPER = 200
@@ -122,6 +123,12 @@ class UnrealRenderLayerEnum(EnumBase):
     tangent = 'tangent'
     basecolor = 'basecolor'
 
+    vertices = 'vertices'
+    skeleton = 'skeleton'
+    actor_infos = 'actor_infos'
+    camera_params = 'camera_params'
+    audio = 'Audio'
+
 
 @dataclass
 class RenderPass:
@@ -182,8 +189,6 @@ class AntiAliasSetting:
     file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}'
     console_variables: Dict[str, float] = field(default_factory=dict)
     anti_aliasing: AntiAliasSetting = AntiAliasSetting()
-    export_vertices: bool = False
-    export_skeleton: bool = False
     export_audio: bool = False
 
     def __post_init__(self):
@@ -193,3 +198,16 @@ def __post_init__(self):
 
 TransformKeys = Union[List[SequenceTransformKey], SequenceTransformKey]
 MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]]
+color_type = TypedDict(
+    'color',
+    {
+        'name': str,
+        'hex': str,
+        'rgb': Tuple[int, int, int],
+    },
+)
+
+
+######### Constants #########
+MASK_COLOR_FILE = PLUGIN_PYTHON_ROOT / 'data' / 'mask_colors.json'
+mask_colors: List[color_type] = json.loads(MASK_COLOR_FILE.read_text())