diff --git a/README.md b/README.md index 2c3a52de..e9e91822 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ pip install xrfeitoria - `Python >= 3.8` - (optional) `Unreal Engine >= 5.1` - [x] Windows - - [ ] Linux + - [x] Linux - [ ] MacOS - (optional) `Blender >= 3.0` - [x] Windows diff --git a/docs/en/README.md b/docs/en/README.md index 67a15a36..8310e9ee 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -30,6 +30,12 @@ docs/en/make.bat html docs/en/make html ``` +### Build `.pyi` files + +```bash +# for instance +stubgen xrfeitoria/sequence/sequence_base.py --include-private +``` ### ~~Draw UML~~ diff --git a/samples/blender/07_amass.py b/samples/blender/07_amass.py index 1f1534ef..01f111b6 100644 --- a/samples/blender/07_amass.py +++ b/samples/blender/07_amass.py @@ -10,6 +10,7 @@ ** It is recommended to run this script with Blender >= 3.6 ** """ + from pathlib import Path import xrfeitoria as xf @@ -20,26 +21,26 @@ # prepare the assets #################### -root = Path('.cache/sample-amass').resolve() # modify this to your own path +asset_root = Path('.cache/sample-amass').resolve() # modify this to your own path # 1. Download Amass from https://amass.is.tue.mpg.de/download.php # For example, download ACCAD (SMPL-X N) from https://download.is.tue.mpg.de/download.php?domain=amass&sfile=amass_per_dataset/smplx/neutral/mosh_results/ACCAD.tar.bz2 # and use `ACCAD/s001/EricCamper04_stageii.npz` from the uncompressed folder -amass_file = root / 'EricCamper04_stageii.npz' +amass_file = asset_root / 'EricCamper04_stageii.npz' # 2.1 Download SMPL-XL model from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.fbx # or from https://openxlab.org.cn/datasets/OpenXDLab/SynBody/tree/main/Assets # With downloading this, you are agreeing to CC BY-NC-SA 4.0 License (https://creativecommons.org/licenses/by-nc-sa/4.0/). -smpl_xl_file = root / 'SMPL-XL-001.fbx' +smpl_xl_file = asset_root / 'SMPL-XL-001.fbx' # 2.2 Download the meta information from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.npz -smpl_xl_meta_file = root / 'SMPL-XL-001.npz' +smpl_xl_meta_file = asset_root / 'SMPL-XL-001.npz' # 3. Define the output file path -seq_name = 'seq_amass' -output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender' / Path(__file__).stem -output_path.mkdir(parents=True, exist_ok=True) -saved_humandata_file = output_path / 'output.npz' -saved_blend_file = output_path / 'output.blend' +seq_name = Path(__file__).stem +output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender' +seq_dir = output_path / seq_name +saved_humandata_file = seq_dir / 'smplx' / 'output.npz' +saved_blend_file = seq_dir / 'output.blend' @remote_blender() @@ -56,6 +57,7 @@ def main(background: bool = False): motion = load_amass_motion(amass_file) motion.convert_fps(30) # convert the motion from 120fps (amass) to 30fps + motion.cut_motion(end_frame=10) # cut the motion to 10 frames, for demonstration purpose motion_data = motion.get_motion_data() # modify this to your blender executable path @@ -63,7 +65,7 @@ def main(background: bool = False): exec_path='C:/Program Files/Blender Foundation/Blender 3.6/blender.exe', background=background ) - with xf_runner.Sequence.new(seq_name=seq_name, seq_length=motion.n_frames) as seq: + with xf_runner.sequence(seq_name=seq_name, seq_length=motion.n_frames) as seq: # Import SMPL-XL model actor = xf_runner.Actor.import_from_file(smpl_xl_file) apply_scale(actor.name) # SMPL-XL model is imported with scale, we need to apply scale to it diff --git a/samples/unreal/07_amass.py b/samples/unreal/07_amass.py index e0e8ae22..7d2774f5 100644 --- a/samples/unreal/07_amass.py +++ b/samples/unreal/07_amass.py @@ -8,36 +8,37 @@ SMPL-XL: a parametric human model based on SMPL-X in a layered representation, introduced in https://synbody.github.io/ Amass: a large database of human motion, introduced in https://amass.is.tue.mpg.de/ """ + from pathlib import Path import xrfeitoria as xf from xrfeitoria.data_structure.models import RenderPass from xrfeitoria.utils import setup_logger -from xrfeitoria.utils.anim import dump_humandata, load_amass_motion +from xrfeitoria.utils.anim import dump_humandata, load_amass_motion, refine_smpl_x_from_actor_info from ..config import unreal_exec, unreal_project # prepare the assets #################### -root = Path('.cache/sample-amass').resolve() # modify this to your own path +assets_root = Path('.cache/sample-amass').resolve() # modify this to your own path # 1. Download Amass from https://amass.is.tue.mpg.de/download.php # For example, download ACCAD (SMPL-X N) from https://download.is.tue.mpg.de/download.php?domain=amass&sfile=amass_per_dataset/smplx/neutral/mosh_results/ACCAD.tar.bz2 # and use `ACCAD/s001/EricCamper04_stageii.npz` from the uncompressed folder -amass_file = root / 'EricCamper04_stageii.npz' +amass_file = assets_root / 'EricCamper04_stageii.npz' # 2.1 Download SMPL-XL model from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.fbx # or from https://openxlab.org.cn/datasets/OpenXDLab/SynBody/tree/main/Assets # With downloading this, you are agreeing to CC BY-NC-SA 4.0 License (https://creativecommons.org/licenses/by-nc-sa/4.0/). -smpl_xl_file = root / 'SMPL-XL-001.fbx' +smpl_xl_file = assets_root / 'SMPL-XL-001.fbx' # 2.2 Download the meta information from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.npz -smpl_xl_meta_file = root / 'SMPL-XL-001.npz' +smpl_xl_meta_file = assets_root / 'SMPL-XL-001.npz' # 3. Define the output file path -seq_name = 'seq_amass' -output_path = Path(__file__).resolve().parents[2] / 'output/samples/unreal' / Path(__file__).stem -output_path.mkdir(parents=True, exist_ok=True) -saved_humandata_file = output_path / 'output.npz' +seq_name = Path(__file__).stem +output_path = Path(__file__).resolve().parents[2] / 'output/samples/unreal' +seq_dir = output_path / seq_name +saved_humandata_file = seq_dir / 'smplx' / 'output.npz' def main(background: bool = False): @@ -45,27 +46,31 @@ def main(background: bool = False): motion = load_amass_motion(amass_file) motion.convert_fps(30) # convert the motion from 120fps (amass) to 30fps + motion.cut_motion(end_frame=10) # cut the motion to 10 frames, for demonstration purpose motion_data = motion.get_motion_data() xf_runner = xf.init_unreal(exec_path=unreal_exec, project_path=unreal_project, background=background) # Import SMPL-XL model - actor_path = xf_runner.utils.import_asset(smpl_xl_file) + actor_path = xf_runner.utils.import_asset(smpl_xl_file, replace=False) - with xf_runner.Sequence.new(seq_name=seq_name, level='/Game/Levels/Playground', seq_length=motion.n_frames) as seq: + with xf_runner.sequence( + seq_name=seq_name, level='/Game/Levels/Playground', seq_length=motion.n_frames, replace=True + ) as seq: seq.show() # Spawn the actor, and add motion data as FK animation actor = seq.spawn_actor( actor_asset_path=actor_path, - location=(0, 0, 0), + location=(3, 0, 0), rotation=(0, 0, 0), stencil_value=1, motion_data=motion_data, ) + actor_name = actor.name camera = seq.spawn_camera( - location=(0, 2.5, 0.6), + location=(3, 2.5, 0.6), rotation=(0, 0, -90), ) @@ -73,17 +78,26 @@ def main(background: bool = False): seq.add_to_renderer( output_path=output_path, resolution=(1920, 1080), - render_passes=[RenderPass('img', 'png')], + render_passes=[RenderPass('img', 'jpg')], + export_skeleton=True, + export_vertices=True, ) # Save the motion data as annotation in humandata format defined in https://github.com/open-mmlab/mmhuman3d/blob/main/docs/human_data.md - dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file) + dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file, actor_name=actor_name) # render xf_runner.render() + # refine smplx parameters + refine_smpl_x_from_actor_info( + smpl_x_file=saved_humandata_file, + actor_info_file=seq_dir / 'actor_infos' / f'{actor_name}.npz', + replace_smpl_x_file=True, + ) + logger.info('🎉 [bold green]Success!') - output_img = output_path / seq_name / 'img' / camera.name / '0000.png' + output_img = seq_dir / 'img' / camera.name / '0000.png' if output_img.exists(): logger.info(f'Check the output in "{output_img.as_posix()}"') if not background: diff --git a/samples/utils.py b/samples/utils.py index aa5e66d0..be275212 100644 --- a/samples/utils.py +++ b/samples/utils.py @@ -38,7 +38,10 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat logger.info('Visualizing vertices') # fixed file structure img_path = seq_output_path / 'img' / camera_name / f'{frame_idx:04d}.png' - camera_param_json = seq_output_path / 'camera_params' / f'{camera_name}.json' + if (seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json').exists(): + camera_param_json = seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json' + else: + camera_param_json = seq_output_path / 'camera_params' / f'{camera_name}.json' # load img and camera parameters img = np.array(Image.open(img_path.as_posix())) @@ -58,7 +61,7 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat # draw vertices on image img = projector.draw_points3d(verts[frame_idx], camera_param, image=img, color=colors[idx]) - save_path = vis_dir / f'{frame_idx:04d}-overlap.png' + save_path = vis_dir / f'{camera_name}-{frame_idx:04d}-overlap.png' Image.fromarray(img).save(save_path.as_posix()) logger.info(f'Original image: "{img_path.as_posix()}"') logger.info(f'Overlap image saved to: "{save_path.as_posix()}"') diff --git a/src/XRFeitoriaBpy/core/factory.py b/src/XRFeitoriaBpy/core/factory.py index 31d537a0..b91e2d80 100644 --- a/src/XRFeitoriaBpy/core/factory.py +++ b/src/XRFeitoriaBpy/core/factory.py @@ -1113,6 +1113,7 @@ def apply_motion_data_to_action( motion_data: 'List[MotionFrame]', action: 'bpy.types.Action', scale: float = 1.0, + is_first_frame_as_origin: bool = True, ) -> None: """Apply motion data in dict to object. @@ -1121,9 +1122,8 @@ def apply_motion_data_to_action( containing rotation (quaternion) and location. action (bpy.types.Action): Action. scale (float, optional): Scale of movement in location of animation. Defaults to 1.0. + is_first_frame_as_origin (bool, optional): Whether to set the first frame as the origin. Defaults to True. """ - import numpy as np - num_frames = len(motion_data) fcurves_map = {(fc.data_path, fc.array_index): fc for fc in action.fcurves} @@ -1137,9 +1137,8 @@ def _get_fcurve(data_path: str, index: int): return fcurve # Set keyframes - frames_iter = range(num_frames) loc0 = [0, 0, 0] - for f in frames_iter: + for f in range(num_frames): for bone_name in motion_data[0].keys(): # rotation_quaternion data_path = f'pose.bones["{bone_name}"].rotation_quaternion' @@ -1154,25 +1153,33 @@ def _get_fcurve(data_path: str, index: int): if 'location' in motion: data_path = f'pose.bones["{bone_name}"].location' location_ = motion['location'] - if f < 1: - loc0 = location_ - location_ = np.zeros(3) - else: - location_ = np.subtract(location_, loc0) * scale + if is_first_frame_as_origin: + if f < 1: + loc0 = location_ + location_ = np.zeros(3) + else: + location_ = np.subtract(location_, loc0) * scale for idx, val in enumerate(location_): fcurve = _get_fcurve(data_path=data_path, index=idx) # fcurve.keyframe_points[f].co = (f, val) fcurve.keyframe_points.insert(frame=f, value=val, options={'FAST'}) - def apply_motion_data_to_actor(motion_data: 'List[MotionFrame]', actor_name: str) -> None: + def apply_motion_data_to_actor( + motion_data: 'List[MotionFrame]', + actor_name: str, + is_first_frame_as_origin: bool = True, + ) -> None: """Applies motion data to a given actor. Args: motion_data: A list of dictionaries containing motion data (quaternion) for the actor. actor_name: The name of the actor to apply the motion data to. + is_first_frame_as_origin: Whether to set the first frame as the origin. """ action = bpy.data.actions.new('Action') - XRFeitoriaBlenderFactory.apply_motion_data_to_action(motion_data=motion_data, action=action) + XRFeitoriaBlenderFactory.apply_motion_data_to_action( + motion_data=motion_data, action=action, is_first_frame_as_origin=is_first_frame_as_origin + ) XRFeitoriaBlenderFactory.apply_action_to_actor(action, actor=bpy.data.objects[actor_name]) def apply_shape_keys_to_mesh(shape_keys: 'List[Dict[str, float]]', mesh_name: str) -> None: diff --git a/src/XRFeitoriaUnreal/Content/Python/constants.py b/src/XRFeitoriaUnreal/Content/Python/constants.py index 5191b026..2616a5a3 100644 --- a/src/XRFeitoriaUnreal/Content/Python/constants.py +++ b/src/XRFeitoriaUnreal/Content/Python/constants.py @@ -1,7 +1,8 @@ +import json from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, TypedDict, Union import unreal @@ -19,7 +20,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]: PLUGIN_NAME = 'XRFeitoriaUnreal' MATERIAL_PATHS = { - 'depth': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_depth_EXR', + 'depth': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_depth', 'mask': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_mask_MRQ', 'flow': f'/{PLUGIN_NAME}/Materials/PPM_velocity', 'diffuse': f'/{PLUGIN_NAME}/Materials/PPM_diffusecolor', @@ -41,7 +42,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]: ENGINE_MAJOR_VERSION = int(unreal.SystemLibrary.get_engine_version().split('.')[0]) ENGINE_MINOR_VERSION = int(unreal.SystemLibrary.get_engine_version().split('.')[1]) DEFAULT_PATH = f'/Game/{PLUGIN_NAME}' -DEFAULT_SEQUENCE_PATH = f'{DEFAULT_PATH}/Sequences' +DEFAULT_SEQUENCE_DIR = f'{DEFAULT_PATH}/Sequences' DEFAULT_ASSET_PATH = f'{DEFAULT_PATH}/Assets' DEFAULT_SEQUENCE_DATA_ASSET = f'/{PLUGIN_NAME}/DefaultSequenceData' MRQ_JOB_UPPER = 200 @@ -122,6 +123,12 @@ class UnrealRenderLayerEnum(EnumBase): tangent = 'tangent' basecolor = 'basecolor' + vertices = 'vertices' + skeleton = 'skeleton' + actor_infos = 'actor_infos' + camera_params = 'camera_params' + audio = 'Audio' + @dataclass class RenderPass: @@ -182,8 +189,6 @@ class AntiAliasSetting: file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}' console_variables: Dict[str, float] = field(default_factory=dict) anti_aliasing: AntiAliasSetting = AntiAliasSetting() - export_vertices: bool = False - export_skeleton: bool = False export_audio: bool = False def __post_init__(self): @@ -193,3 +198,16 @@ def __post_init__(self): TransformKeys = Union[List[SequenceTransformKey], SequenceTransformKey] MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]] +color_type = TypedDict( + 'color', + { + 'name': str, + 'hex': str, + 'rgb': Tuple[int, int, int], + }, +) + + +######### Constants ######### +MASK_COLOR_FILE = PLUGIN_PYTHON_ROOT / 'data' / 'mask_colors.json' +mask_colors: List[color_type] = json.loads(MASK_COLOR_FILE.read_text()) diff --git a/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py b/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py index 4acfd0c9..35f2c8cd 100644 --- a/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py +++ b/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py @@ -99,21 +99,6 @@ def set_render_all_cameras(movie_preset: unreal.MoviePipelineMasterConfig, enabl ) camera_setting.render_all_cameras = enable - @staticmethod - def set_export_vertices(movie_preset: unreal.MoviePipelineMasterConfig, enable: bool = True) -> None: - export_setting: unreal.MoviePipelineMeshOperator = movie_preset.find_or_add_setting_by_class( - unreal.MoviePipelineMeshOperator - ) - export_setting.static_mesh_operator_option.save_vertices_position = enable - export_setting.skeletal_mesh_operator_option.save_vertices_position = enable - - @staticmethod - def set_export_skeleton(movie_preset: unreal.MoviePipelineMasterConfig, enable: bool = True) -> None: - export_setting: unreal.MoviePipelineMeshOperator = movie_preset.find_or_add_setting_by_class( - unreal.MoviePipelineMeshOperator - ) - export_setting.skeletal_mesh_operator_option.save_skeleton_position = enable - @staticmethod def set_export_audio(movie_preset: unreal.MoviePipelineMasterConfig) -> None: export_setting: unreal.MoviePipelineWaveOutput = movie_preset.find_or_add_setting_by_class( @@ -277,8 +262,6 @@ def create_movie_preset( output_path: Optional[str] = None, anti_alias: RenderJobUnreal.AntiAliasSetting = RenderJobUnreal.AntiAliasSetting(), console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0.0}, - export_vertices: bool = False, - export_skeleton: bool = False, export_audio: bool = False, ) -> unreal.MoviePipelineMasterConfig: """ @@ -296,6 +279,7 @@ def create_movie_preset( output_path (str): Path of the output, e.g. 'E:/output' anti_alias (dict): Anti-alias settings. console_variables (bool): Console variables. + export_audio (bool): Whether to export audio. Returns: unreal.MoviePipelineMasterConfig: The created movie preset. @@ -308,8 +292,6 @@ def create_movie_preset( cls.add_anti_alias(movie_preset, anti_alias) cls.add_console_command(movie_preset, console_variables) cls.set_render_all_cameras(movie_preset, enable=True) - cls.set_export_vertices(movie_preset, enable=export_vertices) - cls.set_export_skeleton(movie_preset, enable=export_skeleton) if export_audio: cls.set_export_audio(movie_preset) @@ -396,8 +378,6 @@ def add_job_to_queue(cls, job: RenderJobUnreal) -> bool: output_path=job.output_path, anti_alias=job.anti_aliasing, console_variables=job.console_variables, - export_vertices=job.export_vertices, - export_skeleton=job.export_skeleton, export_audio=job.export_audio, ) new_job.set_configuration(movie_preset) @@ -542,7 +522,7 @@ def main(): sequence_path='/Game/Sequences/NewSequence', resolution=[1920, 1080], output_path='E:/Datasets/tmp', - file_name_format='{sequence_name}/{render_pass}/{frame_number}', + file_name_format='{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables={'r.MotionBlurQuality': 0.0}, anti_alias={'enable': False}, ) diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index 6a667efa..dd80a18f 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -4,7 +4,7 @@ import utils_actor from constants import ( DEFAULT_SEQUENCE_DATA_ASSET, - DEFAULT_SEQUENCE_PATH, + DEFAULT_SEQUENCE_DIR, ENGINE_MAJOR_VERSION, ENGINE_MINOR_VERSION, MotionFrame, @@ -14,7 +14,6 @@ data_asset_suffix, ) from utils import add_levels, get_levels, get_soft_object_path, get_world, new_world, save_current_level -from utils_actor import get_actor_mesh_component EditorLevelSequenceSub = SubSystem.EditorLevelSequenceSub EditorAssetSub = SubSystem.EditorAssetSub @@ -82,11 +81,62 @@ def find_binding_by_name(sequence: unreal.LevelSequence, name: str) -> unreal.Se return binding for binding in sequence.get_bindings(): + binding: unreal.MovieSceneBindingProxy if binding.get_name() == name: return binding raise RuntimeError(f'Failed to find binding: {name}') +def find_binding_by_class( + sequence: unreal.LevelSequence, actor_class: Type[unreal.Actor] +) -> Optional[unreal.MovieSceneBindingProxy]: + """Finds a Sequencer binding for the specified actor class in the given Level + Sequence. + + Args: + sequence (unreal.LevelSequence): The Level Sequence to search for the binding. + actor_class (Type[unreal.Actor]): The class of the actor to find or create the binding for. + + Returns: + Optional[unreal.MovieSceneBindingProxy]: The Sequencer binding for the actor class, or None if not found. + """ + bound_objects: List[unreal.SequencerBoundObjects] = unreal.SequencerTools.get_bound_objects( + get_world(), sequence, sequence.get_bindings(), sequence.get_playback_range() + ) + + for bound_object in bound_objects: + if len(bound_object.bound_objects) == 0: + continue + if bound_object.bound_objects[0].static_class() == actor_class.static_class(): + return bound_object.binding_proxy + return None + + +def find_or_create_binding_by_class( + sequence: unreal.LevelSequence, actor_class: Type[unreal.Actor], spawn_in_sequence: bool = True +) -> unreal.MovieSceneBindingProxy: + """Finds or creates a Sequencer binding for the specified actor class in the given + Level Sequence. + + Args: + sequence (unreal.LevelSequence): The Level Sequence to search for the binding. + actor_class (Type[unreal.Actor]): The class of the actor to find or create the binding for. + spawn_in_sequence (bool, optional): Whether to spawn the actor in the sequence if it doesn't exist. + If False, the actor will be spawned in the world but not in the sequence. Defaults to True. + + Returns: + unreal.MovieSceneBindingProxy: The Sequencer binding for the actor class. + """ + binding = find_binding_by_class(sequence, actor_class) + if binding is not None: + return binding + + if spawn_in_sequence: + return sequence.add_spawnable_from_class(actor_class) + else: + return sequence.add_possessable(actor_class) + + def get_time(sequence: unreal.LevelSequence, frame: int) -> unreal.FrameNumber: """Initialize a FrameNumber from the given time and then convert it to a FrameTime with no sub-frame. @@ -704,7 +754,7 @@ def add_actor_to_sequence( actor_binding = sequence.add_possessable(actor) # mesh_component = actor.skeletal_mesh_component - mesh_component = get_actor_mesh_component(actor) + mesh_component = utils_actor.get_actor_mesh_component(actor) mesh_component_binding = sequence.add_possessable(mesh_component) # set stencil value @@ -765,12 +815,12 @@ def add_spawnable_actor_to_sequence( # add actor to sequence actor_binding = sequence.add_spawnable_from_instance(actor_asset) - actor = get_spawnable_actor_from_binding(sequence, actor_binding) + actor: unreal.Actor = get_spawnable_actor_from_binding(sequence, actor_binding) actor_binding.set_name(actor_name) actor.set_actor_label(actor_name) # mesh_component = actor.skeletal_mesh_component - mesh_component = get_actor_mesh_component(actor) + mesh_component = utils_actor.get_actor_mesh_component(actor) mesh_component_binding = sequence.add_possessable(mesh_component) # set stencil value @@ -858,6 +908,41 @@ def generate_sequence( return new_sequence +def get_camera_param(camera: unreal.CameraActor) -> Dict[str, Any]: + """Get camera parameters. + + Args: + camera (unreal.CameraActor): The camera actor. + + Returns: + Dict[str, Any]: A dictionary containing the camera parameters. + """ + return { + 'location': camera.get_actor_location().to_tuple(), + 'rotation': camera.get_actor_rotation().to_tuple(), + 'fov': camera.camera_component.get_editor_property('FieldOfView'), + } + + +def get_actor_param(actor: unreal.Actor) -> Dict[str, Any]: + """Get actor parameters. + + Args: + actor (unreal.Actor): The actor. + + Returns: + Dict[str, Any]: A dictionary containing the actor parameters. + """ + stencil_value = utils_actor.get_actor_mesh_component(actor).get_editor_property('custom_depth_stencil_value') + return { + 'location': actor.get_actor_location().to_tuple(), + 'rotation': actor.get_actor_rotation().to_tuple(), + 'scale': actor.get_actor_scale3d().to_tuple(), + 'mask_color': utils_actor.get_mask_color(stencil_value=stencil_value), + 'stencil_value': stencil_value, + } + + class Sequence: map_path = None sequence_path = None @@ -938,7 +1023,7 @@ def new( map_path = EditorLevelSub.get_current_level().get_path_name().split('.')[0] assert unreal.EditorAssetLibrary.does_asset_exist(map_path), f'Map `{map_path}` does not exist' if seq_dir is None: - seq_dir = DEFAULT_SEQUENCE_PATH + seq_dir = DEFAULT_SEQUENCE_DIR seq_path = f'{seq_dir}/{seq_name}' data_asset_path = f'{seq_path}{data_asset_suffix}' @@ -972,6 +1057,7 @@ def new( def show(cls) -> None: assert cls.sequence is not None, 'Sequence not initialized' unreal.LevelSequenceEditorBlueprintLibrary.open_level_sequence(cls.sequence) + unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(0) @staticmethod def new_data_asset( @@ -1068,6 +1154,16 @@ def set_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int for section in master_track.get_sections(): section.set_end_frame(end_frame) + @classmethod + def get_playback(cls) -> Tuple[int, int]: + """Get the playback range of the sequence. + + Returns: + Tuple[int, int]: The start frame and end frame of the playback range. + """ + assert cls.sequence is not None, 'Sequence not initialized' + return cls.sequence.get_playback_start(), cls.sequence.get_playback_end() + # ------ add actor and camera -------- # @classmethod @@ -1093,7 +1189,7 @@ def add_camera( camera_transform_keys=transform_keys, camera_fov=fov, ) - # cls.bindings[camera_name] = bindings + cls.bindings[camera_name] = bindings else: camera = utils_actor.get_actor_by_name(camera_name) bindings = add_camera_to_sequence( @@ -1102,7 +1198,7 @@ def add_camera( camera_transform_keys=transform_keys, camera_fov=fov, ) - # cls.bindings[camera_name] = bindings + cls.bindings[camera_name] = bindings @classmethod def add_actor( @@ -1144,7 +1240,7 @@ def add_actor( actor_transform_keys=transform_keys, actor_stencil_value=stencil_value, ) - # cls.bindings[actor_name] = bindings + cls.bindings[actor_name] = bindings else: actor = utils_actor.get_actor_by_name(actor_name) @@ -1156,7 +1252,7 @@ def add_actor( animation_asset=animation_asset, motion_data=motion_data, ) - # cls.bindings[actor_name] = bindings + cls.bindings[actor_name] = bindings @classmethod def add_audio( @@ -1181,10 +1277,50 @@ def add_audio( bindings = add_audio_to_sequence( sequence=cls.sequence, audio_asset=audio_asset, start_frame=start_frame, end_frame=end_frame ) - # cls.bindings[audio_asset.get_name()] = bindings + cls.bindings[audio_asset.get_name()] = bindings + + @classmethod + def add_annotator( + cls, + save_dir: str, + resolution: Tuple[int, int], + export_vertices: bool, + export_skeleton: bool, + ): + actor_binding = find_binding_by_class(cls.sequence, unreal.Annotator) + if actor_binding is not None: + actor_binding.remove() + actor_binding = cls.sequence.add_spawnable_from_class(unreal.Annotator) + + add_property_bool_track_to_binding( + binding=actor_binding, property_name='bSaveVerticesPosition', property_value=export_vertices + ) + add_property_bool_track_to_binding( + binding=actor_binding, property_name='bSaveSkeletonPosition', property_value=export_skeleton + ) + add_property_string_track_to_binding( + binding=actor_binding, property_name='DirectorySequence', property_value=save_dir + ) + add_property_int_track_to_binding(binding=actor_binding, property_name='Width', property_value=resolution[0]) + add_property_int_track_to_binding(binding=actor_binding, property_name='Height', property_value=resolution[1]) + + +def test(): + Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') + Sequence.save_camera_params(save_dir='E:/tmp') if __name__ == '__main__': + Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') + + Sequence.new('/Game/NewMap', 'test1') + Sequence.spawn_camera(transform_keys=SequenceTransformKey(frame=0, location=(0, 0, 0), rotation=(0, 0, 0))) + Sequence.spawn_actor( + '/Game/StarterContent/Props/SM_Chair', + transform_keys=SequenceTransformKey(frame=0, location=(0, 0, 0), rotation=(0, 0, 0)), + ) + Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') + Sequence.new('/Game/NewMap', 'test1') Sequence.spawn_camera(transform_keys=SequenceTransformKey(frame=0, location=(0, 0, 0), rotation=(0, 0, 0))) Sequence.spawn_actor( diff --git a/src/XRFeitoriaUnreal/Content/Python/utils.py b/src/XRFeitoriaUnreal/Content/Python/utils.py index 45b7aef1..1f3e66a7 100644 --- a/src/XRFeitoriaUnreal/Content/Python/utils.py +++ b/src/XRFeitoriaUnreal/Content/Python/utils.py @@ -51,6 +51,44 @@ def timer(self, deltaTime): self.func(*self.args, **self.kwargs) +class LoaderTimer: + """A decorator to load assets before running the main function. + + example_usage: + >>> @LoaderTimer + >>> def main(): + >>> ... + Caution: Function decorated by this decorator cannot return anything. + """ + + def __init__(self, func): + self.func = func + self.args: Tuple = None + self.kwargs: Dict = None + self.time_to_wait = 1 + self.time_spent = 0 + + self.tickhandle = unreal.register_slate_pre_tick_callback(self.timer) + unreal.log_warning('registered tick handle') + + def __call__(self, *args, **kwargs): + if args is None: + args = [] + self.args = args + + if kwargs is None: + kwargs = {} + self.kwargs = kwargs + + def timer(self, deltaTime): + if self.time_spent < self.time_to_wait: + self.time_spent += deltaTime + else: + unreal.log_warning('[Timer] ready!') + unreal.unregister_slate_pre_tick_callback(self.tickhandle) + self.func(*self.args, **self.kwargs) + + def timer_func(func): # This function shows the execution time of # the function object passed @@ -69,7 +107,10 @@ def wrap_func(*args, **kwargs): def import_asset( - path: Union[str, List[str]], dst_dir_in_engine: Optional[str] = None, replace: bool = True + path: Union[str, List[str]], + dst_dir_in_engine: Optional[str] = None, + replace: bool = True, + with_parent_dir: bool = True, ) -> List[str]: """Import assets to the default asset path. @@ -78,6 +119,8 @@ def import_asset( dst_dir_in_engine (str, optional): destination directory in the engine. Defaults to None falls back to DEFAULT_ASSET_PATH. replace (bool, optional): whether to replace the existing asset. Defaults to True. + with_parent_dir (bool, optional): whether to create a parent directory that contains the imported asset. + If False, the imported asset will be in `dst_dir_in_engine` directly. Defaults to True. Returns: List[str]: a list of paths to the imported assets, e.g. ["/Game/XRFeitoriaUnreal/Assets/SMPL_XL"] @@ -90,44 +133,57 @@ def import_asset( paths = path.copy() asset_paths = [] - for path in paths: - assert Path(path).exists(), f'File does not exist: {path}' - name = Path(path).stem - dst_dir = unreal.Paths.combine([dst_dir_in_engine, name]) - dst_path = unreal.Paths.combine([dst_dir, name]) # check if asset exists - if unreal.EditorAssetLibrary.does_asset_exist(dst_path) and not replace: - asset_paths.append(dst_path) - continue - - unreal.log(f'Importing asset: {path}') - if path.lower().endswith('.fbx'): - asset_tools = unreal.AssetToolsHelpers.get_asset_tools() - import_options = unreal.FbxImportUI() - import_options.set_editor_property('import_animations', True) - - import_task = unreal.AssetImportTask() - import_task.set_editor_property('automated', True) - import_task.set_editor_property('destination_name', '') - import_task.set_editor_property('destination_path', dst_dir) - import_task.set_editor_property('filename', path) - import_task.set_editor_property('replace_existing', replace) - import_task.set_editor_property('options', import_options) - - import_tasks = [import_task] - asset_tools.import_asset_tasks(import_tasks) - asset_paths.extend( - [path.partition('.')[0] for path in import_task.get_editor_property('imported_object_paths')] - ) - else: - assetsTools = unreal.AssetToolsHelpers.get_asset_tools() - assetImportData = unreal.AutomatedAssetImportData() - assetImportData.destination_path = dst_dir - assetImportData.filenames = [path] - assetImportData.replace_existing = replace - assets: List[unreal.Object] = assetsTools.import_assets_automated(assetImportData) - asset_paths.extend([asset.get_path_name().partition('.')[0] for asset in assets]) - unreal.EditorAssetLibrary.save_directory(dst_dir, False, True) # save assets - unreal.log(f'Imported asset: {path}') + with unreal.ScopedSlowTask(len(paths), 'Importing assets') as slow_task: + slow_task.make_dialog(True) + for path in paths: + assert Path(path).exists(), f'File does not exist: {path}' + # update progress bar + slow_task.enter_progress_frame(1, f'Importing assets: {path}') + if slow_task.should_cancel(): + unreal.log('Importing assets cancelled') + break + + # get destination directory + name = Path(path).stem + dst_dir = dst_dir_in_engine + if with_parent_dir: + dst_dir = unreal.Paths.combine([dst_dir, name]) + + # check if asset exists + dst_path = unreal.Paths.combine([dst_dir, name]) # check if asset exists + if unreal.EditorAssetLibrary.does_asset_exist(dst_path) and not replace: + asset_paths.append(dst_path) + continue + + unreal.log(f'Importing asset: {path}') + if path.lower().endswith('.fbx'): + asset_tools = unreal.AssetToolsHelpers.get_asset_tools() + import_options = unreal.FbxImportUI() + import_options.set_editor_property('import_animations', True) + + import_task = unreal.AssetImportTask() + import_task.set_editor_property('automated', True) + import_task.set_editor_property('destination_name', '') + import_task.set_editor_property('destination_path', dst_dir) + import_task.set_editor_property('filename', path) + import_task.set_editor_property('replace_existing', replace) + import_task.set_editor_property('options', import_options) + + import_tasks = [import_task] + asset_tools.import_asset_tasks(import_tasks) + asset_paths.extend( + [path.partition('.')[0] for path in import_task.get_editor_property('imported_object_paths')] + ) + else: + assetsTools = unreal.AssetToolsHelpers.get_asset_tools() + assetImportData = unreal.AutomatedAssetImportData() + assetImportData.destination_path = dst_dir + assetImportData.filenames = [path] + assetImportData.replace_existing = replace + assets: List[unreal.Object] = assetsTools.import_assets_automated(assetImportData) + asset_paths.extend([asset.get_path_name().partition('.')[0] for asset in assets]) + unreal.EditorAssetLibrary.save_directory(dst_dir, False, True) # save assets + unreal.log(f'Imported asset: {path}') return asset_paths diff --git a/src/XRFeitoriaUnreal/Content/Python/utils_actor.py b/src/XRFeitoriaUnreal/Content/Python/utils_actor.py index 3c3bf56b..d323e2f0 100644 --- a/src/XRFeitoriaUnreal/Content/Python/utils_actor.py +++ b/src/XRFeitoriaUnreal/Content/Python/utils_actor.py @@ -1,22 +1,9 @@ -import json -from pathlib import Path -from typing import Dict, List, Tuple, TypedDict, Union +from typing import Dict, List, Tuple, Union import unreal -from constants import SubSystem +from constants import SubSystem, mask_colors from utils import get_world -root = Path(__file__).parent.resolve() -color_type = TypedDict( - 'color', - { - 'name': str, - 'hex': str, - 'rgb': Tuple[int, int, int], - }, -) -mask_colors: List[color_type] = json.loads((root / 'data' / 'mask_colors.json').read_text()) - def get_stencil_value(actor: unreal.Actor) -> int: # skeletal mesh component diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp index c239aab9..8cf12e4b 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp @@ -2,6 +2,15 @@ #include "Annotator.h" +#include "SequencerBindingProxy.h" +#include "SequencerScriptingRange.h" +#include "SequencerSettings.h" +#include "SequencerTools.h" + +#include "EngineUtils.h" +#include "XF_BlueprintFunctionLibrary.h" + + // Sets default values AAnnotator::AAnnotator() { @@ -10,6 +19,300 @@ AAnnotator::AAnnotator() } +void AAnnotator::Initialize() +{ + if (bInitialized) return; + + // Get Playing LevelSequenceActor + for (TActorIterator ActorItr(GetWorld()); ActorItr; ++ActorItr) + { + ALevelSequenceActor* ALevelSequenceActor = *ActorItr; + bool bValid = ALevelSequenceActor && ALevelSequenceActor->GetSequencePlayer() && ALevelSequenceActor->GetSequencePlayer()->IsPlaying(); + if (!bValid) return; + + LevelSequenceActor = ALevelSequenceActor; + LevelSequencePlayer = ALevelSequenceActor->GetSequencePlayer(); + UE_LOG(LogXF, Log, TEXT("Detected LevelSequenceActor: %s"), *LevelSequenceActor->GetName()); + } + ULevelSequence* LevelSequence = LevelSequenceActor->GetSequence(); + UMovieScene* MovieScene = LevelSequence->GetMovieScene(); + + // Get All Bound Objects + TMap BoundObjects; + for (int idx = 0; idx < MovieScene->GetSpawnableCount(); idx++) + { + FMovieSceneSpawnable spawnable = MovieScene->GetSpawnable(idx); + FGuid guid = spawnable.GetGuid(); + FString name = spawnable.GetName(); + + TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); + if (boundObjects.Num() == 0) continue; + BoundObjects.Add(name, boundObjects[0]); + } + for (int idx = 0; idx < MovieScene->GetPossessableCount(); idx++) + { + FMovieScenePossessable possessable = MovieScene->GetPossessable(idx); + FGuid guid = possessable.GetGuid(); + FString name = possessable.GetName(); + + TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); + if (boundObjects.Num() == 0) continue; + BoundObjects.Add(name, boundObjects[0]); + } + UE_LOG(LogXF, Log, TEXT("Detected %d bound objects"), BoundObjects.Num()); + + // Get CameraActors, StaticMeshComponents, SkeletalMeshComponents from LevelSequence + for (TPair pair : BoundObjects) + { + FString name = pair.Key; + UObject* BoundObject = pair.Value; + + // loop over bound objects + if (BoundObject->IsA(ACameraActor::StaticClass())) + { + ACameraActor* Camera = Cast(BoundObject); + CameraActors.Add(name, Camera); + } + else if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) + { + ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); + SkeletalMeshComponents.Add(name, SkeletalMeshActor->GetSkeletalMeshComponent()); + } + else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) + { + AStaticMeshActor* StaticMeshActor = Cast(BoundObject); + StaticMeshComponents.Add(name, StaticMeshActor->GetStaticMeshComponent()); + } + } + UE_LOG(LogXF, Log, TEXT("Detected %d CameraActors, %d StaticMeshComponents, %d SkeletalMeshComponents"), + CameraActors.Num(), StaticMeshComponents.Num(), SkeletalMeshComponents.Num()); + + // Save Skeleton Names (only save on the first frame) + if (bSaveSkeletonPosition) + { + for (TPair pair : SkeletalMeshComponents) + { + FString MeshName = pair.Key; + USkeletalMeshComponent* SkeletalMeshComponent = pair.Value; + + TArray SkeletonPositions; + TArray SkeletonNames; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( + SkeletalMeshComponent, SkeletonPositions, SkeletonNames); + + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get skeleton positions")); + continue; + } + + TArray SkeletonNamesString; + for (FName name : SkeletonNames) SkeletonNamesString.Add(name.ToString()); + FString BoneNamePath = FPaths::Combine( + DirectorySequence, + NameSkeleton, + MeshName + "_BoneName.txt" + ); // {seq_dir}/{skeleton}/{actor_name}_BoneName.txt + FFileHelper::SaveStringArrayToFile(SkeletonNamesString, *BoneNamePath); + } + } + + // Finish Initialization + bInitialized = true; +} + +void AAnnotator::ExportCameraParameters(int FrameNumber) +{ + if (!bInitialized || CameraActors.Num() == 0) return; + for (TPair pair : CameraActors) + { + FString CameraName = pair.Key; + ACameraActor* Camera = pair.Value; + + FVector CamLocation = Camera->GetActorLocation(); + FRotator CamRotation = Camera->GetActorRotation(); + float FOV = Camera->GetCameraComponent()->FieldOfView; + + TArray CamInfo; + CamInfo.Add(CamLocation.X); + CamInfo.Add(CamLocation.Y); + CamInfo.Add(CamLocation.Z); + CamInfo.Add(CamRotation.Roll); + CamInfo.Add(CamRotation.Pitch); + CamInfo.Add(CamRotation.Yaw); + CamInfo.Add(FOV); + CamInfo.Add(Width); + CamInfo.Add(Height); + + FString CameraTransformPath = FPaths::Combine( + DirectorySequence, // seq_dir + NameCameraParams, // camera_params + CameraName, // camera_name + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" // frame_idx + ); // {seq_dir}/{camera_params}/{camera_name}/{frame_idx}.dat + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); + } +} + +void AAnnotator::ExportStaticMeshParameters(int FrameNumber) +{ + for (TPair pair : StaticMeshComponents) + { + FString MeshName = pair.Key; + UStaticMeshComponent* StaticMeshComponent = pair.Value; + + // Save Actor Info (location, rotation, stencil value) + { + FVector ActorLocation = StaticMeshComponent->GetOwner()->GetActorLocation(); + FRotator ActorRotation = StaticMeshComponent->GetOwner()->GetActorRotation(); + int StencilValue = StaticMeshComponent->CustomDepthStencilValue; + + TArray ActorInfo; + ActorInfo.Add(ActorLocation.X); + ActorInfo.Add(ActorLocation.Y); + ActorInfo.Add(ActorLocation.Z); + ActorInfo.Add(ActorRotation.Roll); + ActorInfo.Add(ActorRotation.Pitch); + ActorInfo.Add(ActorRotation.Yaw); + ActorInfo.Add(StencilValue); + + FString ActorInfoPath = FPaths::Combine( + DirectorySequence, + NameActorInfos, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ); // {seq_dir}/{actor_params}/{actor_name}/{frame_idx}.dat + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(ActorInfo, ActorInfoPath); + } + + // Save Vertex Positions + if (bSaveVerticesPosition) + { + // Get Vertex Positions + TArray VertexPositions; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetStaticMeshVertexLocations(StaticMeshComponent, LODIndexToSave, VertexPositions); + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); + continue; + } + + TArray VertexPositionsFloat; + for (FVector position : VertexPositions) + { + VertexPositionsFloat.Add(position.X); + VertexPositionsFloat.Add(position.Y); + VertexPositionsFloat.Add(position.Z); + } + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( + VertexPositionsFloat, + FPaths::Combine( + DirectorySequence, + NameVertices, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ) + ); + } + } +} + +void AAnnotator::ExportSkeletalMeshParameters(int FrameNumber) +{ + for (TPair pair : SkeletalMeshComponents) + { + FString MeshName = pair.Key; + USkeletalMeshComponent* SkeletalMeshComponent = pair.Value; + + // Save Actor Info (location, rotation, stencil value) + { + FVector ActorLocation = SkeletalMeshComponent->GetOwner()->GetActorLocation(); + FRotator ActorRotation = SkeletalMeshComponent->GetOwner()->GetActorRotation(); + int StencilValue = SkeletalMeshComponent->CustomDepthStencilValue; + + TArray ActorInfo; + ActorInfo.Add(ActorLocation.X); + ActorInfo.Add(ActorLocation.Y); + ActorInfo.Add(ActorLocation.Z); + ActorInfo.Add(ActorRotation.Roll); + ActorInfo.Add(ActorRotation.Pitch); + ActorInfo.Add(ActorRotation.Yaw); + ActorInfo.Add(StencilValue); + + FString ActorInfoPath = FPaths::Combine( + DirectorySequence, + NameActorInfos, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ); // {seq_dir}/{actor_params}/{actor_name}/{frame_idx}.dat + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(ActorInfo, ActorInfoPath); + } + + // Save Vertex Positions + if (bSaveVerticesPosition) + { + // Get Vertex Positions (with LOD) + TArray VertexPositions; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshVertexLocationsByLODIndex(SkeletalMeshComponent, LODIndexToSave, VertexPositions); + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); + continue; + } + + TArray VertexPositionsFloat; + for (FVector position : VertexPositions) + { + VertexPositionsFloat.Add(position.X); + VertexPositionsFloat.Add(position.Y); + VertexPositionsFloat.Add(position.Z); + } + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( + VertexPositionsFloat, + FPaths::Combine( + DirectorySequence, + NameVertices, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ) + ); + } + + // Save Skeleton Positions + if (bSaveSkeletonPosition) + { + TArray SkeletonPositions; + TArray SkeletonNames; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( + SkeletalMeshComponent, SkeletonPositions, SkeletonNames); + + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get skeleton positions")); + continue; + } + + // Skeleton Positions + TArray SkeletonPositionsFloat; + for (FVector position : SkeletonPositions) + { + SkeletonPositionsFloat.Add(position.X); + SkeletonPositionsFloat.Add(position.Y); + SkeletonPositionsFloat.Add(position.Z); + } + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( + SkeletonPositionsFloat, + FPaths::Combine( + DirectorySequence, + NameSkeleton, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ) + ); + } + } +} + // Called when the game starts or when spawned void AAnnotator::BeginPlay() { @@ -22,4 +325,11 @@ void AAnnotator::Tick(float DeltaTime) { Super::Tick(DeltaTime); + Initialize(); + if (!bInitialized) return; + + int FrameNum = LevelSequencePlayer->GetCurrentTime().Time.GetFrame().Value; + ExportCameraParameters(FrameNum); + ExportSkeletalMeshParameters(FrameNum); + ExportStaticMeshParameters(FrameNum); } diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp index 4722f5f5..766ff224 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp @@ -6,8 +6,11 @@ void UCustomMoviePipelineDeferredPass::SetupImpl(const MoviePipeline::FMoviePipelineRenderPassInitSettings& InPassInitSettings) { - UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 + UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif check(OutputSettings); AdditionalPostProcessMaterials.Empty(); diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp index 8220f5a9..896109c8 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp @@ -2,36 +2,36 @@ #include "CustomMoviePipelineOutput.h" -#include "ImageWriteTask.h" #include "ImagePixelData.h" #include "ImageWriteQueue.h" #include "ImageWriteStream.h" +#include "ImageWriteTask.h" -#include "Modules/ModuleManager.h" #include "Containers/UnrealString.h" -#include "Misc/StringFormatArg.h" #include "Misc/FileHelper.h" #include "Misc/FrameRate.h" #include "Misc/Paths.h" +#include "Misc/StringFormatArg.h" +#include "Modules/ModuleManager.h" // #include "HAL/PlatformFilemanager.h" // #include "HAL/PlatformTime.h" +#include "Animation/SkeletalMeshActor.h" #include "Camera/CameraActor.h" #include "Camera/CameraComponent.h" #include "Engine/StaticMeshActor.h" -#include "Animation/SkeletalMeshActor.h" #include "MoviePipeline.h" -#include "MoviePipelineOutputSetting.h" #include "MoviePipelineBurnInSetting.h" -#include "MoviePipelineOutputBase.h" #include "MoviePipelineImageQuantization.h" -#include "MoviePipelineWidgetRenderSetting.h" +#include "MoviePipelineOutputBase.h" +#include "MoviePipelineOutputSetting.h" #include "MoviePipelineUtils.h" -#include "MovieRenderTileImage.h" +#include "MoviePipelineWidgetRenderSetting.h" #include "MovieRenderOverlappedImage.h" #include "MovieRenderPipelineCoreModule.h" +#include "MovieRenderTileImage.h" #include "XF_BlueprintFunctionLibrary.h" @@ -41,191 +41,18 @@ DECLARE_CYCLE_STAT(TEXT("ImgSeqOutput_RecieveImageData"), STAT_ImgSeqRecieveImageData, STATGROUP_MoviePipeline); -void UCustomMoviePipelineOutput::SetupForPipelineImpl(UMoviePipeline* InPipeline) -{ - if (InPipeline) - { - InPipeline->SetFlushDiskWritesPerShot(true); - } - - ULevelSequence* LevelSequence = GetPipeline()->GetTargetSequence(); - UMovieSceneSequence* MovieSceneSequence = GetPipeline()->GetTargetSequence(); - UMovieScene* MovieScene = LevelSequence->GetMovieScene(); - TArray bindings = MovieScene->GetBindings(); - - TArray bindingProxies; - for (FMovieSceneBinding binding : bindings) - { - FGuid guid = binding.GetObjectGuid(); - bindingProxies.Add(FSequencerBindingProxy(guid, MovieSceneSequence)); - } - - boundObjects = USequencerToolsFunctionLibrary::GetBoundObjects( - GetPipeline()->GetWorld(), - LevelSequence, - bindingProxies, - FSequencerScriptingRange::FromNative( - MovieScene->GetPlaybackRange(), - MovieScene->GetDisplayRate() - ) - ); - - for (FSequencerBoundObjects boundObject : boundObjects) - { - // loop over bound objects - UObject* BoundObject = boundObject.BoundObjects[0]; // only have one item - if (BoundObject->IsA(ACameraActor::StaticClass())) - { - ACameraActor* Camera = Cast(BoundObject); - Cameras.Add(Camera); - } - else if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) - { - ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); - SkeletalMeshComponents.Add(SkeletalMeshActor->GetSkeletalMeshComponent()); - } - else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) - { - AStaticMeshActor* StaticMeshActor = Cast(BoundObject); - StaticMeshComponents.Add(StaticMeshActor->GetStaticMeshComponent()); - } - else if (BoundObject->IsA(USkeletalMeshComponent::StaticClass())) - { - USkeletalMeshComponent* SkeletalMeshComponent = Cast(BoundObject); - // check if it's already in the list - bool bFound = false; - for (USkeletalMeshComponent* SkeletalMeshComponentInList : SkeletalMeshComponents) - { - if (SkeletalMeshComponentInList == SkeletalMeshComponent) - { - bFound = true; - break; - } - } - if (!bFound) SkeletalMeshComponents.Add(SkeletalMeshComponent); - } - else if (BoundObject->IsA(UStaticMeshComponent::StaticClass())) - { - UStaticMeshComponent* StaticMeshComponent = Cast(BoundObject); - // check if it's already in the list - bool bFound = false; - for (UStaticMeshComponent* StaticMeshComponentInList : StaticMeshComponents) - { - if (StaticMeshComponentInList == StaticMeshComponent) - { - bFound = true; - break; - } - } - if (!bFound) - StaticMeshComponents.Add(StaticMeshComponent); - } - } -} - void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) { - if (bIsFirstFrame) - { - // Get Output Setting + // Get Output Setting + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - check(OutputSettings); - int ResolutionX = OutputSettings->OutputResolution.X; - int ResolutionY = OutputSettings->OutputResolution.Y; - - // Save Camera Transform (KRT) - for (ACameraActor* Camera : Cameras) - { - FVector CamLocation = Camera->GetActorLocation(); - FRotator CamRotation = Camera->GetActorRotation(); - float FOV = Camera->GetCameraComponent()->FieldOfView; - - TArray CamInfo; - CamInfo.Add(CamLocation.X); - CamInfo.Add(CamLocation.Y); - CamInfo.Add(CamLocation.Z); - CamInfo.Add(CamRotation.Roll); - CamInfo.Add(CamRotation.Pitch); - CamInfo.Add(CamRotation.Yaw); - CamInfo.Add(FOV); - CamInfo.Add(ResolutionX); - CamInfo.Add(ResolutionY); - - // Actor in level - FString CameraNameFromLabel = Camera->GetActorNameOrLabel(); - // Actor spawned from sequence - FString CameraNameFromName = Camera->GetFName().GetPlainNameString(); - // XXX: Hardcode way to Judge which name is correct, need to be improved - // Should ref to - // GetPipeline()->ResolveFilenameFormatArguments(FileNameFormatString, FormatOverrides, OutputData.FilePath, FinalFormatArgs, &Payload->SampleState.OutputState); - // using {camera_name} - - bool bIsCameraInLevel = CameraNameFromName.StartsWith("CameraActor") || CameraNameFromName.StartsWith("CineCameraActor"); - FString CameraName = bIsCameraInLevel ? CameraNameFromLabel : CameraNameFromName; - - FString CameraTransformPath = GetOutputPath( - DirectoryCameraInfo / CameraName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryCameraInfo/{camera_name}/{frame_idx}.dat - CameraTransformPath = FPaths::SetExtension( - FPaths::GetPath(CameraTransformPath), - FPaths::GetExtension(CameraTransformPath) - ); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); - } - - // Save Actor Info (stencil value) - for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) - { - // Actor in level - FString MeshNameFromLabel = SkeletalMeshComponent->GetOwner()->GetActorNameOrLabel(); - // Actor spawned from sequence - FString MeshNameFromName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - // Judge which name is correct - FString MeshName = MeshNameFromName.StartsWith("SkeletalMesh") ? MeshNameFromLabel : MeshNameFromName; - - int StencilValue = SkeletalMeshComponent->CustomDepthStencilValue; - - FString ActorInfoPath = GetOutputPath( - DirectoryActorInfo / MeshName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryActorInfo/{actor_name}/{frame_idx}.dat - ActorInfoPath = FPaths::SetExtension( - FPaths::GetPath(ActorInfoPath), - FPaths::GetExtension(ActorInfoPath) - ); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatToByteFile(StencilValue, ActorInfoPath); - } - - for (UStaticMeshComponent* StaticMeshComponent : StaticMeshComponents) - { - // Actor in level - FString MeshNameFromLabel = StaticMeshComponent->GetOwner()->GetActorNameOrLabel(); - // Actor spawned from sequence - FString MeshNameFromName = StaticMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - // Judge which name is correct - FString MeshName = MeshNameFromName.StartsWith("StaticMesh") ? MeshNameFromLabel : MeshNameFromName; - - int StencilValue = StaticMeshComponent->CustomDepthStencilValue; - - FString ActorInfoPath = GetOutputPath( - DirectoryActorInfo / MeshName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryActorInfo/{actor_name}/{frame_idx}.dat - ActorInfoPath = FPaths::SetExtension( - FPaths::GetPath(ActorInfoPath), - FPaths::GetExtension(ActorInfoPath) - ); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatToByteFile(StencilValue, ActorInfoPath); - } - - bIsFirstFrame = false; - } + UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif + check(OutputSettings); SCOPE_CYCLE_COUNTER(STAT_ImgSeqRecieveImageData); @@ -235,13 +62,6 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp TArray CompositedPasses; MoviePipeline::GetPassCompositeData(InMergedOutputFrame, CompositedPasses); - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - check(OutputSettings); - - UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - FString OutputDirectory = OutputSettings->OutputDirectory.Path; for (TPair>& RenderPassData : InMergedOutputFrame->ImageOutputData) @@ -441,31 +261,3 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp GetPipeline()->AddOutputFuture(ImageWriteQueue->Enqueue(MoveTemp(TileImageTask)), OutputData); } } - -FString UCustomMoviePipelineOutput::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) -{ - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - check(OutputSettings); - FString OutputDirectory = OutputSettings->OutputDirectory.Path; - FString FileNameFormatString = OutputSettings->FileNameFormat; - - FString OutputPath; - FMoviePipelineFormatArgs Args; - TMap FormatOverrides; - FormatOverrides.Add(TEXT("camera_name"), ""); - FormatOverrides.Add(TEXT("render_pass"), PassName); - FormatOverrides.Add(TEXT("ext"), Ext); - GetPipeline()->ResolveFilenameFormatArguments( - OutputDirectory / FileNameFormatString, FormatOverrides, OutputPath, Args, InOutputState); - - if (FPaths::IsRelative(OutputPath)) - { - OutputPath = FPaths::ConvertRelativePathToFull(OutputPath); - } - - // Replace any double slashes with single slashes. - OutputPath.ReplaceInline(TEXT("//"), TEXT("/")); - - return OutputPath; -} diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp deleted file mode 100644 index b491c499..00000000 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright OpenXRLab 2023-2024. All Rights Reserved. - -#include "MoviePipelineMeshOperator.h" -#include "XF_BlueprintFunctionLibrary.h" -#include "Engine/StaticMeshActor.h" -#include "Animation/SkeletalMeshActor.h" -#include "LevelSequenceEditorBlueprintLibrary.h" -#include "MovieSceneObjectBindingID.h" -#include "Camera/CameraActor.h" -#include "Camera/CameraComponent.h" -#include "Misc/FileHelper.h" -#include "MovieRenderPipelineCoreModule.h" // For logs - - -void UMoviePipelineMeshOperator::SetupForPipelineImpl(UMoviePipeline* InPipeline) -{ - if (InPipeline) - { - InPipeline->SetFlushDiskWritesPerShot(true); - } - - ULevelSequence* LevelSequence = GetPipeline()->GetTargetSequence(); - UMovieSceneSequence* MovieSceneSequence = GetPipeline()->GetTargetSequence(); - UMovieScene* MovieScene = LevelSequence->GetMovieScene(); - - TMap bindingMap; - for (int idx = 0; idx < MovieScene->GetSpawnableCount(); idx++) - { - FMovieSceneSpawnable spawnable = MovieScene->GetSpawnable(idx); - FGuid guid = spawnable.GetGuid(); - FString name = spawnable.GetName(); - - bindingMap.Add(name, guid); - } - - for (int idx = 0; idx < MovieScene->GetPossessableCount(); idx++) - { - FMovieScenePossessable possessable = MovieScene->GetPossessable(idx); - FGuid guid = possessable.GetGuid(); - FString name = possessable.GetName(); - - bindingMap.Add(name, guid); - } - - for (TPair pair : bindingMap) - { - FString name = pair.Key; - FGuid guid = pair.Value; - - TArray _boundObjects_ = USequencerToolsFunctionLibrary::GetBoundObjects( - GetPipeline()->GetWorld(), - LevelSequence, - TArray({ FSequencerBindingProxy(guid, MovieSceneSequence) }), - FSequencerScriptingRange::FromNative( - MovieScene->GetPlaybackRange(), - MovieScene->GetDisplayRate() - ) - ); - - UObject* BoundObject = _boundObjects_[0].BoundObjects[0]; // only have one item - if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) - { - ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); - SkeletalMeshComponents.Add(name, SkeletalMeshActor->GetSkeletalMeshComponent()); - } - else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) - { - AStaticMeshActor* StaticMeshActor = Cast(BoundObject); - StaticMeshComponents.Add(name, StaticMeshActor->GetStaticMeshComponent()); - } - //else if (BoundObject->IsA(USkeletalMeshComponent::StaticClass())) - //{ - // USkeletalMeshComponent* SkeletalMeshComponent = Cast(BoundObject); - // // check if it's already in the list - // bool bFound = false; - // for (TPair SKMPair : SkeletalMeshComponents) - // { - // USkeletalMeshComponent* SkeletalMeshComponentInList = SKMPair.Value; - // if (SkeletalMeshComponentInList == SkeletalMeshComponent) - // { - // bFound = true; - // break; - // } - // } - // if (!bFound) SkeletalMeshComponents.Add(name, SkeletalMeshComponent); - //} - //else if (BoundObject->IsA(UStaticMeshComponent::StaticClass())) - //{ - // UStaticMeshComponent* StaticMeshComponent = Cast(BoundObject); - // // check if it's already in the list - // bool bFound = false; - // for (TPair SKMPair : StaticMeshComponents) - // { - // UStaticMeshComponent* StaticMeshComponentInList = SKMPair.Value; - // if (StaticMeshComponentInList == StaticMeshComponent) - // { - // bFound = true; - // break; - // } - // } - // if (!bFound) StaticMeshComponents.Add(name, StaticMeshComponent); - //} - } -} - -void UMoviePipelineMeshOperator::OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) -{ - for (TPair SKMPair : SkeletalMeshComponents) - { - // loop over Skeletal mesh components - if (!SkeletalMeshOperatorOption.bEnabled) continue; - - FString MeshName = SKMPair.Key; - USkeletalMeshComponent* SkeletalMeshComponent = SKMPair.Value; - - //// Actor in level - //FString MeshNameFromLabel = SkeletalMeshComponent->GetOwner()->GetActorNameOrLabel(); - //// Actor spawned from sequence - //FString MeshNameFromName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - //// Judge which name is correct - //FString MeshName = MeshNameFromName.StartsWith("SkeletalMesh") ? MeshNameFromLabel : MeshNameFromName; - - if (SkeletalMeshOperatorOption.bSaveVerticesPosition) - { - // Get Vertex Positions (with LOD) - TArray VertexPositions; - bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshVertexLocationsByLODIndex( - SkeletalMeshComponent, - SkeletalMeshOperatorOption.LODIndex, - VertexPositions - ); - if (!isSuccess) - { - UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); - continue; - } - TArray VertexPositionsFloat; - for (FVector position : VertexPositions) - { - VertexPositionsFloat.Add(position.X); - VertexPositionsFloat.Add(position.Y); - VertexPositionsFloat.Add(position.Z); - } - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - VertexPositionsFloat, GetOutputPath( - SkeletalMeshOperatorOption.DirectoryVertices / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - } - - if (SkeletalMeshOperatorOption.bSaveSkeletonPosition) - { - TArray SkeletonPositions; - TArray SkeletonNames; - bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( - SkeletalMeshComponent, SkeletonPositions, SkeletonNames); - - // Skeleton Names (only save on the first frame) - TArray SkeletonNamesString; - for (FName name : SkeletonNames) SkeletonNamesString.Add(name.ToString()); - FString BoneNamePath = GetOutputPath( - SkeletalMeshOperatorOption.DirectorySkeleton / MeshName, "txt", &InMergedOutputFrame->FrameOutputState); - // save to DirectorySkeleton / BoneName.txt - BoneNamePath = FPaths::Combine( - FPaths::GetPath(BoneNamePath), - FPaths::SetExtension("BoneName", FPaths::GetExtension(BoneNamePath)) - ); - if (bIsFirstFrame) FFileHelper::SaveStringArrayToFile(SkeletonNamesString, *BoneNamePath); - - // Skeleton Positions - TArray SkeletonPositionsFloat; - for (FVector position : SkeletonPositions) - { - SkeletonPositionsFloat.Add(position.X); - SkeletonPositionsFloat.Add(position.Y); - SkeletonPositionsFloat.Add(position.Z); - } - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - SkeletonPositionsFloat, GetOutputPath( - SkeletalMeshOperatorOption.DirectorySkeleton / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - } - - //if (SkeletalMeshOperatorOption.bSaveOcclusionRate || SkeletalMeshOperatorOption.bSaveOcclusionResult) - //{ - // - // float non_occlusion_rate; - // float self_occlusion_rate; - // float inter_occlusion_rate; - // TArray SkeletonPositions; - // TArray SkeletonNames; - // TArray SkeletonOcclusion; - // float MeshThickness = 5.0f; - // bool isSuccess = UXF_BlueprintFunctionLibrary::DetectInterOcclusionSkeleton( - // SkeletalMeshComponent, - // Camera, - // non_occlusion_rate, - // self_occlusion_rate, - // inter_occlusion_rate, - // SkeletonPositions, - // SkeletonNames, - // SkeletonOcclusion, - // MeshThickness, - // false - // ); - // // Occlusion Int - // TArray SkeletonOcclusionInt; - // for (EOcclusion occlusion : SkeletonOcclusion) SkeletonOcclusionInt.Add((uint8)occlusion); - // FFileHelper::SaveArrayToFile( - // SkeletonOcclusionInt, *GetOutputPath("Occlusion" / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - // - // // Occlusion rate - // TArray OcclusionRate; - // OcclusionRate.Add(non_occlusion_rate); - // OcclusionRate.Add(self_occlusion_rate); - // OcclusionRate.Add(inter_occlusion_rate); - // UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - // OcclusionRate, *GetOutputPath("OcclusionRate" / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - // // TODO: export to npz - //} - } - for (TPair SKMPair : StaticMeshComponents) - { - // loop over static mesh components - if (!StaticMeshOperatorOption.bEnabled) continue; - - UStaticMeshComponent* StaticMeshComponent = SKMPair.Value; - FString MeshName = SKMPair.Key; - - //// Actor in level - //FString MeshNameFromLabel = StaticMeshComponent->GetOwner()->GetActorNameOrLabel(); - //// Actor spawned from sequence - //FString MeshNameFromName = StaticMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - //// Judge which name is correct - //FString MeshName = MeshNameFromName.StartsWith("StaticMesh") ? MeshNameFromLabel : MeshNameFromName; - - if (StaticMeshOperatorOption.bSaveVerticesPosition) - { - // Get Vertex Positions (with LOD) - TArray VertexPositions; - bool isSuccess = UXF_BlueprintFunctionLibrary::GetStaticMeshVertexLocations( - StaticMeshComponent, - StaticMeshOperatorOption.LODIndex, - VertexPositions - ); - if (!isSuccess) - { - UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); - continue; - } - TArray VertexPositionsFloat; - for (FVector position : VertexPositions) - { - VertexPositionsFloat.Add(position.X); - VertexPositionsFloat.Add(position.Y); - VertexPositionsFloat.Add(position.Z); - } - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - VertexPositionsFloat, GetOutputPath( - StaticMeshOperatorOption.DirectoryVertices / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - } - } - - if (bIsFirstFrame) bIsFirstFrame = false; -} - - -void UMoviePipelineMeshOperator::BeginExportImpl() -{ - FCoreDelegates::OnEndFrame.RemoveAll(this); - UE_LOG(LogMovieRenderPipelineIO, Log, TEXT("Mesh Operator Ended.")); -} - -FString UMoviePipelineMeshOperator::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) -{ - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - check(OutputSettings); - FString OutputDirectory = OutputSettings->OutputDirectory.Path; - FString FileNameFormatString = OutputSettings->FileNameFormat; - - FString OutputPath; - FMoviePipelineFormatArgs Args; - TMap FormatOverrides; - FormatOverrides.Add(TEXT("camera_name"), ""); - FormatOverrides.Add(TEXT("render_pass"), PassName); - FormatOverrides.Add(TEXT("ext"), Ext); - GetPipeline()->ResolveFilenameFormatArguments( - OutputDirectory / FileNameFormatString, FormatOverrides, OutputPath, Args, InOutputState); - - if (FPaths::IsRelative(OutputPath)) - { - OutputPath = FPaths::ConvertRelativePathToFull(OutputPath); - } - - // Replace any double slashes with single slashes. - OutputPath.ReplaceInline(TEXT("//"), TEXT("/")); - - return OutputPath; -} diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp index 1cd89d7e..2880c307 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp @@ -3,27 +3,27 @@ #include "XF_BlueprintFunctionLibrary.h" +#include "Camera/CameraActor.h" +#include "EditorFramework/AssetImportData.h" +#include "Engine/ObjectLibrary.h" +#include "GameFramework/Actor.h" +#include "GenericPlatform/GenericPlatformFile.h" #include "iostream" -#include -#include -#include "Misc/Paths.h" +#include "Math/Rotator.h" +#include "Math/Vector.h" #include "Misc/FileHelper.h" #include "Misc/MessageDialog.h" -#include "Engine/ObjectLibrary.h" -#include "EditorFramework/AssetImportData.h" -#include "GenericPlatform/GenericPlatformFile.h" -#include "Rendering/SkeletalMeshRenderData.h" +#include "Misc/Paths.h" #include "Rendering/MultiSizeIndexContainer.h" #include "Rendering/SkeletalMeshLODRenderData.h" -#include "GameFramework/Actor.h" -#include "Math/Vector.h" -#include "Math/Rotator.h" -#include "Camera/CameraActor.h" +#include "Rendering/SkeletalMeshRenderData.h" +#include +#include -#include "Kismet/KismetSystemLibrary.h" -#include "Kismet/KismetStringLibrary.h" -#include "DrawDebugHelpers.h" #include "AssetRegistry/AssetRegistryModule.h" +#include "DrawDebugHelpers.h" +#include "Kismet/KismetStringLibrary.h" +#include "Kismet/KismetSystemLibrary.h" #include "PhysicsAssetUtils.h" #include "PhysicsEngine/PhysicsAsset.h" @@ -39,9 +39,6 @@ #endif -DEFINE_LOG_CATEGORY(LogXF); - - bool UXF_BlueprintFunctionLibrary::FileExists(FString Path) { IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile(); diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp index 3327cead..6a5726dd 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp @@ -1,12 +1,13 @@ // Copyright OpenXRLab 2023-2024. All Rights Reserved. #include "XRFeitoriaUnreal.h" + +#include "CustomMoviePipelineDeferredPass.h" +#include "CustomMoviePipelineOutput.h" #include "Engine/RendererSettings.h" -#include "MovieRenderPipelineSettings.h" #include "MoviePipelineCameraSetting.h" +#include "MovieRenderPipelineSettings.h" #include "Settings/EditorProjectSettings.h" -#include "CustomMoviePipelineOutput.h" -#include "CustomMoviePipelineDeferredPass.h" #define LOCTEXT_NAMESPACE "FXRFeitoriaUnrealModule" diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h index e94377c4..59522e68 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h @@ -3,53 +3,36 @@ #pragma once #include "CoreMinimal.h" + +#include "LevelSequence.h" +#include "LevelSequenceActor.h" +#include "LevelSequencePlayer.h" +#include "MovieScene.h" +#include "MovieSceneBinding.h" +#include "MovieSceneBindingProxy.h" + #include "GameFramework/Actor.h" -#include "Annotator.generated.h" +#include "Animation/SkeletalMeshActor.h" +#include "Camera/CameraActor.h" +#include "Camera/CameraComponent.h" +#include "Engine/StaticMeshActor.h" -UENUM(BlueprintType) -enum class ECustomPPMType : uint8 -{ - None, - SemanticSegmentation, - DepthMap, - NormalMap, - Roughtness, - Specular, - Tangent, - OpticaFlow, - Metallic, - Diffuse, - Basecolor, - Custom -}; +#include "Annotator.generated.h" -UCLASS() +UCLASS(Blueprintable) class XRFEITORIAUNREAL_API AAnnotator : public AActor { GENERATED_BODY() -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations|Stencil Value") - bool bSignStencilValue = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations|Stencil Value", - meta = (EditCondition = "bSignStencilValue", EditConditionHides)) - bool bManualSignStencilValue = false; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations|Stencil Value", - meta = (EditCondition = "bManualSignStencilValue", EditConditionHides)) - TArray SegmentObjects; - - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations") - ECustomPPMType PreviewAnnotationType = ECustomPPMType::None; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations", - meta = (EditCondition = "PreviewAnnotationType == ECustomPPMType::Custom", EditConditionHides)) - UMaterialInterface* CustomPostprocessMaterial; - public: // Sets default values for this actor's properties AAnnotator(); + // Initialize the Annotator + void Initialize(); + void ExportCameraParameters(int FrameNumber); + void ExportStaticMeshParameters(int FrameNumber); + void ExportSkeletalMeshParameters(int FrameNumber); protected: // Called when the game starts or when spawned @@ -59,4 +42,34 @@ class XRFEITORIAUNREAL_API AAnnotator : public AActor // Called every frame virtual void Tick(float DeltaTime) override; +public: + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString DirectorySequence; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString NameActorInfos = "actor_infos"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString NameCameraParams = "camera_params"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString NameVertices = "vertices"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString NameSkeleton = "skeleton"; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator|Resolution") + int Width = 1920; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator|Resolution") + int Height = 1080; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + bool bSaveSkeletonPosition = false; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + bool bSaveVerticesPosition = false; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + int32 LODIndexToSave = 0; + +private: + ALevelSequenceActor* LevelSequenceActor; + ULevelSequencePlayer* LevelSequencePlayer; + TMap CameraActors; + TMap StaticMeshComponents; + TMap SkeletalMeshComponents; + bool bInitialized = false; }; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h index 136e76c1..014a0153 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h @@ -2,8 +2,8 @@ #pragma once -#include "MoviePipelineDeferredPasses.h" #include "Misc/StringFormatArg.h" +#include "MoviePipelineDeferredPasses.h" #include "Runtime/Launch/Resources/Version.h" #if WITH_UNREALEXR @@ -128,7 +128,6 @@ class XRFEITORIAUNREAL_API UCustomMoviePipelineOutput : public UMoviePipelineIma { OutputFormat = EImageFormat::PNG; } - virtual void SetupForPipelineImpl(UMoviePipeline* InPipeline); virtual void OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) override; public: @@ -143,20 +142,4 @@ class XRFEITORIAUNREAL_API UCustomMoviePipelineOutput : public UMoviePipelineIma UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "RenderPasses|Additional") TArray AdditionalRenderPasses; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "RenderPasses|Camera") - FString DirectoryActorInfo = "actor_infos"; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "RenderPasses|Camera") - FString DirectoryCameraInfo = "camera_params"; - -private: - FString GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState); - -private: - TArray boundObjects; - TArray Cameras; - TArray StaticMeshComponents; - TArray SkeletalMeshComponents; - bool bIsFirstFrame = true; }; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h deleted file mode 100644 index af17eb01..00000000 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright OpenXRLab 2023-2024. All Rights Reserved. - -#pragma once - -#include "MoviePipeline.h" -#include "MoviePipelineOutputSetting.h" -#include "MoviePipelineMasterConfig.h" -#include "Misc/FileHelper.h" -// #include "HAL/PlatformFilemanager.h" - -#if WITH_EDITOR -#include "MovieSceneExportMetadata.h" -#include "MovieSceneToolHelpers.h" -#include "MovieScene.h" -#endif - -#include "SequencerTools.h" -#include "SequencerSettings.h" -#include "SequencerBindingProxy.h" -#include "SequencerScriptingRange.h" - -#include "Kismet/KismetMathLibrary.h" -#include "Kismet/KismetStringLibrary.h" - -#include "LevelSequence.h" -#include "Components/StaticMeshComponent.h" -#include "Components/SkeletalMeshComponent.h" - -#include "CoreMinimal.h" -#include "MoviePipelineOutputBase.h" -#include "MovieRenderPipelineDataTypes.h" -#include "MoviePipelineMeshOperator.generated.h" - -/** - * - */ - -USTRUCT(BlueprintType) -struct XRFEITORIAUNREAL_API FMeshOperatorOption -{ - GENERATED_BODY() - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bEnabled = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bSaveVerticesPosition = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionResult = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionRate = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - FString DirectoryVertices = "vertices"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - int32 LODIndex = 0; -}; - - -USTRUCT(BlueprintType) -struct XRFEITORIAUNREAL_API FSkeletalMeshOperatorOption -{ - GENERATED_BODY() - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bEnabled = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bSaveVerticesPosition = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bSaveSkeletonPosition = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionResult = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionRate = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - FString DirectoryVertices = "vertices"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - FString DirectorySkeleton = "skeleton"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - int32 LODIndex = 0; -}; - -UCLASS(Blueprintable) -class XRFEITORIAUNREAL_API UMoviePipelineMeshOperator : public UMoviePipelineOutputBase -{ - GENERATED_BODY() -public: -#if WITH_EDITOR - virtual FText GetDisplayText() const override { return NSLOCTEXT("MovieRenderPipeline", "MeshOperator_DisplayText", "Mesh Operator"); } -#endif - virtual void SetupForPipelineImpl(UMoviePipeline* InPipeline); - virtual void OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) override; - virtual void BeginExportImpl() override; -private: - FString GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState); - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mesh Operator") - FMeshOperatorOption StaticMeshOperatorOption = FMeshOperatorOption(); - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mesh Operator") - FSkeletalMeshOperatorOption SkeletalMeshOperatorOption = FSkeletalMeshOperatorOption(); - -private: - TMap StaticMeshComponents; - TMap SkeletalMeshComponents; - bool bIsFirstFrame = true; -}; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h index 41910f62..e7bc47c2 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h @@ -1,18 +1,20 @@ // Copyright OpenXRLab 2023-2024. All Rights Reserved. #pragma once + +#include "Components/PostProcessComponent.h" +#include "Components/SceneCaptureComponent2D.h" +#include "Engine/TextureRenderTarget2D.h" #include "Serialization/Archive.h" -#include "Serialization/BufferArchive.h" #include "Serialization/ArchiveSaveCompressedProxy.h" -#include "Engine/TextureRenderTarget2D.h" -#include "Components/SceneCaptureComponent2D.h" -#include "Components/PostProcessComponent.h" +#include "Serialization/BufferArchive.h" #include "CoreMinimal.h" #include "Kismet/BlueprintFunctionLibrary.h" #include "XF_BlueprintFunctionLibrary.generated.h" DECLARE_LOG_CATEGORY_EXTERN(LogXF, Log, All); +DEFINE_LOG_CATEGORY(LogXF); /** * diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs index a0ea0481..5e5347de 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs @@ -36,6 +36,7 @@ public XRFeitoriaUnreal(ReadOnlyTargetRules Target) : base(Target) "UEOpenExrRTTI", // Needed for EXR metadata "ImageWrapper", "CinematicCamera", // For metadata + "MovieRenderPipelineCore", "MovieRenderPipelineSettings", // For settings "MovieRenderPipelineRenderPasses", "MovieRenderPipelineEditor", diff --git a/tests/unreal/sequence.py b/tests/unreal/sequence.py index 32118ed5..e38bb59a 100644 --- a/tests/unreal/sequence.py +++ b/tests/unreal/sequence.py @@ -23,67 +23,77 @@ def new_seq(xf_runner: XRFeitoriaUnreal, level_path: str, seq_name: str): kc_path = xf_runner.utils.import_asset(path=kc_fbx) - with xf_runner.Sequence.new(level=level_path, seq_name=seq_name, seq_length=30, replace=True) as seq: - seq.show() - seq.spawn_camera(location=(-5, 0, 1), rotation=(0, 0, 0), fov=90.0, camera_name='Camera') - seq.spawn_camera_with_keys( - transform_keys=[ - SeqTransKey(frame=0, location=(-2, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), - SeqTransKey(frame=30, location=(-5, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), - ], - fov=90.0, - camera_name='Camera2', - ) - seq.spawn_actor( - actor_asset_path='/Engine/BasicShapes/Cube', - actor_name='Actor', - location=[3, 0, 0], - rotation=[0, 0, 0], - stencil_value=2, - ) - seq.spawn_actor_with_keys( - actor_asset_path='/Engine/BasicShapes/Cone', - transform_keys=[ - SeqTransKey(frame=0, location=(-1, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), - SeqTransKey(frame=30, location=(0, 3, 5), rotation=(0, 0, 360), interpolation='AUTO'), - ], - actor_name='Actor2', - stencil_value=3, - ) - seq.spawn_actor( - actor_asset_path='/Engine/BasicShapes/Cylinder', - location=[0, 0, 0], - rotation=[0, 0, 0], - stencil_value=4, - ) - - seq.spawn_actor_with_keys( - actor_asset_path=kc_path, - transform_keys=[ - SeqTransKey( - frame=0, location=(0, 0, 0), rotation=(0, 0, 0), scale=(0.05, 0.05, 0.05), interpolation='AUTO' - ), - SeqTransKey(frame=5, location=(2, 0, 3), rotation=(0, 180, 0), interpolation='AUTO'), - SeqTransKey(frame=10, location=(0, 3, 0), rotation=(180, 0, 0), interpolation='AUTO'), - SeqTransKey(frame=15, location=(0, 0, 3), rotation=(0, 0, 180), interpolation='AUTO'), - SeqTransKey(frame=20, location=(0, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), - ], - actor_name='KoupenChan', - stencil_value=5, - ) - - seq.add_to_renderer( - output_path=output_path, - resolution=(1920, 1080), - render_passes=[ - RenderPass('img', 'png'), - RenderPass('mask', 'exr'), - ], - export_vertices=True, - export_skeleton=True, - ) - - xf_runner.utils.save_current_level() + seq = xf_runner.sequence(level=level_path, seq_name=seq_name, seq_length=30, replace=True) + seq.spawn_camera_with_keys( + transform_keys=[ + SeqTransKey(frame=0, location=(0, 3, 1), rotation=(0, 0, -90), interpolation='AUTO'), + SeqTransKey(frame=30, location=(-3, 2, 2), rotation=(0, 0, -45), interpolation='AUTO'), + ], + fov=90.0, + camera_name='camera', + ) + camera2 = xf_runner.Camera.spawn(camera_name='camera2') + seq.use_camera_with_keys( + camera=camera2, + transform_keys=[ + SeqTransKey(frame=0, location=(-2, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), + SeqTransKey(frame=30, location=(-5, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), + ], + fov=90.0, + ) + seq.spawn_actor( + actor_asset_path='/Engine/BasicShapes/Cube', + actor_name='Actor', + location=[3, 0, 0], + rotation=[0, 0, 0], + stencil_value=2, + ) + seq.spawn_actor_with_keys( + actor_asset_path='/Engine/BasicShapes/Cone', + transform_keys=[ + SeqTransKey(frame=0, location=(-1, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), + SeqTransKey(frame=30, location=(0, 3, 5), rotation=(0, 0, 360), interpolation='AUTO'), + ], + actor_name='Actor2', + stencil_value=3, + ) + seq.spawn_actor( + actor_asset_path='/Engine/BasicShapes/Cylinder', + location=[0, 0, 0], + rotation=[0, 0, 0], + stencil_value=4, + ) + + seq.spawn_actor_with_keys( + actor_asset_path=kc_path, + transform_keys=[ + SeqTransKey( + frame=0, location=(0, 0, 0), rotation=(0, 0, 0), scale=(0.05, 0.05, 0.05), interpolation='AUTO' + ), + SeqTransKey(frame=5, location=(2, 0, 3), rotation=(0, 180, 0), interpolation='AUTO'), + SeqTransKey(frame=10, location=(0, 3, 0), rotation=(180, 0, 0), interpolation='AUTO'), + SeqTransKey(frame=15, location=(0, 0, 3), rotation=(0, 0, 180), interpolation='AUTO'), + SeqTransKey(frame=20, location=(0, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), + ], + actor_name='KoupenChan', + stencil_value=5, + ) + + seq.add_to_renderer( + output_path=output_path, + resolution=(1920, 1080), + render_passes=[ + RenderPass('img', 'png'), + RenderPass('mask', 'exr'), + ], + export_vertices=True, + export_skeleton=True, + ) + + xf_runner.utils.save_current_level() + + seq.save() + seq.close() def sequence_test(debug: bool = False, background: bool = False): @@ -105,7 +115,7 @@ def sequence_test(debug: bool = False, background: bool = False): for frame_idx in range(0, 30, 5): visualize_vertices( - camera_name='Camera', + camera_name='camera', actor_names=['KoupenChan'], seq_output_path=output_path / seq_name, frame_idx=frame_idx, diff --git a/xrfeitoria/camera/camera_base.py b/xrfeitoria/camera/camera_base.py index 89cc5a6c..a2531b68 100644 --- a/xrfeitoria/camera/camera_base.py +++ b/xrfeitoria/camera/camera_base.py @@ -15,6 +15,10 @@ class CameraBase(ABC, ObjectBase): _object_utils = ObjectUtilsBase + @classmethod + def from_param(cls, param): + ... + @classmethod def spawn( cls, diff --git a/xrfeitoria/camera/camera_parameter.py b/xrfeitoria/camera/camera_parameter.py index a63aab7d..64a1f708 100644 --- a/xrfeitoria/camera/camera_parameter.py +++ b/xrfeitoria/camera/camera_parameter.py @@ -8,7 +8,8 @@ from xrprimer.data_structure.camera import PinholeCameraParameter from xrprimer.transform.convention.camera import convert_camera_parameter -from ..data_structure.constants import PathLike +from ..data_structure.constants import PathLike, Vector +from ..utils.converter import ConverterUnreal class CameraParameter(PinholeCameraParameter): @@ -110,6 +111,38 @@ def get_extrinsic(self) -> List: """ return self.extrinsic.tolist() + def model_dump(self) -> dict: + """Dump camera parameters to a dict.""" + return { + 'class_name': 'PinholeCameraParameter', + 'convention': self.convention, + 'extrinsic_r': self.extrinsic_r.tolist(), + 'extrinsic_t': self.extrinsic_t.tolist(), + 'height': self.height, + 'width': self.width, + 'intrinsic': self.intrinsic.tolist(), + 'name': self.name, + 'world2cam': self.world2cam, + } + + @classmethod + def from_dict(cls, data: dict) -> 'CameraParameter': + """Construct a camera parameter data structure from a dict. + + Args: + data (dict): The camera parameter data. + + Returns: + CameraParameter: An instance of CameraParameter class. + """ + return cls( + K=data['intrinsic'], + R=data['extrinsic_r'], + T=data['extrinsic_t'], + convention=data['convention'], + world2cam=data['world2cam'], + ) + @classmethod def fromfile(cls, file: PathLike) -> 'CameraParameter': """Construct a camera parameter data structure from a json file. @@ -141,9 +174,30 @@ def from_bin(cls, file: PathLike) -> 'CameraParameter': rotation = dat[3:6] camera_fov = dat[6] image_size = (dat[7], dat[8]) # (width, height) + return cls.from_unreal_convention(location, rotation, camera_fov, image_size) + + @classmethod + def from_unreal_convention( + cls, + location: Vector, + rotation: Vector, + fov: float, + image_size: Tuple[int, int], # (width, height) + ) -> 'CameraParameter': + """Converts camera parameters from Unreal Engine convention to CameraParameter + object. + Args: + location (Vector): The camera location in Unreal Engine convention. + rotation (Vector): The camera rotation in Unreal Engine convention. + fov (float): The camera field of view in degrees. + image_size (Tuple[int, int]): The size of the camera image in pixels (width, height). + + Returns: + CameraParameter: The converted camera parameters. + """ # intrinsic matrix K - fov = math.radians(camera_fov) + fov = math.radians(fov) focal = max(image_size) / 2 / math.tan(fov / 2) fx = fy = focal K = np.array( @@ -155,9 +209,8 @@ def from_bin(cls, file: PathLike) -> 'CameraParameter': ) # extrinsic matrix RT - x, y, z = -rotation[1], -rotation[2], -rotation[0] - R = rotation_matrix([x, y, z], order='xyz', degrees=True) - _T = np.array([location[1], -location[2], location[0]]) / 100.0 # unit: meter + R = ConverterUnreal.rotation_camera_from_ue(rotation, degrees=True) + _T = ConverterUnreal.location_from_ue(location) T = -R @ _T # construct camera parameter @@ -198,129 +251,3 @@ def convert_convention(self, dst: str): def __repr__(self) -> str: return f'CameraParameter(T={self.extrinsic_t}, convention="{self.convention}", world2cam={self.world2cam})' - - -def rotation_matrix(angles: Tuple[float, float, float], order='xyz', degrees: bool = True) -> npt.NDArray[np.float32]: - """ - Args: - angles (Tuple[float, float, float]): Rotation angles in degrees or radians. - order (str, optional): Rotation order. Defaults to 'xyz'. - degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. - Returns: - ndarray: Rotation matrix 3x3. - - Examples: - >>> rotation_matrix((0, 0, 0), order='xyz') - - References: - https://programming-surgeon.com/en/euler-angle-python-en/ - """ - if degrees: - angles = np.deg2rad(angles) - - theta1, theta2, theta3 = angles - c1 = np.cos(theta1) - s1 = np.sin(theta1) - c2 = np.cos(theta2) - s2 = np.sin(theta2) - c3 = np.cos(theta3) - s3 = np.sin(theta3) - - if order == 'xzx': - matrix = np.array( - [ - [c2, -c3 * s2, s2 * s3], - [c1 * s2, c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3], - [s1 * s2, c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3], - ] - ) - elif order == 'xyx': - matrix = np.array( - [ - [c2, s2 * s3, c3 * s2], - [s1 * s2, c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1], - [-c1 * s2, c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3], - ] - ) - elif order == 'yxy': - matrix = np.array( - [ - [c1 * c3 - c2 * s1 * s3, s1 * s2, c1 * s3 + c2 * c3 * s1], - [s2 * s3, c2, -c3 * s2], - [-c3 * s1 - c1 * c2 * s3, c1 * s2, c1 * c2 * c3 - s1 * s3], - ] - ) - elif order == 'yzy': - matrix = np.array( - [ - [c1 * c2 * c3 - s1 * s3, -c1 * s2, c3 * s1 + c1 * c2 * s3], - [c3 * s2, c2, s2 * s3], - [-c1 * s3 - c2 * c3 * s1, s1 * s2, c1 * c3 - c2 * s1 * s3], - ] - ) - elif order == 'zyz': - matrix = np.array( - [ - [c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3, c1 * s2], - [c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3, s1 * s2], - [-c3 * s2, s2 * s3, c2], - ] - ) - elif order == 'zxz': - matrix = np.array( - [ - [c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1, s1 * s2], - [c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3, -c1 * s2], - [s2 * s3, c3 * s2, c2], - ] - ) - elif order == 'xyz': - matrix = np.array( - [ - [c2 * c3, -c2 * s3, s2], - [c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1], - [s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2], - ] - ) - elif order == 'xzy': - matrix = np.array( - [ - [c2 * c3, -s2, c2 * s3], - [s1 * s3 + c1 * c3 * s2, c1 * c2, c1 * s2 * s3 - c3 * s1], - [c3 * s1 * s2 - c1 * s3, c2 * s1, c1 * c3 + s1 * s2 * s3], - ] - ) - elif order == 'yxz': - matrix = np.array( - [ - [c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3, c2 * s1], - [c2 * s3, c2 * c3, -s2], - [c1 * s2 * s3 - c3 * s1, c1 * c3 * s2 + s1 * s3, c1 * c2], - ] - ) - elif order == 'yzx': - matrix = np.array( - [ - [c1 * c2, s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3], - [s2, c2 * c3, -c2 * s3], - [-c2 * s1, c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3], - ] - ) - elif order == 'zyx': - matrix = np.array( - [ - [c1 * c2, c1 * s2 * s3 - c3 * s1, s1 * s3 + c1 * c3 * s2], - [c2 * s1, c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3], - [-s2, c2 * s3, c2 * c3], - ] - ) - elif order == 'zxy': - matrix = np.array( - [ - [c1 * c3 - s1 * s2 * s3, -c2 * s1, c1 * s3 + c3 * s1 * s2], - [c3 * s1 + c1 * s2 * s3, c1 * c2, s1 * s3 - c1 * c3 * s2], - [-c2 * s3, s2, c2 * c3], - ] - ) - - return matrix diff --git a/xrfeitoria/data_structure/constants.py b/xrfeitoria/data_structure/constants.py index 0266d34d..ed5b7359 100644 --- a/xrfeitoria/data_structure/constants.py +++ b/xrfeitoria/data_structure/constants.py @@ -10,6 +10,14 @@ PathLike = Union[str, Path] MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]] actor_info_type = TypedDict('actor_info', {'actor_name': str, 'mask_color': Tuple[int, int, int]}) +color_type = TypedDict( + 'color', + { + 'name': str, + 'hex': str, + 'rgb': Tuple[int, int, int], + }, +) ##### Package Constants ##### @@ -17,7 +25,7 @@ plugin_name_blender = 'XRFeitoriaBpy' plugin_name_unreal = 'XRFeitoriaUnreal' plugin_name_pattern = '{plugin_name}-{plugin_version}-{engine_version}-{platform}' -xf_obj_name = '[XF]{obj_type}-{obj_idx:03d}' +xf_obj_name = 'XF-{obj_type}-{obj_idx:03d}' ##### Path Constants ##### @@ -39,7 +47,7 @@ default_path_unreal = f'/Game/{plugin_name_unreal}' default_asset_path_unreal = f'{default_path_unreal}/Assets' -default_sequence_path_unreal = f'{default_path_unreal}/Sequences' +default_sequence_dir_unreal = f'{default_path_unreal}/Sequences' ##### Enum Constants ##### diff --git a/xrfeitoria/data_structure/models.py b/xrfeitoria/data_structure/models.py index 4156f11c..f2d337cb 100644 --- a/xrfeitoria/data_structure/models.py +++ b/xrfeitoria/data_structure/models.py @@ -248,8 +248,6 @@ class AntiAliasSetting(BaseModel): anti_aliasing: AntiAliasSetting = Field( default=AntiAliasSetting(), description='Anti aliasing setting of the render job.' ) - export_vertices: bool = Field(default=False, description='Whether to export vertices of the render job.') - export_skeleton: bool = Field(default=False, description='Whether to export skeleton of the render job.') export_audio: bool = Field(default=False, description='Whether to export audio of the render job.') class Config: diff --git a/xrfeitoria/factory.py b/xrfeitoria/factory.py index b778ace5..2ad0a1cc 100644 --- a/xrfeitoria/factory.py +++ b/xrfeitoria/factory.py @@ -1,9 +1,14 @@ -from typing import Optional +from typing import TYPE_CHECKING, Optional from . import _tls from .data_structure.constants import EngineEnum, PathLike, default_level_blender from .utils import setup_logger +if TYPE_CHECKING: + from typing_extensions import deprecated +else: + deprecated = lambda *args, **kwargs: (lambda func: func) + __all__ = ['init_blender', 'init_unreal'] @@ -51,7 +56,7 @@ def __init__( from .actor.actor_blender import ActorBlender, ShapeBlenderWrapper # isort:skip from .material.material_blender import MaterialBlender # isort:skip from .renderer.renderer_blender import RendererBlender # isort:skip - from .sequence.sequence_wrapper import SequenceWrapperBlender, sequence_wrapper_blender # isort:skip + from .sequence.sequence_wrapper import sequence_wrapper_blender # isort:skip from .utils.runner import BlenderRPCRunner # isort:skip from .utils.functions import blender_functions # isort:skip @@ -63,8 +68,8 @@ def __init__( self.Shape = ShapeBlenderWrapper self.Renderer = RendererBlender self.render = self.Renderer.render_jobs + # self.Sequence = SequenceWrapperBlender self.sequence = sequence_wrapper_blender - self.Sequence = SequenceWrapperBlender self.utils = blender_functions self._rpc_runner = BlenderRPCRunner( engine_exec=engine_exec, @@ -76,6 +81,13 @@ def __init__( new_process=new_process, ) + @property + @deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) + def Sequence(self): + from .sequence.sequence_wrapper import SequenceWrapperBlender + + return SequenceWrapperBlender + class XRFeitoriaUnreal: """Factory class contains all the classes and functions for Unreal. @@ -119,7 +131,7 @@ def __init__( from .camera.camera_unreal import CameraUnreal # isort:skip from .actor.actor_unreal import ActorUnreal, ShapeUnrealWrapper # isort:skip from .renderer.renderer_unreal import RendererUnreal # isort:skip - from .sequence.sequence_wrapper import SequenceWrapperUnreal, sequence_wrapper_unreal # isort:skip + from .sequence.sequence_wrapper import sequence_wrapper_unreal # isort:skip from .utils.runner import UnrealRPCRunner # isort:skip from .utils.functions import unreal_functions # isort:skip @@ -130,8 +142,8 @@ def __init__( self.Shape = ShapeUnrealWrapper self.Renderer = RendererUnreal self.render = self.Renderer.render_jobs + # self.Sequence = SequenceWrapperUnreal self.sequence = sequence_wrapper_unreal - self.Sequence = SequenceWrapperUnreal self.utils = unreal_functions self._rpc_runner = UnrealRPCRunner( engine_exec=engine_exec, @@ -143,6 +155,13 @@ def __init__( new_process=new_process, ) + @property + @deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) + def Sequence(self): + from .sequence.sequence_wrapper import SequenceWrapperUnreal + + return SequenceWrapperUnreal + class init_blender(XRFeitoriaBlender): """Initialize Blender with ``XRFeitoria``, which would start Blender as RPC server, diff --git a/xrfeitoria/object/object_utils.py b/xrfeitoria/object/object_utils.py index 7ee5eae5..21e407e4 100644 --- a/xrfeitoria/object/object_utils.py +++ b/xrfeitoria/object/object_utils.py @@ -411,7 +411,7 @@ def _generate_obj_name_in_engine(obj_type: 'Literal["camera", "actor"]') -> str: Returns: str: Name of the new object. """ - objs = [obj for obj in bpy.data.objects if obj_type in obj.name and obj.name.startswith(xf_obj_name[:4])] + objs = [obj for obj in bpy.data.objects if obj_type in obj.name and obj.name.startswith(xf_obj_name[:3])] # return f'[XF]{obj_type}-{collection.name}-{(len(objs)+1):03}' return xf_obj_name.format(obj_type=obj_type, obj_idx=(len(objs) + 1)) @@ -670,7 +670,7 @@ def _generate_obj_name_in_engine(obj_type: 'Literal["camera", "actor"]') -> str: actors = [ actor for actor in actors - if obj_type in actor.get_actor_label() and actor.get_actor_label().startswith(xf_obj_name[:4]) + if obj_type in actor.get_actor_label() and actor.get_actor_label().startswith(xf_obj_name[:3]) ] return xf_obj_name.format(obj_type=obj_type, obj_idx=(len(actors) + 1)) diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 872702c5..64d15d79 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -1,4 +1,3 @@ -import json import shutil import socket from pathlib import Path @@ -6,8 +5,9 @@ from loguru import logger -from ..data_structure.constants import PathLike, RenderOutputEnumUnreal, actor_info_type +from ..data_structure.constants import PathLike, RenderOutputEnumUnreal from ..rpc import remote_unreal +from ..utils.converter import ConverterUnreal from ..utils.functions import unreal_functions from .renderer_base import RendererBase, render_status @@ -41,8 +41,6 @@ def add_job( file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0}, anti_aliasing: 'Optional[RenderJob.AntiAliasSetting]' = None, - export_vertices: bool = False, - export_skeleton: bool = False, export_audio: bool = False, ) -> None: """Add a rendering job to the renderer queue. @@ -57,8 +55,6 @@ def add_job( console_variables (Dict[str, float], optional): Console variables to set. Defaults to ``{'r.MotionBlurQuality': 0}``. Ref to :ref:`FAQ-console-variables` for details. anti_aliasing (Optional[RenderJobUnreal.AntiAliasSetting], optional): Anti aliasing setting. Defaults to None. - export_vertices (bool, optional): Whether to export vertices. Defaults to False. - export_skeleton (bool, optional): Whether to export skeleton. Defaults to False. export_audio (bool, optional): Whether to export audio. Defaults to False. Note: @@ -84,8 +80,6 @@ def add_job( file_name_format=file_name_format, console_variables=console_variables, anti_aliasing=anti_aliasing, - export_vertices=export_vertices, - export_skeleton=export_skeleton, export_audio=export_audio, ) cls._add_job_in_engine(job.model_dump(mode='json')) @@ -201,6 +195,56 @@ def convert_camera(camera_file: Path) -> None: cam_param.dump(camera_file.with_suffix('.json').as_posix()) camera_file.unlink() + def convert_actor_infos(folder: Path) -> None: + """Convert stencil value from `.dat` to `.npz`. Merge all actor info files + into one. + + actor_info files are in the format of: + ``` + { + 'location': np.ndarray, # shape: (frame, 3) + 'rotation': np.ndarray, # shape: (frame, 3, 3) + 'stencil_value': np.ndarray, # shape: (frame,) + 'mask_color': np.ndarray, # shape: (frame, 3) + } + ``` + + Args: + folder (Path): Path to the folder containing actor info files. + """ + # Get all files in the folder and sort them + actor_info_files = sorted(folder.glob('*.dat')) + if not actor_info_files: + return + # Read all actor info files into a list + location = [] + rotation = [] + stencil_value = [] + mask_color = [] + for actor_info_file in actor_info_files: + with open(actor_info_file, 'rb') as f: + dat = np.frombuffer(f.read(), np.float32).reshape(7) + location.append(ConverterUnreal.location_from_ue(dat[:3])) + rotation.append(ConverterUnreal.rotation_from_ue(dat[3:6])) + stencil_value.append(int(dat[6])) + mask_color.append(unreal_functions.get_mask_color(int(dat[6]))) + + location = np.stack(location) # shape: (frame, 3) + rotation = np.stack(rotation) # shape: (frame, 3, 3) + stencil_value = np.array(stencil_value) # shape: (frame,) + mask_color = np.array(mask_color) # shape: (frame, 3) + + # Save the actor infos in a compressed `.npz` file + np.savez_compressed( + file=folder.with_suffix('.npz'), + location=location, + rotation=rotation, + stencil_value=stencil_value, + mask_color=mask_color, + ) + # Remove the folder + shutil.rmtree(folder) + def convert_vertices(folder: Path) -> None: """Convert vertices from `.dat` to `.npz`. Merge all vertices files into one `.npz` file with structure of: {'verts': np.ndarray, 'faces': None} @@ -210,50 +254,22 @@ def convert_vertices(folder: Path) -> None: """ # Get all vertices files in the folder and sort them vertices_files = sorted(folder.glob('*.dat')) - # Read all vertices files into a list - vertices = [ - np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) for vertices_file in vertices_files - ] - if not vertices: + if not vertices_files: return - - # Stack all vertices into one array with shape (frame, verts, 3) - vertices = np.stack(vertices) - # Convert convention from unreal to opencv, [x, y, z] -> [y, -z, x] - vertices = np.stack([vertices[:, :, 1], -vertices[:, :, 2], vertices[:, :, 0]], axis=-1) - vertices /= 100 # convert from cm to m - + # Read all vertices files into an ndarray, shape: (frame, vertex, 3) + vertices = np.stack( + [ + np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) + for vertices_file in vertices_files + ] + ) + # Convert from ue camera space to opencv camera space convention + vertices = ConverterUnreal.location_from_ue(vertices) # Save the vertices in a compressed `.npz` file np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) # Remove the folder shutil.rmtree(folder) - def convert_actor_infos(folder: Path) -> None: - """Convert stencil value from `.dat` to `.json`. - - Args: - folder (Path): Path to the folder contains ``actor_infos``. - """ - # Get all stencil value files in the folder and sort them - actor_info_files = sorted(folder.glob('*.dat')) - # Read all actor info files into a list - actor_infos: List[actor_info_type] = [] - for actor_info_file in actor_info_files: - stencil_value = np.frombuffer(actor_info_file.read_bytes(), np.float32) - stencil_value = int(stencil_value) - mask_color = unreal_functions.get_mask_color(stencil_value) - actor_infos.append({'actor_name': actor_info_file.stem, 'mask_color': mask_color}) - - if not actor_infos: - return - - # Save the actor infos in a `.json` file - with (folder.parent / f'{folder.name}.json').open('w') as f: - json.dump(actor_infos, f, indent=4) - - # Remove the folder - shutil.rmtree(folder) - console = get_console() try: spinner: Spinner = console._live.renderable @@ -265,32 +281,30 @@ def convert_actor_infos(folder: Path) -> None: for idx, job in enumerate(cls.render_queue): seq_name = job.sequence_path.split('/')[-1] seq_path = Path(job.output_path).resolve() / seq_name + file_name_format = job.file_name_format # TODO: use this to rename the files + if file_name_format != '{sequence_name}/{render_pass}/{camera_name}/{frame_number}': # XXX: hard-coded + logger.warning( + 'The `file_name_format` in renderer is not the default value, which may cause some issues in post-processing. ' + ) text = f'job {idx + 1}/{len(cls.render_queue)}: seq_name="{seq_name}", post-processing...' spinner.update(text=text) # 1. convert camera parameters from `.bat` to `.json` with xrprimer - # glob camera files in {seq_path}/{cam_param_dir}/* - camera_files = sorted(seq_path.glob(f'{RenderOutputEnumUnreal.camera_params.value}/*.dat')) - for camera_file in camera_files: + for camera_file in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.camera_params.value}/*/*.dat')): convert_camera(camera_file) # 2. convert actor infos from `.dat` to `.json` - convert_actor_infos(folder=seq_path / RenderOutputEnumUnreal.actor_infos.value) + for actor_info_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.actor_infos.value}/*')): + convert_actor_infos(actor_info_folder) # 3. convert vertices from `.dat` to `.npz` - if job.export_vertices: - # glob actors in {seq_path}/vertices/* - actor_folders = sorted(seq_path.glob(f'{RenderOutputEnumUnreal.vertices.value}/*')) - for actor_folder in actor_folders: - convert_vertices(actor_folder) + for actor_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.vertices.value}/*')): + convert_vertices(actor_folder) # 4. convert skeleton from `.dat` to `.json` - if job.export_skeleton: - # glob actors in {seq_path}/skeleton/* - actor_folders = sorted(seq_path.glob(f'{RenderOutputEnumUnreal.skeleton.value}/*')) - for actor_folder in actor_folders: - convert_vertices(actor_folder) + for actor_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.skeleton.value}/*')): + convert_vertices(actor_folder) @staticmethod def _add_job_in_engine(job: 'Dict[str, Any]') -> None: diff --git a/xrfeitoria/sequence/sequence_base.pyi b/xrfeitoria/sequence/sequence_base.pyi index 4e6198e0..c82f1eda 100644 --- a/xrfeitoria/sequence/sequence_base.pyi +++ b/xrfeitoria/sequence/sequence_base.pyi @@ -1,6 +1,8 @@ from abc import ABC -from typing import List, Optional, Tuple, Union +from typing import List, Literal, Optional, Tuple, Union +from ..actor.actor_base import ActorBase +from ..camera.camera_base import CameraBase from ..data_structure.constants import EngineEnum, PathLike, Vector from ..data_structure.models import RenderPass, TransformKeys @@ -11,10 +13,10 @@ class SequenceBase(ABC): def _new( cls, seq_name: str, - level: Union[str, List[str]], - seq_fps: int = ..., - seq_length: int = ..., - replace: bool = ..., + level: Optional[str] = None, + seq_fps: int = 60, + seq_length: int = 1, + replace: bool = False, **kwargs, ) -> None: ... @classmethod @@ -22,86 +24,79 @@ class SequenceBase(ABC): @classmethod def close(cls) -> None: ... @classmethod - def save(cls) -> None: ... - @classmethod - def show(cls) -> None: ... - @classmethod def import_actor( cls, file_path: PathLike, - actor_name: Optional[str] = ..., - location: Vector = ..., - rotation: Vector = ..., - scale: Vector = ..., - stencil_value: int = ..., - ) -> ...: ... + actor_name: Optional[str] = None, + location: Vector = None, + rotation: Vector = None, + scale: Vector = None, + stencil_value: int = 1, + ) -> ActorBase: ... @classmethod def spawn_camera( - cls, - location: Vector, - rotation: Vector, - fov: float = ..., - camera_name: str = ..., - ) -> ...: ... + cls, location: Vector = None, rotation: Vector = None, fov: float = 90.0, camera_name: Optional[str] = None + ) -> CameraBase: ... @classmethod def spawn_camera_with_keys( cls, transform_keys: TransformKeys, - fov: float = ..., - camera_name: str = ..., - ) -> ...: ... + fov: float = 90.0, + camera_name: str = None, + ) -> CameraBase: ... + @classmethod def use_camera( cls, - camera, - location: Optional[Vector] = ..., - rotation: Optional[Vector] = ..., - fov: float = ..., + camera: CameraBase, + location: Optional[Vector] = None, + rotation: Optional[Vector] = None, + fov: float = None, ) -> None: ... @classmethod def use_camera_with_keys( cls, - camera, + camera: CameraBase, transform_keys: TransformKeys, - fov: float = ..., + fov: float = None, ) -> None: ... @classmethod def use_actor( cls, - actor, - location: Optional[Vector] = ..., - rotation: Optional[Vector] = ..., - scale: Optional[Vector] = ..., - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., + actor: ActorBase, + location: Optional[Vector] = None, + rotation: Optional[Vector] = None, + scale: Optional[Vector] = None, + stencil_value: int = None, + anim_asset_path: Optional[str] = None, ) -> None: ... @classmethod def use_actor_with_keys( cls, - actor, + actor: ActorBase, transform_keys: TransformKeys, - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., + stencil_value: int = None, + anim_asset_path: Optional[str] = None, ) -> None: ... @classmethod def spawn_shape( cls, - shape_type: str, - location: Vector = ..., - rotation: Vector = ..., - scale: Vector = ..., - shape_name: str = ..., - stencil_value: int = ..., + type: Literal['plane', 'cube', 'sphere', 'cylinder', 'cone'], + shape_name: str = None, + location: Vector = None, + rotation: Vector = None, + scale: Vector = None, + stencil_value: int = 1, **kwargs, - ) -> ...: ... + ) -> ActorBase: ... @classmethod def spawn_shape_with_keys( cls, transform_keys: TransformKeys, - shape_type: str, - shape_name: str = ..., - stencil_value: int = ..., + type: Literal['plane', 'cube', 'sphere', 'cylinder', 'cone'], + shape_name: str = None, + stencil_value: int = 1, **kwargs, - ) -> ...: ... + ) -> ActorBase: ... @classmethod def add_to_renderer( cls, diff --git a/xrfeitoria/sequence/sequence_unreal.py b/xrfeitoria/sequence/sequence_unreal.py index b5d059e9..53fff8db 100644 --- a/xrfeitoria/sequence/sequence_unreal.py +++ b/xrfeitoria/sequence/sequence_unreal.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Literal, Optional, Tuple, Union +from typing import Dict, List, Literal, Optional, Tuple, TypedDict, Union from loguru import logger @@ -24,6 +24,16 @@ except (ImportError, ModuleNotFoundError): pass +dict_process_dir = TypedDict( + 'dict_process_dir', + { + 'camera_dir': str, + 'actor_infos_dir': str, + 'vertices_dir': str, + 'skeleton_dir': str, + }, +) + @remote_unreal(dec_class=True, suffix='_in_engine') class SequenceUnreal(SequenceBase): @@ -53,7 +63,7 @@ def show(cls) -> None: def add_to_renderer( cls, output_path: PathLike, - resolution: Tuple[int, int], + resolution: Tuple[int, int], # (width, height) render_passes: 'List[RenderPass]', file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0}, @@ -69,7 +79,7 @@ def add_to_renderer( Args: output_path (PathLike): The path where the rendered output will be saved. - resolution (Tuple[int, int]): The resolution of the output. + resolution (Tuple[int, int]): The resolution of the output. (width, height) render_passes (List[RenderPass]): The list of render passes to be rendered. file_name_format (str, optional): The format of the output file name. Defaults to ``{sequence_name}/{render_pass}/{camera_name}/{frame_number}``. @@ -97,6 +107,14 @@ def add_to_renderer( sequence_path = SequenceUnreal._get_seq_path_in_engine() if anti_aliasing is None: anti_aliasing = RenderJobUnreal.AntiAliasSetting() + + cls._preprocess_before_render( + save_dir=f'{output_path}/{cls.name}', + resolution=resolution, + export_vertices=export_vertices, + export_skeleton=export_skeleton, + ) + cls._renderer.add_job( map_path=map_path, sequence_path=sequence_path, @@ -106,10 +124,9 @@ def add_to_renderer( file_name_format=file_name_format, console_variables=console_variables, anti_aliasing=anti_aliasing, - export_vertices=export_vertices, - export_skeleton=export_skeleton, export_audio=export_audio, ) + logger.info( f'[cyan]Added[/cyan] sequence "{cls.name}" to [bold]`Renderer`[/bold] ' f'(jobs to render: {len(cls._renderer.render_queue)})' @@ -254,6 +271,15 @@ def set_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int """ cls._set_playback_in_engine(start_frame=start_frame, end_frame=end_frame) + @classmethod + def get_playback(cls) -> Tuple[int, int]: + """Get the playback range for the sequence. + + Returns: + Tuple[int, int]: The start and end frame of the playback range. + """ + return cls._get_playback_in_engine() + @classmethod def set_camera_cut_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: """Set the playback range for the sequence. @@ -280,13 +306,24 @@ def _open(cls, seq_name: str, seq_dir: 'Optional[str]' = None) -> None: cls.name = seq_name logger.info(f'>>>> [cyan]Opened[/cyan] sequence "{cls.name}" >>>>') + @classmethod + def _preprocess_before_render( + cls, + save_dir: str, + resolution: Tuple[int, int], + export_vertices: bool, + export_skeleton: bool, + ) -> None: + # add annotator for saving camera parameters, actor infos, vertices, and skeleton + cls._add_annotator_in_engine(save_dir, resolution, export_vertices, export_skeleton) + ##################################### ###### RPC METHODS (Private) ######## ##################################### @staticmethod - def _get_default_seq_path_in_engine() -> str: - return XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_PATH + def _get_default_seq_dir_in_engine() -> str: + return XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_DIR @staticmethod def _get_seq_info_in_engine( @@ -295,8 +332,8 @@ def _get_seq_info_in_engine( map_path: 'Optional[str]' = None, ) -> 'Tuple[str, str]': _suffix = XRFeitoriaUnrealFactory.constants.data_asset_suffix - default_sequence_path = XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_PATH - seq_dir = seq_dir or default_sequence_path # default sequence path + default_sequence_dir = XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_DIR + seq_dir = seq_dir or default_sequence_dir # default sequence directory if map_path is None: seq_data_path = f'{seq_dir}/{seq_name}{_suffix}' unreal_functions.check_asset_in_engine(seq_data_path, raise_error=True) @@ -375,6 +412,10 @@ def _show_seq_in_engine() -> None: def _set_playback_in_engine(start_frame: 'Optional[int]' = None, end_frame: 'Optional[int]' = None) -> None: XRFeitoriaUnrealFactory.Sequence.set_playback(start_frame=start_frame, end_frame=end_frame) + @staticmethod + def _get_playback_in_engine() -> 'Tuple[int, int]': + return XRFeitoriaUnrealFactory.Sequence.get_playback() + @staticmethod def _set_camera_cut_player_in_engine( start_frame: 'Optional[int]' = None, end_frame: 'Optional[int]' = None @@ -526,3 +567,18 @@ def _add_audio_in_engine( start_frame=start_frame, end_frame=end_frame, ) + + # ------ render -------- # + @staticmethod + def _add_annotator_in_engine( + save_dir: str, + resolution: 'Tuple[int, int]', + export_vertices: bool, + export_skeleton: bool, + ) -> None: + XRFeitoriaUnrealFactory.Sequence.add_annotator( + save_dir=save_dir, + resolution=resolution, + export_vertices=export_vertices, + export_skeleton=export_skeleton, + ) diff --git a/xrfeitoria/sequence/sequence_unreal.pyi b/xrfeitoria/sequence/sequence_unreal.pyi index 2038e6e4..c0b1381f 100644 --- a/xrfeitoria/sequence/sequence_unreal.pyi +++ b/xrfeitoria/sequence/sequence_unreal.pyi @@ -1,4 +1,6 @@ -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Literal, Optional, Tuple, Union + +from typing_extensions import TypedDict from ..actor.actor_unreal import ActorUnreal from ..camera.camera_unreal import CameraUnreal @@ -6,20 +8,23 @@ from ..data_structure.constants import MotionFrame, PathLike, Vector from ..data_structure.models import RenderJobUnreal, RenderPass, TransformKeys from ..object.object_utils import ObjectUtilsUnreal from ..renderer.renderer_unreal import RendererUnreal -from ..utils.functions import unreal_functions from .sequence_base import SequenceBase +class dict_process_dir(TypedDict): + camera_dir: str + actor_infos_dir: str + vertices_dir: str + skeleton_dir: str + class SequenceUnreal(SequenceBase): + _actor = ActorUnreal + _camera = CameraUnreal + _object_utils = ObjectUtilsUnreal + _renderer = RendererUnreal @classmethod - def import_actor( - cls, - file_path: PathLike, - actor_name: Optional[str] = ..., - location: Vector = ..., - rotation: Vector = ..., - scale: Vector = ..., - stencil_value: int = ..., - ) -> ActorUnreal: ... + def save(cls) -> None: ... + @classmethod + def show(cls) -> None: ... @classmethod def add_to_renderer( cls, @@ -28,16 +33,12 @@ class SequenceUnreal(SequenceBase): render_passes: List[RenderPass], file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0}, - anti_aliasing: 'Optional[RenderJobUnreal.AntiAliasSetting]' = None, + anti_aliasing: Optional[RenderJobUnreal.AntiAliasSetting] = None, export_vertices: bool = False, export_skeleton: bool = False, export_audio: bool = False, ) -> None: ... @classmethod - def spawn_camera( - cls, location: Vector, rotation: Vector, fov: float = ..., camera_name: str = ... - ) -> CameraUnreal: ... - @classmethod def spawn_actor( cls, actor_asset_path: str, @@ -60,37 +61,8 @@ class SequenceUnreal(SequenceBase): motion_data: Optional[List[MotionFrame]] = None, ) -> ActorUnreal: ... @classmethod - def use_camera( - cls, camera: CameraUnreal, location: Optional[Vector] = ..., rotation: Optional[Vector] = ..., fov: float = ... - ) -> None: ... - @classmethod - def use_camera_with_keys( - cls, camera: CameraUnreal, transform_keys: TransformKeys, fov: float = ... - ) -> CameraUnreal: ... - @classmethod - def use_actor( - cls, - actor: ActorUnreal, - location: Optional[Vector] = ..., - rotation: Optional[Vector] = ..., - scale: Optional[Vector] = ..., - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., - ) -> None: ... - @classmethod - def use_actor_with_keys( - cls, - actor: ActorUnreal, - transform_keys: TransformKeys, - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., - ) -> None: ... - @classmethod def add_audio( - cls, - audio_asset_path: str, - start_frame: Optional[int] = None, - end_frame: Optional[int] = None, + cls, audio_asset_path: str, start_frame: Optional[int] = None, end_frame: Optional[int] = None ) -> None: ... @classmethod def get_map_path(cls) -> str: ... @@ -99,8 +71,22 @@ class SequenceUnreal(SequenceBase): @classmethod def set_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: ... @classmethod + def get_playback(cls) -> Tuple[int, int]: ... + @classmethod def set_camera_cut_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: ... @classmethod - def _open(cls, seq_name: str, seq_dir: 'Optional[str]' = ...) -> None: ... + def _open(cls, seq_name: str, seq_dir: Optional[str] = None) -> None: ... + @classmethod + def _preprocess_before_render( + cls, save_dir: str, resolution: Tuple[int, int], export_vertices: bool, export_skeleton: bool + ) -> None: ... + @staticmethod + def _get_default_seq_dir_in_engine() -> str: ... + @staticmethod + def _get_seq_info_in_engine( + seq_name: str, seq_dir: Optional[str] = None, map_path: Optional[str] = None + ) -> Tuple[str, str]: ... + @staticmethod + def _get_map_path_in_engine() -> str: ... @staticmethod - def _get_default_seq_path_in_engine() -> str: ... + def _get_seq_path_in_engine() -> str: ... diff --git a/xrfeitoria/sequence/sequence_wrapper.py b/xrfeitoria/sequence/sequence_wrapper.py index 4dd0d09b..6f44c5de 100644 --- a/xrfeitoria/sequence/sequence_wrapper.py +++ b/xrfeitoria/sequence/sequence_wrapper.py @@ -4,6 +4,8 @@ from contextlib import contextmanager from typing import ContextManager, List, Optional, Tuple, Union +from typing_extensions import deprecated + from ..data_structure.constants import default_level_blender from ..utils.functions import blender_functions, unreal_functions from .sequence_base import SequenceBase @@ -13,6 +15,7 @@ __all__ = ['sequence_wrapper_blender', 'sequence_wrapper_unreal'] +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperBlender: """Sequence utils class.""" @@ -72,6 +75,7 @@ def open(cls, seq_name: str) -> ContextManager[SequenceBase]: cls._seq.close() +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperUnreal: """Sequence utils class for Unreal.""" @@ -96,6 +100,7 @@ def open(cls, seq_name: str, seq_dir: 'Optional[str]' = None) -> ContextManager[ SequenceUnreal: Sequence object. """ cls._seq._open(seq_name=seq_name, seq_dir=seq_dir) + cls._seq.show() yield cls._seq cls._seq.save() cls._seq.close() @@ -133,6 +138,7 @@ def new( replace=replace, seq_dir=seq_dir, ) + cls._seq.show() yield cls._seq cls._seq.save() cls._seq.close() @@ -172,22 +178,23 @@ def sequence_wrapper_unreal( seq_length: int = 1, replace: bool = False, ) -> Union[SequenceUnreal, ContextManager[SequenceUnreal]]: - """Create a new sequence and close the sequence after exiting it. + """Create a new sequence, open it in editor, and close the sequence after exiting + it. Args: seq_name (str): The name of the sequence. - seq_dir (Optional[str], optional): The directory where the sequence is located. Defaults to None. + seq_dir (Optional[str], optional): The directory where the sequence is located. Defaults to None. Falls back to the default sequence path (/Game/XRFeitoriaUnreal/Sequences). level (Optional[str], optional): The level to associate the sequence with. Defaults to None. seq_fps (int, optional): The frames per second of the sequence. Defaults to 30. - seq_length (int, optional): The length of the sequence in seconds. Defaults to 1. + seq_length (int, optional): The length of the sequence in frames. Defaults to 1. replace (bool, optional): Whether to replace an existing sequence with the same name. Defaults to False. Returns: SequenceUnreal: The created SequenceUnreal object. """ - default_sequence_path = SequenceUnreal._get_default_seq_path_in_engine() - seq_dir = seq_dir or default_sequence_path + default_sequence_dir = SequenceUnreal._get_default_seq_dir_in_engine() + seq_dir = seq_dir or default_sequence_dir if ( unreal_functions.check_asset_in_engine(f'{seq_dir}/{seq_name}') and unreal_functions.check_asset_in_engine(f'{seq_dir}/{seq_name}_data') @@ -203,4 +210,6 @@ def sequence_wrapper_unreal( seq_length=seq_length, replace=replace, ) + # Open the sequence in editor, for letting `get_bound_objects` work + SequenceUnreal.show() return SequenceUnreal() diff --git a/xrfeitoria/sequence/sequence_wrapper.pyi b/xrfeitoria/sequence/sequence_wrapper.pyi index d4a6075d..5827a0a4 100644 --- a/xrfeitoria/sequence/sequence_wrapper.pyi +++ b/xrfeitoria/sequence/sequence_wrapper.pyi @@ -1,18 +1,25 @@ from typing import ContextManager, List, Optional, Union +from typing_extensions import deprecated + from ..data_structure.constants import default_level_blender from .sequence_blender import SequenceBlender as SequenceBlender from .sequence_unreal import SequenceUnreal as SequenceUnreal +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperBlender: + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def new( cls, seq_name: str, level: str = ..., seq_fps: int = ..., seq_length: int = ..., replace: bool = ... ) -> ContextManager[SequenceBlender]: ... + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def open(cls, seq_name: str) -> ContextManager[SequenceBlender]: ... +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperUnreal: + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def new( cls, @@ -23,6 +30,7 @@ class SequenceWrapperUnreal: replace: bool = ..., seq_dir: Optional[str] = ..., ) -> ContextManager[SequenceUnreal]: ... + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def open(cls, seq_name: str, seq_dir: Optional[str] = ...) -> ContextManager[SequenceUnreal]: ... diff --git a/xrfeitoria/utils/anim/__init__.py b/xrfeitoria/utils/anim/__init__.py index 9e9e61a3..94332d75 100644 --- a/xrfeitoria/utils/anim/__init__.py +++ b/xrfeitoria/utils/anim/__init__.py @@ -1 +1,7 @@ -from .utils import dump_humandata, load_amass_motion, load_humandata_motion +from .utils import ( + dump_humandata, + load_amass_motion, + load_humandata_motion, + refine_smpl_x, + refine_smpl_x_from_actor_info, +) diff --git a/xrfeitoria/utils/anim/constants.py b/xrfeitoria/utils/anim/constants.py index 7b211d13..8fc6d7d5 100644 --- a/xrfeitoria/utils/anim/constants.py +++ b/xrfeitoria/utils/anim/constants.py @@ -1,5 +1,5 @@ -SMPL_BODY_BONES = [ - 'pelvis', +SMPL_JOINT_NAMES = [ + 'pelvis', # global orientation 'left_hip', 'right_hip', 'spine1', @@ -28,10 +28,11 @@ 'jaw', # 'left_eyeball', # 'right_eyeball', - 'left_eye_smplhf', - 'right_eye_smplhf', + # + # 'left_eye_smplhf', + # 'right_eye_smplhf', ] -SMPL_FINGER_BONES = [ +SMPL_FINGER_JOINTS = [ # left hand 'left_index1', 'left_index2', @@ -67,7 +68,7 @@ ] SMPLX_JOINT_NAMES = [ - 'pelvis', + 'pelvis', # global orientation 'left_hip', 'right_hip', 'spine1', diff --git a/xrfeitoria/utils/anim/motion.py b/xrfeitoria/utils/anim/motion.py index 7f7d5f1e..daccd2c1 100644 --- a/xrfeitoria/utils/anim/motion.py +++ b/xrfeitoria/utils/anim/motion.py @@ -1,4 +1,5 @@ """Motion data structure and related functions.""" + from collections import OrderedDict from functools import partial from pathlib import Path @@ -8,9 +9,10 @@ from scipy.spatial.transform import Rotation as spRotation from ...data_structure.constants import MotionFrame, PathLike +from ..converter import ConverterMotion from .constants import ( - NUM_SMPLX_BODYJOINTS, SMPL_IDX_TO_JOINTS, + SMPL_JOINT_NAMES, SMPL_PARENT_IDX, SMPLX_HAND_POSES, SMPLX_IDX_TO_JOINTS, @@ -24,59 +26,6 @@ __all__ = ['Motion', 'SMPLMotion', 'SMPLXMotion', 'get_humandata'] -class Converter: - @classmethod - def vec_humandata2smplx(cls, vector: np.ndarray) -> np.ndarray: - """From humandata transl (in **OpenCV space**) to SMPLX armature's **pelvis - local space** in Blender. (The pelvis local space is designed to be the same - with **SMPL space**.) - - [right, front, up]: (-x, -z, -y) ==> (-x, z, y) - - Args: - vector (np.ndarray): of shape (N, 3) or (3,) - - Returns: - np.ndarray: of shape (N, 3) or (3,) - """ - if vector.shape == (3,): - vector = np.array([vector[0], -vector[1], -vector[2]], dtype=vector.dtype) - elif vector.ndim == 2 and vector.shape[1] == 3: - vector = np.array([vector[:, 0], -vector[:, 1], -vector[:, 2]]).T - else: - raise ValueError(f'vector.shape={vector.shape}') - return vector - - @classmethod - def vec_smplx2humandata(cls, vector: np.ndarray) -> np.ndarray: - # vice versa - return cls.vec_humandata2smplx(vector) - - @classmethod - def vec_amass2humandata(cls, vector: np.ndarray) -> np.ndarray: - """From amass transl (pelvis's local space) to humandata transl (in **OpenCV - space**) - - [right, front, up]: (x, y, z) ==> (-x, -z, -y) - - (CAUTION: we can see amass animation actors face back - in blender via the smplx add-on) - - Args: - vector (np.ndarray): of shape (N, 3) or (3,) - - Returns: - np.ndarray: of shape (N, 3) or (3,) - """ - if vector.shape == (3,): - vector = np.array([-vector[0], -vector[2], -vector[1]], dtype=vector.dtype) - elif vector.ndim == 2 and vector.shape[1] == 3: - vector = np.array([-vector[:, 0], -vector[:, 2], -vector[:, 1]]).T - else: - raise ValueError(f'vector.shape={vector.shape}') - return vector - - class Motion: """Wrap motion data. Provide methods to get transform info for 3D calculations. @@ -244,6 +193,42 @@ def sample_motion(self, n_frames: int): self.smplx_data[k] = v[indices] self.insert_rest_pose() + def cut_motion(self, start_frame: Optional[int] = None, end_frame: Optional[int] = None): + """Cut the motion sequence to a given number of frames (to [start_frame, + end_frame]) + + Args: + start_frame (Optional[int], optional): The start frame to cut to. Defaults to None. + end_frame (Optional[int], optional): The end frame to cut to. Defaults to None. + + Raises: + AssertionError: If the start frame is less than 0. + AssertionError: If the end frame is greater than the number of frames in the motion sequence. + AssertionError: If the start frame is greater than or equal to the end frame. + """ + if start_frame is None: + start_frame = 0 + if end_frame is None: + end_frame = self.n_frames + + assert start_frame >= 0, f'start_frame={start_frame}' + assert end_frame <= self.n_frames, f'end_frame={end_frame} should be less than n_frames={self.n_frames}' + assert start_frame < end_frame, f'start_frame={start_frame} should be less than end_frame={end_frame}' + n_frames = end_frame - start_frame + + self.transl = self.transl[start_frame:end_frame] + self.body_poses = self.body_poses[start_frame:end_frame] + self.global_orient = self.global_orient[start_frame:end_frame] + self.n_frames = n_frames + if hasattr(self, 'smpl_data'): + for k, v in self.smpl_data.items(): + if k != 'betas': + self.smpl_data[k] = v[start_frame:end_frame] + if hasattr(self, 'smplx_data'): + for k, v in self.smplx_data.items(): + if k != 'betas': + self.smplx_data[k] = v[start_frame:end_frame] + def cut_transl(self): """Cut the transl to zero. @@ -314,7 +299,7 @@ class SMPLMotion(Motion): NAME_TO_SMPL_IDX = OrderedDict([(v, k) for k, v in SMPL_IDX_TO_NAME.items() if v]) NAMES = [x for x in SMPL_IDX_TO_NAME.values() if x] PARENTS = list(SMPL_PARENT_IDX) - BONE_NAMES = SMPLX_JOINT_NAMES[1 : NUM_SMPLX_BODYJOINTS + 1] + BONE_NAMES = SMPL_JOINT_NAMES BONE_NAME_TO_IDX: Dict[str, int] = {bone_name: idx for idx, bone_name in enumerate(BONE_NAMES)} # In order to make the smpl head up to +z @@ -337,7 +322,7 @@ def from_smpl_data( fps: float = 30.0, insert_rest_pose: bool = False, global_orient_adj: Optional[spRotation] = GLOBAL_ORIENT_ADJUSTMENT, - vector_convertor: Optional[ConverterType] = Converter.vec_humandata2smplx, + vector_convertor: Optional[ConverterType] = ConverterMotion.vec_humandata2smplx, ) -> 'SMPLMotion': """Create SMPLMotion instance from smpl_data. @@ -380,27 +365,80 @@ def from_smpl_data( # Create instance transl_bl = smpl_data['transl'] - n_frames = transl_bl.shape[0] - body_poses_bl = np.concatenate( - [smpl_data[key] for key in ('global_orient', 'body_pose')], - axis=1, - dtype=np.float32, - ).reshape([n_frames, -1, 3]) # - Adjust in order to make the smpl head up to +z if global_orient_adj is not None: - body_poses_bl[:, 0, :] = (global_orient_adj * spRotation.from_rotvec(body_poses_bl[:, 0, :])).as_rotvec() + global_orient_bl = spRotation.from_rotvec(smpl_data['global_orient']) + smpl_data['global_orient'] = (global_orient_adj * global_orient_bl).as_rotvec() if insert_rest_pose: - body_poses_bl[0, 0, :] = 0.0 - + smpl_data['global_orient'][0] = 0 # - Convert from humandata to smplx pelvis local space in blender if vector_convertor is not None: transl_bl = vector_convertor(transl_bl) smpl_data['transl'] = transl_bl - instance = cls(transl=transl_bl, body_poses=body_poses_bl, fps=fps) + # Concatenate all the poses + body_pose_keys = ('global_orient', 'body_pose') + body_poses_bl = [smpl_data[key] for key in body_pose_keys] + n_frames = transl_bl.shape[0] + body_poses_bl = np.concatenate(body_poses_bl, axis=1, dtype=np.float32).reshape([n_frames, -1, 3]) + + instance = SMPLMotion(transl=transl_bl, body_poses=body_poses_bl, fps=fps) instance.smpl_data = smpl_data return instance + @classmethod + def from_amass_data(cls, amass_data, insert_rest_pose: bool) -> 'SMPLMotion': + """Create a Motion instance from AMASS data (SMPL) + + Args: + amass_data (dict): A dictionary containing the AMASS data. + insert_rest_pose (bool): Whether to insert a rest pose at the beginning of the motion. + + Returns: + SMPLMotion: A SMPLMotion instance containing the AMASS data. + """ + fps = 120 + + betas = amass_data['betas'][:10] + transl = amass_data['trans'] + global_orient = amass_data['poses'][:, :3] + body_pose = amass_data['poses'][:, 3:66] + # left_hand_pose = amass_data['poses'][:, 66 : 66 + 45] + # right_hand_pose = amass_data['poses'][:, 66 + 45 :] + # n_frames = global_orient.shape[0] + # expression = np.zeros([n_frames, 10], dtype=np.float32) + + # motions in AMASS dataset are -y up, rotate it to +y up + amass2humandata_adj = spRotation.from_euler('xyz', np.deg2rad([90, 180, 0])) + global_orient = (amass2humandata_adj * spRotation.from_rotvec(global_orient)).as_rotvec() # type: ignore + # transl_0 = transl[0, :] + # transl = amass2humandata_adj.apply(transl - transl_0) + transl_0 + transl = ConverterMotion.vec_amass2humandata(transl) + # TODO: all axis offset + height_offset = transl[0, 1] + + smpl_data = { + 'betas': betas, + 'transl': transl, + 'global_orient': global_orient, + 'body_pose': body_pose, + } + if insert_rest_pose: + for key, arr in smpl_data.items(): + arr = arr.astype(np.float32) + if key != 'betas': + arr = np.insert(arr, 0, 0, axis=0) + if key == 'global_orient': + # make 0-th frame has the same orient with humandata + arr[0, :] = [np.pi, 0, 0] + elif key == 'transl': + arr[1:, 1] -= height_offset + # TODO: handle pelvis height, get pelvis_height, and set frame-0 as T-pose + # arr[0, 1] = pelvis_height + smpl_data[key] = arr + + return cls.from_smpl_data(smpl_data, insert_rest_pose=False, fps=fps) + def _get_bone_rotvec(self, bone_name, frame=0) -> np.ndarray: idx = self._bone2idx(bone_name) if idx == 0: @@ -461,7 +499,7 @@ def dump_humandata( } """ humandata = get_humandata( - smpl_x_data=self.smplx_data, + smpl_x_data=self.smpl_data, smpl_x_type='smpl', betas=betas, meta=meta, @@ -518,7 +556,7 @@ def from_smplx_data( insert_rest_pose: bool = False, flat_hand_mean: bool = False, global_orient_adj: Optional[spRotation] = GLOBAL_ORIENT_ADJUSTMENT, - vector_convertor: Optional[Callable[[np.ndarray], np.ndarray]] = Converter.vec_humandata2smplx, + vector_convertor: Optional[Callable[[np.ndarray], np.ndarray]] = ConverterMotion.vec_humandata2smplx, ) -> 'SMPLXMotion': """Create SMPLXMotion instance from smplx_data. @@ -647,7 +685,7 @@ def from_amass_data(cls, amass_data, insert_rest_pose: bool, flat_hand_mean: boo global_orient = (amass2humandata_adj * spRotation.from_rotvec(global_orient)).as_rotvec() # type: ignore # transl_0 = transl[0, :] # transl = amass2humandata_adj.apply(transl - transl_0) + transl_0 - transl = Converter.vec_amass2humandata(transl) + transl = ConverterMotion.vec_amass2humandata(transl) # TODO: all axis offset height_offset = transl[0, 1] diff --git a/xrfeitoria/utils/anim/utils.py b/xrfeitoria/utils/anim/utils.py index 84ac0b3e..b3dc548b 100644 --- a/xrfeitoria/utils/anim/utils.py +++ b/xrfeitoria/utils/anim/utils.py @@ -1,29 +1,35 @@ """Utilities for animation data loading and dumping.""" + from pathlib import Path -from typing import Union +from typing import Optional, Union import numpy as np +from scipy.spatial.transform import Rotation as spRotation +from ...actor.actor_base import ActorBase from ...data_structure.constants import PathLike -from .motion import Motion, SMPLMotion, SMPLXMotion +from .motion import SMPLMotion, SMPLXMotion -def load_amass_motion(input_amass_smplx_path: PathLike) -> SMPLXMotion: +def load_amass_motion(input_amass_smpl_x_path: PathLike, is_smplx: bool = True) -> Union[SMPLMotion, SMPLXMotion]: """Load AMASS SMPLX motion data. Only for SMPLX motion for now. Args: - input_amass_smplx_path (PathLike): Path to AMASS SMPLX motion data. + input_amass_smpl_x_path (PathLike): Path to AMASS SMPL/SMPLX motion data. Returns: - Motion: Motion data, which consists of data read from AMASS file. + Union[SMPLMotion, SMPLXMotion]: Motion data, which consists of data read from AMASS file. """ - input_amass_smplx_path = Path(input_amass_smplx_path).resolve() - if not input_amass_smplx_path.exists(): - raise ValueError(f'Not exist: {input_amass_smplx_path}') + input_amass_smpl_x_path = Path(input_amass_smpl_x_path).resolve() + if not input_amass_smpl_x_path.exists(): + raise ValueError(f'Not exist: {input_amass_smpl_x_path}') # Use AMASS motion # src_actor_name = "SMPLX" - amass_smplx_data = np.load(input_amass_smplx_path, allow_pickle=True) - src_motion = SMPLXMotion.from_amass_data(amass_smplx_data, insert_rest_pose=True) + amass_smpl_x_data = np.load(input_amass_smpl_x_path, allow_pickle=True) + if is_smplx: + src_motion = SMPLXMotion.from_amass_data(amass_smpl_x_data, insert_rest_pose=True) + else: + src_motion = SMPLMotion.from_amass_data(amass_smpl_x_data, insert_rest_pose=True) return src_motion @@ -54,14 +60,20 @@ def load_humandata_motion(input_humandata_path: PathLike) -> Union[SMPLMotion, S return src_motion -def dump_humandata(motion: SMPLXMotion, save_filepath: PathLike, meta_filepath: PathLike) -> None: +def dump_humandata( + motion: Union[SMPLMotion, SMPLXMotion], + save_filepath: PathLike, + meta_filepath: PathLike, + actor_name: Optional[str] = None, +) -> None: """Dump human data to a file. This function must be associate with a meta file provided by SMPL-XL. Args: - motion (SMPLXMotion): Motion data to dump. + motion (Union[SMPLMotion, SMPLXMotion]): The motion data to be dumped. save_filepath (PathLike): The file path to save the dumped data. meta_filepath (PathLike): The file path to the meta information, storing the parameters of the SMPL-XL model. + actor_name (Optional[str], optional): The name of the actor. Defaults to None. Note: HumanData is a structure of smpl/smplx data defined in https://github.com/open-mmlab/mmhuman3d/blob/main/docs/human_data.md @@ -84,19 +96,86 @@ def dump_humandata(motion: SMPLXMotion, save_filepath: PathLike, meta_filepath: 'right_hand_pose': right_hand_pose, # (n_frames, 45) 'expression': expression, # (n_frames, 10) }, - 'meta': {'gender': 'neutral'}, # optional + 'meta': {'gender': 'neutral', 'actor_name': '(XF)actor-001'}, # optional } """ meta_info = np.load(meta_filepath, allow_pickle=True) - smplx = meta_info['smplx'].item() + if 'smplx' in meta_info.keys(): + smpl_x = meta_info['smplx'].item() + elif 'smpl' in meta_info.keys(): + smpl_x = meta_info['smpl'].item() + + _meta_ = meta_info['meta'].item() + if actor_name: + _meta_['actor_name'] = actor_name + motion.dump_humandata( filepath=save_filepath, - betas=smplx['betas'], - meta=meta_info['meta'].item(), - global_orient_offset=smplx['global_orient'], - transl_offset=smplx['transl'], - root_location_t0=smplx['root_location_t0'], - pelvis_location_t0=smplx['pelvis_location_t0'], + betas=smpl_x['betas'], + meta=_meta_, + global_orient_offset=smpl_x['global_orient'], + transl_offset=smpl_x['transl'], + root_location_t0=smpl_x['root_location_t0'], + pelvis_location_t0=smpl_x['pelvis_location_t0'], + ) + + +def refine_smpl_x( + smpl_x_file: Path, + replace_smpl_x_file: bool = False, + offset_location: np.ndarray = np.zeros(3), + offset_rotation: np.ndarray = np.eye(3), +) -> None: + """Refine translation and rotation of SMPL-X parameters.""" + + # Load SMPL-X data + smpl_x_file = Path(smpl_x_file) + data = dict(np.load(smpl_x_file, allow_pickle=True)) + if 'smplx' in data.keys(): + smpl_x_type = 'smplx' + elif 'smpl' in data.keys(): + smpl_x_type = 'smpl' + else: + raise ValueError(f'Unknown keys in {smpl_x_file}: {data.keys()}') + + # Convert offset_rotation + if offset_rotation.shape == (3, 3): + offset_rotation = spRotation.from_matrix(offset_rotation) + elif offset_rotation.shape == (4,): + offset_rotation = spRotation.from_quat(offset_rotation) + else: + raise ValueError('Please convert offset_rotation to 3x3 matrix or 4-dim quaternion.') + + smpl_x_data = data[smpl_x_type].item() + global_orient = smpl_x_data['global_orient'] + global_orient = spRotation.from_rotvec(global_orient) + global_orient = (offset_rotation * global_orient).as_rotvec() + transl = smpl_x_data['transl'] + transl = offset_rotation.apply(transl - transl[0]) + transl[0] + transl += offset_location + + smpl_x_data['global_orient'] = global_orient.astype(np.float32) + smpl_x_data['transl'] = transl.astype(np.float32) + data[smpl_x_type] = smpl_x_data + + if replace_smpl_x_file: + np.savez(smpl_x_file, **data) + else: + np.savez(smpl_x_file.parent / f'{smpl_x_file.stem}_refined.npz', **data) + + +def refine_smpl_x_from_actor_info(smpl_x_file: Path, actor_info_file: Path, replace_smpl_x_file: bool = False): + """Refine translation and rotation of SMPL-X parameters from actor info file.""" + actor_info = np.load(actor_info_file, allow_pickle=True) + location = actor_info['location'] + rotation = actor_info['rotation'] + assert np.all(location == location[0]) and np.all(rotation == rotation[0]) + + refine_smpl_x( + smpl_x_file=smpl_x_file, + replace_smpl_x_file=replace_smpl_x_file, + offset_location=location[0], + offset_rotation=rotation[0], ) diff --git a/xrfeitoria/utils/converter.py b/xrfeitoria/utils/converter.py new file mode 100644 index 00000000..03db09bc --- /dev/null +++ b/xrfeitoria/utils/converter.py @@ -0,0 +1,239 @@ +"""Converter for different spaces.""" + +from typing import Union + +import numpy as np + +from ..data_structure.constants import Vector + + +def rotation_matrix(angles: Union[Vector, np.ndarray], order='xyz', degrees: bool = True) -> np.ndarray: + """ + Args: + angles (Tuple[float, float, float]): Rotation angles in degrees or radians. + order (str, optional): Rotation order. Defaults to 'xyz'. + degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. + Returns: + ndarray: Rotation matrix 3x3. + + Examples: + >>> rotation_matrix((0, 0, 0), order='xyz') + + References: + https://programming-surgeon.com/en/euler-angle-python-en/ + """ + if degrees: + angles = np.deg2rad(angles) + + theta1, theta2, theta3 = angles + c1 = np.cos(theta1) + s1 = np.sin(theta1) + c2 = np.cos(theta2) + s2 = np.sin(theta2) + c3 = np.cos(theta3) + s3 = np.sin(theta3) + + if order == 'xzx': + matrix = np.array( + [ + [c2, -c3 * s2, s2 * s3], + [c1 * s2, c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3], + [s1 * s2, c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3], + ] + ) + elif order == 'xyx': + matrix = np.array( + [ + [c2, s2 * s3, c3 * s2], + [s1 * s2, c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1], + [-c1 * s2, c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3], + ] + ) + elif order == 'yxy': + matrix = np.array( + [ + [c1 * c3 - c2 * s1 * s3, s1 * s2, c1 * s3 + c2 * c3 * s1], + [s2 * s3, c2, -c3 * s2], + [-c3 * s1 - c1 * c2 * s3, c1 * s2, c1 * c2 * c3 - s1 * s3], + ] + ) + elif order == 'yzy': + matrix = np.array( + [ + [c1 * c2 * c3 - s1 * s3, -c1 * s2, c3 * s1 + c1 * c2 * s3], + [c3 * s2, c2, s2 * s3], + [-c1 * s3 - c2 * c3 * s1, s1 * s2, c1 * c3 - c2 * s1 * s3], + ] + ) + elif order == 'zyz': + matrix = np.array( + [ + [c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3, c1 * s2], + [c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3, s1 * s2], + [-c3 * s2, s2 * s3, c2], + ] + ) + elif order == 'zxz': + matrix = np.array( + [ + [c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1, s1 * s2], + [c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3, -c1 * s2], + [s2 * s3, c3 * s2, c2], + ] + ) + elif order == 'xyz': + matrix = np.array( + [ + [c2 * c3, -c2 * s3, s2], + [c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1], + [s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2], + ] + ) + elif order == 'xzy': + matrix = np.array( + [ + [c2 * c3, -s2, c2 * s3], + [s1 * s3 + c1 * c3 * s2, c1 * c2, c1 * s2 * s3 - c3 * s1], + [c3 * s1 * s2 - c1 * s3, c2 * s1, c1 * c3 + s1 * s2 * s3], + ] + ) + elif order == 'yxz': + matrix = np.array( + [ + [c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3, c2 * s1], + [c2 * s3, c2 * c3, -s2], + [c1 * s2 * s3 - c3 * s1, c1 * c3 * s2 + s1 * s3, c1 * c2], + ] + ) + elif order == 'yzx': + matrix = np.array( + [ + [c1 * c2, s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3], + [s2, c2 * c3, -c2 * s3], + [-c2 * s1, c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3], + ] + ) + elif order == 'zyx': + matrix = np.array( + [ + [c1 * c2, c1 * s2 * s3 - c3 * s1, s1 * s3 + c1 * c3 * s2], + [c2 * s1, c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3], + [-s2, c2 * s3, c2 * c3], + ] + ) + elif order == 'zxy': + matrix = np.array( + [ + [c1 * c3 - s1 * s2 * s3, -c2 * s1, c1 * s3 + c3 * s1 * s2], + [c3 * s1 + c1 * s2 * s3, c1 * c2, s1 * s3 - c1 * c3 * s2], + [-c2 * s3, s2, c2 * c3], + ] + ) + + return matrix + + +class ConverterMotion: + @classmethod + def vec_humandata2smplx(cls, vector: np.ndarray) -> np.ndarray: + """From humandata transl (in **OpenCV space**) to SMPLX armature's **pelvis + local space** in Blender. (The pelvis local space is designed to be the same + with **SMPL space**.) + + [right, front, up]: (-x, -z, -y) ==> (-x, z, y) + + Args: + vector (np.ndarray): of shape (N, 3) or (3,) + + Returns: + np.ndarray: of shape (N, 3) or (3,) + """ + if vector.shape == (3,): + ret = np.array([vector[0], -vector[1], -vector[2]], dtype=vector.dtype) + elif vector.ndim == 2 and vector.shape[1] == 3: + ret = np.array([vector[:, 0], -vector[:, 1], -vector[:, 2]]).T + else: + raise ValueError(f'vector.shape={vector.shape}') + return ret + + @classmethod + def vec_smplx2humandata(cls, vector: np.ndarray) -> np.ndarray: + # vice versa + return cls.vec_humandata2smplx(vector) + + @classmethod + def vec_amass2humandata(cls, vector: np.ndarray) -> np.ndarray: + """From amass transl (pelvis's local space) to humandata transl (in **OpenCV + space**) + + [right, front, up]: (x, y, z) ==> (-x, -z, -y) + + (CAUTION: we can see amass animation actors face back + in blender via the smplx add-on) + + Args: + vector (np.ndarray): of shape (N, 3) or (3,) + + Returns: + np.ndarray: of shape (N, 3) or (3,) + """ + if vector.shape == (3,): + vector = np.array([-vector[0], -vector[2], -vector[1]], dtype=vector.dtype) + elif vector.ndim == 2 and vector.shape[1] == 3: + vector = np.array([-vector[:, 0], -vector[:, 2], -vector[:, 1]]).T + else: + raise ValueError(f'vector.shape={vector.shape}') + return vector + + +class ConverterUnreal: + UNITS_SCALE = 100.0 # 1 meter = 100 cm + ROTATION_OFFSET = [0, 0, -90.0] # (x, y, z) in degrees, around z-axis (left-handed) + + @classmethod + def rotation_camera_from_ue(cls, euler, degrees=True) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention. + Note: convert to left-handed + + Args: + euler (np.ndarray): of shape (3,) + degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. + + Returns: + np.ndarray: Rotation matrix 3x3. + """ + x, y, z = -euler[1], -euler[2], -euler[0] + return rotation_matrix([x, y, z], order='xyz', degrees=degrees) + + @classmethod + def rotation_from_ue(cls, euler, offset=ROTATION_OFFSET, degrees=True) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention. + + Args: + euler (np.ndarray): of shape (3,) + offset (np.ndarray, optional): of shape (3,). Defaults to ROTATION_OFFSET [0, 0, -90.0]. + degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. + + Returns: + np.ndarray: Rotation matrix 3x3. + """ + _euler = np.array(euler) + np.array(offset) + return rotation_matrix(_euler, 'zxy', degrees=degrees) + + @classmethod + def location_from_ue(cls, vector: np.ndarray) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention. + + [right, front, up]: (x, y, z) ==> (y, -z, x) + + Args: + vector (np.ndarray): of shape (3,) or (... , 3) + + Returns: + np.ndarray: of shape (3,) or (... , 3) + """ + if vector.shape == (3,): + ret = np.array([vector[1], -vector[2], vector[0]]) / cls.UNITS_SCALE + elif vector.ndim >= 2 and vector.shape[-1] == 3: + ret = np.stack([vector[..., 1], -vector[..., 2], vector[..., 0]], axis=-1) / cls.UNITS_SCALE + return ret diff --git a/xrfeitoria/utils/functions/blender_functions.py b/xrfeitoria/utils/functions/blender_functions.py index 321352e4..49d2fc6a 100644 --- a/xrfeitoria/utils/functions/blender_functions.py +++ b/xrfeitoria/utils/functions/blender_functions.py @@ -67,14 +67,21 @@ def import_file(file_path: 'PathLike') -> None: @remote_blender() -def apply_motion_data_to_actor(motion_data: 'List[MotionFrame]', actor_name: str) -> None: +def apply_motion_data_to_actor( + motion_data: 'List[MotionFrame]', + actor_name: str, + is_first_frame_as_origin: bool = True, +) -> None: """Applies motion data to a given actor in Blender. Args: motion_data (List[MotionFrame]): A list of dictionaries containing motion data for the actor. actor_name (str): The name of the actor to apply the motion data to. + is_first_frame_as_origin (bool, optional): Whether to set the first frame as the origin. Defaults to True. """ - XRFeitoriaBlenderFactory.apply_motion_data_to_actor(motion_data=motion_data, actor_name=actor_name) + XRFeitoriaBlenderFactory.apply_motion_data_to_actor( + motion_data=motion_data, actor_name=actor_name, is_first_frame_as_origin=is_first_frame_as_origin + ) @remote_blender() @@ -130,8 +137,11 @@ def save_blend(save_path: 'PathLike' = None, pack: bool = False): if save_path is None: save_path = bpy.data.filepath + # path.resolve() would do Network Drive Handling like: + # X:/path/to/file.blend -> //xxx.xxx.xxx.xxx/Drive/path/to/file.blend + # which made Blender failed to save (Windows) + save_path = Path(save_path).absolute() # set suffix to .blend - save_path = Path(save_path).resolve() if save_path.suffix != '.blend': save_path = save_path.with_suffix('.blend') diff --git a/xrfeitoria/utils/functions/unreal_functions.py b/xrfeitoria/utils/functions/unreal_functions.py index 1e226fb3..3b1b994d 100644 --- a/xrfeitoria/utils/functions/unreal_functions.py +++ b/xrfeitoria/utils/functions/unreal_functions.py @@ -1,8 +1,10 @@ """Remote functions for unreal.""" +import json +from functools import lru_cache from typing import List, Optional, Tuple, Union -from ...data_structure.constants import Vector +from ...data_structure.constants import Vector, color_type from ...rpc import remote_unreal try: @@ -11,19 +13,35 @@ except ImportError: pass +# Constants +mask_colors: List[color_type] = [] + @remote_unreal() +def get_mask_color_file() -> str: + """Returns the path of the mask color file. + + Returns: + str: The path of the mask color file. + """ + return XRFeitoriaUnrealFactory.constants.MASK_COLOR_FILE.as_posix() + + +@lru_cache def get_mask_color(stencil_value: int) -> 'Tuple[int, int, int]': - """Get mask color from stencil value. + """Retrieves the RGB color value associated with the given stencil value. Args: - stencil_value (int): stencil value + stencil_value (int): The stencil value for which to retrieve the color. Returns: - Tuple[int, int, int]: mask color. (r, g, b) in [0, 255] + Tuple[int, int, int]: The RGB color value associated with the stencil value. """ - # TODO: move this to local, not remote - return XRFeitoriaUnrealFactory.utils_actor.get_mask_color(stencil_value) + global mask_colors + if len(mask_colors) == 0: + with open(get_mask_color_file(), 'r') as f: + mask_colors = json.load(f) + return mask_colors[stencil_value]['rgb'] @remote_unreal() @@ -70,7 +88,10 @@ def save_current_level(asset_path: 'Optional[str]' = None) -> None: @remote_unreal() def import_asset( - path: 'Union[str, List[str]]', dst_dir_in_engine: 'Optional[str]' = None, replace: bool = True + path: 'Union[str, List[str]]', + dst_dir_in_engine: 'Optional[str]' = None, + replace: bool = True, + with_parent_dir: bool = True, ) -> 'Union[str, List[str]]': """Import assets to the default asset path. @@ -79,11 +100,15 @@ def import_asset( dst_dir_in_engine (Optional[str], optional): destination directory in the engine. Defaults to None falls back to '/Game/XRFeitoriaUnreal/Assets' replace (bool, optional): whether to replace the existing asset. Defaults to True. + with_parent_dir (bool, optional): whether to create a parent directory that contains the imported asset. + If False, the imported asset will be in `dst_dir_in_engine` directly. Defaults to True. Returns: Union[str, List[str]]: a path or a list of paths to the imported assets, e.g. "/Game/XRFeitoriaUnreal/Assets/SMPL_XL" """ - paths = XRFeitoriaUnrealFactory.utils.import_asset(path, dst_dir_in_engine, replace=replace) + paths = XRFeitoriaUnrealFactory.utils.import_asset( + path, dst_dir_in_engine, replace=replace, with_parent_dir=with_parent_dir + ) if len(paths) == 1: return paths[0] return paths diff --git a/xrfeitoria/utils/tools.py b/xrfeitoria/utils/tools.py index 1ec42253..e04ba2b4 100644 --- a/xrfeitoria/utils/tools.py +++ b/xrfeitoria/utils/tools.py @@ -7,6 +7,7 @@ import loguru from loguru import logger +from rich import get_console from rich.console import Console from rich.progress import ( BarColumn, @@ -92,13 +93,13 @@ def setup_logging( logger.remove() # remove default logger # logger.add(sink=lambda msg: rprint(msg, end=''), level=level, format=cls.logger_format) - c = Console( - width=sys.maxsize, # disable wrapping - log_time=False, - log_path=False, - log_time_format='', - ) - logger.add(sink=lambda msg: c.print(msg, end=''), level=level, format=cls.logger_format) + # c = Console( + # width=sys.maxsize, # disable wrapping + # log_time=False, + # log_path=False, + # log_time_format='', + # ) + logger.add(sink=lambda msg: get_console().print(msg, end=''), level=level, format=cls.logger_format) if log_path: # add file logger log_path = Path(log_path).resolve() diff --git a/xrfeitoria/utils/viewer.py b/xrfeitoria/utils/viewer.py index 31fc8363..b6f2b65a 100644 --- a/xrfeitoria/utils/viewer.py +++ b/xrfeitoria/utils/viewer.py @@ -1,5 +1,7 @@ """Utils for loading images and annotations.""" + import os +from functools import lru_cache from pathlib import Path from typing import List, Tuple, Union @@ -72,20 +74,27 @@ def get_flow(self) -> np.ndarray: img = flow_vis.flow_to_color(flow, convert_to_bgr=False) return img - def get_depth(self, depth_rescale: float = 1.0) -> np.ndarray: + def get_depth(self, inverse: bool = False, depth_rescale: float = 1.0) -> np.ndarray: """Get depth in `.exr` format. Args: - depth_rescale (float, optional): scaling the depth - to map it into (0, 255). Depth values great than - `depth_rescale` will be clipped. Defaults to 1.0. + inverse (bool, optional): whether to inverse the depth. + If True, white (255) represents the farthest, and black (0) represents the nearest. + if False, white (255) represents the nearest, and black (0) represents the farthest. + Defaults to False. + depth_rescale (float, optional): scaling the depth to map it into (0, 255). + ``depth = depth / depth_rescale``. + Depth values greater than `depth_rescale` will be clipped. Defaults to 1.0. Returns: np.ndarray: depth data of shape (H, W, 3) """ depth = self.exr_mat img = self.float2int(depth / depth_rescale) - img[img == 0] = 255 + if inverse: + img = 255 - img + else: + img[img == 0] = 255 return img @@ -120,6 +129,17 @@ def __init__(self, sequence_dir: PathLike) -> None: """ self.sequence_dir = Path(sequence_dir) + @property + @lru_cache + def camera_names(self) -> List[str]: + camera_folders = list(self.sequence_dir.glob(f'{self.IMG}/*')) + return [camera_folder.name for camera_folder in camera_folders] + + @property + @lru_cache + def frame_num(self) -> int: + return len(list(self.sequence_dir.glob(f'{self.IMG}/{self.camera_names[0]}/*'))) + def get_img(self, camera_name: str, frame: int) -> np.ndarray: """Get rgb image of the given frame ('img/{frame:04d}.*') @@ -189,15 +209,19 @@ def get_mask(self, camera_name: str, frame: int) -> np.ndarray: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) return img - def get_depth(self, camera_name: str, frame: int, depth_rescale=1.0) -> np.ndarray: + def get_depth(self, camera_name: str, frame: int, inverse: bool = False, depth_rescale: float = 1.0) -> np.ndarray: """Get depth of the given frame ('depth/{frame:04d}.*') Args: camera_name (str): the camera name frame (int): the frame number - depth_rescale (float, optional): scaling the depth - to map it into (0, 255). Depth values great than - `depth_rescale` will be clipped. Defaults to 1.0. + inverse (bool, optional): whether to inverse the depth. + If True, white (255) represents the farthest, and black (0) represents the nearest. + if False, white (255) represents the nearest, and black (0) represents the farthest. + Defaults to False. + depth_rescale (float, optional): scaling the depth to map it into (0, 255). + ``depth = depth / depth_rescale``. + Depth values greater than `depth_rescale` will be clipped. Defaults to 1.0. Returns: np.ndarray: depth of shape (H, W, 3) @@ -209,7 +233,7 @@ def get_depth(self, camera_name: str, frame: int, depth_rescale=1.0) -> np.ndarr if not file_path.exists(): raise ValueError(f'Depth of {frame}-frame not found: {file_path}') if file_path.suffix == '.exr': - return ExrReader(file_path).get_depth(depth_rescale=depth_rescale) + return ExrReader(file_path).get_depth(inverse=inverse, depth_rescale=depth_rescale) else: img = cv2.imread(str(file_path)) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) @@ -221,9 +245,6 @@ def get_flow(self, camera_name: str, frame: int) -> np.ndarray: Args: camera_name (str): the camera name frame (int): the frame number - depth_rescale (float, optional): scaling the depth - to map it into (0, 255). Depth values great than - `depth_rescale` will be clipped. Defaults to 1.0. Returns: np.ndarray: optical flow of shape (H, W, 3)