From e11a905919f3de4f1570857a707b157de8955165 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 29 Jan 2024 11:12:07 +0800 Subject: [PATCH 01/27] remove `MoviePipelineMasterConfig` --- .../Private/CustomMoviePipelineDeferredPass.cpp | 4 ++-- .../Private/CustomMoviePipelineOutput.cpp | 16 ++++++++-------- .../Private/MoviePipelineMeshOperator.cpp | 4 ++-- .../Public/MoviePipelineMeshOperator.h | 6 +++++- .../XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs | 1 + 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp index 4722f5f5..581b2c8d 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp @@ -6,8 +6,8 @@ void UCustomMoviePipelineDeferredPass::SetupImpl(const MoviePipeline::FMoviePipelineRenderPassInitSettings& InPassInitSettings) { - UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + // UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); check(OutputSettings); AdditionalPostProcessMaterials.Empty(); diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp index 8220f5a9..11ae11f3 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp @@ -129,8 +129,8 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp if (bIsFirstFrame) { // Get Output Setting - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); check(OutputSettings); int ResolutionX = OutputSettings->OutputResolution.X; int ResolutionY = OutputSettings->OutputResolution.Y; @@ -235,12 +235,12 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp TArray CompositedPasses; MoviePipeline::GetPassCompositeData(InMergedOutputFrame, CompositedPasses); - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); check(OutputSettings); - UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + // UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); FString OutputDirectory = OutputSettings->OutputDirectory.Path; @@ -444,8 +444,8 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp FString UCustomMoviePipelineOutput::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) { - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); check(OutputSettings); FString OutputDirectory = OutputSettings->OutputDirectory.Path; FString FileNameFormatString = OutputSettings->FileNameFormat; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp index b491c499..629ab995 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp @@ -270,8 +270,8 @@ void UMoviePipelineMeshOperator::BeginExportImpl() FString UMoviePipelineMeshOperator::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) { - //UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); check(OutputSettings); FString OutputDirectory = OutputSettings->OutputDirectory.Path; FString FileNameFormatString = OutputSettings->FileNameFormat; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h index af17eb01..3ac58aa7 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h @@ -4,7 +4,6 @@ #include "MoviePipeline.h" #include "MoviePipelineOutputSetting.h" -#include "MoviePipelineMasterConfig.h" #include "Misc/FileHelper.h" // #include "HAL/PlatformFilemanager.h" @@ -14,6 +13,11 @@ #include "MovieScene.h" #endif +#if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION == 0 + #include "MoviePipelineMasterConfig.h" +#endif + + #include "SequencerTools.h" #include "SequencerSettings.h" #include "SequencerBindingProxy.h" diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs index a0ea0481..5e5347de 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/XRFeitoriaUnreal.Build.cs @@ -36,6 +36,7 @@ public XRFeitoriaUnreal(ReadOnlyTargetRules Target) : base(Target) "UEOpenExrRTTI", // Needed for EXR metadata "ImageWrapper", "CinematicCamera", // For metadata + "MovieRenderPipelineCore", "MovieRenderPipelineSettings", // For settings "MovieRenderPipelineRenderPasses", "MovieRenderPipelineEditor", From d86e8bb7a63ff081e1cfc2c06c1efccd6b2e100d Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 29 Jan 2024 14:44:48 +0800 Subject: [PATCH 02/27] fix bug --- .../CustomMoviePipelineDeferredPass.cpp | 7 +++-- .../Private/CustomMoviePipelineOutput.cpp | 26 ++++++++++++------- .../Private/MoviePipelineMeshOperator.cpp | 7 +++-- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp index 581b2c8d..766ff224 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineDeferredPass.cpp @@ -6,8 +6,11 @@ void UCustomMoviePipelineDeferredPass::SetupImpl(const MoviePipeline::FMoviePipelineRenderPassInitSettings& InPassInitSettings) { - UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - // UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 + UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UCustomMoviePipelineOutput* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif check(OutputSettings); AdditionalPostProcessMaterials.Empty(); diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp index 11ae11f3..2181f1e1 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp @@ -129,8 +129,11 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp if (bIsFirstFrame) { // Get Output Setting - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif check(OutputSettings); int ResolutionX = OutputSettings->OutputResolution.X; int ResolutionY = OutputSettings->OutputResolution.Y; @@ -235,13 +238,15 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp TArray CompositedPasses; MoviePipeline::GetPassCompositeData(InMergedOutputFrame, CompositedPasses); - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif check(OutputSettings); - UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - // UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - FString OutputDirectory = OutputSettings->OutputDirectory.Path; for (TPair>& RenderPassData : InMergedOutputFrame->ImageOutputData) @@ -444,8 +449,11 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp FString UCustomMoviePipelineOutput::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) { - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif check(OutputSettings); FString OutputDirectory = OutputSettings->OutputDirectory.Path; FString FileNameFormatString = OutputSettings->FileNameFormat; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp index 629ab995..ce8ac058 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp @@ -270,8 +270,11 @@ void UMoviePipelineMeshOperator::BeginExportImpl() FString UMoviePipelineMeshOperator::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) { - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - // UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); + #else + UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); + #endif check(OutputSettings); FString OutputDirectory = OutputSettings->OutputDirectory.Path; FString FileNameFormatString = OutputSettings->FileNameFormat; From 9e174893312c9484e56bcfdbe43d16c777d8ee81 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 29 Jan 2024 14:45:13 +0800 Subject: [PATCH 03/27] add `deprecated` decorator --- xrfeitoria/sequence/sequence_wrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xrfeitoria/sequence/sequence_wrapper.py b/xrfeitoria/sequence/sequence_wrapper.py index 4dd0d09b..b77a22de 100644 --- a/xrfeitoria/sequence/sequence_wrapper.py +++ b/xrfeitoria/sequence/sequence_wrapper.py @@ -4,6 +4,8 @@ from contextlib import contextmanager from typing import ContextManager, List, Optional, Tuple, Union +from typing_extensions import deprecated + from ..data_structure.constants import default_level_blender from ..utils.functions import blender_functions, unreal_functions from .sequence_base import SequenceBase @@ -13,6 +15,7 @@ __all__ = ['sequence_wrapper_blender', 'sequence_wrapper_unreal'] +@deprecated('Use `sequence` function instead.', category=DeprecationWarning) class SequenceWrapperBlender: """Sequence utils class.""" @@ -72,6 +75,7 @@ def open(cls, seq_name: str) -> ContextManager[SequenceBase]: cls._seq.close() +@deprecated('Use `sequence` function instead.', category=DeprecationWarning) class SequenceWrapperUnreal: """Sequence utils class for Unreal.""" From 64488cd5bc111dc28123506577bfac3b3deebc06 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 29 Jan 2024 18:40:33 +0800 Subject: [PATCH 04/27] Add path handling for saving blend files --- xrfeitoria/utils/functions/blender_functions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xrfeitoria/utils/functions/blender_functions.py b/xrfeitoria/utils/functions/blender_functions.py index 321352e4..5655beb6 100644 --- a/xrfeitoria/utils/functions/blender_functions.py +++ b/xrfeitoria/utils/functions/blender_functions.py @@ -1,5 +1,6 @@ """Remote functions for blender.""" +import os from pathlib import Path from typing import Dict, List, Literal, Optional, Tuple @@ -130,8 +131,11 @@ def save_blend(save_path: 'PathLike' = None, pack: bool = False): if save_path is None: save_path = bpy.data.filepath + # path.resolve() would do Network Drive Handling like: + # X:/path/to/file.blend -> //xxx.xxx.xxx.xxx/Drive/path/to/file.blend + # which made Blender failed to save (Windows) + save_path = Path(os.path.abspath(save_path)) # set suffix to .blend - save_path = Path(save_path).resolve() if save_path.suffix != '.blend': save_path = save_path.with_suffix('.blend') From a3ebea532b7f14395500a20bac98e15a1fb182d2 Mon Sep 17 00:00:00 2001 From: Haiyi Date: Tue, 6 Feb 2024 18:08:44 +0800 Subject: [PATCH 05/27] [Update] misc about camera (#2) - [UE] optimize import assets - [UE] save_camera_param - [Blender] set_first_frame_as_origin in animation apply --- docs/en/README.md | 6 + samples/utils.py | 4 +- src/XRFeitoriaBpy/core/factory.py | 29 ++-- .../Content/Python/constants.py | 1 + .../Content/Python/sequence.py | 94 +++++++++++- src/XRFeitoriaUnreal/Content/Python/utils.py | 134 +++++++++++++----- .../Private/CustomMoviePipelineOutput.cpp | 118 ++++++++------- tests/unreal/sequence.py | 15 +- xrfeitoria/camera/camera_base.py | 4 + xrfeitoria/camera/camera_parameter.py | 25 +++- xrfeitoria/renderer/renderer_unreal.py | 2 +- xrfeitoria/sequence/sequence_base.pyi | 95 ++++++------- xrfeitoria/sequence/sequence_unreal.py | 62 +++++++- xrfeitoria/sequence/sequence_unreal.pyi | 72 ++++------ xrfeitoria/sequence/sequence_wrapper.py | 4 + .../utils/functions/blender_functions.py | 11 +- .../utils/functions/unreal_functions.py | 11 +- xrfeitoria/utils/tools.py | 15 +- 18 files changed, 469 insertions(+), 233 deletions(-) diff --git a/docs/en/README.md b/docs/en/README.md index 67a15a36..8310e9ee 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -30,6 +30,12 @@ docs/en/make.bat html docs/en/make html ``` +### Build `.pyi` files + +```bash +# for instance +stubgen xrfeitoria/sequence/sequence_base.py --include-private +``` ### ~~Draw UML~~ diff --git a/samples/utils.py b/samples/utils.py index aa5e66d0..915b4357 100644 --- a/samples/utils.py +++ b/samples/utils.py @@ -38,7 +38,7 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat logger.info('Visualizing vertices') # fixed file structure img_path = seq_output_path / 'img' / camera_name / f'{frame_idx:04d}.png' - camera_param_json = seq_output_path / 'camera_params' / f'{camera_name}.json' + camera_param_json = seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json' # load img and camera parameters img = np.array(Image.open(img_path.as_posix())) @@ -58,7 +58,7 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat # draw vertices on image img = projector.draw_points3d(verts[frame_idx], camera_param, image=img, color=colors[idx]) - save_path = vis_dir / f'{frame_idx:04d}-overlap.png' + save_path = vis_dir / f'{camera_name}-{frame_idx:04d}-overlap.png' Image.fromarray(img).save(save_path.as_posix()) logger.info(f'Original image: "{img_path.as_posix()}"') logger.info(f'Overlap image saved to: "{save_path.as_posix()}"') diff --git a/src/XRFeitoriaBpy/core/factory.py b/src/XRFeitoriaBpy/core/factory.py index 31d537a0..b91e2d80 100644 --- a/src/XRFeitoriaBpy/core/factory.py +++ b/src/XRFeitoriaBpy/core/factory.py @@ -1113,6 +1113,7 @@ def apply_motion_data_to_action( motion_data: 'List[MotionFrame]', action: 'bpy.types.Action', scale: float = 1.0, + is_first_frame_as_origin: bool = True, ) -> None: """Apply motion data in dict to object. @@ -1121,9 +1122,8 @@ def apply_motion_data_to_action( containing rotation (quaternion) and location. action (bpy.types.Action): Action. scale (float, optional): Scale of movement in location of animation. Defaults to 1.0. + is_first_frame_as_origin (bool, optional): Whether to set the first frame as the origin. Defaults to True. """ - import numpy as np - num_frames = len(motion_data) fcurves_map = {(fc.data_path, fc.array_index): fc for fc in action.fcurves} @@ -1137,9 +1137,8 @@ def _get_fcurve(data_path: str, index: int): return fcurve # Set keyframes - frames_iter = range(num_frames) loc0 = [0, 0, 0] - for f in frames_iter: + for f in range(num_frames): for bone_name in motion_data[0].keys(): # rotation_quaternion data_path = f'pose.bones["{bone_name}"].rotation_quaternion' @@ -1154,25 +1153,33 @@ def _get_fcurve(data_path: str, index: int): if 'location' in motion: data_path = f'pose.bones["{bone_name}"].location' location_ = motion['location'] - if f < 1: - loc0 = location_ - location_ = np.zeros(3) - else: - location_ = np.subtract(location_, loc0) * scale + if is_first_frame_as_origin: + if f < 1: + loc0 = location_ + location_ = np.zeros(3) + else: + location_ = np.subtract(location_, loc0) * scale for idx, val in enumerate(location_): fcurve = _get_fcurve(data_path=data_path, index=idx) # fcurve.keyframe_points[f].co = (f, val) fcurve.keyframe_points.insert(frame=f, value=val, options={'FAST'}) - def apply_motion_data_to_actor(motion_data: 'List[MotionFrame]', actor_name: str) -> None: + def apply_motion_data_to_actor( + motion_data: 'List[MotionFrame]', + actor_name: str, + is_first_frame_as_origin: bool = True, + ) -> None: """Applies motion data to a given actor. Args: motion_data: A list of dictionaries containing motion data (quaternion) for the actor. actor_name: The name of the actor to apply the motion data to. + is_first_frame_as_origin: Whether to set the first frame as the origin. """ action = bpy.data.actions.new('Action') - XRFeitoriaBlenderFactory.apply_motion_data_to_action(motion_data=motion_data, action=action) + XRFeitoriaBlenderFactory.apply_motion_data_to_action( + motion_data=motion_data, action=action, is_first_frame_as_origin=is_first_frame_as_origin + ) XRFeitoriaBlenderFactory.apply_action_to_actor(action, actor=bpy.data.objects[actor_name]) def apply_shape_keys_to_mesh(shape_keys: 'List[Dict[str, float]]', mesh_name: str) -> None: diff --git a/src/XRFeitoriaUnreal/Content/Python/constants.py b/src/XRFeitoriaUnreal/Content/Python/constants.py index 5191b026..634c6e3d 100644 --- a/src/XRFeitoriaUnreal/Content/Python/constants.py +++ b/src/XRFeitoriaUnreal/Content/Python/constants.py @@ -45,6 +45,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]: DEFAULT_ASSET_PATH = f'{DEFAULT_PATH}/Assets' DEFAULT_SEQUENCE_DATA_ASSET = f'/{PLUGIN_NAME}/DefaultSequenceData' MRQ_JOB_UPPER = 200 +cam_param_dir = 'camera_params' data_asset_suffix = '_data' diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index 6a667efa..b1042681 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -1,3 +1,5 @@ +import json +from pathlib import Path from typing import Any, Dict, List, NoReturn, Optional, Tuple, Type, Union import unreal @@ -8,6 +10,7 @@ ENGINE_MAJOR_VERSION, ENGINE_MINOR_VERSION, MotionFrame, + PathLike, SequenceTransformKey, SubSystem, TransformKeys, @@ -765,7 +768,7 @@ def add_spawnable_actor_to_sequence( # add actor to sequence actor_binding = sequence.add_spawnable_from_instance(actor_asset) - actor = get_spawnable_actor_from_binding(sequence, actor_binding) + actor: unreal.Actor = get_spawnable_actor_from_binding(sequence, actor_binding) actor_binding.set_name(actor_name) actor.set_actor_label(actor_name) @@ -858,6 +861,22 @@ def generate_sequence( return new_sequence +def get_camera_param(camera: unreal.CameraActor) -> Dict[str, Any]: + """Get camera parameters. + + Args: + camera (unreal.CameraActor): The camera actor. + + Returns: + Dict[str, Any]: A dictionary containing the camera parameters. + """ + return { + 'location': camera.get_actor_location().to_tuple(), + 'rotation': camera.get_actor_rotation().to_tuple(), + 'fov': camera.camera_component.get_editor_property('FieldOfView'), + } + + class Sequence: map_path = None sequence_path = None @@ -972,6 +991,7 @@ def new( def show(cls) -> None: assert cls.sequence is not None, 'Sequence not initialized' unreal.LevelSequenceEditorBlueprintLibrary.open_level_sequence(cls.sequence) + unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(0) @staticmethod def new_data_asset( @@ -1093,7 +1113,7 @@ def add_camera( camera_transform_keys=transform_keys, camera_fov=fov, ) - # cls.bindings[camera_name] = bindings + cls.bindings[camera_name] = bindings else: camera = utils_actor.get_actor_by_name(camera_name) bindings = add_camera_to_sequence( @@ -1102,7 +1122,7 @@ def add_camera( camera_transform_keys=transform_keys, camera_fov=fov, ) - # cls.bindings[camera_name] = bindings + cls.bindings[camera_name] = bindings @classmethod def add_actor( @@ -1144,7 +1164,7 @@ def add_actor( actor_transform_keys=transform_keys, actor_stencil_value=stencil_value, ) - # cls.bindings[actor_name] = bindings + cls.bindings[actor_name] = bindings else: actor = utils_actor.get_actor_by_name(actor_name) @@ -1156,7 +1176,7 @@ def add_actor( animation_asset=animation_asset, motion_data=motion_data, ) - # cls.bindings[actor_name] = bindings + cls.bindings[actor_name] = bindings @classmethod def add_audio( @@ -1181,10 +1201,72 @@ def add_audio( bindings = add_audio_to_sequence( sequence=cls.sequence, audio_asset=audio_asset, start_frame=start_frame, end_frame=end_frame ) - # cls.bindings[audio_asset.get_name()] = bindings + cls.bindings[audio_asset.get_name()] = bindings + + @classmethod + def save_camera_params(cls, save_dir: PathLike, per_frame: bool = True): + """Saves the camera parameters of the sequence into json files. + + Args: + save_dir (PathLike): The directory where the camera parameters will be saved. + per_frame (bool, optional): Whether to save camera parameters per frame. Defaults to True. + """ + assert cls.sequence is not None, 'Sequence not initialized' + cls.show() + + save_dir = Path(save_dir) + save_dir.mkdir(parents=True, exist_ok=True) + unreal.log(f'[XRFeitoria] saving camera parameters of sequence to {save_dir}') + + # camera_actors: Dict[str, unreal.CameraActor] = { + # name: binding['camera']['self'] for name, binding in cls.bindings.items() if 'camera' in binding.keys() + # } + + camera_actors = {} + for name, binding in cls.bindings.items(): + if 'camera' not in binding.keys(): + continue + camera_binding = binding['camera']['binding'] + camera = unreal.LevelSequenceEditorBlueprintLibrary.get_bound_objects(get_binding_id(camera_binding))[0] + camera_actors[name] = camera + + def save_camera_param(frame_idx: int) -> Dict[str, Any]: + """Save camera parameters of the given frame to + {save_dir}/{camera_name}/{frame_idx:04d}.json. + + Args: + frame_idx (int): The frame index to save camera parameters for. + """ + unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(frame_idx) + for name, camera in camera_actors.items(): + save_path = save_dir / name / f'{frame_idx:04d}.json' + save_path.parent.mkdir(parents=True, exist_ok=True) + with open(save_path, 'w') as f: + json.dump(get_camera_param(camera), f, indent=4) + + if not per_frame: + save_camera_param(frame_idx=0) + else: + for frame_idx in range(0, cls.sequence.get_playback_end()): + save_camera_param(frame_idx=frame_idx) + + +def test(): + Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') + Sequence.save_camera_params(save_dir='E:/tmp') if __name__ == '__main__': + Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') + + Sequence.new('/Game/NewMap', 'test1') + Sequence.spawn_camera(transform_keys=SequenceTransformKey(frame=0, location=(0, 0, 0), rotation=(0, 0, 0))) + Sequence.spawn_actor( + '/Game/StarterContent/Props/SM_Chair', + transform_keys=SequenceTransformKey(frame=0, location=(0, 0, 0), rotation=(0, 0, 0)), + ) + Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') + Sequence.new('/Game/NewMap', 'test1') Sequence.spawn_camera(transform_keys=SequenceTransformKey(frame=0, location=(0, 0, 0), rotation=(0, 0, 0))) Sequence.spawn_actor( diff --git a/src/XRFeitoriaUnreal/Content/Python/utils.py b/src/XRFeitoriaUnreal/Content/Python/utils.py index 45b7aef1..1f3e66a7 100644 --- a/src/XRFeitoriaUnreal/Content/Python/utils.py +++ b/src/XRFeitoriaUnreal/Content/Python/utils.py @@ -51,6 +51,44 @@ def timer(self, deltaTime): self.func(*self.args, **self.kwargs) +class LoaderTimer: + """A decorator to load assets before running the main function. + + example_usage: + >>> @LoaderTimer + >>> def main(): + >>> ... + Caution: Function decorated by this decorator cannot return anything. + """ + + def __init__(self, func): + self.func = func + self.args: Tuple = None + self.kwargs: Dict = None + self.time_to_wait = 1 + self.time_spent = 0 + + self.tickhandle = unreal.register_slate_pre_tick_callback(self.timer) + unreal.log_warning('registered tick handle') + + def __call__(self, *args, **kwargs): + if args is None: + args = [] + self.args = args + + if kwargs is None: + kwargs = {} + self.kwargs = kwargs + + def timer(self, deltaTime): + if self.time_spent < self.time_to_wait: + self.time_spent += deltaTime + else: + unreal.log_warning('[Timer] ready!') + unreal.unregister_slate_pre_tick_callback(self.tickhandle) + self.func(*self.args, **self.kwargs) + + def timer_func(func): # This function shows the execution time of # the function object passed @@ -69,7 +107,10 @@ def wrap_func(*args, **kwargs): def import_asset( - path: Union[str, List[str]], dst_dir_in_engine: Optional[str] = None, replace: bool = True + path: Union[str, List[str]], + dst_dir_in_engine: Optional[str] = None, + replace: bool = True, + with_parent_dir: bool = True, ) -> List[str]: """Import assets to the default asset path. @@ -78,6 +119,8 @@ def import_asset( dst_dir_in_engine (str, optional): destination directory in the engine. Defaults to None falls back to DEFAULT_ASSET_PATH. replace (bool, optional): whether to replace the existing asset. Defaults to True. + with_parent_dir (bool, optional): whether to create a parent directory that contains the imported asset. + If False, the imported asset will be in `dst_dir_in_engine` directly. Defaults to True. Returns: List[str]: a list of paths to the imported assets, e.g. ["/Game/XRFeitoriaUnreal/Assets/SMPL_XL"] @@ -90,44 +133,57 @@ def import_asset( paths = path.copy() asset_paths = [] - for path in paths: - assert Path(path).exists(), f'File does not exist: {path}' - name = Path(path).stem - dst_dir = unreal.Paths.combine([dst_dir_in_engine, name]) - dst_path = unreal.Paths.combine([dst_dir, name]) # check if asset exists - if unreal.EditorAssetLibrary.does_asset_exist(dst_path) and not replace: - asset_paths.append(dst_path) - continue - - unreal.log(f'Importing asset: {path}') - if path.lower().endswith('.fbx'): - asset_tools = unreal.AssetToolsHelpers.get_asset_tools() - import_options = unreal.FbxImportUI() - import_options.set_editor_property('import_animations', True) - - import_task = unreal.AssetImportTask() - import_task.set_editor_property('automated', True) - import_task.set_editor_property('destination_name', '') - import_task.set_editor_property('destination_path', dst_dir) - import_task.set_editor_property('filename', path) - import_task.set_editor_property('replace_existing', replace) - import_task.set_editor_property('options', import_options) - - import_tasks = [import_task] - asset_tools.import_asset_tasks(import_tasks) - asset_paths.extend( - [path.partition('.')[0] for path in import_task.get_editor_property('imported_object_paths')] - ) - else: - assetsTools = unreal.AssetToolsHelpers.get_asset_tools() - assetImportData = unreal.AutomatedAssetImportData() - assetImportData.destination_path = dst_dir - assetImportData.filenames = [path] - assetImportData.replace_existing = replace - assets: List[unreal.Object] = assetsTools.import_assets_automated(assetImportData) - asset_paths.extend([asset.get_path_name().partition('.')[0] for asset in assets]) - unreal.EditorAssetLibrary.save_directory(dst_dir, False, True) # save assets - unreal.log(f'Imported asset: {path}') + with unreal.ScopedSlowTask(len(paths), 'Importing assets') as slow_task: + slow_task.make_dialog(True) + for path in paths: + assert Path(path).exists(), f'File does not exist: {path}' + # update progress bar + slow_task.enter_progress_frame(1, f'Importing assets: {path}') + if slow_task.should_cancel(): + unreal.log('Importing assets cancelled') + break + + # get destination directory + name = Path(path).stem + dst_dir = dst_dir_in_engine + if with_parent_dir: + dst_dir = unreal.Paths.combine([dst_dir, name]) + + # check if asset exists + dst_path = unreal.Paths.combine([dst_dir, name]) # check if asset exists + if unreal.EditorAssetLibrary.does_asset_exist(dst_path) and not replace: + asset_paths.append(dst_path) + continue + + unreal.log(f'Importing asset: {path}') + if path.lower().endswith('.fbx'): + asset_tools = unreal.AssetToolsHelpers.get_asset_tools() + import_options = unreal.FbxImportUI() + import_options.set_editor_property('import_animations', True) + + import_task = unreal.AssetImportTask() + import_task.set_editor_property('automated', True) + import_task.set_editor_property('destination_name', '') + import_task.set_editor_property('destination_path', dst_dir) + import_task.set_editor_property('filename', path) + import_task.set_editor_property('replace_existing', replace) + import_task.set_editor_property('options', import_options) + + import_tasks = [import_task] + asset_tools.import_asset_tasks(import_tasks) + asset_paths.extend( + [path.partition('.')[0] for path in import_task.get_editor_property('imported_object_paths')] + ) + else: + assetsTools = unreal.AssetToolsHelpers.get_asset_tools() + assetImportData = unreal.AutomatedAssetImportData() + assetImportData.destination_path = dst_dir + assetImportData.filenames = [path] + assetImportData.replace_existing = replace + assets: List[unreal.Object] = assetsTools.import_assets_automated(assetImportData) + asset_paths.extend([asset.get_path_name().partition('.')[0] for asset in assets]) + unreal.EditorAssetLibrary.save_directory(dst_dir, False, True) # save assets + unreal.log(f'Imported asset: {path}') return asset_paths diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp index 2181f1e1..7f8b98cf 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp @@ -34,6 +34,7 @@ #include "MovieRenderPipelineCoreModule.h" #include "XF_BlueprintFunctionLibrary.h" +#include "MoviePipelineMeshOperator.h" #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION == 0 #include "MoviePipelineMasterConfig.h" @@ -126,61 +127,10 @@ void UCustomMoviePipelineOutput::SetupForPipelineImpl(UMoviePipeline* InPipeline void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) { + // Save Actor Info (stencil value) if (bIsFirstFrame) { - // Get Output Setting - #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - #else - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - #endif - check(OutputSettings); - int ResolutionX = OutputSettings->OutputResolution.X; - int ResolutionY = OutputSettings->OutputResolution.Y; - - // Save Camera Transform (KRT) - for (ACameraActor* Camera : Cameras) - { - FVector CamLocation = Camera->GetActorLocation(); - FRotator CamRotation = Camera->GetActorRotation(); - float FOV = Camera->GetCameraComponent()->FieldOfView; - - TArray CamInfo; - CamInfo.Add(CamLocation.X); - CamInfo.Add(CamLocation.Y); - CamInfo.Add(CamLocation.Z); - CamInfo.Add(CamRotation.Roll); - CamInfo.Add(CamRotation.Pitch); - CamInfo.Add(CamRotation.Yaw); - CamInfo.Add(FOV); - CamInfo.Add(ResolutionX); - CamInfo.Add(ResolutionY); - - // Actor in level - FString CameraNameFromLabel = Camera->GetActorNameOrLabel(); - // Actor spawned from sequence - FString CameraNameFromName = Camera->GetFName().GetPlainNameString(); - // XXX: Hardcode way to Judge which name is correct, need to be improved - // Should ref to - // GetPipeline()->ResolveFilenameFormatArguments(FileNameFormatString, FormatOverrides, OutputData.FilePath, FinalFormatArgs, &Payload->SampleState.OutputState); - // using {camera_name} - - bool bIsCameraInLevel = CameraNameFromName.StartsWith("CameraActor") || CameraNameFromName.StartsWith("CineCameraActor"); - FString CameraName = bIsCameraInLevel ? CameraNameFromLabel : CameraNameFromName; - - FString CameraTransformPath = GetOutputPath( - DirectoryCameraInfo / CameraName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryCameraInfo/{camera_name}/{frame_idx}.dat - CameraTransformPath = FPaths::SetExtension( - FPaths::GetPath(CameraTransformPath), - FPaths::GetExtension(CameraTransformPath) - ); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); - } - - // Save Actor Info (stencil value) + // SkeletalMesh for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) { // Actor in level @@ -204,6 +154,7 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp UXF_BlueprintFunctionLibrary::SaveFloatToByteFile(StencilValue, ActorInfoPath); } + // StaticMesh for (UStaticMeshComponent* StaticMeshComponent : StaticMeshComponents) { // Actor in level @@ -230,14 +181,7 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp bIsFirstFrame = false; } - SCOPE_CYCLE_COUNTER(STAT_ImgSeqRecieveImageData); - - check(InMergedOutputFrame); - - // Special case for extracting Burn Ins and Widget Renderer - TArray CompositedPasses; - MoviePipeline::GetPassCompositeData(InMergedOutputFrame, CompositedPasses); - + // Get Output Setting #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); UMoviePipelineColorSetting* ColorSetting = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); @@ -247,6 +191,58 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp #endif check(OutputSettings); + // Save Camera Transform (KRT) + int ResolutionX = OutputSettings->OutputResolution.X; + int ResolutionY = OutputSettings->OutputResolution.Y; + for (ACameraActor* Camera : Cameras) + { + FVector CamLocation = Camera->GetActorLocation(); + FRotator CamRotation = Camera->GetActorRotation(); + float FOV = Camera->GetCameraComponent()->FieldOfView; + + TArray CamInfo; + CamInfo.Add(CamLocation.X); + CamInfo.Add(CamLocation.Y); + CamInfo.Add(CamLocation.Z); + CamInfo.Add(CamRotation.Roll); + CamInfo.Add(CamRotation.Pitch); + CamInfo.Add(CamRotation.Yaw); + CamInfo.Add(FOV); + CamInfo.Add(ResolutionX); + CamInfo.Add(ResolutionY); + + // Actor in level + FString CameraNameFromLabel = Camera->GetActorNameOrLabel(); + // Actor spawned from sequence + FString CameraNameFromName = Camera->GetFName().GetPlainNameString(); + // XXX: Hardcode way to Judge which name is correct, need to be improved + // Should ref to + // GetPipeline()->ResolveFilenameFormatArguments(FileNameFormatString, FormatOverrides, OutputData.FilePath, FinalFormatArgs, &Payload->SampleState.OutputState); + // using {camera_name} + + bool bIsCameraInLevel = CameraNameFromName.StartsWith("CameraActor") || CameraNameFromName.StartsWith("CineCameraActor"); + FString CameraName = bIsCameraInLevel ? CameraNameFromLabel : CameraNameFromName; + + FString CameraTransformPath = GetOutputPath( + DirectoryCameraInfo / CameraName, + "dat", + &InMergedOutputFrame->FrameOutputState + ); // DirectoryCameraInfo/{camera_name}/{frame_idx}.dat + //CameraTransformPath = FPaths::SetExtension( + // FPaths::GetPath(CameraTransformPath), + // FPaths::GetExtension(CameraTransformPath) + //); // get rid of the frame index + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); + } + + SCOPE_CYCLE_COUNTER(STAT_ImgSeqRecieveImageData); + + check(InMergedOutputFrame); + + // Special case for extracting Burn Ins and Widget Renderer + TArray CompositedPasses; + MoviePipeline::GetPassCompositeData(InMergedOutputFrame, CompositedPasses); + FString OutputDirectory = OutputSettings->OutputDirectory.Path; for (TPair>& RenderPassData : InMergedOutputFrame->ImageOutputData) diff --git a/tests/unreal/sequence.py b/tests/unreal/sequence.py index 32118ed5..254434c2 100644 --- a/tests/unreal/sequence.py +++ b/tests/unreal/sequence.py @@ -24,15 +24,22 @@ def new_seq(xf_runner: XRFeitoriaUnreal, level_path: str, seq_name: str): kc_path = xf_runner.utils.import_asset(path=kc_fbx) with xf_runner.Sequence.new(level=level_path, seq_name=seq_name, seq_length=30, replace=True) as seq: - seq.show() - seq.spawn_camera(location=(-5, 0, 1), rotation=(0, 0, 0), fov=90.0, camera_name='Camera') seq.spawn_camera_with_keys( + transform_keys=[ + SeqTransKey(frame=0, location=(0, 3, 1), rotation=(0, 0, -90), interpolation='AUTO'), + SeqTransKey(frame=30, location=(-3, 2, 2), rotation=(0, 0, -45), interpolation='AUTO'), + ], + fov=90.0, + camera_name='camera', + ) + camera2 = xf_runner.Camera.spawn(camera_name='camera2') + seq.use_camera_with_keys( + camera=camera2, transform_keys=[ SeqTransKey(frame=0, location=(-2, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), SeqTransKey(frame=30, location=(-5, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), ], fov=90.0, - camera_name='Camera2', ) seq.spawn_actor( actor_asset_path='/Engine/BasicShapes/Cube', @@ -105,7 +112,7 @@ def sequence_test(debug: bool = False, background: bool = False): for frame_idx in range(0, 30, 5): visualize_vertices( - camera_name='Camera', + camera_name='camera', actor_names=['KoupenChan'], seq_output_path=output_path / seq_name, frame_idx=frame_idx, diff --git a/xrfeitoria/camera/camera_base.py b/xrfeitoria/camera/camera_base.py index 89cc5a6c..a2531b68 100644 --- a/xrfeitoria/camera/camera_base.py +++ b/xrfeitoria/camera/camera_base.py @@ -15,6 +15,10 @@ class CameraBase(ABC, ObjectBase): _object_utils = ObjectUtilsBase + @classmethod + def from_param(cls, param): + ... + @classmethod def spawn( cls, diff --git a/xrfeitoria/camera/camera_parameter.py b/xrfeitoria/camera/camera_parameter.py index a63aab7d..bad87c35 100644 --- a/xrfeitoria/camera/camera_parameter.py +++ b/xrfeitoria/camera/camera_parameter.py @@ -8,7 +8,7 @@ from xrprimer.data_structure.camera import PinholeCameraParameter from xrprimer.transform.convention.camera import convert_camera_parameter -from ..data_structure.constants import PathLike +from ..data_structure.constants import PathLike, Vector class CameraParameter(PinholeCameraParameter): @@ -141,9 +141,30 @@ def from_bin(cls, file: PathLike) -> 'CameraParameter': rotation = dat[3:6] camera_fov = dat[6] image_size = (dat[7], dat[8]) # (width, height) + return cls.from_unreal_convention(location, rotation, camera_fov, image_size) + @classmethod + def from_unreal_convention( + cls, + location: Vector, + rotation: Vector, + fov: float, + image_size: Tuple[int, int], # (width, height) + ) -> 'CameraParameter': + """Converts camera parameters from Unreal Engine convention to CameraParameter + object. + + Args: + location (Vector): The camera location in Unreal Engine convention. + rotation (Vector): The camera rotation in Unreal Engine convention. + fov (float): The camera field of view in degrees. + image_size (Tuple[int, int]): The size of the camera image in pixels (width, height). + + Returns: + CameraParameter: The converted camera parameters. + """ # intrinsic matrix K - fov = math.radians(camera_fov) + fov = math.radians(fov) focal = max(image_size) / 2 / math.tan(fov / 2) fx = fy = focal K = np.array( diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 872702c5..3fc181e9 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -271,7 +271,7 @@ def convert_actor_infos(folder: Path) -> None: # 1. convert camera parameters from `.bat` to `.json` with xrprimer # glob camera files in {seq_path}/{cam_param_dir}/* - camera_files = sorted(seq_path.glob(f'{RenderOutputEnumUnreal.camera_params.value}/*.dat')) + camera_files = sorted((seq_path / RenderOutputEnumUnreal.camera_params.value).rglob('*.dat')) for camera_file in camera_files: convert_camera(camera_file) diff --git a/xrfeitoria/sequence/sequence_base.pyi b/xrfeitoria/sequence/sequence_base.pyi index 4e6198e0..c82f1eda 100644 --- a/xrfeitoria/sequence/sequence_base.pyi +++ b/xrfeitoria/sequence/sequence_base.pyi @@ -1,6 +1,8 @@ from abc import ABC -from typing import List, Optional, Tuple, Union +from typing import List, Literal, Optional, Tuple, Union +from ..actor.actor_base import ActorBase +from ..camera.camera_base import CameraBase from ..data_structure.constants import EngineEnum, PathLike, Vector from ..data_structure.models import RenderPass, TransformKeys @@ -11,10 +13,10 @@ class SequenceBase(ABC): def _new( cls, seq_name: str, - level: Union[str, List[str]], - seq_fps: int = ..., - seq_length: int = ..., - replace: bool = ..., + level: Optional[str] = None, + seq_fps: int = 60, + seq_length: int = 1, + replace: bool = False, **kwargs, ) -> None: ... @classmethod @@ -22,86 +24,79 @@ class SequenceBase(ABC): @classmethod def close(cls) -> None: ... @classmethod - def save(cls) -> None: ... - @classmethod - def show(cls) -> None: ... - @classmethod def import_actor( cls, file_path: PathLike, - actor_name: Optional[str] = ..., - location: Vector = ..., - rotation: Vector = ..., - scale: Vector = ..., - stencil_value: int = ..., - ) -> ...: ... + actor_name: Optional[str] = None, + location: Vector = None, + rotation: Vector = None, + scale: Vector = None, + stencil_value: int = 1, + ) -> ActorBase: ... @classmethod def spawn_camera( - cls, - location: Vector, - rotation: Vector, - fov: float = ..., - camera_name: str = ..., - ) -> ...: ... + cls, location: Vector = None, rotation: Vector = None, fov: float = 90.0, camera_name: Optional[str] = None + ) -> CameraBase: ... @classmethod def spawn_camera_with_keys( cls, transform_keys: TransformKeys, - fov: float = ..., - camera_name: str = ..., - ) -> ...: ... + fov: float = 90.0, + camera_name: str = None, + ) -> CameraBase: ... + @classmethod def use_camera( cls, - camera, - location: Optional[Vector] = ..., - rotation: Optional[Vector] = ..., - fov: float = ..., + camera: CameraBase, + location: Optional[Vector] = None, + rotation: Optional[Vector] = None, + fov: float = None, ) -> None: ... @classmethod def use_camera_with_keys( cls, - camera, + camera: CameraBase, transform_keys: TransformKeys, - fov: float = ..., + fov: float = None, ) -> None: ... @classmethod def use_actor( cls, - actor, - location: Optional[Vector] = ..., - rotation: Optional[Vector] = ..., - scale: Optional[Vector] = ..., - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., + actor: ActorBase, + location: Optional[Vector] = None, + rotation: Optional[Vector] = None, + scale: Optional[Vector] = None, + stencil_value: int = None, + anim_asset_path: Optional[str] = None, ) -> None: ... @classmethod def use_actor_with_keys( cls, - actor, + actor: ActorBase, transform_keys: TransformKeys, - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., + stencil_value: int = None, + anim_asset_path: Optional[str] = None, ) -> None: ... @classmethod def spawn_shape( cls, - shape_type: str, - location: Vector = ..., - rotation: Vector = ..., - scale: Vector = ..., - shape_name: str = ..., - stencil_value: int = ..., + type: Literal['plane', 'cube', 'sphere', 'cylinder', 'cone'], + shape_name: str = None, + location: Vector = None, + rotation: Vector = None, + scale: Vector = None, + stencil_value: int = 1, **kwargs, - ) -> ...: ... + ) -> ActorBase: ... @classmethod def spawn_shape_with_keys( cls, transform_keys: TransformKeys, - shape_type: str, - shape_name: str = ..., - stencil_value: int = ..., + type: Literal['plane', 'cube', 'sphere', 'cylinder', 'cone'], + shape_name: str = None, + stencil_value: int = 1, **kwargs, - ) -> ...: ... + ) -> ActorBase: ... @classmethod def add_to_renderer( cls, diff --git a/xrfeitoria/sequence/sequence_unreal.py b/xrfeitoria/sequence/sequence_unreal.py index b5d059e9..86798fe1 100644 --- a/xrfeitoria/sequence/sequence_unreal.py +++ b/xrfeitoria/sequence/sequence_unreal.py @@ -1,4 +1,6 @@ -from typing import Dict, List, Literal, Optional, Tuple, Union +import json +from pathlib import Path +from typing import Dict, List, Literal, Optional, Tuple, TypedDict, Union from loguru import logger @@ -24,6 +26,8 @@ except (ImportError, ModuleNotFoundError): pass +dict_process_dir = TypedDict('dict_process_dir', {'camera_dir': str, 'vertices_dir': str, 'skeleton_dir': str}) + @remote_unreal(dec_class=True, suffix='_in_engine') class SequenceUnreal(SequenceBase): @@ -49,11 +53,38 @@ def show(cls) -> None: """Show the sequence in the engine.""" cls._show_seq_in_engine() + @classmethod + def _preprocess_before_render( + cls, + save_dir: str, + resolution: Tuple[int, int], + export_vertices: bool, + export_skeleton: bool, + ) -> None: + from ..camera.camera_parameter import CameraParameter + + _dir_ = cls._preprocess_in_engine( + save_dir=save_dir, export_vertices=export_vertices, export_skeleton=export_skeleton + ) + + # convert camera parameters to xrprimer structure + for file in Path(_dir_['camera_dir']).glob('*/*.json'): + data = json.loads(file.read_text()) + cam_param = CameraParameter.from_unreal_convention( + location=data['location'], + rotation=data['rotation'], + fov=data['fov'], + image_size=resolution, + ) + cam_param.dump(file.as_posix()) # replace the original file + + print(_dir_['camera_dir']) + @classmethod def add_to_renderer( cls, output_path: PathLike, - resolution: Tuple[int, int], + resolution: Tuple[int, int], # (width, height) render_passes: 'List[RenderPass]', file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0}, @@ -69,7 +100,7 @@ def add_to_renderer( Args: output_path (PathLike): The path where the rendered output will be saved. - resolution (Tuple[int, int]): The resolution of the output. + resolution (Tuple[int, int]): The resolution of the output. (width, height) render_passes (List[RenderPass]): The list of render passes to be rendered. file_name_format (str, optional): The format of the output file name. Defaults to ``{sequence_name}/{render_pass}/{camera_name}/{frame_number}``. @@ -97,6 +128,14 @@ def add_to_renderer( sequence_path = SequenceUnreal._get_seq_path_in_engine() if anti_aliasing is None: anti_aliasing = RenderJobUnreal.AntiAliasSetting() + + cls._preprocess_before_render( + save_dir=f'{output_path}/{cls.name}', + resolution=resolution, + export_vertices=export_vertices, + export_skeleton=export_skeleton, + ) + cls._renderer.add_job( map_path=map_path, sequence_path=sequence_path, @@ -110,6 +149,7 @@ def add_to_renderer( export_skeleton=export_skeleton, export_audio=export_audio, ) + logger.info( f'[cyan]Added[/cyan] sequence "{cls.name}" to [bold]`Renderer`[/bold] ' f'(jobs to render: {len(cls._renderer.render_queue)})' @@ -314,6 +354,22 @@ def _get_map_path_in_engine() -> str: def _get_seq_path_in_engine() -> str: return XRFeitoriaUnrealFactory.Sequence.sequence_path + @staticmethod + def _preprocess_in_engine( + save_dir: str, + export_vertices: bool = False, + export_skeleton: bool = False, + ) -> 'dict_process_dir': + camera_dir = f'{save_dir}/{XRFeitoriaUnrealFactory.constants.cam_param_dir}' + XRFeitoriaUnrealFactory.Sequence.save_camera_params(save_dir=camera_dir, per_frame=True) + + # TODO: export vertices and skeleton + return { + 'camera_dir': camera_dir, + 'vertices_dir': '', + 'skeleton_dir': '', + } + @staticmethod def _new_seq_in_engine( seq_name: str, diff --git a/xrfeitoria/sequence/sequence_unreal.pyi b/xrfeitoria/sequence/sequence_unreal.pyi index 2038e6e4..2abd87a4 100644 --- a/xrfeitoria/sequence/sequence_unreal.pyi +++ b/xrfeitoria/sequence/sequence_unreal.pyi @@ -1,4 +1,6 @@ -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Literal, Optional, Tuple, Union + +from typing_extensions import TypedDict from ..actor.actor_unreal import ActorUnreal from ..camera.camera_unreal import CameraUnreal @@ -6,20 +8,26 @@ from ..data_structure.constants import MotionFrame, PathLike, Vector from ..data_structure.models import RenderJobUnreal, RenderPass, TransformKeys from ..object.object_utils import ObjectUtilsUnreal from ..renderer.renderer_unreal import RendererUnreal -from ..utils.functions import unreal_functions from .sequence_base import SequenceBase +class dict_process_dir(TypedDict): + camera_dir: str + vertices_dir: str + skeleton_dir: str + class SequenceUnreal(SequenceBase): + _actor = ActorUnreal + _camera = CameraUnreal + _object_utils = ObjectUtilsUnreal + _renderer = RendererUnreal @classmethod - def import_actor( - cls, - file_path: PathLike, - actor_name: Optional[str] = ..., - location: Vector = ..., - rotation: Vector = ..., - scale: Vector = ..., - stencil_value: int = ..., - ) -> ActorUnreal: ... + def save(cls) -> None: ... + @classmethod + def show(cls) -> None: ... + @classmethod + def _preprocess_before_render( + cls, save_dir: str, resolution: Tuple[int, int], export_vertices: bool, export_skeleton: bool + ) -> None: ... @classmethod def add_to_renderer( cls, @@ -28,16 +36,12 @@ class SequenceUnreal(SequenceBase): render_passes: List[RenderPass], file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0}, - anti_aliasing: 'Optional[RenderJobUnreal.AntiAliasSetting]' = None, + anti_aliasing: Optional[RenderJobUnreal.AntiAliasSetting] = None, export_vertices: bool = False, export_skeleton: bool = False, export_audio: bool = False, ) -> None: ... @classmethod - def spawn_camera( - cls, location: Vector, rotation: Vector, fov: float = ..., camera_name: str = ... - ) -> CameraUnreal: ... - @classmethod def spawn_actor( cls, actor_asset_path: str, @@ -60,32 +64,6 @@ class SequenceUnreal(SequenceBase): motion_data: Optional[List[MotionFrame]] = None, ) -> ActorUnreal: ... @classmethod - def use_camera( - cls, camera: CameraUnreal, location: Optional[Vector] = ..., rotation: Optional[Vector] = ..., fov: float = ... - ) -> None: ... - @classmethod - def use_camera_with_keys( - cls, camera: CameraUnreal, transform_keys: TransformKeys, fov: float = ... - ) -> CameraUnreal: ... - @classmethod - def use_actor( - cls, - actor: ActorUnreal, - location: Optional[Vector] = ..., - rotation: Optional[Vector] = ..., - scale: Optional[Vector] = ..., - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., - ) -> None: ... - @classmethod - def use_actor_with_keys( - cls, - actor: ActorUnreal, - transform_keys: TransformKeys, - stencil_value: int = ..., - anim_asset_path: Optional[str] = ..., - ) -> None: ... - @classmethod def add_audio( cls, audio_asset_path: str, @@ -101,6 +79,14 @@ class SequenceUnreal(SequenceBase): @classmethod def set_camera_cut_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: ... @classmethod - def _open(cls, seq_name: str, seq_dir: 'Optional[str]' = ...) -> None: ... + def _open(cls, seq_name: str, seq_dir: Optional[str] = None) -> None: ... @staticmethod def _get_default_seq_path_in_engine() -> str: ... + @staticmethod + def _get_seq_info_in_engine( + seq_name: str, seq_dir: Optional[str] = None, map_path: Optional[str] = None + ) -> Tuple[str, str]: ... + @staticmethod + def _get_map_path_in_engine() -> str: ... + @staticmethod + def _get_seq_path_in_engine() -> str: ... diff --git a/xrfeitoria/sequence/sequence_wrapper.py b/xrfeitoria/sequence/sequence_wrapper.py index b77a22de..78313305 100644 --- a/xrfeitoria/sequence/sequence_wrapper.py +++ b/xrfeitoria/sequence/sequence_wrapper.py @@ -100,6 +100,7 @@ def open(cls, seq_name: str, seq_dir: 'Optional[str]' = None) -> ContextManager[ SequenceUnreal: Sequence object. """ cls._seq._open(seq_name=seq_name, seq_dir=seq_dir) + cls._seq.show() yield cls._seq cls._seq.save() cls._seq.close() @@ -137,6 +138,7 @@ def new( replace=replace, seq_dir=seq_dir, ) + cls._seq.show() yield cls._seq cls._seq.save() cls._seq.close() @@ -207,4 +209,6 @@ def sequence_wrapper_unreal( seq_length=seq_length, replace=replace, ) + # Open the sequence in editor, for letting `get_bound_objects` work + SequenceUnreal.show() return SequenceUnreal() diff --git a/xrfeitoria/utils/functions/blender_functions.py b/xrfeitoria/utils/functions/blender_functions.py index 5655beb6..a7621c16 100644 --- a/xrfeitoria/utils/functions/blender_functions.py +++ b/xrfeitoria/utils/functions/blender_functions.py @@ -68,14 +68,21 @@ def import_file(file_path: 'PathLike') -> None: @remote_blender() -def apply_motion_data_to_actor(motion_data: 'List[MotionFrame]', actor_name: str) -> None: +def apply_motion_data_to_actor( + motion_data: 'List[MotionFrame]', + actor_name: str, + is_first_frame_as_origin: bool = True, +) -> None: """Applies motion data to a given actor in Blender. Args: motion_data (List[MotionFrame]): A list of dictionaries containing motion data for the actor. actor_name (str): The name of the actor to apply the motion data to. + is_first_frame_as_origin (bool, optional): Whether to set the first frame as the origin. Defaults to True. """ - XRFeitoriaBlenderFactory.apply_motion_data_to_actor(motion_data=motion_data, actor_name=actor_name) + XRFeitoriaBlenderFactory.apply_motion_data_to_actor( + motion_data=motion_data, actor_name=actor_name, is_first_frame_as_origin=is_first_frame_as_origin + ) @remote_blender() diff --git a/xrfeitoria/utils/functions/unreal_functions.py b/xrfeitoria/utils/functions/unreal_functions.py index 1e226fb3..51aaaf53 100644 --- a/xrfeitoria/utils/functions/unreal_functions.py +++ b/xrfeitoria/utils/functions/unreal_functions.py @@ -70,7 +70,10 @@ def save_current_level(asset_path: 'Optional[str]' = None) -> None: @remote_unreal() def import_asset( - path: 'Union[str, List[str]]', dst_dir_in_engine: 'Optional[str]' = None, replace: bool = True + path: 'Union[str, List[str]]', + dst_dir_in_engine: 'Optional[str]' = None, + replace: bool = True, + with_parent_dir: bool = True, ) -> 'Union[str, List[str]]': """Import assets to the default asset path. @@ -79,11 +82,15 @@ def import_asset( dst_dir_in_engine (Optional[str], optional): destination directory in the engine. Defaults to None falls back to '/Game/XRFeitoriaUnreal/Assets' replace (bool, optional): whether to replace the existing asset. Defaults to True. + with_parent_dir (bool, optional): whether to create a parent directory that contains the imported asset. + If False, the imported asset will be in `dst_dir_in_engine` directly. Defaults to True. Returns: Union[str, List[str]]: a path or a list of paths to the imported assets, e.g. "/Game/XRFeitoriaUnreal/Assets/SMPL_XL" """ - paths = XRFeitoriaUnrealFactory.utils.import_asset(path, dst_dir_in_engine, replace=replace) + paths = XRFeitoriaUnrealFactory.utils.import_asset( + path, dst_dir_in_engine, replace=replace, with_parent_dir=with_parent_dir + ) if len(paths) == 1: return paths[0] return paths diff --git a/xrfeitoria/utils/tools.py b/xrfeitoria/utils/tools.py index 1ec42253..e04ba2b4 100644 --- a/xrfeitoria/utils/tools.py +++ b/xrfeitoria/utils/tools.py @@ -7,6 +7,7 @@ import loguru from loguru import logger +from rich import get_console from rich.console import Console from rich.progress import ( BarColumn, @@ -92,13 +93,13 @@ def setup_logging( logger.remove() # remove default logger # logger.add(sink=lambda msg: rprint(msg, end=''), level=level, format=cls.logger_format) - c = Console( - width=sys.maxsize, # disable wrapping - log_time=False, - log_path=False, - log_time_format='', - ) - logger.add(sink=lambda msg: c.print(msg, end=''), level=level, format=cls.logger_format) + # c = Console( + # width=sys.maxsize, # disable wrapping + # log_time=False, + # log_path=False, + # log_time_format='', + # ) + logger.add(sink=lambda msg: get_console().print(msg, end=''), level=level, format=cls.logger_format) if log_path: # add file logger log_path = Path(log_path).resolve() From ac47a0619937f080a5b322262815afcf2246bfe9 Mon Sep 17 00:00:00 2001 From: Haiyi Date: Wed, 21 Feb 2024 16:52:01 +0800 Subject: [PATCH 06/27] [Unreal] Change postprocess to preprocess (#3) --- README.md | 2 +- .../Content/Python/constants.py | 9 +- .../Content/Python/custom_movie_pipeline.py | 22 +- .../Content/Python/sequence.py | 156 +++++++-- .../Private/CustomMoviePipelineOutput.cpp | 212 ------------- .../Private/MoviePipelineMeshOperator.cpp | 300 ------------------ .../Public/CustomMoviePipelineOutput.h | 17 - .../Public/MoviePipelineMeshOperator.h | 111 ------- tests/unreal/sequence.py | 139 ++++---- xrfeitoria/camera/camera_parameter.py | 32 ++ xrfeitoria/data_structure/models.py | 2 - xrfeitoria/renderer/renderer_unreal.py | 129 +------- xrfeitoria/sequence/sequence_unreal.py | 77 ++++- xrfeitoria/utils/anim/constants.py | 4 +- 14 files changed, 309 insertions(+), 903 deletions(-) delete mode 100644 src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp delete mode 100644 src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h diff --git a/README.md b/README.md index 2c3a52de..e9e91822 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ pip install xrfeitoria - `Python >= 3.8` - (optional) `Unreal Engine >= 5.1` - [x] Windows - - [ ] Linux + - [x] Linux - [ ] MacOS - (optional) `Blender >= 3.0` - [x] Windows diff --git a/src/XRFeitoriaUnreal/Content/Python/constants.py b/src/XRFeitoriaUnreal/Content/Python/constants.py index 634c6e3d..4b2fdc0f 100644 --- a/src/XRFeitoriaUnreal/Content/Python/constants.py +++ b/src/XRFeitoriaUnreal/Content/Python/constants.py @@ -45,7 +45,6 @@ def get_plugin_path() -> Tuple[Path, Path, Path]: DEFAULT_ASSET_PATH = f'{DEFAULT_PATH}/Assets' DEFAULT_SEQUENCE_DATA_ASSET = f'/{PLUGIN_NAME}/DefaultSequenceData' MRQ_JOB_UPPER = 200 -cam_param_dir = 'camera_params' data_asset_suffix = '_data' @@ -123,6 +122,12 @@ class UnrealRenderLayerEnum(EnumBase): tangent = 'tangent' basecolor = 'basecolor' + vertices = 'vertices' + skeleton = 'skeleton' + actor_infos = 'actor_infos' + camera_params = 'camera_params' + audio = 'Audio' + @dataclass class RenderPass: @@ -183,8 +188,6 @@ class AntiAliasSetting: file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}' console_variables: Dict[str, float] = field(default_factory=dict) anti_aliasing: AntiAliasSetting = AntiAliasSetting() - export_vertices: bool = False - export_skeleton: bool = False export_audio: bool = False def __post_init__(self): diff --git a/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py b/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py index 4acfd0c9..7fddcdb7 100644 --- a/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py +++ b/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py @@ -99,21 +99,6 @@ def set_render_all_cameras(movie_preset: unreal.MoviePipelineMasterConfig, enabl ) camera_setting.render_all_cameras = enable - @staticmethod - def set_export_vertices(movie_preset: unreal.MoviePipelineMasterConfig, enable: bool = True) -> None: - export_setting: unreal.MoviePipelineMeshOperator = movie_preset.find_or_add_setting_by_class( - unreal.MoviePipelineMeshOperator - ) - export_setting.static_mesh_operator_option.save_vertices_position = enable - export_setting.skeletal_mesh_operator_option.save_vertices_position = enable - - @staticmethod - def set_export_skeleton(movie_preset: unreal.MoviePipelineMasterConfig, enable: bool = True) -> None: - export_setting: unreal.MoviePipelineMeshOperator = movie_preset.find_or_add_setting_by_class( - unreal.MoviePipelineMeshOperator - ) - export_setting.skeletal_mesh_operator_option.save_skeleton_position = enable - @staticmethod def set_export_audio(movie_preset: unreal.MoviePipelineMasterConfig) -> None: export_setting: unreal.MoviePipelineWaveOutput = movie_preset.find_or_add_setting_by_class( @@ -277,8 +262,6 @@ def create_movie_preset( output_path: Optional[str] = None, anti_alias: RenderJobUnreal.AntiAliasSetting = RenderJobUnreal.AntiAliasSetting(), console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0.0}, - export_vertices: bool = False, - export_skeleton: bool = False, export_audio: bool = False, ) -> unreal.MoviePipelineMasterConfig: """ @@ -296,6 +279,7 @@ def create_movie_preset( output_path (str): Path of the output, e.g. 'E:/output' anti_alias (dict): Anti-alias settings. console_variables (bool): Console variables. + export_audio (bool): Whether to export audio. Returns: unreal.MoviePipelineMasterConfig: The created movie preset. @@ -308,8 +292,6 @@ def create_movie_preset( cls.add_anti_alias(movie_preset, anti_alias) cls.add_console_command(movie_preset, console_variables) cls.set_render_all_cameras(movie_preset, enable=True) - cls.set_export_vertices(movie_preset, enable=export_vertices) - cls.set_export_skeleton(movie_preset, enable=export_skeleton) if export_audio: cls.set_export_audio(movie_preset) @@ -396,8 +378,6 @@ def add_job_to_queue(cls, job: RenderJobUnreal) -> bool: output_path=job.output_path, anti_alias=job.anti_aliasing, console_variables=job.console_variables, - export_vertices=job.export_vertices, - export_skeleton=job.export_skeleton, export_audio=job.export_audio, ) new_job.set_configuration(movie_preset) diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index b1042681..8d828069 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import Any, Dict, List, NoReturn, Optional, Tuple, Type, Union +import numpy as np import unreal import utils_actor from constants import ( @@ -14,6 +15,7 @@ SequenceTransformKey, SubSystem, TransformKeys, + UnrealRenderLayerEnum, data_asset_suffix, ) from utils import add_levels, get_levels, get_soft_object_path, get_world, new_world, save_current_level @@ -877,6 +879,25 @@ def get_camera_param(camera: unreal.CameraActor) -> Dict[str, Any]: } +def get_actor_param(actor: unreal.Actor) -> Dict[str, Any]: + """Get actor parameters. + + Args: + actor (unreal.Actor): The actor. + + Returns: + Dict[str, Any]: A dictionary containing the actor parameters. + """ + stencil_value = get_actor_mesh_component(actor).get_editor_property('custom_depth_stencil_value') + return { + 'location': actor.get_actor_location().to_tuple(), + 'rotation': actor.get_actor_rotation().to_tuple(), + 'scale': actor.get_actor_scale3d().to_tuple(), + 'mask_color': utils_actor.get_mask_color(stencil_value=stencil_value), + 'stencil_value': stencil_value, + } + + class Sequence: map_path = None sequence_path = None @@ -1088,6 +1109,16 @@ def set_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int for section in master_track.get_sections(): section.set_end_frame(end_frame) + @classmethod + def get_playback(cls) -> Tuple[int, int]: + """Get the playback range of the sequence. + + Returns: + Tuple[int, int]: The start frame and end frame of the playback range. + """ + assert cls.sequence is not None, 'Sequence not initialized' + return cls.sequence.get_playback_start(), cls.sequence.get_playback_end() + # ------ add actor and camera -------- # @classmethod @@ -1204,39 +1235,54 @@ def add_audio( cls.bindings[audio_asset.get_name()] = bindings @classmethod - def save_camera_params(cls, save_dir: PathLike, per_frame: bool = True): - """Saves the camera parameters of the sequence into json files. + def save_params( + cls, + save_dir: PathLike, + per_frame: bool = True, + export_vertices: bool = False, + export_skeleton: bool = False, + frame_idx: Optional[int] = None, + ) -> Dict[str, str]: + """Save parameters of the sequence. Args: - save_dir (PathLike): The directory where the camera parameters will be saved. - per_frame (bool, optional): Whether to save camera parameters per frame. Defaults to True. + save_dir (PathLike): The directory to save the parameters. + per_frame (bool, optional): Whether to save parameters for each frame. Defaults to True. + export_vertices (bool, optional): Whether to export vertices of the mesh. Defaults to False. + export_skeleton (bool, optional): Whether to export skeleton of the skeletal mesh. Defaults to False. + frame_idx (int, optional): The frame index to save parameters for. Defaults to None. Will be used only if `per_frame` is False. + + Returns: + Dict[str, str]: A dictionary containing the paths of the saved parameters. """ assert cls.sequence is not None, 'Sequence not initialized' cls.show() save_dir = Path(save_dir) save_dir.mkdir(parents=True, exist_ok=True) - unreal.log(f'[XRFeitoria] saving camera parameters of sequence to {save_dir}') - - # camera_actors: Dict[str, unreal.CameraActor] = { - # name: binding['camera']['self'] for name, binding in cls.bindings.items() if 'camera' in binding.keys() - # } - camera_actors = {} + # get actors + mesh_actors: Dict[str, unreal.Actor] = {} + camera_actors: Dict[str, unreal.CameraActor] = {} for name, binding in cls.bindings.items(): - if 'camera' not in binding.keys(): - continue - camera_binding = binding['camera']['binding'] - camera = unreal.LevelSequenceEditorBlueprintLibrary.get_bound_objects(get_binding_id(camera_binding))[0] - camera_actors[name] = camera - - def save_camera_param(frame_idx: int) -> Dict[str, Any]: + if 'camera' in binding.keys(): + camera_binding = binding['camera']['binding'] + camera = unreal.LevelSequenceEditorBlueprintLibrary.get_bound_objects(get_binding_id(camera_binding))[0] + camera_actors[name] = camera + if 'actor' in binding.keys(): + actor_binding = binding['actor']['binding'] + actor = unreal.LevelSequenceEditorBlueprintLibrary.get_bound_objects(get_binding_id(actor_binding))[0] + mesh_actors[name] = actor + + # define save functions + def save_camera_param(frame_idx: int, save_root: PathLike) -> Dict[str, Any]: """Save camera parameters of the given frame to - {save_dir}/{camera_name}/{frame_idx:04d}.json. + {save_root}/camera_params/{camera_name}/{frame_idx:04d}.json. Args: frame_idx (int): The frame index to save camera parameters for. """ + save_dir = Path(save_root) / UnrealRenderLayerEnum.camera_params.value unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(frame_idx) for name, camera in camera_actors.items(): save_path = save_dir / name / f'{frame_idx:04d}.json' @@ -1244,11 +1290,77 @@ def save_camera_param(frame_idx: int) -> Dict[str, Any]: with open(save_path, 'w') as f: json.dump(get_camera_param(camera), f, indent=4) - if not per_frame: - save_camera_param(frame_idx=0) + def save_actor_param( + frame_idx: int, + save_root: PathLike, + export_vertices: bool = False, + export_skeleton: bool = False, + ) -> Dict[str, Any]: + """Save actor parameters of the given frame to + {save_root}/{params_type}/{actor_name}/{frame_idx:04d}.json. `params_type` + is one of ['actor_infos', 'vertices', 'skeleton']. + + Args: + frame_idx (int): The frame index to save actor parameters for. + save_root (PathLike): The root directory to save the parameters. + export_vertices (bool, optional): Whether to export vertices of the mesh. Defaults to False. + export_skeleton (bool, optional): Whether to export skeleton of the skeletal mesh. Defaults to False. + """ + # Init directories + actor_infos_dir = Path(save_root) / UnrealRenderLayerEnum.actor_infos.value + vertices_dir = Path(save_root) / UnrealRenderLayerEnum.vertices.value + skeleton_dir = Path(save_root) / UnrealRenderLayerEnum.skeleton.value + + unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(frame_idx) + for name, actor in mesh_actors.items(): + actor_infos_file = actor_infos_dir / name / f'{frame_idx:04d}.json' + actor_infos_file.parent.mkdir(parents=True, exist_ok=True) + with open(actor_infos_file, 'w') as f: + json.dump(get_actor_param(actor), f, indent=4) + + if export_vertices: + vertices_file = vertices_dir / name / f'{frame_idx:04d}.npz' + vertices_file.parent.mkdir(parents=True, exist_ok=True) + if isinstance(actor, unreal.SkeletalMeshActor): + skl_mesh_comp = actor.skeletal_mesh_component + vertices_data = ( + unreal.XF_BlueprintFunctionLibrary.get_skeletal_mesh_vertex_locations_by_lod_index( + skl_mesh_comp, 0 + ) + ) + elif isinstance(actor, unreal.StaticMeshActor): + static_mesh_comp = actor.static_mesh_component + vertices_data = unreal.XF_BlueprintFunctionLibrary.get_static_mesh_vertex_locations( + static_mesh_comp, 0 + ) + vertices_data = np.array([v.to_tuple() for v in vertices_data]) + np.savez_compressed(vertices_file, vertices_data) + + if export_skeleton and isinstance(actor, unreal.SkeletalMeshActor): + skeleton_file = skeleton_dir / name / f'{frame_idx:04d}.npz' + skeleton_file.parent.mkdir(parents=True, exist_ok=True) + vertices_data = unreal.XF_BlueprintFunctionLibrary.get_skeletal_mesh_bone_locations(skl_mesh_comp) + # bone_locations (Array[Vector]): + # bone_names (Array[Name]): + # np.savez_compressed(skeleton_file, skeleton_data) + + # save parameters + if per_frame: + for frame_idx in range(cls.START_FRAME, cls.sequence.get_playback_end()): + save_camera_param(frame_idx, save_root=save_dir) + save_actor_param(frame_idx, save_dir, export_vertices, export_skeleton) else: - for frame_idx in range(0, cls.sequence.get_playback_end()): - save_camera_param(frame_idx=frame_idx) + if frame_idx is None: + frame_idx = cls.START_FRAME + save_camera_param(frame_idx, save_root=save_dir) + save_actor_param(frame_idx, save_dir, export_vertices, export_skeleton) + + return { + 'camera_dir': (save_dir / UnrealRenderLayerEnum.camera_params.value).as_posix(), + 'actor_infos_dir': (save_dir / UnrealRenderLayerEnum.actor_infos.value).as_posix(), + 'vertices_dir': (save_dir / UnrealRenderLayerEnum.vertices.value).as_posix(), + 'skeleton_dir': (save_dir / UnrealRenderLayerEnum.skeleton.value).as_posix(), + } def test(): diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp index 7f8b98cf..433876e1 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp @@ -34,7 +34,6 @@ #include "MovieRenderPipelineCoreModule.h" #include "XF_BlueprintFunctionLibrary.h" -#include "MoviePipelineMeshOperator.h" #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION == 0 #include "MoviePipelineMasterConfig.h" @@ -42,145 +41,9 @@ DECLARE_CYCLE_STAT(TEXT("ImgSeqOutput_RecieveImageData"), STAT_ImgSeqRecieveImageData, STATGROUP_MoviePipeline); -void UCustomMoviePipelineOutput::SetupForPipelineImpl(UMoviePipeline* InPipeline) -{ - if (InPipeline) - { - InPipeline->SetFlushDiskWritesPerShot(true); - } - - ULevelSequence* LevelSequence = GetPipeline()->GetTargetSequence(); - UMovieSceneSequence* MovieSceneSequence = GetPipeline()->GetTargetSequence(); - UMovieScene* MovieScene = LevelSequence->GetMovieScene(); - TArray bindings = MovieScene->GetBindings(); - - TArray bindingProxies; - for (FMovieSceneBinding binding : bindings) - { - FGuid guid = binding.GetObjectGuid(); - bindingProxies.Add(FSequencerBindingProxy(guid, MovieSceneSequence)); - } - - boundObjects = USequencerToolsFunctionLibrary::GetBoundObjects( - GetPipeline()->GetWorld(), - LevelSequence, - bindingProxies, - FSequencerScriptingRange::FromNative( - MovieScene->GetPlaybackRange(), - MovieScene->GetDisplayRate() - ) - ); - - for (FSequencerBoundObjects boundObject : boundObjects) - { - // loop over bound objects - UObject* BoundObject = boundObject.BoundObjects[0]; // only have one item - if (BoundObject->IsA(ACameraActor::StaticClass())) - { - ACameraActor* Camera = Cast(BoundObject); - Cameras.Add(Camera); - } - else if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) - { - ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); - SkeletalMeshComponents.Add(SkeletalMeshActor->GetSkeletalMeshComponent()); - } - else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) - { - AStaticMeshActor* StaticMeshActor = Cast(BoundObject); - StaticMeshComponents.Add(StaticMeshActor->GetStaticMeshComponent()); - } - else if (BoundObject->IsA(USkeletalMeshComponent::StaticClass())) - { - USkeletalMeshComponent* SkeletalMeshComponent = Cast(BoundObject); - // check if it's already in the list - bool bFound = false; - for (USkeletalMeshComponent* SkeletalMeshComponentInList : SkeletalMeshComponents) - { - if (SkeletalMeshComponentInList == SkeletalMeshComponent) - { - bFound = true; - break; - } - } - if (!bFound) SkeletalMeshComponents.Add(SkeletalMeshComponent); - } - else if (BoundObject->IsA(UStaticMeshComponent::StaticClass())) - { - UStaticMeshComponent* StaticMeshComponent = Cast(BoundObject); - // check if it's already in the list - bool bFound = false; - for (UStaticMeshComponent* StaticMeshComponentInList : StaticMeshComponents) - { - if (StaticMeshComponentInList == StaticMeshComponent) - { - bFound = true; - break; - } - } - if (!bFound) - StaticMeshComponents.Add(StaticMeshComponent); - } - } -} - void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) { - // Save Actor Info (stencil value) - if (bIsFirstFrame) - { - // SkeletalMesh - for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) - { - // Actor in level - FString MeshNameFromLabel = SkeletalMeshComponent->GetOwner()->GetActorNameOrLabel(); - // Actor spawned from sequence - FString MeshNameFromName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - // Judge which name is correct - FString MeshName = MeshNameFromName.StartsWith("SkeletalMesh") ? MeshNameFromLabel : MeshNameFromName; - - int StencilValue = SkeletalMeshComponent->CustomDepthStencilValue; - - FString ActorInfoPath = GetOutputPath( - DirectoryActorInfo / MeshName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryActorInfo/{actor_name}/{frame_idx}.dat - ActorInfoPath = FPaths::SetExtension( - FPaths::GetPath(ActorInfoPath), - FPaths::GetExtension(ActorInfoPath) - ); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatToByteFile(StencilValue, ActorInfoPath); - } - - // StaticMesh - for (UStaticMeshComponent* StaticMeshComponent : StaticMeshComponents) - { - // Actor in level - FString MeshNameFromLabel = StaticMeshComponent->GetOwner()->GetActorNameOrLabel(); - // Actor spawned from sequence - FString MeshNameFromName = StaticMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - // Judge which name is correct - FString MeshName = MeshNameFromName.StartsWith("StaticMesh") ? MeshNameFromLabel : MeshNameFromName; - - int StencilValue = StaticMeshComponent->CustomDepthStencilValue; - - FString ActorInfoPath = GetOutputPath( - DirectoryActorInfo / MeshName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryActorInfo/{actor_name}/{frame_idx}.dat - ActorInfoPath = FPaths::SetExtension( - FPaths::GetPath(ActorInfoPath), - FPaths::GetExtension(ActorInfoPath) - ); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatToByteFile(StencilValue, ActorInfoPath); - } - - bIsFirstFrame = false; - } - // Get Output Setting #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); @@ -191,50 +54,6 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp #endif check(OutputSettings); - // Save Camera Transform (KRT) - int ResolutionX = OutputSettings->OutputResolution.X; - int ResolutionY = OutputSettings->OutputResolution.Y; - for (ACameraActor* Camera : Cameras) - { - FVector CamLocation = Camera->GetActorLocation(); - FRotator CamRotation = Camera->GetActorRotation(); - float FOV = Camera->GetCameraComponent()->FieldOfView; - - TArray CamInfo; - CamInfo.Add(CamLocation.X); - CamInfo.Add(CamLocation.Y); - CamInfo.Add(CamLocation.Z); - CamInfo.Add(CamRotation.Roll); - CamInfo.Add(CamRotation.Pitch); - CamInfo.Add(CamRotation.Yaw); - CamInfo.Add(FOV); - CamInfo.Add(ResolutionX); - CamInfo.Add(ResolutionY); - - // Actor in level - FString CameraNameFromLabel = Camera->GetActorNameOrLabel(); - // Actor spawned from sequence - FString CameraNameFromName = Camera->GetFName().GetPlainNameString(); - // XXX: Hardcode way to Judge which name is correct, need to be improved - // Should ref to - // GetPipeline()->ResolveFilenameFormatArguments(FileNameFormatString, FormatOverrides, OutputData.FilePath, FinalFormatArgs, &Payload->SampleState.OutputState); - // using {camera_name} - - bool bIsCameraInLevel = CameraNameFromName.StartsWith("CameraActor") || CameraNameFromName.StartsWith("CineCameraActor"); - FString CameraName = bIsCameraInLevel ? CameraNameFromLabel : CameraNameFromName; - - FString CameraTransformPath = GetOutputPath( - DirectoryCameraInfo / CameraName, - "dat", - &InMergedOutputFrame->FrameOutputState - ); // DirectoryCameraInfo/{camera_name}/{frame_idx}.dat - //CameraTransformPath = FPaths::SetExtension( - // FPaths::GetPath(CameraTransformPath), - // FPaths::GetExtension(CameraTransformPath) - //); // get rid of the frame index - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); - } - SCOPE_CYCLE_COUNTER(STAT_ImgSeqRecieveImageData); check(InMergedOutputFrame); @@ -442,34 +261,3 @@ void UCustomMoviePipelineOutput::OnReceiveImageDataImpl(FMoviePipelineMergerOutp GetPipeline()->AddOutputFuture(ImageWriteQueue->Enqueue(MoveTemp(TileImageTask)), OutputData); } } - -FString UCustomMoviePipelineOutput::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) -{ - #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - #else - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - #endif - check(OutputSettings); - FString OutputDirectory = OutputSettings->OutputDirectory.Path; - FString FileNameFormatString = OutputSettings->FileNameFormat; - - FString OutputPath; - FMoviePipelineFormatArgs Args; - TMap FormatOverrides; - FormatOverrides.Add(TEXT("camera_name"), ""); - FormatOverrides.Add(TEXT("render_pass"), PassName); - FormatOverrides.Add(TEXT("ext"), Ext); - GetPipeline()->ResolveFilenameFormatArguments( - OutputDirectory / FileNameFormatString, FormatOverrides, OutputPath, Args, InOutputState); - - if (FPaths::IsRelative(OutputPath)) - { - OutputPath = FPaths::ConvertRelativePathToFull(OutputPath); - } - - // Replace any double slashes with single slashes. - OutputPath.ReplaceInline(TEXT("//"), TEXT("/")); - - return OutputPath; -} diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp deleted file mode 100644 index ce8ac058..00000000 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/MoviePipelineMeshOperator.cpp +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright OpenXRLab 2023-2024. All Rights Reserved. - -#include "MoviePipelineMeshOperator.h" -#include "XF_BlueprintFunctionLibrary.h" -#include "Engine/StaticMeshActor.h" -#include "Animation/SkeletalMeshActor.h" -#include "LevelSequenceEditorBlueprintLibrary.h" -#include "MovieSceneObjectBindingID.h" -#include "Camera/CameraActor.h" -#include "Camera/CameraComponent.h" -#include "Misc/FileHelper.h" -#include "MovieRenderPipelineCoreModule.h" // For logs - - -void UMoviePipelineMeshOperator::SetupForPipelineImpl(UMoviePipeline* InPipeline) -{ - if (InPipeline) - { - InPipeline->SetFlushDiskWritesPerShot(true); - } - - ULevelSequence* LevelSequence = GetPipeline()->GetTargetSequence(); - UMovieSceneSequence* MovieSceneSequence = GetPipeline()->GetTargetSequence(); - UMovieScene* MovieScene = LevelSequence->GetMovieScene(); - - TMap bindingMap; - for (int idx = 0; idx < MovieScene->GetSpawnableCount(); idx++) - { - FMovieSceneSpawnable spawnable = MovieScene->GetSpawnable(idx); - FGuid guid = spawnable.GetGuid(); - FString name = spawnable.GetName(); - - bindingMap.Add(name, guid); - } - - for (int idx = 0; idx < MovieScene->GetPossessableCount(); idx++) - { - FMovieScenePossessable possessable = MovieScene->GetPossessable(idx); - FGuid guid = possessable.GetGuid(); - FString name = possessable.GetName(); - - bindingMap.Add(name, guid); - } - - for (TPair pair : bindingMap) - { - FString name = pair.Key; - FGuid guid = pair.Value; - - TArray _boundObjects_ = USequencerToolsFunctionLibrary::GetBoundObjects( - GetPipeline()->GetWorld(), - LevelSequence, - TArray({ FSequencerBindingProxy(guid, MovieSceneSequence) }), - FSequencerScriptingRange::FromNative( - MovieScene->GetPlaybackRange(), - MovieScene->GetDisplayRate() - ) - ); - - UObject* BoundObject = _boundObjects_[0].BoundObjects[0]; // only have one item - if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) - { - ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); - SkeletalMeshComponents.Add(name, SkeletalMeshActor->GetSkeletalMeshComponent()); - } - else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) - { - AStaticMeshActor* StaticMeshActor = Cast(BoundObject); - StaticMeshComponents.Add(name, StaticMeshActor->GetStaticMeshComponent()); - } - //else if (BoundObject->IsA(USkeletalMeshComponent::StaticClass())) - //{ - // USkeletalMeshComponent* SkeletalMeshComponent = Cast(BoundObject); - // // check if it's already in the list - // bool bFound = false; - // for (TPair SKMPair : SkeletalMeshComponents) - // { - // USkeletalMeshComponent* SkeletalMeshComponentInList = SKMPair.Value; - // if (SkeletalMeshComponentInList == SkeletalMeshComponent) - // { - // bFound = true; - // break; - // } - // } - // if (!bFound) SkeletalMeshComponents.Add(name, SkeletalMeshComponent); - //} - //else if (BoundObject->IsA(UStaticMeshComponent::StaticClass())) - //{ - // UStaticMeshComponent* StaticMeshComponent = Cast(BoundObject); - // // check if it's already in the list - // bool bFound = false; - // for (TPair SKMPair : StaticMeshComponents) - // { - // UStaticMeshComponent* StaticMeshComponentInList = SKMPair.Value; - // if (StaticMeshComponentInList == StaticMeshComponent) - // { - // bFound = true; - // break; - // } - // } - // if (!bFound) StaticMeshComponents.Add(name, StaticMeshComponent); - //} - } -} - -void UMoviePipelineMeshOperator::OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) -{ - for (TPair SKMPair : SkeletalMeshComponents) - { - // loop over Skeletal mesh components - if (!SkeletalMeshOperatorOption.bEnabled) continue; - - FString MeshName = SKMPair.Key; - USkeletalMeshComponent* SkeletalMeshComponent = SKMPair.Value; - - //// Actor in level - //FString MeshNameFromLabel = SkeletalMeshComponent->GetOwner()->GetActorNameOrLabel(); - //// Actor spawned from sequence - //FString MeshNameFromName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - //// Judge which name is correct - //FString MeshName = MeshNameFromName.StartsWith("SkeletalMesh") ? MeshNameFromLabel : MeshNameFromName; - - if (SkeletalMeshOperatorOption.bSaveVerticesPosition) - { - // Get Vertex Positions (with LOD) - TArray VertexPositions; - bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshVertexLocationsByLODIndex( - SkeletalMeshComponent, - SkeletalMeshOperatorOption.LODIndex, - VertexPositions - ); - if (!isSuccess) - { - UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); - continue; - } - TArray VertexPositionsFloat; - for (FVector position : VertexPositions) - { - VertexPositionsFloat.Add(position.X); - VertexPositionsFloat.Add(position.Y); - VertexPositionsFloat.Add(position.Z); - } - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - VertexPositionsFloat, GetOutputPath( - SkeletalMeshOperatorOption.DirectoryVertices / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - } - - if (SkeletalMeshOperatorOption.bSaveSkeletonPosition) - { - TArray SkeletonPositions; - TArray SkeletonNames; - bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( - SkeletalMeshComponent, SkeletonPositions, SkeletonNames); - - // Skeleton Names (only save on the first frame) - TArray SkeletonNamesString; - for (FName name : SkeletonNames) SkeletonNamesString.Add(name.ToString()); - FString BoneNamePath = GetOutputPath( - SkeletalMeshOperatorOption.DirectorySkeleton / MeshName, "txt", &InMergedOutputFrame->FrameOutputState); - // save to DirectorySkeleton / BoneName.txt - BoneNamePath = FPaths::Combine( - FPaths::GetPath(BoneNamePath), - FPaths::SetExtension("BoneName", FPaths::GetExtension(BoneNamePath)) - ); - if (bIsFirstFrame) FFileHelper::SaveStringArrayToFile(SkeletonNamesString, *BoneNamePath); - - // Skeleton Positions - TArray SkeletonPositionsFloat; - for (FVector position : SkeletonPositions) - { - SkeletonPositionsFloat.Add(position.X); - SkeletonPositionsFloat.Add(position.Y); - SkeletonPositionsFloat.Add(position.Z); - } - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - SkeletonPositionsFloat, GetOutputPath( - SkeletalMeshOperatorOption.DirectorySkeleton / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - } - - //if (SkeletalMeshOperatorOption.bSaveOcclusionRate || SkeletalMeshOperatorOption.bSaveOcclusionResult) - //{ - // - // float non_occlusion_rate; - // float self_occlusion_rate; - // float inter_occlusion_rate; - // TArray SkeletonPositions; - // TArray SkeletonNames; - // TArray SkeletonOcclusion; - // float MeshThickness = 5.0f; - // bool isSuccess = UXF_BlueprintFunctionLibrary::DetectInterOcclusionSkeleton( - // SkeletalMeshComponent, - // Camera, - // non_occlusion_rate, - // self_occlusion_rate, - // inter_occlusion_rate, - // SkeletonPositions, - // SkeletonNames, - // SkeletonOcclusion, - // MeshThickness, - // false - // ); - // // Occlusion Int - // TArray SkeletonOcclusionInt; - // for (EOcclusion occlusion : SkeletonOcclusion) SkeletonOcclusionInt.Add((uint8)occlusion); - // FFileHelper::SaveArrayToFile( - // SkeletonOcclusionInt, *GetOutputPath("Occlusion" / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - // - // // Occlusion rate - // TArray OcclusionRate; - // OcclusionRate.Add(non_occlusion_rate); - // OcclusionRate.Add(self_occlusion_rate); - // OcclusionRate.Add(inter_occlusion_rate); - // UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - // OcclusionRate, *GetOutputPath("OcclusionRate" / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - // // TODO: export to npz - //} - } - for (TPair SKMPair : StaticMeshComponents) - { - // loop over static mesh components - if (!StaticMeshOperatorOption.bEnabled) continue; - - UStaticMeshComponent* StaticMeshComponent = SKMPair.Value; - FString MeshName = SKMPair.Key; - - //// Actor in level - //FString MeshNameFromLabel = StaticMeshComponent->GetOwner()->GetActorNameOrLabel(); - //// Actor spawned from sequence - //FString MeshNameFromName = StaticMeshComponent->GetOwner()->GetFName().GetPlainNameString(); - //// Judge which name is correct - //FString MeshName = MeshNameFromName.StartsWith("StaticMesh") ? MeshNameFromLabel : MeshNameFromName; - - if (StaticMeshOperatorOption.bSaveVerticesPosition) - { - // Get Vertex Positions (with LOD) - TArray VertexPositions; - bool isSuccess = UXF_BlueprintFunctionLibrary::GetStaticMeshVertexLocations( - StaticMeshComponent, - StaticMeshOperatorOption.LODIndex, - VertexPositions - ); - if (!isSuccess) - { - UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); - continue; - } - TArray VertexPositionsFloat; - for (FVector position : VertexPositions) - { - VertexPositionsFloat.Add(position.X); - VertexPositionsFloat.Add(position.Y); - VertexPositionsFloat.Add(position.Z); - } - UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( - VertexPositionsFloat, GetOutputPath( - StaticMeshOperatorOption.DirectoryVertices / MeshName, "dat", &InMergedOutputFrame->FrameOutputState)); - } - } - - if (bIsFirstFrame) bIsFirstFrame = false; -} - - -void UMoviePipelineMeshOperator::BeginExportImpl() -{ - FCoreDelegates::OnEndFrame.RemoveAll(this); - UE_LOG(LogMovieRenderPipelineIO, Log, TEXT("Mesh Operator Ended.")); -} - -FString UMoviePipelineMeshOperator::GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState) -{ - #if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION <2 - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelineMasterConfig()->FindSetting(); - #else - UMoviePipelineOutputSetting* OutputSettings = GetPipeline()->GetPipelinePrimaryConfig()->FindSetting(); - #endif - check(OutputSettings); - FString OutputDirectory = OutputSettings->OutputDirectory.Path; - FString FileNameFormatString = OutputSettings->FileNameFormat; - - FString OutputPath; - FMoviePipelineFormatArgs Args; - TMap FormatOverrides; - FormatOverrides.Add(TEXT("camera_name"), ""); - FormatOverrides.Add(TEXT("render_pass"), PassName); - FormatOverrides.Add(TEXT("ext"), Ext); - GetPipeline()->ResolveFilenameFormatArguments( - OutputDirectory / FileNameFormatString, FormatOverrides, OutputPath, Args, InOutputState); - - if (FPaths::IsRelative(OutputPath)) - { - OutputPath = FPaths::ConvertRelativePathToFull(OutputPath); - } - - // Replace any double slashes with single slashes. - OutputPath.ReplaceInline(TEXT("//"), TEXT("/")); - - return OutputPath; -} diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h index 136e76c1..73101af1 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h @@ -128,7 +128,6 @@ class XRFEITORIAUNREAL_API UCustomMoviePipelineOutput : public UMoviePipelineIma { OutputFormat = EImageFormat::PNG; } - virtual void SetupForPipelineImpl(UMoviePipeline* InPipeline); virtual void OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) override; public: @@ -143,20 +142,4 @@ class XRFEITORIAUNREAL_API UCustomMoviePipelineOutput : public UMoviePipelineIma UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "RenderPasses|Additional") TArray AdditionalRenderPasses; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "RenderPasses|Camera") - FString DirectoryActorInfo = "actor_infos"; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "RenderPasses|Camera") - FString DirectoryCameraInfo = "camera_params"; - -private: - FString GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState); - -private: - TArray boundObjects; - TArray Cameras; - TArray StaticMeshComponents; - TArray SkeletalMeshComponents; - bool bIsFirstFrame = true; }; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h deleted file mode 100644 index 3ac58aa7..00000000 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/MoviePipelineMeshOperator.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright OpenXRLab 2023-2024. All Rights Reserved. - -#pragma once - -#include "MoviePipeline.h" -#include "MoviePipelineOutputSetting.h" -#include "Misc/FileHelper.h" -// #include "HAL/PlatformFilemanager.h" - -#if WITH_EDITOR -#include "MovieSceneExportMetadata.h" -#include "MovieSceneToolHelpers.h" -#include "MovieScene.h" -#endif - -#if ENGINE_MAJOR_VERSION == 5 && ENGINE_MINOR_VERSION == 0 - #include "MoviePipelineMasterConfig.h" -#endif - - -#include "SequencerTools.h" -#include "SequencerSettings.h" -#include "SequencerBindingProxy.h" -#include "SequencerScriptingRange.h" - -#include "Kismet/KismetMathLibrary.h" -#include "Kismet/KismetStringLibrary.h" - -#include "LevelSequence.h" -#include "Components/StaticMeshComponent.h" -#include "Components/SkeletalMeshComponent.h" - -#include "CoreMinimal.h" -#include "MoviePipelineOutputBase.h" -#include "MovieRenderPipelineDataTypes.h" -#include "MoviePipelineMeshOperator.generated.h" - -/** - * - */ - -USTRUCT(BlueprintType) -struct XRFEITORIAUNREAL_API FMeshOperatorOption -{ - GENERATED_BODY() - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bEnabled = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bSaveVerticesPosition = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionResult = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionRate = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - FString DirectoryVertices = "vertices"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - int32 LODIndex = 0; -}; - - -USTRUCT(BlueprintType) -struct XRFEITORIAUNREAL_API FSkeletalMeshOperatorOption -{ - GENERATED_BODY() - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bEnabled = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bSaveVerticesPosition = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - bool bSaveSkeletonPosition = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionResult = true; - //UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - // bool bSaveOcclusionRate = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - FString DirectoryVertices = "vertices"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - FString DirectorySkeleton = "skeleton"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Occlusion Checker") - int32 LODIndex = 0; -}; - -UCLASS(Blueprintable) -class XRFEITORIAUNREAL_API UMoviePipelineMeshOperator : public UMoviePipelineOutputBase -{ - GENERATED_BODY() -public: -#if WITH_EDITOR - virtual FText GetDisplayText() const override { return NSLOCTEXT("MovieRenderPipeline", "MeshOperator_DisplayText", "Mesh Operator"); } -#endif - virtual void SetupForPipelineImpl(UMoviePipeline* InPipeline); - virtual void OnReceiveImageDataImpl(FMoviePipelineMergerOutputFrame* InMergedOutputFrame) override; - virtual void BeginExportImpl() override; -private: - FString GetOutputPath(FString PassName, FString Ext, const FMoviePipelineFrameOutputState* InOutputState); - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mesh Operator") - FMeshOperatorOption StaticMeshOperatorOption = FMeshOperatorOption(); - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mesh Operator") - FSkeletalMeshOperatorOption SkeletalMeshOperatorOption = FSkeletalMeshOperatorOption(); - -private: - TMap StaticMeshComponents; - TMap SkeletalMeshComponents; - bool bIsFirstFrame = true; -}; diff --git a/tests/unreal/sequence.py b/tests/unreal/sequence.py index 254434c2..e38bb59a 100644 --- a/tests/unreal/sequence.py +++ b/tests/unreal/sequence.py @@ -23,74 +23,77 @@ def new_seq(xf_runner: XRFeitoriaUnreal, level_path: str, seq_name: str): kc_path = xf_runner.utils.import_asset(path=kc_fbx) - with xf_runner.Sequence.new(level=level_path, seq_name=seq_name, seq_length=30, replace=True) as seq: - seq.spawn_camera_with_keys( - transform_keys=[ - SeqTransKey(frame=0, location=(0, 3, 1), rotation=(0, 0, -90), interpolation='AUTO'), - SeqTransKey(frame=30, location=(-3, 2, 2), rotation=(0, 0, -45), interpolation='AUTO'), - ], - fov=90.0, - camera_name='camera', - ) - camera2 = xf_runner.Camera.spawn(camera_name='camera2') - seq.use_camera_with_keys( - camera=camera2, - transform_keys=[ - SeqTransKey(frame=0, location=(-2, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), - SeqTransKey(frame=30, location=(-5, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), - ], - fov=90.0, - ) - seq.spawn_actor( - actor_asset_path='/Engine/BasicShapes/Cube', - actor_name='Actor', - location=[3, 0, 0], - rotation=[0, 0, 0], - stencil_value=2, - ) - seq.spawn_actor_with_keys( - actor_asset_path='/Engine/BasicShapes/Cone', - transform_keys=[ - SeqTransKey(frame=0, location=(-1, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), - SeqTransKey(frame=30, location=(0, 3, 5), rotation=(0, 0, 360), interpolation='AUTO'), - ], - actor_name='Actor2', - stencil_value=3, - ) - seq.spawn_actor( - actor_asset_path='/Engine/BasicShapes/Cylinder', - location=[0, 0, 0], - rotation=[0, 0, 0], - stencil_value=4, - ) - - seq.spawn_actor_with_keys( - actor_asset_path=kc_path, - transform_keys=[ - SeqTransKey( - frame=0, location=(0, 0, 0), rotation=(0, 0, 0), scale=(0.05, 0.05, 0.05), interpolation='AUTO' - ), - SeqTransKey(frame=5, location=(2, 0, 3), rotation=(0, 180, 0), interpolation='AUTO'), - SeqTransKey(frame=10, location=(0, 3, 0), rotation=(180, 0, 0), interpolation='AUTO'), - SeqTransKey(frame=15, location=(0, 0, 3), rotation=(0, 0, 180), interpolation='AUTO'), - SeqTransKey(frame=20, location=(0, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), - ], - actor_name='KoupenChan', - stencil_value=5, - ) - - seq.add_to_renderer( - output_path=output_path, - resolution=(1920, 1080), - render_passes=[ - RenderPass('img', 'png'), - RenderPass('mask', 'exr'), - ], - export_vertices=True, - export_skeleton=True, - ) - - xf_runner.utils.save_current_level() + seq = xf_runner.sequence(level=level_path, seq_name=seq_name, seq_length=30, replace=True) + seq.spawn_camera_with_keys( + transform_keys=[ + SeqTransKey(frame=0, location=(0, 3, 1), rotation=(0, 0, -90), interpolation='AUTO'), + SeqTransKey(frame=30, location=(-3, 2, 2), rotation=(0, 0, -45), interpolation='AUTO'), + ], + fov=90.0, + camera_name='camera', + ) + camera2 = xf_runner.Camera.spawn(camera_name='camera2') + seq.use_camera_with_keys( + camera=camera2, + transform_keys=[ + SeqTransKey(frame=0, location=(-2, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), + SeqTransKey(frame=30, location=(-5, 0, 1), rotation=(0, 0, 0), interpolation='AUTO'), + ], + fov=90.0, + ) + seq.spawn_actor( + actor_asset_path='/Engine/BasicShapes/Cube', + actor_name='Actor', + location=[3, 0, 0], + rotation=[0, 0, 0], + stencil_value=2, + ) + seq.spawn_actor_with_keys( + actor_asset_path='/Engine/BasicShapes/Cone', + transform_keys=[ + SeqTransKey(frame=0, location=(-1, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), + SeqTransKey(frame=30, location=(0, 3, 5), rotation=(0, 0, 360), interpolation='AUTO'), + ], + actor_name='Actor2', + stencil_value=3, + ) + seq.spawn_actor( + actor_asset_path='/Engine/BasicShapes/Cylinder', + location=[0, 0, 0], + rotation=[0, 0, 0], + stencil_value=4, + ) + + seq.spawn_actor_with_keys( + actor_asset_path=kc_path, + transform_keys=[ + SeqTransKey( + frame=0, location=(0, 0, 0), rotation=(0, 0, 0), scale=(0.05, 0.05, 0.05), interpolation='AUTO' + ), + SeqTransKey(frame=5, location=(2, 0, 3), rotation=(0, 180, 0), interpolation='AUTO'), + SeqTransKey(frame=10, location=(0, 3, 0), rotation=(180, 0, 0), interpolation='AUTO'), + SeqTransKey(frame=15, location=(0, 0, 3), rotation=(0, 0, 180), interpolation='AUTO'), + SeqTransKey(frame=20, location=(0, 0, 0), rotation=(0, 0, 0), interpolation='AUTO'), + ], + actor_name='KoupenChan', + stencil_value=5, + ) + + seq.add_to_renderer( + output_path=output_path, + resolution=(1920, 1080), + render_passes=[ + RenderPass('img', 'png'), + RenderPass('mask', 'exr'), + ], + export_vertices=True, + export_skeleton=True, + ) + + xf_runner.utils.save_current_level() + + seq.save() + seq.close() def sequence_test(debug: bool = False, background: bool = False): diff --git a/xrfeitoria/camera/camera_parameter.py b/xrfeitoria/camera/camera_parameter.py index bad87c35..2eea659c 100644 --- a/xrfeitoria/camera/camera_parameter.py +++ b/xrfeitoria/camera/camera_parameter.py @@ -110,6 +110,38 @@ def get_extrinsic(self) -> List: """ return self.extrinsic.tolist() + def model_dump(self) -> dict: + """Dump camera parameters to a dict.""" + return { + 'class_name': 'PinholeCameraParameter', + 'convention': self.convention, + 'extrinsic_r': self.extrinsic_r.tolist(), + 'extrinsic_t': self.extrinsic_t.tolist(), + 'height': self.height, + 'width': self.width, + 'intrinsic': self.intrinsic.tolist(), + 'name': self.name, + 'world2cam': self.world2cam, + } + + @classmethod + def from_dict(cls, data: dict) -> 'CameraParameter': + """Construct a camera parameter data structure from a dict. + + Args: + data (dict): The camera parameter data. + + Returns: + CameraParameter: An instance of CameraParameter class. + """ + return cls( + K=data['intrinsic'], + R=data['extrinsic_r'], + T=data['extrinsic_t'], + convention=data['convention'], + world2cam=data['world2cam'], + ) + @classmethod def fromfile(cls, file: PathLike) -> 'CameraParameter': """Construct a camera parameter data structure from a json file. diff --git a/xrfeitoria/data_structure/models.py b/xrfeitoria/data_structure/models.py index 4156f11c..f2d337cb 100644 --- a/xrfeitoria/data_structure/models.py +++ b/xrfeitoria/data_structure/models.py @@ -248,8 +248,6 @@ class AntiAliasSetting(BaseModel): anti_aliasing: AntiAliasSetting = Field( default=AntiAliasSetting(), description='Anti aliasing setting of the render job.' ) - export_vertices: bool = Field(default=False, description='Whether to export vertices of the render job.') - export_skeleton: bool = Field(default=False, description='Whether to export skeleton of the render job.') export_audio: bool = Field(default=False, description='Whether to export audio of the render job.') class Config: diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 3fc181e9..ccf46994 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -1,14 +1,11 @@ -import json -import shutil import socket from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from loguru import logger -from ..data_structure.constants import PathLike, RenderOutputEnumUnreal, actor_info_type +from ..data_structure.constants import PathLike from ..rpc import remote_unreal -from ..utils.functions import unreal_functions from .renderer_base import RendererBase, render_status try: @@ -41,8 +38,6 @@ def add_job( file_name_format: str = '{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables: Dict[str, float] = {'r.MotionBlurQuality': 0}, anti_aliasing: 'Optional[RenderJob.AntiAliasSetting]' = None, - export_vertices: bool = False, - export_skeleton: bool = False, export_audio: bool = False, ) -> None: """Add a rendering job to the renderer queue. @@ -57,8 +52,6 @@ def add_job( console_variables (Dict[str, float], optional): Console variables to set. Defaults to ``{'r.MotionBlurQuality': 0}``. Ref to :ref:`FAQ-console-variables` for details. anti_aliasing (Optional[RenderJobUnreal.AntiAliasSetting], optional): Anti aliasing setting. Defaults to None. - export_vertices (bool, optional): Whether to export vertices. Defaults to False. - export_skeleton (bool, optional): Whether to export skeleton. Defaults to False. export_audio (bool, optional): Whether to export audio. Defaults to False. Note: @@ -84,8 +77,6 @@ def add_job( file_name_format=file_name_format, console_variables=console_variables, anti_aliasing=anti_aliasing, - export_vertices=export_vertices, - export_skeleton=export_skeleton, export_audio=export_audio, ) cls._add_job_in_engine(job.model_dump(mode='json')) @@ -171,127 +162,9 @@ def render_jobs(cls) -> None: # cls.clear() server.close() - cls._post_process() - # clear render queue cls.clear() - @classmethod - def _post_process(cls) -> None: - """Post-processes the rendered output by: - - converting camera parameters: from `.dat` to `.json` - - convert actor infos: from `.dat` to `.json` - - convert vertices: from `.dat` to `.npz` - - convert skeleton: from `.dat` to `.npz` - - This method is called after rendering is complete. - """ - import numpy as np # isort:skip - from rich import get_console # isort:skip - from rich.spinner import Spinner # isort:skip - from ..camera.camera_parameter import CameraParameter # isort:skip - - def convert_camera(camera_file: Path) -> None: - """Convert camera parameters from `.dat` to `.json` with `xrprimer`. - - Args: - camera_file (Path): Path to the camera file. - """ - cam_param = CameraParameter.from_bin(camera_file) - cam_param.dump(camera_file.with_suffix('.json').as_posix()) - camera_file.unlink() - - def convert_vertices(folder: Path) -> None: - """Convert vertices from `.dat` to `.npz`. Merge all vertices files into one - `.npz` file with structure of: {'verts': np.ndarray, 'faces': None} - - Args: - folder (Path): Path to the folder containing vertices files. - """ - # Get all vertices files in the folder and sort them - vertices_files = sorted(folder.glob('*.dat')) - # Read all vertices files into a list - vertices = [ - np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) for vertices_file in vertices_files - ] - if not vertices: - return - - # Stack all vertices into one array with shape (frame, verts, 3) - vertices = np.stack(vertices) - # Convert convention from unreal to opencv, [x, y, z] -> [y, -z, x] - vertices = np.stack([vertices[:, :, 1], -vertices[:, :, 2], vertices[:, :, 0]], axis=-1) - vertices /= 100 # convert from cm to m - - # Save the vertices in a compressed `.npz` file - np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) - # Remove the folder - shutil.rmtree(folder) - - def convert_actor_infos(folder: Path) -> None: - """Convert stencil value from `.dat` to `.json`. - - Args: - folder (Path): Path to the folder contains ``actor_infos``. - """ - # Get all stencil value files in the folder and sort them - actor_info_files = sorted(folder.glob('*.dat')) - # Read all actor info files into a list - actor_infos: List[actor_info_type] = [] - for actor_info_file in actor_info_files: - stencil_value = np.frombuffer(actor_info_file.read_bytes(), np.float32) - stencil_value = int(stencil_value) - mask_color = unreal_functions.get_mask_color(stencil_value) - actor_infos.append({'actor_name': actor_info_file.stem, 'mask_color': mask_color}) - - if not actor_infos: - return - - # Save the actor infos in a `.json` file - with (folder.parent / f'{folder.name}.json').open('w') as f: - json.dump(actor_infos, f, indent=4) - - # Remove the folder - shutil.rmtree(folder) - - console = get_console() - try: - spinner: Spinner = console._live.renderable - except AttributeError: - status = console.status('[bold green]:rocket: Rendering...[/bold green]') - status.start() - spinner: Spinner = status.renderable - - for idx, job in enumerate(cls.render_queue): - seq_name = job.sequence_path.split('/')[-1] - seq_path = Path(job.output_path).resolve() / seq_name - - text = f'job {idx + 1}/{len(cls.render_queue)}: seq_name="{seq_name}", post-processing...' - spinner.update(text=text) - - # 1. convert camera parameters from `.bat` to `.json` with xrprimer - # glob camera files in {seq_path}/{cam_param_dir}/* - camera_files = sorted((seq_path / RenderOutputEnumUnreal.camera_params.value).rglob('*.dat')) - for camera_file in camera_files: - convert_camera(camera_file) - - # 2. convert actor infos from `.dat` to `.json` - convert_actor_infos(folder=seq_path / RenderOutputEnumUnreal.actor_infos.value) - - # 3. convert vertices from `.dat` to `.npz` - if job.export_vertices: - # glob actors in {seq_path}/vertices/* - actor_folders = sorted(seq_path.glob(f'{RenderOutputEnumUnreal.vertices.value}/*')) - for actor_folder in actor_folders: - convert_vertices(actor_folder) - - # 4. convert skeleton from `.dat` to `.json` - if job.export_skeleton: - # glob actors in {seq_path}/skeleton/* - actor_folders = sorted(seq_path.glob(f'{RenderOutputEnumUnreal.skeleton.value}/*')) - for actor_folder in actor_folders: - convert_vertices(actor_folder) - @staticmethod def _add_job_in_engine(job: 'Dict[str, Any]') -> None: _job = XRFeitoriaUnrealFactory.constants.RenderJobUnreal(**job) diff --git a/xrfeitoria/sequence/sequence_unreal.py b/xrfeitoria/sequence/sequence_unreal.py index 86798fe1..78de2de6 100644 --- a/xrfeitoria/sequence/sequence_unreal.py +++ b/xrfeitoria/sequence/sequence_unreal.py @@ -26,7 +26,15 @@ except (ImportError, ModuleNotFoundError): pass -dict_process_dir = TypedDict('dict_process_dir', {'camera_dir': str, 'vertices_dir': str, 'skeleton_dir': str}) +dict_process_dir = TypedDict( + 'dict_process_dir', + { + 'camera_dir': str, + 'actor_infos_dir': str, + 'vertices_dir': str, + 'skeleton_dir': str, + }, +) @remote_unreal(dec_class=True, suffix='_in_engine') @@ -63,11 +71,16 @@ def _preprocess_before_render( ) -> None: from ..camera.camera_parameter import CameraParameter - _dir_ = cls._preprocess_in_engine( - save_dir=save_dir, export_vertices=export_vertices, export_skeleton=export_skeleton - ) + for frame_idx in range(*cls.get_playback()): + _dir_ = cls._preprocess_in_engine( + save_dir=save_dir, + per_frame=False, + export_vertices=export_vertices, + export_skeleton=export_skeleton, + frame_idx=frame_idx, + ) - # convert camera parameters to xrprimer structure + # 1. convert camera parameters to xrprimer structure for file in Path(_dir_['camera_dir']).glob('*/*.json'): data = json.loads(file.read_text()) cam_param = CameraParameter.from_unreal_convention( @@ -78,7 +91,10 @@ def _preprocess_before_render( ) cam_param.dump(file.as_posix()) # replace the original file - print(_dir_['camera_dir']) + # TODO: + # 2. convert actor infos from `.dat` to `.json` + # 3. convert vertices from `.dat` to `.npz` + # 4. convert skeleton from `.dat` to `.json` @classmethod def add_to_renderer( @@ -129,6 +145,12 @@ def add_to_renderer( if anti_aliasing is None: anti_aliasing = RenderJobUnreal.AntiAliasSetting() + msg = 'Preprocessing before rendering, including exporting camera parameters' + if export_vertices: + msg += ', vertices' + if export_skeleton: + msg += ', skeleton' + logger.info(msg) cls._preprocess_before_render( save_dir=f'{output_path}/{cls.name}', resolution=resolution, @@ -145,8 +167,6 @@ def add_to_renderer( file_name_format=file_name_format, console_variables=console_variables, anti_aliasing=anti_aliasing, - export_vertices=export_vertices, - export_skeleton=export_skeleton, export_audio=export_audio, ) @@ -294,6 +314,15 @@ def set_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int """ cls._set_playback_in_engine(start_frame=start_frame, end_frame=end_frame) + @classmethod + def get_playback(cls) -> Tuple[int, int]: + """Get the playback range for the sequence. + + Returns: + Tuple[int, int]: The start and end frame of the playback range. + """ + return cls._get_playback_in_engine() + @classmethod def set_camera_cut_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: """Set the playback range for the sequence. @@ -357,18 +386,30 @@ def _get_seq_path_in_engine() -> str: @staticmethod def _preprocess_in_engine( save_dir: str, + per_frame: bool = False, export_vertices: bool = False, export_skeleton: bool = False, + frame_idx: 'Optional[int]' = None, ) -> 'dict_process_dir': - camera_dir = f'{save_dir}/{XRFeitoriaUnrealFactory.constants.cam_param_dir}' - XRFeitoriaUnrealFactory.Sequence.save_camera_params(save_dir=camera_dir, per_frame=True) + """Preprocesses the sequence in the Unreal Engine. - # TODO: export vertices and skeleton - return { - 'camera_dir': camera_dir, - 'vertices_dir': '', - 'skeleton_dir': '', - } + Args: + save_dir (str): The directory to save the processed sequence. + per_frame (bool, optional): Whether to process the sequence per frame. Defaults to False. + export_vertices (bool, optional): Whether to export the vertices. Defaults to False. + export_skeleton (bool, optional): Whether to export the skeleton. Defaults to False. + frame_idx (Optional[int], optional): The index of the frame to process. Defaults to None. + + Returns: + dict_process_dir: The directory paths of the saved data. + """ + return XRFeitoriaUnrealFactory.Sequence.save_params( + save_dir=save_dir, + per_frame=per_frame, + export_vertices=export_vertices, + export_skeleton=export_skeleton, + frame_idx=frame_idx, + ) @staticmethod def _new_seq_in_engine( @@ -431,6 +472,10 @@ def _show_seq_in_engine() -> None: def _set_playback_in_engine(start_frame: 'Optional[int]' = None, end_frame: 'Optional[int]' = None) -> None: XRFeitoriaUnrealFactory.Sequence.set_playback(start_frame=start_frame, end_frame=end_frame) + @staticmethod + def _get_playback_in_engine() -> 'Tuple[int, int]': + return XRFeitoriaUnrealFactory.Sequence.get_playback() + @staticmethod def _set_camera_cut_player_in_engine( start_frame: 'Optional[int]' = None, end_frame: 'Optional[int]' = None diff --git a/xrfeitoria/utils/anim/constants.py b/xrfeitoria/utils/anim/constants.py index 7b211d13..44dbf423 100644 --- a/xrfeitoria/utils/anim/constants.py +++ b/xrfeitoria/utils/anim/constants.py @@ -1,5 +1,5 @@ SMPL_BODY_BONES = [ - 'pelvis', + 'pelvis', # global orientation 'left_hip', 'right_hip', 'spine1', @@ -67,7 +67,7 @@ ] SMPLX_JOINT_NAMES = [ - 'pelvis', + 'pelvis', # global orientation 'left_hip', 'right_hip', 'spine1', From 7a21781e99e825bf4174fc059cfcec63edae989d Mon Sep 17 00:00:00 2001 From: wangfanzhou Date: Wed, 28 Feb 2024 23:15:33 +0800 Subject: [PATCH 07/27] fix bugs when loading or dumping `smpl` data --- xrfeitoria/utils/anim/motion.py | 7 ++++--- xrfeitoria/utils/anim/utils.py | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/xrfeitoria/utils/anim/motion.py b/xrfeitoria/utils/anim/motion.py index 7f7d5f1e..b8a54b15 100644 --- a/xrfeitoria/utils/anim/motion.py +++ b/xrfeitoria/utils/anim/motion.py @@ -1,4 +1,5 @@ """Motion data structure and related functions.""" + from collections import OrderedDict from functools import partial from pathlib import Path @@ -314,11 +315,11 @@ class SMPLMotion(Motion): NAME_TO_SMPL_IDX = OrderedDict([(v, k) for k, v in SMPL_IDX_TO_NAME.items() if v]) NAMES = [x for x in SMPL_IDX_TO_NAME.values() if x] PARENTS = list(SMPL_PARENT_IDX) - BONE_NAMES = SMPLX_JOINT_NAMES[1 : NUM_SMPLX_BODYJOINTS + 1] + BONE_NAMES = SMPLX_JOINT_NAMES[0:NUM_SMPLX_BODYJOINTS] BONE_NAME_TO_IDX: Dict[str, int] = {bone_name: idx for idx, bone_name in enumerate(BONE_NAMES)} # In order to make the smpl head up to +z - GLOBAL_ORIENT_ADJUSTMENT = spRotation.from_euler('xyz', np.deg2rad([180, 0, 0])) + GLOBAL_ORIENT_ADJUSTMENT = spRotation.from_euler('xyz', np.deg2rad([0, 0, 0])) def __init__( self, @@ -461,7 +462,7 @@ def dump_humandata( } """ humandata = get_humandata( - smpl_x_data=self.smplx_data, + smpl_x_data=self.smpl_data, smpl_x_type='smpl', betas=betas, meta=meta, diff --git a/xrfeitoria/utils/anim/utils.py b/xrfeitoria/utils/anim/utils.py index 84ac0b3e..5b18d8a8 100644 --- a/xrfeitoria/utils/anim/utils.py +++ b/xrfeitoria/utils/anim/utils.py @@ -1,4 +1,5 @@ """Utilities for animation data loading and dumping.""" + from pathlib import Path from typing import Union @@ -88,7 +89,10 @@ def dump_humandata(motion: SMPLXMotion, save_filepath: PathLike, meta_filepath: } """ meta_info = np.load(meta_filepath, allow_pickle=True) - smplx = meta_info['smplx'].item() + if 'smplx' in meta_info.keys(): + smplx = meta_info['smplx'].item() + elif 'smpl' in meta_info.keys(): + smplx = meta_info['smpl'].item() motion.dump_humandata( filepath=save_filepath, betas=smplx['betas'], From 658c037fcb1d736f919c6cf4592f0162af2404f6 Mon Sep 17 00:00:00 2001 From: meihaiyi Date: Fri, 1 Mar 2024 16:55:38 +0800 Subject: [PATCH 08/27] Update SMPL joint names and global orientation adjustment --- xrfeitoria/utils/anim/constants.py | 9 +++++---- xrfeitoria/utils/anim/motion.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/xrfeitoria/utils/anim/constants.py b/xrfeitoria/utils/anim/constants.py index 44dbf423..8fc6d7d5 100644 --- a/xrfeitoria/utils/anim/constants.py +++ b/xrfeitoria/utils/anim/constants.py @@ -1,4 +1,4 @@ -SMPL_BODY_BONES = [ +SMPL_JOINT_NAMES = [ 'pelvis', # global orientation 'left_hip', 'right_hip', @@ -28,10 +28,11 @@ 'jaw', # 'left_eyeball', # 'right_eyeball', - 'left_eye_smplhf', - 'right_eye_smplhf', + # + # 'left_eye_smplhf', + # 'right_eye_smplhf', ] -SMPL_FINGER_BONES = [ +SMPL_FINGER_JOINTS = [ # left hand 'left_index1', 'left_index2', diff --git a/xrfeitoria/utils/anim/motion.py b/xrfeitoria/utils/anim/motion.py index b8a54b15..13273b0d 100644 --- a/xrfeitoria/utils/anim/motion.py +++ b/xrfeitoria/utils/anim/motion.py @@ -10,8 +10,8 @@ from ...data_structure.constants import MotionFrame, PathLike from .constants import ( - NUM_SMPLX_BODYJOINTS, SMPL_IDX_TO_JOINTS, + SMPL_JOINT_NAMES, SMPL_PARENT_IDX, SMPLX_HAND_POSES, SMPLX_IDX_TO_JOINTS, @@ -315,11 +315,11 @@ class SMPLMotion(Motion): NAME_TO_SMPL_IDX = OrderedDict([(v, k) for k, v in SMPL_IDX_TO_NAME.items() if v]) NAMES = [x for x in SMPL_IDX_TO_NAME.values() if x] PARENTS = list(SMPL_PARENT_IDX) - BONE_NAMES = SMPLX_JOINT_NAMES[0:NUM_SMPLX_BODYJOINTS] + BONE_NAMES = SMPL_JOINT_NAMES BONE_NAME_TO_IDX: Dict[str, int] = {bone_name: idx for idx, bone_name in enumerate(BONE_NAMES)} # In order to make the smpl head up to +z - GLOBAL_ORIENT_ADJUSTMENT = spRotation.from_euler('xyz', np.deg2rad([0, 0, 0])) + GLOBAL_ORIENT_ADJUSTMENT = spRotation.from_euler('xyz', np.deg2rad([180, 0, 0])) def __init__( self, From 7a10812b35f203a127572ca88711f66ea3103614 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Tue, 12 Mar 2024 16:03:38 +0800 Subject: [PATCH 09/27] Fix bugs for SMPL motion --- samples/blender/07_amass.py | 5 +- xrfeitoria/utils/anim/motion.py | 108 +++++++++++++++++++++++++++++--- xrfeitoria/utils/anim/utils.py | 39 ++++++------ 3 files changed, 122 insertions(+), 30 deletions(-) diff --git a/samples/blender/07_amass.py b/samples/blender/07_amass.py index 1f1534ef..602fd8ad 100644 --- a/samples/blender/07_amass.py +++ b/samples/blender/07_amass.py @@ -10,6 +10,7 @@ ** It is recommended to run this script with Blender >= 3.6 ** """ + from pathlib import Path import xrfeitoria as xf @@ -38,8 +39,8 @@ seq_name = 'seq_amass' output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender' / Path(__file__).stem output_path.mkdir(parents=True, exist_ok=True) -saved_humandata_file = output_path / 'output.npz' -saved_blend_file = output_path / 'output.blend' +saved_humandata_file = output_path / seq_name / 'output.npz' +saved_blend_file = output_path / seq_name / 'output.blend' @remote_blender() diff --git a/xrfeitoria/utils/anim/motion.py b/xrfeitoria/utils/anim/motion.py index 13273b0d..d8bc0104 100644 --- a/xrfeitoria/utils/anim/motion.py +++ b/xrfeitoria/utils/anim/motion.py @@ -245,6 +245,41 @@ def sample_motion(self, n_frames: int): self.smplx_data[k] = v[indices] self.insert_rest_pose() + def cut_motion(self, start_frame: Optional[int] = None, end_frame: Optional[int] = None): + """Cut the motion sequence to a given number of frames. + + Args: + start_frame (Optional[int], optional): The start frame to cut to. Defaults to None. + end_frame (Optional[int], optional): The end frame to cut to. Defaults to None. + + Raises: + AssertionError: If the start frame is less than 0. + AssertionError: If the end frame is greater than the number of frames in the motion sequence. + AssertionError: If the start frame is greater than or equal to the end frame. + """ + if start_frame is None: + start_frame = 0 + if end_frame is None: + end_frame = self.n_frames + + assert start_frame >= 0, f'start_frame={start_frame}' + assert end_frame <= self.n_frames, f'end_frame={end_frame} should be less than n_frames={self.n_frames}' + assert start_frame < end_frame, f'start_frame={start_frame} should be less than end_frame={end_frame}' + n_frames = end_frame - start_frame + + self.transl = self.transl[start_frame:end_frame] + self.body_poses = self.body_poses[start_frame:end_frame] + self.global_orient = self.global_orient[start_frame:end_frame] + self.n_frames = n_frames + if hasattr(self, 'smpl_data'): + for k, v in self.smpl_data.items(): + if k != 'betas': + self.smpl_data[k] = v[start_frame:end_frame] + if hasattr(self, 'smplx_data'): + for k, v in self.smplx_data.items(): + if k != 'betas': + self.smplx_data[k] = v[start_frame:end_frame] + def cut_transl(self): """Cut the transl to zero. @@ -381,27 +416,80 @@ def from_smpl_data( # Create instance transl_bl = smpl_data['transl'] - n_frames = transl_bl.shape[0] - body_poses_bl = np.concatenate( - [smpl_data[key] for key in ('global_orient', 'body_pose')], - axis=1, - dtype=np.float32, - ).reshape([n_frames, -1, 3]) # - Adjust in order to make the smpl head up to +z if global_orient_adj is not None: - body_poses_bl[:, 0, :] = (global_orient_adj * spRotation.from_rotvec(body_poses_bl[:, 0, :])).as_rotvec() + global_orient_bl = spRotation.from_rotvec(smpl_data['global_orient']) + smpl_data['global_orient'] = (global_orient_adj * global_orient_bl).as_rotvec() if insert_rest_pose: - body_poses_bl[0, 0, :] = 0.0 - + smpl_data['global_orient'][0] = 0 # - Convert from humandata to smplx pelvis local space in blender if vector_convertor is not None: transl_bl = vector_convertor(transl_bl) smpl_data['transl'] = transl_bl - instance = cls(transl=transl_bl, body_poses=body_poses_bl, fps=fps) + # Concatenate all the poses + body_pose_keys = ('global_orient', 'body_pose') + body_poses_bl = [smpl_data[key] for key in body_pose_keys] + n_frames = transl_bl.shape[0] + body_poses_bl = np.concatenate(body_poses_bl, axis=1, dtype=np.float32).reshape([n_frames, -1, 3]) + + instance = SMPLMotion(transl=transl_bl, body_poses=body_poses_bl, fps=fps) instance.smpl_data = smpl_data return instance + @classmethod + def from_amass_data(cls, amass_data, insert_rest_pose: bool) -> 'SMPLMotion': + """Create a Motion instance from AMASS data (SMPL) + + Args: + amass_data (dict): A dictionary containing the AMASS data. + insert_rest_pose (bool): Whether to insert a rest pose at the beginning of the motion. + + Returns: + SMPLMotion: A SMPLMotion instance containing the AMASS data. + """ + fps = 120 + + betas = amass_data['betas'][:10] + transl = amass_data['trans'] + global_orient = amass_data['poses'][:, :3] + body_pose = amass_data['poses'][:, 3:66] + # left_hand_pose = amass_data['poses'][:, 66 : 66 + 45] + # right_hand_pose = amass_data['poses'][:, 66 + 45 :] + # n_frames = global_orient.shape[0] + # expression = np.zeros([n_frames, 10], dtype=np.float32) + + # motions in AMASS dataset are -y up, rotate it to +y up + amass2humandata_adj = spRotation.from_euler('xyz', np.deg2rad([90, 180, 0])) + global_orient = (amass2humandata_adj * spRotation.from_rotvec(global_orient)).as_rotvec() # type: ignore + # transl_0 = transl[0, :] + # transl = amass2humandata_adj.apply(transl - transl_0) + transl_0 + transl = Converter.vec_amass2humandata(transl) + # TODO: all axis offset + height_offset = transl[0, 1] + + smpl_data = { + 'betas': betas, + 'transl': transl, + 'global_orient': global_orient, + 'body_pose': body_pose, + } + if insert_rest_pose: + for key, arr in smpl_data.items(): + arr = arr.astype(np.float32) + if key != 'betas': + arr = np.insert(arr, 0, 0, axis=0) + if key == 'global_orient': + # make 0-th frame has the same orient with humandata + arr[0, :] = [np.pi, 0, 0] + elif key == 'transl': + arr[1:, 1] -= height_offset + # TODO: handle pelvis height, get pelvis_height, and set frame-0 as T-pose + # arr[0, 1] = pelvis_height + smpl_data[key] = arr + + return cls.from_smpl_data(smpl_data, insert_rest_pose=False, fps=fps) + def _get_bone_rotvec(self, bone_name, frame=0) -> np.ndarray: idx = self._bone2idx(bone_name) if idx == 0: diff --git a/xrfeitoria/utils/anim/utils.py b/xrfeitoria/utils/anim/utils.py index 5b18d8a8..0e5b4afa 100644 --- a/xrfeitoria/utils/anim/utils.py +++ b/xrfeitoria/utils/anim/utils.py @@ -6,25 +6,28 @@ import numpy as np from ...data_structure.constants import PathLike -from .motion import Motion, SMPLMotion, SMPLXMotion +from .motion import SMPLMotion, SMPLXMotion -def load_amass_motion(input_amass_smplx_path: PathLike) -> SMPLXMotion: +def load_amass_motion(input_amass_smpl_x_path: PathLike, is_smplx: bool = True) -> Union[SMPLMotion, SMPLXMotion]: """Load AMASS SMPLX motion data. Only for SMPLX motion for now. Args: - input_amass_smplx_path (PathLike): Path to AMASS SMPLX motion data. + input_amass_smpl_x_path (PathLike): Path to AMASS SMPL/SMPLX motion data. Returns: - Motion: Motion data, which consists of data read from AMASS file. + Union[SMPLMotion, SMPLXMotion]: Motion data, which consists of data read from AMASS file. """ - input_amass_smplx_path = Path(input_amass_smplx_path).resolve() - if not input_amass_smplx_path.exists(): - raise ValueError(f'Not exist: {input_amass_smplx_path}') + input_amass_smpl_x_path = Path(input_amass_smpl_x_path).resolve() + if not input_amass_smpl_x_path.exists(): + raise ValueError(f'Not exist: {input_amass_smpl_x_path}') # Use AMASS motion # src_actor_name = "SMPLX" - amass_smplx_data = np.load(input_amass_smplx_path, allow_pickle=True) - src_motion = SMPLXMotion.from_amass_data(amass_smplx_data, insert_rest_pose=True) + amass_smpl_x_data = np.load(input_amass_smpl_x_path, allow_pickle=True) + if is_smplx: + src_motion = SMPLXMotion.from_amass_data(amass_smpl_x_data, insert_rest_pose=True) + else: + src_motion = SMPLMotion.from_amass_data(amass_smpl_x_data, insert_rest_pose=True) return src_motion @@ -55,12 +58,12 @@ def load_humandata_motion(input_humandata_path: PathLike) -> Union[SMPLMotion, S return src_motion -def dump_humandata(motion: SMPLXMotion, save_filepath: PathLike, meta_filepath: PathLike) -> None: +def dump_humandata(motion: Union[SMPLMotion, SMPLXMotion], save_filepath: PathLike, meta_filepath: PathLike) -> None: """Dump human data to a file. This function must be associate with a meta file provided by SMPL-XL. Args: - motion (SMPLXMotion): Motion data to dump. + motion (Union[SMPLMotion, SMPLXMotion]): The motion data to be dumped. save_filepath (PathLike): The file path to save the dumped data. meta_filepath (PathLike): The file path to the meta information, storing the parameters of the SMPL-XL model. @@ -90,17 +93,17 @@ def dump_humandata(motion: SMPLXMotion, save_filepath: PathLike, meta_filepath: """ meta_info = np.load(meta_filepath, allow_pickle=True) if 'smplx' in meta_info.keys(): - smplx = meta_info['smplx'].item() + smpl_x = meta_info['smplx'].item() elif 'smpl' in meta_info.keys(): - smplx = meta_info['smpl'].item() + smpl_x = meta_info['smpl'].item() motion.dump_humandata( filepath=save_filepath, - betas=smplx['betas'], + betas=smpl_x['betas'], meta=meta_info['meta'].item(), - global_orient_offset=smplx['global_orient'], - transl_offset=smplx['transl'], - root_location_t0=smplx['root_location_t0'], - pelvis_location_t0=smplx['pelvis_location_t0'], + global_orient_offset=smpl_x['global_orient'], + transl_offset=smpl_x['transl'], + root_location_t0=smpl_x['root_location_t0'], + pelvis_location_t0=smpl_x['pelvis_location_t0'], ) From b7a4cf6877bbbd63cd749c83d6005d500a638cd8 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Fri, 15 Mar 2024 18:18:10 +0800 Subject: [PATCH 10/27] Add Annotator which can export: - camera_params - vertices - skeleton --- .../XRFeitoriaUnreal/Private/Annotator.cpp | 324 ++++++++++++++++++ .../Private/CustomMoviePipelineOutput.cpp | 16 +- .../Private/XF_BlueprintFunctionLibrary.cpp | 31 +- .../Private/XRFeitoriaUnreal.cpp | 7 +- .../XRFeitoriaUnreal/Public/Annotator.h | 79 +++-- .../Public/CustomMoviePipelineOutput.h | 2 +- .../Public/XF_BlueprintFunctionLibrary.h | 10 +- 7 files changed, 401 insertions(+), 68 deletions(-) diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp index c239aab9..2082729b 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp @@ -2,6 +2,15 @@ #include "Annotator.h" +#include "SequencerBindingProxy.h" +#include "SequencerScriptingRange.h" +#include "SequencerSettings.h" +#include "SequencerTools.h" + +#include "EngineUtils.h" +#include "XF_BlueprintFunctionLibrary.h" + + // Sets default values AAnnotator::AAnnotator() { @@ -10,6 +19,314 @@ AAnnotator::AAnnotator() } +void AAnnotator::Initialize() +{ + if (bInitialized) return; + + // Get Playing LevelSequenceActor + for (TActorIterator ActorItr(GetWorld()); ActorItr; ++ActorItr) + { + ALevelSequenceActor* ALevelSequenceActor = *ActorItr; + bool bValid = ALevelSequenceActor && ALevelSequenceActor->GetSequencePlayer() && ALevelSequenceActor->GetSequencePlayer()->IsPlaying(); + if (!bValid) return; + + LevelSequenceActor = ALevelSequenceActor; + LevelSequencePlayer = ALevelSequenceActor->GetSequencePlayer(); + UE_LOG(LogXF, Log, TEXT("Detected LevelSequenceActor: %s"), *LevelSequenceActor->GetName()); + } + ULevelSequence* LevelSequence = LevelSequenceActor->GetSequence(); + + // Get All Bound Objects + const TArray& ObjectBindings = LevelSequence->GetMovieScene()->GetBindings(); + TArray bindingProxies; + for (FMovieSceneBinding binding : ObjectBindings) + { + FGuid guid = binding.GetObjectGuid(); + bindingProxies.Add(FSequencerBindingProxy(guid, LevelSequence)); + } + TArray boundObjects = USequencerToolsFunctionLibrary::GetBoundObjects( + GetWorld(), + LevelSequence, + bindingProxies, + FSequencerScriptingRange::FromNative( + LevelSequence->GetMovieScene()->GetPlaybackRange(), + LevelSequence->GetMovieScene()->GetDisplayRate() + ) + ); + UE_LOG(LogXF, Log, TEXT("Detected %d bound objects"), boundObjects.Num()); + + // Get CameraActors, StaticMeshComponents, SkeletalMeshComponents from LevelSequence + for (FSequencerBoundObjects boundObject : boundObjects) + { + // loop over bound objects + UObject* BoundObject = boundObject.BoundObjects[0]; // only have one item + if (BoundObject->IsA(ACameraActor::StaticClass())) + { + ACameraActor* Camera = Cast(BoundObject); + CameraActors.Add(Camera); + } + else if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) + { + ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); + SkeletalMeshComponents.Add(SkeletalMeshActor->GetSkeletalMeshComponent()); + } + else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) + { + AStaticMeshActor* StaticMeshActor = Cast(BoundObject); + StaticMeshComponents.Add(StaticMeshActor->GetStaticMeshComponent()); + } + else if (BoundObject->IsA(USkeletalMeshComponent::StaticClass())) + { + USkeletalMeshComponent* SkeletalMeshComponent = Cast(BoundObject); + // check if it's already in the list + bool bFound = false; + for (USkeletalMeshComponent* SkeletalMeshComponentInList : SkeletalMeshComponents) + { + if (SkeletalMeshComponentInList == SkeletalMeshComponent) + { + bFound = true; + break; + } + } + if (!bFound) SkeletalMeshComponents.Add(SkeletalMeshComponent); + } + else if (BoundObject->IsA(UStaticMeshComponent::StaticClass())) + { + UStaticMeshComponent* StaticMeshComponent = Cast(BoundObject); + // check if it's already in the list + bool bFound = false; + for (UStaticMeshComponent* StaticMeshComponentInList : StaticMeshComponents) + { + if (StaticMeshComponentInList == StaticMeshComponent) + { + bFound = true; + break; + } + } + if (!bFound) + StaticMeshComponents.Add(StaticMeshComponent); + } + } + UE_LOG(LogXF, Log, TEXT("Detected %d CameraActors, %d StaticMeshComponents, %d SkeletalMeshComponents"), + CameraActors.Num(), StaticMeshComponents.Num(), SkeletalMeshComponents.Num()); + + // Save Skeleton Names (only save on the first frame) + if (bSaveSkeletonPosition) + { + for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) + { + FString MeshName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); + TArray SkeletonPositions; + TArray SkeletonNames; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( + SkeletalMeshComponent, SkeletonPositions, SkeletonNames); + + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get skeleton positions")); + continue; + } + + TArray SkeletonNamesString; + for (FName name : SkeletonNames) SkeletonNamesString.Add(name.ToString()); + FString BoneNamePath = FPaths::Combine( + DirectorySequence, + DirectorySkeleton, + MeshName + "_BoneName.txt" + ); // {seq_dir}/{skeleton}/{actor_name}_BoneName.txt + FFileHelper::SaveStringArrayToFile(SkeletonNamesString, *BoneNamePath); + } + } + + // Finish Initialization + bInitialized = true; +} + +void AAnnotator::ExportCameraParameters(int FrameNumber) +{ + if (!bInitialized || CameraActors.Num() == 0) return; + for (ACameraActor* Camera : CameraActors) + { + FVector CamLocation = Camera->GetActorLocation(); + FRotator CamRotation = Camera->GetActorRotation(); + float FOV = Camera->GetCameraComponent()->FieldOfView; + + TArray CamInfo; + CamInfo.Add(CamLocation.X); + CamInfo.Add(CamLocation.Y); + CamInfo.Add(CamLocation.Z); + CamInfo.Add(CamRotation.Roll); + CamInfo.Add(CamRotation.Pitch); + CamInfo.Add(CamRotation.Yaw); + CamInfo.Add(FOV); + + FString CameraTransformPath = FPaths::Combine( + DirectorySequence, // seq_dir + DirectoryCameraParams, // camera_params + Camera->GetFName().GetPlainNameString(), // camera_name + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" // frame_idx + ); // {seq_dir}/{camera_params}/{camera_name}/{frame_idx}.dat + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); + } +} + +void AAnnotator::ExportStaticMeshParameters(int FrameNumber) +{ + for (UStaticMeshComponent* StaticMeshComponent : StaticMeshComponents) + { + FString MeshName = StaticMeshComponent->GetOwner()->GetFName().GetPlainNameString(); + + // Save Actor Info (location, rotation, stencil value) + { + FVector ActorLocation = StaticMeshComponent->GetOwner()->GetActorLocation(); + FRotator ActorRotation = StaticMeshComponent->GetOwner()->GetActorRotation(); + int StencilValue = StaticMeshComponent->CustomDepthStencilValue; + + TArray ActorInfo; + ActorInfo.Add(ActorLocation.X); + ActorInfo.Add(ActorLocation.Y); + ActorInfo.Add(ActorLocation.Z); + ActorInfo.Add(ActorRotation.Roll); + ActorInfo.Add(ActorRotation.Pitch); + ActorInfo.Add(ActorRotation.Yaw); + ActorInfo.Add(StencilValue); + + FString ActorInfoPath = FPaths::Combine( + DirectorySequence, + DirectoryActorParams, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ); // {seq_dir}/{actor_params}/{actor_name}/{frame_idx}.dat + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(ActorInfo, ActorInfoPath); + } + + // Save Vertex Positions + if (bSaveVerticesPosition) + { + // Get Vertex Positions + TArray VertexPositions; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetStaticMeshVertexLocations(StaticMeshComponent, LODIndexToSave, VertexPositions); + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); + continue; + } + + TArray VertexPositionsFloat; + for (FVector position : VertexPositions) + { + VertexPositionsFloat.Add(position.X); + VertexPositionsFloat.Add(position.Y); + VertexPositionsFloat.Add(position.Z); + } + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( + VertexPositionsFloat, + FPaths::Combine( + DirectorySequence, + DirectoryVertices, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ) + ); + } + } +} + +void AAnnotator::ExportSkeletalMeshParameters(int FrameNumber) +{ + for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) + { + FString MeshName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); + + // Save Actor Info (location, rotation, stencil value) + { + FVector ActorLocation = SkeletalMeshComponent->GetOwner()->GetActorLocation(); + FRotator ActorRotation = SkeletalMeshComponent->GetOwner()->GetActorRotation(); + int StencilValue = SkeletalMeshComponent->CustomDepthStencilValue; + + TArray ActorInfo; + ActorInfo.Add(ActorLocation.X); + ActorInfo.Add(ActorLocation.Y); + ActorInfo.Add(ActorLocation.Z); + ActorInfo.Add(ActorRotation.Roll); + ActorInfo.Add(ActorRotation.Pitch); + ActorInfo.Add(ActorRotation.Yaw); + ActorInfo.Add(StencilValue); + + FString ActorInfoPath = FPaths::Combine( + DirectorySequence, + DirectoryActorParams, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ); // {seq_dir}/{actor_params}/{actor_name}/{frame_idx}.dat + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(ActorInfo, ActorInfoPath); + } + + // Save Vertex Positions + if (bSaveVerticesPosition) + { + // Get Vertex Positions (with LOD) + TArray VertexPositions; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshVertexLocationsByLODIndex(SkeletalMeshComponent, LODIndexToSave, VertexPositions); + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get vertex positions")); + continue; + } + + TArray VertexPositionsFloat; + for (FVector position : VertexPositions) + { + VertexPositionsFloat.Add(position.X); + VertexPositionsFloat.Add(position.Y); + VertexPositionsFloat.Add(position.Z); + } + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( + VertexPositionsFloat, + FPaths::Combine( + DirectorySequence, + DirectoryVertices, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ) + ); + } + + // Save Skeleton Positions + if (bSaveSkeletonPosition) + { + TArray SkeletonPositions; + TArray SkeletonNames; + bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( + SkeletalMeshComponent, SkeletonPositions, SkeletonNames); + + if (!isSuccess) + { + UE_LOG(LogMovieRenderPipeline, Error, TEXT("Failed to get skeleton positions")); + continue; + } + + // Skeleton Positions + TArray SkeletonPositionsFloat; + for (FVector position : SkeletonPositions) + { + SkeletonPositionsFloat.Add(position.X); + SkeletonPositionsFloat.Add(position.Y); + SkeletonPositionsFloat.Add(position.Z); + } + UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile( + SkeletonPositionsFloat, + FPaths::Combine( + DirectorySequence, + DirectorySkeleton, + MeshName, + FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" + ) + ); + } + } +} + // Called when the game starts or when spawned void AAnnotator::BeginPlay() { @@ -22,4 +339,11 @@ void AAnnotator::Tick(float DeltaTime) { Super::Tick(DeltaTime); + Initialize(); + if (!bInitialized) return; + + int FrameNum = LevelSequencePlayer->GetCurrentTime().Time.GetFrame().Value; + ExportCameraParameters(FrameNum); + ExportSkeletalMeshParameters(FrameNum); + ExportStaticMeshParameters(FrameNum); } diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp index 433876e1..896109c8 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/CustomMoviePipelineOutput.cpp @@ -2,36 +2,36 @@ #include "CustomMoviePipelineOutput.h" -#include "ImageWriteTask.h" #include "ImagePixelData.h" #include "ImageWriteQueue.h" #include "ImageWriteStream.h" +#include "ImageWriteTask.h" -#include "Modules/ModuleManager.h" #include "Containers/UnrealString.h" -#include "Misc/StringFormatArg.h" #include "Misc/FileHelper.h" #include "Misc/FrameRate.h" #include "Misc/Paths.h" +#include "Misc/StringFormatArg.h" +#include "Modules/ModuleManager.h" // #include "HAL/PlatformFilemanager.h" // #include "HAL/PlatformTime.h" +#include "Animation/SkeletalMeshActor.h" #include "Camera/CameraActor.h" #include "Camera/CameraComponent.h" #include "Engine/StaticMeshActor.h" -#include "Animation/SkeletalMeshActor.h" #include "MoviePipeline.h" -#include "MoviePipelineOutputSetting.h" #include "MoviePipelineBurnInSetting.h" -#include "MoviePipelineOutputBase.h" #include "MoviePipelineImageQuantization.h" -#include "MoviePipelineWidgetRenderSetting.h" +#include "MoviePipelineOutputBase.h" +#include "MoviePipelineOutputSetting.h" #include "MoviePipelineUtils.h" -#include "MovieRenderTileImage.h" +#include "MoviePipelineWidgetRenderSetting.h" #include "MovieRenderOverlappedImage.h" #include "MovieRenderPipelineCoreModule.h" +#include "MovieRenderTileImage.h" #include "XF_BlueprintFunctionLibrary.h" diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp index 1cd89d7e..2880c307 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XF_BlueprintFunctionLibrary.cpp @@ -3,27 +3,27 @@ #include "XF_BlueprintFunctionLibrary.h" +#include "Camera/CameraActor.h" +#include "EditorFramework/AssetImportData.h" +#include "Engine/ObjectLibrary.h" +#include "GameFramework/Actor.h" +#include "GenericPlatform/GenericPlatformFile.h" #include "iostream" -#include -#include -#include "Misc/Paths.h" +#include "Math/Rotator.h" +#include "Math/Vector.h" #include "Misc/FileHelper.h" #include "Misc/MessageDialog.h" -#include "Engine/ObjectLibrary.h" -#include "EditorFramework/AssetImportData.h" -#include "GenericPlatform/GenericPlatformFile.h" -#include "Rendering/SkeletalMeshRenderData.h" +#include "Misc/Paths.h" #include "Rendering/MultiSizeIndexContainer.h" #include "Rendering/SkeletalMeshLODRenderData.h" -#include "GameFramework/Actor.h" -#include "Math/Vector.h" -#include "Math/Rotator.h" -#include "Camera/CameraActor.h" +#include "Rendering/SkeletalMeshRenderData.h" +#include +#include -#include "Kismet/KismetSystemLibrary.h" -#include "Kismet/KismetStringLibrary.h" -#include "DrawDebugHelpers.h" #include "AssetRegistry/AssetRegistryModule.h" +#include "DrawDebugHelpers.h" +#include "Kismet/KismetStringLibrary.h" +#include "Kismet/KismetSystemLibrary.h" #include "PhysicsAssetUtils.h" #include "PhysicsEngine/PhysicsAsset.h" @@ -39,9 +39,6 @@ #endif -DEFINE_LOG_CATEGORY(LogXF); - - bool UXF_BlueprintFunctionLibrary::FileExists(FString Path) { IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile(); diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp index 3327cead..6a5726dd 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/XRFeitoriaUnreal.cpp @@ -1,12 +1,13 @@ // Copyright OpenXRLab 2023-2024. All Rights Reserved. #include "XRFeitoriaUnreal.h" + +#include "CustomMoviePipelineDeferredPass.h" +#include "CustomMoviePipelineOutput.h" #include "Engine/RendererSettings.h" -#include "MovieRenderPipelineSettings.h" #include "MoviePipelineCameraSetting.h" +#include "MovieRenderPipelineSettings.h" #include "Settings/EditorProjectSettings.h" -#include "CustomMoviePipelineOutput.h" -#include "CustomMoviePipelineDeferredPass.h" #define LOCTEXT_NAMESPACE "FXRFeitoriaUnrealModule" diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h index e94377c4..0a6d0f22 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h @@ -3,25 +3,21 @@ #pragma once #include "CoreMinimal.h" + +#include "LevelSequence.h" +#include "LevelSequenceActor.h" +#include "LevelSequencePlayer.h" +#include "MovieScene.h" +#include "MovieSceneBinding.h" +#include "MovieSceneBindingProxy.h" + #include "GameFramework/Actor.h" -#include "Annotator.generated.h" +#include "Animation/SkeletalMeshActor.h" +#include "Camera/CameraActor.h" +#include "Camera/CameraComponent.h" +#include "Engine/StaticMeshActor.h" -UENUM(BlueprintType) -enum class ECustomPPMType : uint8 -{ - None, - SemanticSegmentation, - DepthMap, - NormalMap, - Roughtness, - Specular, - Tangent, - OpticaFlow, - Metallic, - Diffuse, - Basecolor, - Custom -}; +#include "Annotator.generated.h" UCLASS() @@ -29,27 +25,14 @@ class XRFEITORIAUNREAL_API AAnnotator : public AActor { GENERATED_BODY() -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations|Stencil Value") - bool bSignStencilValue = true; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations|Stencil Value", - meta = (EditCondition = "bSignStencilValue", EditConditionHides)) - bool bManualSignStencilValue = false; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations|Stencil Value", - meta = (EditCondition = "bManualSignStencilValue", EditConditionHides)) - TArray SegmentObjects; - - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations") - ECustomPPMType PreviewAnnotationType = ECustomPPMType::None; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Annotations", - meta = (EditCondition = "PreviewAnnotationType == ECustomPPMType::Custom", EditConditionHides)) - UMaterialInterface* CustomPostprocessMaterial; - public: // Sets default values for this actor's properties AAnnotator(); + // Initialize the Annotator + void Initialize(); + void ExportCameraParameters(int FrameNumber); + void ExportStaticMeshParameters(int FrameNumber); + void ExportSkeletalMeshParameters(int FrameNumber); protected: // Called when the game starts or when spawned @@ -59,4 +42,30 @@ class XRFEITORIAUNREAL_API AAnnotator : public AActor // Called every frame virtual void Tick(float DeltaTime) override; +public: + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString DirectorySequence; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString DirectoryActorParams = "actor_params"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString DirectoryCameraParams = "camera_params"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString DirectoryVertices = "vertices"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + FString DirectorySkeleton = "skeleton"; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + bool bSaveSkeletonPosition = false; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + bool bSaveVerticesPosition = false; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") + int32 LODIndexToSave = 0; + +private: + ALevelSequenceActor* LevelSequenceActor; + ULevelSequencePlayer* LevelSequencePlayer; + TArray CameraActors; + TArray StaticMeshComponents; + TArray SkeletalMeshComponents; + bool bInitialized = false; }; diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h index 73101af1..014a0153 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/CustomMoviePipelineOutput.h @@ -2,8 +2,8 @@ #pragma once -#include "MoviePipelineDeferredPasses.h" #include "Misc/StringFormatArg.h" +#include "MoviePipelineDeferredPasses.h" #include "Runtime/Launch/Resources/Version.h" #if WITH_UNREALEXR diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h index 41910f62..e7bc47c2 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/XF_BlueprintFunctionLibrary.h @@ -1,18 +1,20 @@ // Copyright OpenXRLab 2023-2024. All Rights Reserved. #pragma once + +#include "Components/PostProcessComponent.h" +#include "Components/SceneCaptureComponent2D.h" +#include "Engine/TextureRenderTarget2D.h" #include "Serialization/Archive.h" -#include "Serialization/BufferArchive.h" #include "Serialization/ArchiveSaveCompressedProxy.h" -#include "Engine/TextureRenderTarget2D.h" -#include "Components/SceneCaptureComponent2D.h" -#include "Components/PostProcessComponent.h" +#include "Serialization/BufferArchive.h" #include "CoreMinimal.h" #include "Kismet/BlueprintFunctionLibrary.h" #include "XF_BlueprintFunctionLibrary.generated.h" DECLARE_LOG_CATEGORY_EXTERN(LogXF, Log, All); +DEFINE_LOG_CATEGORY(LogXF); /** * From 48d0c26b9e00540379f955b20fbd4761c8a40f00 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Sat, 16 Mar 2024 00:50:52 +0800 Subject: [PATCH 11/27] Add Annotator when `seq.add_to_renderer` --- .../Content/Python/sequence.py | 21 +++++ .../XRFeitoriaUnreal/Private/Annotator.cpp | 2 + .../XRFeitoriaUnreal/Public/Annotator.h | 6 +- xrfeitoria/sequence/sequence_unreal.py | 89 ++++++------------- 4 files changed, 54 insertions(+), 64 deletions(-) diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index 8d828069..367014ba 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -1234,6 +1234,27 @@ def add_audio( ) cls.bindings[audio_asset.get_name()] = bindings + @classmethod + def add_annotator( + cls, + save_dir: str, + resolution: Tuple[int, int], + export_vertices: bool, + export_skeleton: bool, + ): + actor_binding = cls.sequence.add_spawnable_from_class(unreal.Annotator) + add_property_bool_track_to_binding( + binding=actor_binding, property_name='bSaveVerticesPosition', property_value=export_vertices + ) + add_property_bool_track_to_binding( + binding=actor_binding, property_name='bSaveSkeletonPosition', property_value=export_skeleton + ) + add_property_string_track_to_binding( + binding=actor_binding, property_name='DirectorySequence', property_value=save_dir + ) + add_property_int_track_to_binding(binding=actor_binding, property_name='Width', property_value=resolution[0]) + add_property_int_track_to_binding(binding=actor_binding, property_name='Height', property_value=resolution[1]) + @classmethod def save_params( cls, diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp index 2082729b..c381a403 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp @@ -159,6 +159,8 @@ void AAnnotator::ExportCameraParameters(int FrameNumber) CamInfo.Add(CamRotation.Pitch); CamInfo.Add(CamRotation.Yaw); CamInfo.Add(FOV); + CamInfo.Add(Width); + CamInfo.Add(Height); FString CameraTransformPath = FPaths::Combine( DirectorySequence, // seq_dir diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h index 0a6d0f22..52326d26 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h @@ -20,7 +20,7 @@ #include "Annotator.generated.h" -UCLASS() +UCLASS(Blueprintable) class XRFEITORIAUNREAL_API AAnnotator : public AActor { GENERATED_BODY() @@ -54,6 +54,10 @@ class XRFEITORIAUNREAL_API AAnnotator : public AActor UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") FString DirectorySkeleton = "skeleton"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator|Resolution") + int Width = 1920; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator|Resolution") + int Height = 1080; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") bool bSaveSkeletonPosition = false; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") diff --git a/xrfeitoria/sequence/sequence_unreal.py b/xrfeitoria/sequence/sequence_unreal.py index 78de2de6..c25180c9 100644 --- a/xrfeitoria/sequence/sequence_unreal.py +++ b/xrfeitoria/sequence/sequence_unreal.py @@ -61,41 +61,6 @@ def show(cls) -> None: """Show the sequence in the engine.""" cls._show_seq_in_engine() - @classmethod - def _preprocess_before_render( - cls, - save_dir: str, - resolution: Tuple[int, int], - export_vertices: bool, - export_skeleton: bool, - ) -> None: - from ..camera.camera_parameter import CameraParameter - - for frame_idx in range(*cls.get_playback()): - _dir_ = cls._preprocess_in_engine( - save_dir=save_dir, - per_frame=False, - export_vertices=export_vertices, - export_skeleton=export_skeleton, - frame_idx=frame_idx, - ) - - # 1. convert camera parameters to xrprimer structure - for file in Path(_dir_['camera_dir']).glob('*/*.json'): - data = json.loads(file.read_text()) - cam_param = CameraParameter.from_unreal_convention( - location=data['location'], - rotation=data['rotation'], - fov=data['fov'], - image_size=resolution, - ) - cam_param.dump(file.as_posix()) # replace the original file - - # TODO: - # 2. convert actor infos from `.dat` to `.json` - # 3. convert vertices from `.dat` to `.npz` - # 4. convert skeleton from `.dat` to `.json` - @classmethod def add_to_renderer( cls, @@ -349,6 +314,17 @@ def _open(cls, seq_name: str, seq_dir: 'Optional[str]' = None) -> None: cls.name = seq_name logger.info(f'>>>> [cyan]Opened[/cyan] sequence "{cls.name}" >>>>') + @classmethod + def _preprocess_before_render( + cls, + save_dir: str, + resolution: Tuple[int, int], + export_vertices: bool, + export_skeleton: bool, + ) -> None: + # add annotator for saving camera parameters, actor infos, vertices, and skeleton + cls._add_annotator_in_engine(save_dir, resolution, export_vertices, export_skeleton) + ##################################### ###### RPC METHODS (Private) ######## ##################################### @@ -383,34 +359,6 @@ def _get_map_path_in_engine() -> str: def _get_seq_path_in_engine() -> str: return XRFeitoriaUnrealFactory.Sequence.sequence_path - @staticmethod - def _preprocess_in_engine( - save_dir: str, - per_frame: bool = False, - export_vertices: bool = False, - export_skeleton: bool = False, - frame_idx: 'Optional[int]' = None, - ) -> 'dict_process_dir': - """Preprocesses the sequence in the Unreal Engine. - - Args: - save_dir (str): The directory to save the processed sequence. - per_frame (bool, optional): Whether to process the sequence per frame. Defaults to False. - export_vertices (bool, optional): Whether to export the vertices. Defaults to False. - export_skeleton (bool, optional): Whether to export the skeleton. Defaults to False. - frame_idx (Optional[int], optional): The index of the frame to process. Defaults to None. - - Returns: - dict_process_dir: The directory paths of the saved data. - """ - return XRFeitoriaUnrealFactory.Sequence.save_params( - save_dir=save_dir, - per_frame=per_frame, - export_vertices=export_vertices, - export_skeleton=export_skeleton, - frame_idx=frame_idx, - ) - @staticmethod def _new_seq_in_engine( seq_name: str, @@ -627,3 +575,18 @@ def _add_audio_in_engine( start_frame=start_frame, end_frame=end_frame, ) + + # ------ render -------- # + @staticmethod + def _add_annotator_in_engine( + save_dir: str, + resolution: 'Tuple[int, int]', + export_vertices: bool, + export_skeleton: bool, + ) -> None: + XRFeitoriaUnrealFactory.Sequence.add_annotator( + save_dir=save_dir, + resolution=resolution, + export_vertices=export_vertices, + export_skeleton=export_skeleton, + ) From d80e75308cf9b58cf77ee905edf9031fbecebb98 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Sat, 16 Mar 2024 01:08:48 +0800 Subject: [PATCH 12/27] postprocess --- xrfeitoria/renderer/renderer_unreal.py | 116 ++++++++++++++++++++++++- xrfeitoria/sequence/sequence_unreal.py | 8 -- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index ccf46994..364f8fef 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -1,11 +1,14 @@ +import json +import shutil import socket from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from loguru import logger -from ..data_structure.constants import PathLike +from ..data_structure.constants import PathLike, RenderOutputEnumUnreal, actor_info_type from ..rpc import remote_unreal +from ..utils.functions import unreal_functions from .renderer_base import RendererBase, render_status try: @@ -162,9 +165,120 @@ def render_jobs(cls) -> None: # cls.clear() server.close() + cls._post_process() + # clear render queue cls.clear() + @classmethod + def _post_process(cls) -> None: + """Post-processes the rendered output by: + - converting camera parameters: from `.dat` to `.json` + - convert actor infos: from `.dat` to `.json` + - convert vertices: from `.dat` to `.npz` + - convert skeleton: from `.dat` to `.npz` + + This method is called after rendering is complete. + """ + import numpy as np # isort:skip + from rich import get_console # isort:skip + from rich.spinner import Spinner # isort:skip + from ..camera.camera_parameter import CameraParameter # isort:skip + + def convert_camera(camera_file: Path) -> None: + """Convert camera parameters from `.dat` to `.json` with `xrprimer`. + + Args: + camera_file (Path): Path to the camera file. + """ + cam_param = CameraParameter.from_bin(camera_file) + cam_param.dump(camera_file.with_suffix('.json').as_posix()) + camera_file.unlink() + + def convert_vertices(folder: Path) -> None: + """Convert vertices from `.dat` to `.npz`. Merge all vertices files into one + `.npz` file with structure of: {'verts': np.ndarray, 'faces': None} + + Args: + folder (Path): Path to the folder containing vertices files. + """ + # Get all vertices files in the folder and sort them + vertices_files = sorted(folder.glob('*.dat')) + # Read all vertices files into a list + vertices = [ + np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) for vertices_file in vertices_files + ] + if not vertices: + return + + # Stack all vertices into one array with shape (frame, verts, 3) + vertices = np.stack(vertices) + # Convert convention from unreal to opencv, [x, y, z] -> [y, -z, x] + vertices = np.stack([vertices[:, :, 1], -vertices[:, :, 2], vertices[:, :, 0]], axis=-1) + vertices /= 100 # convert from cm to m + + # Save the vertices in a compressed `.npz` file + np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) + # Remove the folder + shutil.rmtree(folder) + + def convert_actor_infos(folder: Path) -> None: + """Convert stencil value from `.dat` to `.json`. + + Args: + folder (Path): Path to the folder contains ``actor_infos``. + """ + # Get all stencil value files in the folder and sort them + actor_info_files = sorted(folder.glob('*.dat')) + # Read all actor info files into a list + actor_infos: List[actor_info_type] = [] + for actor_info_file in actor_info_files: + stencil_value = np.frombuffer(actor_info_file.read_bytes(), np.float32) + stencil_value = int(stencil_value) + mask_color = unreal_functions.get_mask_color(stencil_value) + actor_infos.append({'actor_name': actor_info_file.stem, 'mask_color': mask_color}) + + if not actor_infos: + return + + # Save the actor infos in a `.json` file + with (folder.parent / f'{folder.name}.json').open('w') as f: + json.dump(actor_infos, f, indent=4) + + # Remove the folder + shutil.rmtree(folder) + + console = get_console() + try: + spinner: Spinner = console._live.renderable + except AttributeError: + status = console.status('[bold green]:rocket: Rendering...[/bold green]') + status.start() + spinner: Spinner = status.renderable + + for idx, job in enumerate(cls.render_queue): + seq_name = job.sequence_path.split('/')[-1] + seq_path = Path(job.output_path).resolve() / seq_name + file_name_format = job.file_name_format # TODO: use this to rename the files + + text = f'job {idx + 1}/{len(cls.render_queue)}: seq_name="{seq_name}", post-processing...' + spinner.update(text=text) + + # 1. convert camera parameters from `.bat` to `.json` with xrprimer + for camera_file in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.camera_params.value}/*/*.dat')): + convert_camera(camera_file) + + # 2. convert actor infos from `.dat` to `.json` + convert_actor_infos(folder=seq_path / RenderOutputEnumUnreal.actor_infos.value) + + # 3. convert vertices from `.dat` to `.npz` + for actor_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.vertices.value}/*')): + convert_vertices(actor_folder) + + # 4. convert skeleton from `.dat` to `.json` + for actor_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.skeleton.value}/*')): + convert_vertices(actor_folder) + @staticmethod def _add_job_in_engine(job: 'Dict[str, Any]') -> None: _job = XRFeitoriaUnrealFactory.constants.RenderJobUnreal(**job) diff --git a/xrfeitoria/sequence/sequence_unreal.py b/xrfeitoria/sequence/sequence_unreal.py index c25180c9..69efbb03 100644 --- a/xrfeitoria/sequence/sequence_unreal.py +++ b/xrfeitoria/sequence/sequence_unreal.py @@ -1,5 +1,3 @@ -import json -from pathlib import Path from typing import Dict, List, Literal, Optional, Tuple, TypedDict, Union from loguru import logger @@ -110,12 +108,6 @@ def add_to_renderer( if anti_aliasing is None: anti_aliasing = RenderJobUnreal.AntiAliasSetting() - msg = 'Preprocessing before rendering, including exporting camera parameters' - if export_vertices: - msg += ', vertices' - if export_skeleton: - msg += ', skeleton' - logger.info(msg) cls._preprocess_before_render( save_dir=f'{output_path}/{cls.name}', resolution=resolution, From 47a8654ea4ff81b711bfd4ca862e75fa9a158994 Mon Sep 17 00:00:00 2001 From: meihaiyi Date: Sun, 17 Mar 2024 01:26:42 +0800 Subject: [PATCH 13/27] make spawnable actor work --- .../Content/Python/sequence.py | 54 +++++++++++++++++++ .../XRFeitoriaUnreal/Private/Annotator.cpp | 33 +++++------- .../XRFeitoriaUnreal/Public/Annotator.h | 8 +-- 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index 367014ba..903e25f6 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -87,11 +87,62 @@ def find_binding_by_name(sequence: unreal.LevelSequence, name: str) -> unreal.Se return binding for binding in sequence.get_bindings(): + binding: unreal.MovieSceneBindingProxy if binding.get_name() == name: return binding raise RuntimeError(f'Failed to find binding: {name}') +def find_binding_by_class( + sequence: unreal.LevelSequence, actor_class: Type[unreal.Actor] +) -> Optional[unreal.MovieSceneBindingProxy]: + """ + Finds a Sequencer binding for the specified actor class in the given Level Sequence. + + Args: + sequence (unreal.LevelSequence): The Level Sequence to search for the binding. + actor_class (Type[unreal.Actor]): The class of the actor to find or create the binding for. + + Returns: + Optional[unreal.MovieSceneBindingProxy]: The Sequencer binding for the actor class, or None if not found. + + """ + bound_objects: List[unreal.SequencerBoundObjects] = unreal.SequencerTools.get_bound_objects( + get_world(), sequence, sequence.get_bindings(), sequence.get_playback_range() + ) + + for bound_object in bound_objects: + if bound_object.bound_objects[0].static_class() == actor_class.static_class(): + return bound_object.binding_proxy + return None + + +def find_or_create_binding_by_class( + sequence: unreal.LevelSequence, actor_class: Type[unreal.Actor], spawn_in_sequence: bool = True +) -> unreal.MovieSceneBindingProxy: + """ + Finds or creates a Sequencer binding for the specified actor class in the given Level Sequence. + + Args: + sequence (unreal.LevelSequence): The Level Sequence to search for the binding. + actor_class (Type[unreal.Actor]): The class of the actor to find or create the binding for. + spawn_in_sequence (bool, optional): Whether to spawn the actor in the sequence if it doesn't exist. + If False, the actor will be spawned in the world but not in the sequence. Defaults to True. + + Returns: + unreal.MovieSceneBindingProxy: The Sequencer binding for the actor class. + + """ + binding = find_binding_by_class(sequence, actor_class) + if binding is not None: + return binding + + if spawn_in_sequence: + return sequence.add_spawnable_from_class(actor_class) + else: + return sequence.add_possessable(actor_class) + + def get_time(sequence: unreal.LevelSequence, frame: int) -> unreal.FrameNumber: """Initialize a FrameNumber from the given time and then convert it to a FrameTime with no sub-frame. @@ -1242,7 +1293,10 @@ def add_annotator( export_vertices: bool, export_skeleton: bool, ): + actor_binding = find_binding_by_class(cls.sequence, unreal.Annotator) + actor_binding.remove() actor_binding = cls.sequence.add_spawnable_from_class(unreal.Annotator) + add_property_bool_track_to_binding( binding=actor_binding, property_name='bSaveVerticesPosition', property_value=export_vertices ) diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp index c381a403..81fd0a86 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp @@ -37,29 +37,20 @@ void AAnnotator::Initialize() ULevelSequence* LevelSequence = LevelSequenceActor->GetSequence(); // Get All Bound Objects + TArray BoundObjects; const TArray& ObjectBindings = LevelSequence->GetMovieScene()->GetBindings(); - TArray bindingProxies; for (FMovieSceneBinding binding : ObjectBindings) { FGuid guid = binding.GetObjectGuid(); - bindingProxies.Add(FSequencerBindingProxy(guid, LevelSequence)); + TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); + BoundObjects.Append(boundObjects); } - TArray boundObjects = USequencerToolsFunctionLibrary::GetBoundObjects( - GetWorld(), - LevelSequence, - bindingProxies, - FSequencerScriptingRange::FromNative( - LevelSequence->GetMovieScene()->GetPlaybackRange(), - LevelSequence->GetMovieScene()->GetDisplayRate() - ) - ); - UE_LOG(LogXF, Log, TEXT("Detected %d bound objects"), boundObjects.Num()); + UE_LOG(LogXF, Log, TEXT("Detected %d bound objects"), BoundObjects.Num()); // Get CameraActors, StaticMeshComponents, SkeletalMeshComponents from LevelSequence - for (FSequencerBoundObjects boundObject : boundObjects) + for (UObject* BoundObject : BoundObjects) { // loop over bound objects - UObject* BoundObject = boundObject.BoundObjects[0]; // only have one item if (BoundObject->IsA(ACameraActor::StaticClass())) { ACameraActor* Camera = Cast(BoundObject); @@ -131,7 +122,7 @@ void AAnnotator::Initialize() for (FName name : SkeletonNames) SkeletonNamesString.Add(name.ToString()); FString BoneNamePath = FPaths::Combine( DirectorySequence, - DirectorySkeleton, + NameSkeleton, MeshName + "_BoneName.txt" ); // {seq_dir}/{skeleton}/{actor_name}_BoneName.txt FFileHelper::SaveStringArrayToFile(SkeletonNamesString, *BoneNamePath); @@ -164,7 +155,7 @@ void AAnnotator::ExportCameraParameters(int FrameNumber) FString CameraTransformPath = FPaths::Combine( DirectorySequence, // seq_dir - DirectoryCameraParams, // camera_params + NameCameraParams, // camera_params Camera->GetFName().GetPlainNameString(), // camera_name FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" // frame_idx ); // {seq_dir}/{camera_params}/{camera_name}/{frame_idx}.dat @@ -195,7 +186,7 @@ void AAnnotator::ExportStaticMeshParameters(int FrameNumber) FString ActorInfoPath = FPaths::Combine( DirectorySequence, - DirectoryActorParams, + NameActorInfos, MeshName, FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" ); // {seq_dir}/{actor_params}/{actor_name}/{frame_idx}.dat @@ -225,7 +216,7 @@ void AAnnotator::ExportStaticMeshParameters(int FrameNumber) VertexPositionsFloat, FPaths::Combine( DirectorySequence, - DirectoryVertices, + NameVertices, MeshName, FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" ) @@ -257,7 +248,7 @@ void AAnnotator::ExportSkeletalMeshParameters(int FrameNumber) FString ActorInfoPath = FPaths::Combine( DirectorySequence, - DirectoryActorParams, + NameActorInfos, MeshName, FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" ); // {seq_dir}/{actor_params}/{actor_name}/{frame_idx}.dat @@ -287,7 +278,7 @@ void AAnnotator::ExportSkeletalMeshParameters(int FrameNumber) VertexPositionsFloat, FPaths::Combine( DirectorySequence, - DirectoryVertices, + NameVertices, MeshName, FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" ) @@ -320,7 +311,7 @@ void AAnnotator::ExportSkeletalMeshParameters(int FrameNumber) SkeletonPositionsFloat, FPaths::Combine( DirectorySequence, - DirectorySkeleton, + NameSkeleton, MeshName, FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" ) diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h index 52326d26..f7b79e8d 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h @@ -46,13 +46,13 @@ class XRFEITORIAUNREAL_API AAnnotator : public AActor UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") FString DirectorySequence; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") - FString DirectoryActorParams = "actor_params"; + FString NameActorInfos = "actor_infos"; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") - FString DirectoryCameraParams = "camera_params"; + FString NameCameraParams = "camera_params"; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") - FString DirectoryVertices = "vertices"; + FString NameVertices = "vertices"; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator") - FString DirectorySkeleton = "skeleton"; + FString NameSkeleton = "skeleton"; UPROPERTY(EditAnywhere, BlueprintReadWrite, Interp, Category = "Annotator|Resolution") int Width = 1920; From 839bf71df58efe5e85070baa91883a85f37db58c Mon Sep 17 00:00:00 2001 From: meihaiyi Date: Sun, 17 Mar 2024 01:26:59 +0800 Subject: [PATCH 14/27] fix bug of the name of actors --- .../XRFeitoriaUnreal/Private/Annotator.cpp | 87 +++++++++---------- .../XRFeitoriaUnreal/Public/Annotator.h | 6 +- 2 files changed, 42 insertions(+), 51 deletions(-) diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp index 81fd0a86..2b8b1b72 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp @@ -35,67 +35,51 @@ void AAnnotator::Initialize() UE_LOG(LogXF, Log, TEXT("Detected LevelSequenceActor: %s"), *LevelSequenceActor->GetName()); } ULevelSequence* LevelSequence = LevelSequenceActor->GetSequence(); + UMovieScene* MovieScene = LevelSequence->GetMovieScene(); // Get All Bound Objects - TArray BoundObjects; - const TArray& ObjectBindings = LevelSequence->GetMovieScene()->GetBindings(); - for (FMovieSceneBinding binding : ObjectBindings) + TMap BoundObjects; + for (int idx = 0; idx < MovieScene->GetSpawnableCount(); idx++) { - FGuid guid = binding.GetObjectGuid(); + FMovieSceneSpawnable spawnable = MovieScene->GetSpawnable(idx); + FGuid guid = spawnable.GetGuid(); + FString name = spawnable.GetName(); + + TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); + BoundObjects.Add(name, boundObjects[0]); + } + for (int idx = 0; idx < MovieScene->GetPossessableCount(); idx++) + { + FMovieScenePossessable possessable = MovieScene->GetPossessable(idx); + FGuid guid = possessable.GetGuid(); + FString name = possessable.GetName(); + TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); - BoundObjects.Append(boundObjects); + BoundObjects.Add(name, boundObjects[0]); } UE_LOG(LogXF, Log, TEXT("Detected %d bound objects"), BoundObjects.Num()); // Get CameraActors, StaticMeshComponents, SkeletalMeshComponents from LevelSequence - for (UObject* BoundObject : BoundObjects) + for (TPair pair : BoundObjects) { + FString name = pair.Key; + UObject* BoundObject = pair.Value; + // loop over bound objects if (BoundObject->IsA(ACameraActor::StaticClass())) { ACameraActor* Camera = Cast(BoundObject); - CameraActors.Add(Camera); + CameraActors.Add(name, Camera); } else if (BoundObject->IsA(ASkeletalMeshActor::StaticClass())) { ASkeletalMeshActor* SkeletalMeshActor = Cast(BoundObject); - SkeletalMeshComponents.Add(SkeletalMeshActor->GetSkeletalMeshComponent()); + SkeletalMeshComponents.Add(name, SkeletalMeshActor->GetSkeletalMeshComponent()); } else if (BoundObject->IsA(AStaticMeshActor::StaticClass())) { AStaticMeshActor* StaticMeshActor = Cast(BoundObject); - StaticMeshComponents.Add(StaticMeshActor->GetStaticMeshComponent()); - } - else if (BoundObject->IsA(USkeletalMeshComponent::StaticClass())) - { - USkeletalMeshComponent* SkeletalMeshComponent = Cast(BoundObject); - // check if it's already in the list - bool bFound = false; - for (USkeletalMeshComponent* SkeletalMeshComponentInList : SkeletalMeshComponents) - { - if (SkeletalMeshComponentInList == SkeletalMeshComponent) - { - bFound = true; - break; - } - } - if (!bFound) SkeletalMeshComponents.Add(SkeletalMeshComponent); - } - else if (BoundObject->IsA(UStaticMeshComponent::StaticClass())) - { - UStaticMeshComponent* StaticMeshComponent = Cast(BoundObject); - // check if it's already in the list - bool bFound = false; - for (UStaticMeshComponent* StaticMeshComponentInList : StaticMeshComponents) - { - if (StaticMeshComponentInList == StaticMeshComponent) - { - bFound = true; - break; - } - } - if (!bFound) - StaticMeshComponents.Add(StaticMeshComponent); + StaticMeshComponents.Add(name, StaticMeshActor->GetStaticMeshComponent()); } } UE_LOG(LogXF, Log, TEXT("Detected %d CameraActors, %d StaticMeshComponents, %d SkeletalMeshComponents"), @@ -104,9 +88,11 @@ void AAnnotator::Initialize() // Save Skeleton Names (only save on the first frame) if (bSaveSkeletonPosition) { - for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) + for (TPair pair : SkeletalMeshComponents) { - FString MeshName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); + FString MeshName = pair.Key; + USkeletalMeshComponent* SkeletalMeshComponent = pair.Value; + TArray SkeletonPositions; TArray SkeletonNames; bool isSuccess = UXF_BlueprintFunctionLibrary::GetSkeletalMeshBoneLocations( @@ -136,8 +122,11 @@ void AAnnotator::Initialize() void AAnnotator::ExportCameraParameters(int FrameNumber) { if (!bInitialized || CameraActors.Num() == 0) return; - for (ACameraActor* Camera : CameraActors) + for (TPair pair : CameraActors) { + FString CameraName = pair.Key; + ACameraActor* Camera = pair.Value; + FVector CamLocation = Camera->GetActorLocation(); FRotator CamRotation = Camera->GetActorRotation(); float FOV = Camera->GetCameraComponent()->FieldOfView; @@ -156,7 +145,7 @@ void AAnnotator::ExportCameraParameters(int FrameNumber) FString CameraTransformPath = FPaths::Combine( DirectorySequence, // seq_dir NameCameraParams, // camera_params - Camera->GetFName().GetPlainNameString(), // camera_name + CameraName, // camera_name FString::Printf(TEXT("%04d"), FrameNumber) + ".dat" // frame_idx ); // {seq_dir}/{camera_params}/{camera_name}/{frame_idx}.dat UXF_BlueprintFunctionLibrary::SaveFloatArrayToByteFile(CamInfo, CameraTransformPath); @@ -165,9 +154,10 @@ void AAnnotator::ExportCameraParameters(int FrameNumber) void AAnnotator::ExportStaticMeshParameters(int FrameNumber) { - for (UStaticMeshComponent* StaticMeshComponent : StaticMeshComponents) + for (TPair pair : StaticMeshComponents) { - FString MeshName = StaticMeshComponent->GetOwner()->GetFName().GetPlainNameString(); + FString MeshName = pair.Key; + UStaticMeshComponent* StaticMeshComponent = pair.Value; // Save Actor Info (location, rotation, stencil value) { @@ -227,9 +217,10 @@ void AAnnotator::ExportStaticMeshParameters(int FrameNumber) void AAnnotator::ExportSkeletalMeshParameters(int FrameNumber) { - for (USkeletalMeshComponent* SkeletalMeshComponent : SkeletalMeshComponents) + for (TPair pair : SkeletalMeshComponents) { - FString MeshName = SkeletalMeshComponent->GetOwner()->GetFName().GetPlainNameString(); + FString MeshName = pair.Key; + USkeletalMeshComponent* SkeletalMeshComponent = pair.Value; // Save Actor Info (location, rotation, stencil value) { diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h index f7b79e8d..59522e68 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Public/Annotator.h @@ -68,8 +68,8 @@ class XRFEITORIAUNREAL_API AAnnotator : public AActor private: ALevelSequenceActor* LevelSequenceActor; ULevelSequencePlayer* LevelSequencePlayer; - TArray CameraActors; - TArray StaticMeshComponents; - TArray SkeletalMeshComponents; + TMap CameraActors; + TMap StaticMeshComponents; + TMap SkeletalMeshComponents; bool bInitialized = false; }; From 7a3bf9cddb66d793deb44d2d2ce5eb41cea0617c Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 10:53:19 +0800 Subject: [PATCH 15/27] pre commit --- src/XRFeitoriaUnreal/Content/Python/sequence.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index 903e25f6..ccee1419 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -96,8 +96,8 @@ def find_binding_by_name(sequence: unreal.LevelSequence, name: str) -> unreal.Se def find_binding_by_class( sequence: unreal.LevelSequence, actor_class: Type[unreal.Actor] ) -> Optional[unreal.MovieSceneBindingProxy]: - """ - Finds a Sequencer binding for the specified actor class in the given Level Sequence. + """Finds a Sequencer binding for the specified actor class in the given Level + Sequence. Args: sequence (unreal.LevelSequence): The Level Sequence to search for the binding. @@ -105,7 +105,6 @@ def find_binding_by_class( Returns: Optional[unreal.MovieSceneBindingProxy]: The Sequencer binding for the actor class, or None if not found. - """ bound_objects: List[unreal.SequencerBoundObjects] = unreal.SequencerTools.get_bound_objects( get_world(), sequence, sequence.get_bindings(), sequence.get_playback_range() @@ -120,8 +119,8 @@ def find_binding_by_class( def find_or_create_binding_by_class( sequence: unreal.LevelSequence, actor_class: Type[unreal.Actor], spawn_in_sequence: bool = True ) -> unreal.MovieSceneBindingProxy: - """ - Finds or creates a Sequencer binding for the specified actor class in the given Level Sequence. + """Finds or creates a Sequencer binding for the specified actor class in the given + Level Sequence. Args: sequence (unreal.LevelSequence): The Level Sequence to search for the binding. @@ -131,7 +130,6 @@ def find_or_create_binding_by_class( Returns: unreal.MovieSceneBindingProxy: The Sequencer binding for the actor class. - """ binding = find_binding_by_class(sequence, actor_class) if binding is not None: From 5065932546e68077b16baead3f385962ef2a4d92 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 14:10:35 +0800 Subject: [PATCH 16/27] bug fix --- .../Content/Python/constants.py | 2 +- .../Content/Python/sequence.py | 9 ++++++--- .../XRFeitoriaUnreal/Private/Annotator.cpp | 2 ++ xrfeitoria/data_structure/constants.py | 2 +- xrfeitoria/sequence/sequence_unreal.py | 8 ++++---- xrfeitoria/sequence/sequence_unreal.pyi | 18 +++++++++--------- xrfeitoria/sequence/sequence_wrapper.py | 11 ++++++----- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/XRFeitoriaUnreal/Content/Python/constants.py b/src/XRFeitoriaUnreal/Content/Python/constants.py index 4b2fdc0f..07b1a7cf 100644 --- a/src/XRFeitoriaUnreal/Content/Python/constants.py +++ b/src/XRFeitoriaUnreal/Content/Python/constants.py @@ -41,7 +41,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]: ENGINE_MAJOR_VERSION = int(unreal.SystemLibrary.get_engine_version().split('.')[0]) ENGINE_MINOR_VERSION = int(unreal.SystemLibrary.get_engine_version().split('.')[1]) DEFAULT_PATH = f'/Game/{PLUGIN_NAME}' -DEFAULT_SEQUENCE_PATH = f'{DEFAULT_PATH}/Sequences' +DEFAULT_SEQUENCE_DIR = f'{DEFAULT_PATH}/Sequences' DEFAULT_ASSET_PATH = f'{DEFAULT_PATH}/Assets' DEFAULT_SEQUENCE_DATA_ASSET = f'/{PLUGIN_NAME}/DefaultSequenceData' MRQ_JOB_UPPER = 200 diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index ccee1419..5030ee4f 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -7,7 +7,7 @@ import utils_actor from constants import ( DEFAULT_SEQUENCE_DATA_ASSET, - DEFAULT_SEQUENCE_PATH, + DEFAULT_SEQUENCE_DIR, ENGINE_MAJOR_VERSION, ENGINE_MINOR_VERSION, MotionFrame, @@ -111,6 +111,8 @@ def find_binding_by_class( ) for bound_object in bound_objects: + if len(bound_object.bound_objects) == 0: + continue if bound_object.bound_objects[0].static_class() == actor_class.static_class(): return bound_object.binding_proxy return None @@ -1027,7 +1029,7 @@ def new( map_path = EditorLevelSub.get_current_level().get_path_name().split('.')[0] assert unreal.EditorAssetLibrary.does_asset_exist(map_path), f'Map `{map_path}` does not exist' if seq_dir is None: - seq_dir = DEFAULT_SEQUENCE_PATH + seq_dir = DEFAULT_SEQUENCE_DIR seq_path = f'{seq_dir}/{seq_name}' data_asset_path = f'{seq_path}{data_asset_suffix}' @@ -1292,7 +1294,8 @@ def add_annotator( export_skeleton: bool, ): actor_binding = find_binding_by_class(cls.sequence, unreal.Annotator) - actor_binding.remove() + if actor_binding is not None: + actor_binding.remove() actor_binding = cls.sequence.add_spawnable_from_class(unreal.Annotator) add_property_bool_track_to_binding( diff --git a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp index 2b8b1b72..8cf12e4b 100644 --- a/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp +++ b/src/XRFeitoriaUnreal/Source/XRFeitoriaUnreal/Private/Annotator.cpp @@ -46,6 +46,7 @@ void AAnnotator::Initialize() FString name = spawnable.GetName(); TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); + if (boundObjects.Num() == 0) continue; BoundObjects.Add(name, boundObjects[0]); } for (int idx = 0; idx < MovieScene->GetPossessableCount(); idx++) @@ -55,6 +56,7 @@ void AAnnotator::Initialize() FString name = possessable.GetName(); TArray boundObjects = LevelSequencePlayer->GetBoundObjects(FMovieSceneObjectBindingID(guid)); + if (boundObjects.Num() == 0) continue; BoundObjects.Add(name, boundObjects[0]); } UE_LOG(LogXF, Log, TEXT("Detected %d bound objects"), BoundObjects.Num()); diff --git a/xrfeitoria/data_structure/constants.py b/xrfeitoria/data_structure/constants.py index 0266d34d..2527bbee 100644 --- a/xrfeitoria/data_structure/constants.py +++ b/xrfeitoria/data_structure/constants.py @@ -39,7 +39,7 @@ default_path_unreal = f'/Game/{plugin_name_unreal}' default_asset_path_unreal = f'{default_path_unreal}/Assets' -default_sequence_path_unreal = f'{default_path_unreal}/Sequences' +default_sequence_dir_unreal = f'{default_path_unreal}/Sequences' ##### Enum Constants ##### diff --git a/xrfeitoria/sequence/sequence_unreal.py b/xrfeitoria/sequence/sequence_unreal.py index 69efbb03..53fff8db 100644 --- a/xrfeitoria/sequence/sequence_unreal.py +++ b/xrfeitoria/sequence/sequence_unreal.py @@ -322,8 +322,8 @@ def _preprocess_before_render( ##################################### @staticmethod - def _get_default_seq_path_in_engine() -> str: - return XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_PATH + def _get_default_seq_dir_in_engine() -> str: + return XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_DIR @staticmethod def _get_seq_info_in_engine( @@ -332,8 +332,8 @@ def _get_seq_info_in_engine( map_path: 'Optional[str]' = None, ) -> 'Tuple[str, str]': _suffix = XRFeitoriaUnrealFactory.constants.data_asset_suffix - default_sequence_path = XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_PATH - seq_dir = seq_dir or default_sequence_path # default sequence path + default_sequence_dir = XRFeitoriaUnrealFactory.constants.DEFAULT_SEQUENCE_DIR + seq_dir = seq_dir or default_sequence_dir # default sequence directory if map_path is None: seq_data_path = f'{seq_dir}/{seq_name}{_suffix}' unreal_functions.check_asset_in_engine(seq_data_path, raise_error=True) diff --git a/xrfeitoria/sequence/sequence_unreal.pyi b/xrfeitoria/sequence/sequence_unreal.pyi index 2abd87a4..c0b1381f 100644 --- a/xrfeitoria/sequence/sequence_unreal.pyi +++ b/xrfeitoria/sequence/sequence_unreal.pyi @@ -12,6 +12,7 @@ from .sequence_base import SequenceBase class dict_process_dir(TypedDict): camera_dir: str + actor_infos_dir: str vertices_dir: str skeleton_dir: str @@ -25,10 +26,6 @@ class SequenceUnreal(SequenceBase): @classmethod def show(cls) -> None: ... @classmethod - def _preprocess_before_render( - cls, save_dir: str, resolution: Tuple[int, int], export_vertices: bool, export_skeleton: bool - ) -> None: ... - @classmethod def add_to_renderer( cls, output_path: PathLike, @@ -65,10 +62,7 @@ class SequenceUnreal(SequenceBase): ) -> ActorUnreal: ... @classmethod def add_audio( - cls, - audio_asset_path: str, - start_frame: Optional[int] = None, - end_frame: Optional[int] = None, + cls, audio_asset_path: str, start_frame: Optional[int] = None, end_frame: Optional[int] = None ) -> None: ... @classmethod def get_map_path(cls) -> str: ... @@ -77,11 +71,17 @@ class SequenceUnreal(SequenceBase): @classmethod def set_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: ... @classmethod + def get_playback(cls) -> Tuple[int, int]: ... + @classmethod def set_camera_cut_playback(cls, start_frame: Optional[int] = None, end_frame: Optional[int] = None) -> None: ... @classmethod def _open(cls, seq_name: str, seq_dir: Optional[str] = None) -> None: ... + @classmethod + def _preprocess_before_render( + cls, save_dir: str, resolution: Tuple[int, int], export_vertices: bool, export_skeleton: bool + ) -> None: ... @staticmethod - def _get_default_seq_path_in_engine() -> str: ... + def _get_default_seq_dir_in_engine() -> str: ... @staticmethod def _get_seq_info_in_engine( seq_name: str, seq_dir: Optional[str] = None, map_path: Optional[str] = None diff --git a/xrfeitoria/sequence/sequence_wrapper.py b/xrfeitoria/sequence/sequence_wrapper.py index 78313305..b5104e14 100644 --- a/xrfeitoria/sequence/sequence_wrapper.py +++ b/xrfeitoria/sequence/sequence_wrapper.py @@ -178,22 +178,23 @@ def sequence_wrapper_unreal( seq_length: int = 1, replace: bool = False, ) -> Union[SequenceUnreal, ContextManager[SequenceUnreal]]: - """Create a new sequence and close the sequence after exiting it. + """Create a new sequence, open it in editor, and close the sequence after exiting + it. Args: seq_name (str): The name of the sequence. - seq_dir (Optional[str], optional): The directory where the sequence is located. Defaults to None. + seq_dir (Optional[str], optional): The directory where the sequence is located. Defaults to None. Falls back to the default sequence path (/Game/XRFeitoriaUnreal/Sequences). level (Optional[str], optional): The level to associate the sequence with. Defaults to None. seq_fps (int, optional): The frames per second of the sequence. Defaults to 30. - seq_length (int, optional): The length of the sequence in seconds. Defaults to 1. + seq_length (int, optional): The length of the sequence in frames. Defaults to 1. replace (bool, optional): Whether to replace an existing sequence with the same name. Defaults to False. Returns: SequenceUnreal: The created SequenceUnreal object. """ - default_sequence_path = SequenceUnreal._get_default_seq_path_in_engine() - seq_dir = seq_dir or default_sequence_path + default_sequence_dir = SequenceUnreal._get_default_seq_dir_in_engine() + seq_dir = seq_dir or default_sequence_dir if ( unreal_functions.check_asset_in_engine(f'{seq_dir}/{seq_name}') and unreal_functions.check_asset_in_engine(f'{seq_dir}/{seq_name}_data') From 4c3f1771b233f66110e80a6e99b5d94d4b81e4d7 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 15:43:02 +0800 Subject: [PATCH 17/27] Add color_type TypedDict and update mask_colors loading --- .../Content/Python/constants.py | 16 +- .../Content/Python/sequence.py | 140 +---------- .../Content/Python/utils_actor.py | 17 +- xrfeitoria/camera/camera_parameter.py | 127 +--------- xrfeitoria/data_structure/constants.py | 8 + xrfeitoria/renderer/renderer_unreal.py | 46 ++-- xrfeitoria/utils/anim/motion.py | 62 +---- xrfeitoria/utils/converter.py | 218 ++++++++++++++++++ .../utils/functions/unreal_functions.py | 30 ++- 9 files changed, 302 insertions(+), 362 deletions(-) create mode 100644 xrfeitoria/utils/converter.py diff --git a/src/XRFeitoriaUnreal/Content/Python/constants.py b/src/XRFeitoriaUnreal/Content/Python/constants.py index 07b1a7cf..97e0c662 100644 --- a/src/XRFeitoriaUnreal/Content/Python/constants.py +++ b/src/XRFeitoriaUnreal/Content/Python/constants.py @@ -1,7 +1,8 @@ +import json from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, TypedDict, Union import unreal @@ -197,3 +198,16 @@ def __post_init__(self): TransformKeys = Union[List[SequenceTransformKey], SequenceTransformKey] MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]] +color_type = TypedDict( + 'color', + { + 'name': str, + 'hex': str, + 'rgb': Tuple[int, int, int], + }, +) + + +######### Constants ######### +MASK_COLOR_FILE = PLUGIN_PYTHON_ROOT / 'data' / 'mask_colors.json' +mask_colors: List[color_type] = json.loads(MASK_COLOR_FILE.read_text()) diff --git a/src/XRFeitoriaUnreal/Content/Python/sequence.py b/src/XRFeitoriaUnreal/Content/Python/sequence.py index 5030ee4f..dd80a18f 100644 --- a/src/XRFeitoriaUnreal/Content/Python/sequence.py +++ b/src/XRFeitoriaUnreal/Content/Python/sequence.py @@ -1,8 +1,5 @@ -import json -from pathlib import Path from typing import Any, Dict, List, NoReturn, Optional, Tuple, Type, Union -import numpy as np import unreal import utils_actor from constants import ( @@ -11,15 +8,12 @@ ENGINE_MAJOR_VERSION, ENGINE_MINOR_VERSION, MotionFrame, - PathLike, SequenceTransformKey, SubSystem, TransformKeys, - UnrealRenderLayerEnum, data_asset_suffix, ) from utils import add_levels, get_levels, get_soft_object_path, get_world, new_world, save_current_level -from utils_actor import get_actor_mesh_component EditorLevelSequenceSub = SubSystem.EditorLevelSequenceSub EditorAssetSub = SubSystem.EditorAssetSub @@ -760,7 +754,7 @@ def add_actor_to_sequence( actor_binding = sequence.add_possessable(actor) # mesh_component = actor.skeletal_mesh_component - mesh_component = get_actor_mesh_component(actor) + mesh_component = utils_actor.get_actor_mesh_component(actor) mesh_component_binding = sequence.add_possessable(mesh_component) # set stencil value @@ -826,7 +820,7 @@ def add_spawnable_actor_to_sequence( actor.set_actor_label(actor_name) # mesh_component = actor.skeletal_mesh_component - mesh_component = get_actor_mesh_component(actor) + mesh_component = utils_actor.get_actor_mesh_component(actor) mesh_component_binding = sequence.add_possessable(mesh_component) # set stencil value @@ -939,7 +933,7 @@ def get_actor_param(actor: unreal.Actor) -> Dict[str, Any]: Returns: Dict[str, Any]: A dictionary containing the actor parameters. """ - stencil_value = get_actor_mesh_component(actor).get_editor_property('custom_depth_stencil_value') + stencil_value = utils_actor.get_actor_mesh_component(actor).get_editor_property('custom_depth_stencil_value') return { 'location': actor.get_actor_location().to_tuple(), 'rotation': actor.get_actor_rotation().to_tuple(), @@ -1310,134 +1304,6 @@ def add_annotator( add_property_int_track_to_binding(binding=actor_binding, property_name='Width', property_value=resolution[0]) add_property_int_track_to_binding(binding=actor_binding, property_name='Height', property_value=resolution[1]) - @classmethod - def save_params( - cls, - save_dir: PathLike, - per_frame: bool = True, - export_vertices: bool = False, - export_skeleton: bool = False, - frame_idx: Optional[int] = None, - ) -> Dict[str, str]: - """Save parameters of the sequence. - - Args: - save_dir (PathLike): The directory to save the parameters. - per_frame (bool, optional): Whether to save parameters for each frame. Defaults to True. - export_vertices (bool, optional): Whether to export vertices of the mesh. Defaults to False. - export_skeleton (bool, optional): Whether to export skeleton of the skeletal mesh. Defaults to False. - frame_idx (int, optional): The frame index to save parameters for. Defaults to None. Will be used only if `per_frame` is False. - - Returns: - Dict[str, str]: A dictionary containing the paths of the saved parameters. - """ - assert cls.sequence is not None, 'Sequence not initialized' - cls.show() - - save_dir = Path(save_dir) - save_dir.mkdir(parents=True, exist_ok=True) - - # get actors - mesh_actors: Dict[str, unreal.Actor] = {} - camera_actors: Dict[str, unreal.CameraActor] = {} - for name, binding in cls.bindings.items(): - if 'camera' in binding.keys(): - camera_binding = binding['camera']['binding'] - camera = unreal.LevelSequenceEditorBlueprintLibrary.get_bound_objects(get_binding_id(camera_binding))[0] - camera_actors[name] = camera - if 'actor' in binding.keys(): - actor_binding = binding['actor']['binding'] - actor = unreal.LevelSequenceEditorBlueprintLibrary.get_bound_objects(get_binding_id(actor_binding))[0] - mesh_actors[name] = actor - - # define save functions - def save_camera_param(frame_idx: int, save_root: PathLike) -> Dict[str, Any]: - """Save camera parameters of the given frame to - {save_root}/camera_params/{camera_name}/{frame_idx:04d}.json. - - Args: - frame_idx (int): The frame index to save camera parameters for. - """ - save_dir = Path(save_root) / UnrealRenderLayerEnum.camera_params.value - unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(frame_idx) - for name, camera in camera_actors.items(): - save_path = save_dir / name / f'{frame_idx:04d}.json' - save_path.parent.mkdir(parents=True, exist_ok=True) - with open(save_path, 'w') as f: - json.dump(get_camera_param(camera), f, indent=4) - - def save_actor_param( - frame_idx: int, - save_root: PathLike, - export_vertices: bool = False, - export_skeleton: bool = False, - ) -> Dict[str, Any]: - """Save actor parameters of the given frame to - {save_root}/{params_type}/{actor_name}/{frame_idx:04d}.json. `params_type` - is one of ['actor_infos', 'vertices', 'skeleton']. - - Args: - frame_idx (int): The frame index to save actor parameters for. - save_root (PathLike): The root directory to save the parameters. - export_vertices (bool, optional): Whether to export vertices of the mesh. Defaults to False. - export_skeleton (bool, optional): Whether to export skeleton of the skeletal mesh. Defaults to False. - """ - # Init directories - actor_infos_dir = Path(save_root) / UnrealRenderLayerEnum.actor_infos.value - vertices_dir = Path(save_root) / UnrealRenderLayerEnum.vertices.value - skeleton_dir = Path(save_root) / UnrealRenderLayerEnum.skeleton.value - - unreal.LevelSequenceEditorBlueprintLibrary.set_current_time(frame_idx) - for name, actor in mesh_actors.items(): - actor_infos_file = actor_infos_dir / name / f'{frame_idx:04d}.json' - actor_infos_file.parent.mkdir(parents=True, exist_ok=True) - with open(actor_infos_file, 'w') as f: - json.dump(get_actor_param(actor), f, indent=4) - - if export_vertices: - vertices_file = vertices_dir / name / f'{frame_idx:04d}.npz' - vertices_file.parent.mkdir(parents=True, exist_ok=True) - if isinstance(actor, unreal.SkeletalMeshActor): - skl_mesh_comp = actor.skeletal_mesh_component - vertices_data = ( - unreal.XF_BlueprintFunctionLibrary.get_skeletal_mesh_vertex_locations_by_lod_index( - skl_mesh_comp, 0 - ) - ) - elif isinstance(actor, unreal.StaticMeshActor): - static_mesh_comp = actor.static_mesh_component - vertices_data = unreal.XF_BlueprintFunctionLibrary.get_static_mesh_vertex_locations( - static_mesh_comp, 0 - ) - vertices_data = np.array([v.to_tuple() for v in vertices_data]) - np.savez_compressed(vertices_file, vertices_data) - - if export_skeleton and isinstance(actor, unreal.SkeletalMeshActor): - skeleton_file = skeleton_dir / name / f'{frame_idx:04d}.npz' - skeleton_file.parent.mkdir(parents=True, exist_ok=True) - vertices_data = unreal.XF_BlueprintFunctionLibrary.get_skeletal_mesh_bone_locations(skl_mesh_comp) - # bone_locations (Array[Vector]): - # bone_names (Array[Name]): - # np.savez_compressed(skeleton_file, skeleton_data) - - # save parameters - if per_frame: - for frame_idx in range(cls.START_FRAME, cls.sequence.get_playback_end()): - save_camera_param(frame_idx, save_root=save_dir) - save_actor_param(frame_idx, save_dir, export_vertices, export_skeleton) - else: - if frame_idx is None: - frame_idx = cls.START_FRAME - save_camera_param(frame_idx, save_root=save_dir) - save_actor_param(frame_idx, save_dir, export_vertices, export_skeleton) - - return { - 'camera_dir': (save_dir / UnrealRenderLayerEnum.camera_params.value).as_posix(), - 'actor_infos_dir': (save_dir / UnrealRenderLayerEnum.actor_infos.value).as_posix(), - 'vertices_dir': (save_dir / UnrealRenderLayerEnum.vertices.value).as_posix(), - 'skeleton_dir': (save_dir / UnrealRenderLayerEnum.skeleton.value).as_posix(), - } - def test(): Sequence.open('/Game/Levels/SequenceTest', '/Game/XRFeitoriaUnreal/Sequences/seq_test') diff --git a/src/XRFeitoriaUnreal/Content/Python/utils_actor.py b/src/XRFeitoriaUnreal/Content/Python/utils_actor.py index 3c3bf56b..d323e2f0 100644 --- a/src/XRFeitoriaUnreal/Content/Python/utils_actor.py +++ b/src/XRFeitoriaUnreal/Content/Python/utils_actor.py @@ -1,22 +1,9 @@ -import json -from pathlib import Path -from typing import Dict, List, Tuple, TypedDict, Union +from typing import Dict, List, Tuple, Union import unreal -from constants import SubSystem +from constants import SubSystem, mask_colors from utils import get_world -root = Path(__file__).parent.resolve() -color_type = TypedDict( - 'color', - { - 'name': str, - 'hex': str, - 'rgb': Tuple[int, int, int], - }, -) -mask_colors: List[color_type] = json.loads((root / 'data' / 'mask_colors.json').read_text()) - def get_stencil_value(actor: unreal.Actor) -> int: # skeletal mesh component diff --git a/xrfeitoria/camera/camera_parameter.py b/xrfeitoria/camera/camera_parameter.py index 2eea659c..e137329e 100644 --- a/xrfeitoria/camera/camera_parameter.py +++ b/xrfeitoria/camera/camera_parameter.py @@ -9,6 +9,7 @@ from xrprimer.transform.convention.camera import convert_camera_parameter from ..data_structure.constants import PathLike, Vector +from ..utils.converter import rotation_matrix class CameraParameter(PinholeCameraParameter): @@ -251,129 +252,3 @@ def convert_convention(self, dst: str): def __repr__(self) -> str: return f'CameraParameter(T={self.extrinsic_t}, convention="{self.convention}", world2cam={self.world2cam})' - - -def rotation_matrix(angles: Tuple[float, float, float], order='xyz', degrees: bool = True) -> npt.NDArray[np.float32]: - """ - Args: - angles (Tuple[float, float, float]): Rotation angles in degrees or radians. - order (str, optional): Rotation order. Defaults to 'xyz'. - degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. - Returns: - ndarray: Rotation matrix 3x3. - - Examples: - >>> rotation_matrix((0, 0, 0), order='xyz') - - References: - https://programming-surgeon.com/en/euler-angle-python-en/ - """ - if degrees: - angles = np.deg2rad(angles) - - theta1, theta2, theta3 = angles - c1 = np.cos(theta1) - s1 = np.sin(theta1) - c2 = np.cos(theta2) - s2 = np.sin(theta2) - c3 = np.cos(theta3) - s3 = np.sin(theta3) - - if order == 'xzx': - matrix = np.array( - [ - [c2, -c3 * s2, s2 * s3], - [c1 * s2, c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3], - [s1 * s2, c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3], - ] - ) - elif order == 'xyx': - matrix = np.array( - [ - [c2, s2 * s3, c3 * s2], - [s1 * s2, c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1], - [-c1 * s2, c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3], - ] - ) - elif order == 'yxy': - matrix = np.array( - [ - [c1 * c3 - c2 * s1 * s3, s1 * s2, c1 * s3 + c2 * c3 * s1], - [s2 * s3, c2, -c3 * s2], - [-c3 * s1 - c1 * c2 * s3, c1 * s2, c1 * c2 * c3 - s1 * s3], - ] - ) - elif order == 'yzy': - matrix = np.array( - [ - [c1 * c2 * c3 - s1 * s3, -c1 * s2, c3 * s1 + c1 * c2 * s3], - [c3 * s2, c2, s2 * s3], - [-c1 * s3 - c2 * c3 * s1, s1 * s2, c1 * c3 - c2 * s1 * s3], - ] - ) - elif order == 'zyz': - matrix = np.array( - [ - [c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3, c1 * s2], - [c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3, s1 * s2], - [-c3 * s2, s2 * s3, c2], - ] - ) - elif order == 'zxz': - matrix = np.array( - [ - [c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1, s1 * s2], - [c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3, -c1 * s2], - [s2 * s3, c3 * s2, c2], - ] - ) - elif order == 'xyz': - matrix = np.array( - [ - [c2 * c3, -c2 * s3, s2], - [c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1], - [s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2], - ] - ) - elif order == 'xzy': - matrix = np.array( - [ - [c2 * c3, -s2, c2 * s3], - [s1 * s3 + c1 * c3 * s2, c1 * c2, c1 * s2 * s3 - c3 * s1], - [c3 * s1 * s2 - c1 * s3, c2 * s1, c1 * c3 + s1 * s2 * s3], - ] - ) - elif order == 'yxz': - matrix = np.array( - [ - [c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3, c2 * s1], - [c2 * s3, c2 * c3, -s2], - [c1 * s2 * s3 - c3 * s1, c1 * c3 * s2 + s1 * s3, c1 * c2], - ] - ) - elif order == 'yzx': - matrix = np.array( - [ - [c1 * c2, s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3], - [s2, c2 * c3, -c2 * s3], - [-c2 * s1, c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3], - ] - ) - elif order == 'zyx': - matrix = np.array( - [ - [c1 * c2, c1 * s2 * s3 - c3 * s1, s1 * s3 + c1 * c3 * s2], - [c2 * s1, c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3], - [-s2, c2 * s3, c2 * c3], - ] - ) - elif order == 'zxy': - matrix = np.array( - [ - [c1 * c3 - s1 * s2 * s3, -c2 * s1, c1 * s3 + c3 * s1 * s2], - [c3 * s1 + c1 * s2 * s3, c1 * c2, s1 * s3 - c1 * c3 * s2], - [-c2 * s3, s2, c2 * c3], - ] - ) - - return matrix diff --git a/xrfeitoria/data_structure/constants.py b/xrfeitoria/data_structure/constants.py index 2527bbee..707fad99 100644 --- a/xrfeitoria/data_structure/constants.py +++ b/xrfeitoria/data_structure/constants.py @@ -10,6 +10,14 @@ PathLike = Union[str, Path] MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]] actor_info_type = TypedDict('actor_info', {'actor_name': str, 'mask_color': Tuple[int, int, int]}) +color_type = TypedDict( + 'color', + { + 'name': str, + 'hex': str, + 'rgb': Tuple[int, int, int], + }, +) ##### Package Constants ##### diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 364f8fef..1fe0e882 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -204,18 +204,18 @@ def convert_vertices(folder: Path) -> None: """ # Get all vertices files in the folder and sort them vertices_files = sorted(folder.glob('*.dat')) + if not vertices_files: + return # Read all vertices files into a list vertices = [ np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) for vertices_file in vertices_files ] - if not vertices: - return # Stack all vertices into one array with shape (frame, verts, 3) vertices = np.stack(vertices) # Convert convention from unreal to opencv, [x, y, z] -> [y, -z, x] vertices = np.stack([vertices[:, :, 1], -vertices[:, :, 2], vertices[:, :, 0]], axis=-1) - vertices /= 100 # convert from cm to m + vertices /= 100.0 # convert from cm to m # Save the vertices in a compressed `.npz` file np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) @@ -223,28 +223,33 @@ def convert_vertices(folder: Path) -> None: shutil.rmtree(folder) def convert_actor_infos(folder: Path) -> None: - """Convert stencil value from `.dat` to `.json`. + """Convert stencil value from `.dat` to `.npz`. Merge all actor info files + into one. Args: - folder (Path): Path to the folder contains ``actor_infos``. + folder (Path): Path to the folder containing actor info files. """ - # Get all stencil value files in the folder and sort them + # Get all files in the folder and sort them actor_info_files = sorted(folder.glob('*.dat')) + if not actor_info_files: + return # Read all actor info files into a list - actor_infos: List[actor_info_type] = [] + location = [] + rotation = [] + mask_color = [] for actor_info_file in actor_info_files: - stencil_value = np.frombuffer(actor_info_file.read_bytes(), np.float32) - stencil_value = int(stencil_value) - mask_color = unreal_functions.get_mask_color(stencil_value) - actor_infos.append({'actor_name': actor_info_file.stem, 'mask_color': mask_color}) - - if not actor_infos: - return - - # Save the actor infos in a `.json` file - with (folder.parent / f'{folder.name}.json').open('w') as f: - json.dump(actor_infos, f, indent=4) - + with open(actor_info_file, 'rb') as f: + dat = np.frombuffer(f.read(), np.float32).reshape(7) + location.append(dat[:3]) + rotation.append(dat[3:6]) + mask_color.append(unreal_functions.get_mask_color(int(dat[6]))) + + location = np.array(location) / 100.0 # convert from cm to m + rotation = np.array(rotation) + mask_color = np.array(mask_color) + + # Save the actor infos in a compressed `.npz` file + np.savez_compressed(folder.with_suffix('.npz'), location=location, rotation=rotation, mask_color=mask_color) # Remove the folder shutil.rmtree(folder) @@ -269,7 +274,8 @@ def convert_actor_infos(folder: Path) -> None: convert_camera(camera_file) # 2. convert actor infos from `.dat` to `.json` - convert_actor_infos(folder=seq_path / RenderOutputEnumUnreal.actor_infos.value) + for actor_info_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.actor_infos.value}/*')): + convert_actor_infos(actor_info_folder) # 3. convert vertices from `.dat` to `.npz` for actor_folder in sorted(seq_path.glob(f'{RenderOutputEnumUnreal.vertices.value}/*')): diff --git a/xrfeitoria/utils/anim/motion.py b/xrfeitoria/utils/anim/motion.py index d8bc0104..46443d05 100644 --- a/xrfeitoria/utils/anim/motion.py +++ b/xrfeitoria/utils/anim/motion.py @@ -9,6 +9,7 @@ from scipy.spatial.transform import Rotation as spRotation from ...data_structure.constants import MotionFrame, PathLike +from ..converter import ConverterMotion from .constants import ( SMPL_IDX_TO_JOINTS, SMPL_JOINT_NAMES, @@ -25,59 +26,6 @@ __all__ = ['Motion', 'SMPLMotion', 'SMPLXMotion', 'get_humandata'] -class Converter: - @classmethod - def vec_humandata2smplx(cls, vector: np.ndarray) -> np.ndarray: - """From humandata transl (in **OpenCV space**) to SMPLX armature's **pelvis - local space** in Blender. (The pelvis local space is designed to be the same - with **SMPL space**.) - - [right, front, up]: (-x, -z, -y) ==> (-x, z, y) - - Args: - vector (np.ndarray): of shape (N, 3) or (3,) - - Returns: - np.ndarray: of shape (N, 3) or (3,) - """ - if vector.shape == (3,): - vector = np.array([vector[0], -vector[1], -vector[2]], dtype=vector.dtype) - elif vector.ndim == 2 and vector.shape[1] == 3: - vector = np.array([vector[:, 0], -vector[:, 1], -vector[:, 2]]).T - else: - raise ValueError(f'vector.shape={vector.shape}') - return vector - - @classmethod - def vec_smplx2humandata(cls, vector: np.ndarray) -> np.ndarray: - # vice versa - return cls.vec_humandata2smplx(vector) - - @classmethod - def vec_amass2humandata(cls, vector: np.ndarray) -> np.ndarray: - """From amass transl (pelvis's local space) to humandata transl (in **OpenCV - space**) - - [right, front, up]: (x, y, z) ==> (-x, -z, -y) - - (CAUTION: we can see amass animation actors face back - in blender via the smplx add-on) - - Args: - vector (np.ndarray): of shape (N, 3) or (3,) - - Returns: - np.ndarray: of shape (N, 3) or (3,) - """ - if vector.shape == (3,): - vector = np.array([-vector[0], -vector[2], -vector[1]], dtype=vector.dtype) - elif vector.ndim == 2 and vector.shape[1] == 3: - vector = np.array([-vector[:, 0], -vector[:, 2], -vector[:, 1]]).T - else: - raise ValueError(f'vector.shape={vector.shape}') - return vector - - class Motion: """Wrap motion data. Provide methods to get transform info for 3D calculations. @@ -373,7 +321,7 @@ def from_smpl_data( fps: float = 30.0, insert_rest_pose: bool = False, global_orient_adj: Optional[spRotation] = GLOBAL_ORIENT_ADJUSTMENT, - vector_convertor: Optional[ConverterType] = Converter.vec_humandata2smplx, + vector_convertor: Optional[ConverterType] = ConverterMotion.vec_humandata2smplx, ) -> 'SMPLMotion': """Create SMPLMotion instance from smpl_data. @@ -464,7 +412,7 @@ def from_amass_data(cls, amass_data, insert_rest_pose: bool) -> 'SMPLMotion': global_orient = (amass2humandata_adj * spRotation.from_rotvec(global_orient)).as_rotvec() # type: ignore # transl_0 = transl[0, :] # transl = amass2humandata_adj.apply(transl - transl_0) + transl_0 - transl = Converter.vec_amass2humandata(transl) + transl = ConverterMotion.vec_amass2humandata(transl) # TODO: all axis offset height_offset = transl[0, 1] @@ -607,7 +555,7 @@ def from_smplx_data( insert_rest_pose: bool = False, flat_hand_mean: bool = False, global_orient_adj: Optional[spRotation] = GLOBAL_ORIENT_ADJUSTMENT, - vector_convertor: Optional[Callable[[np.ndarray], np.ndarray]] = Converter.vec_humandata2smplx, + vector_convertor: Optional[Callable[[np.ndarray], np.ndarray]] = ConverterMotion.vec_humandata2smplx, ) -> 'SMPLXMotion': """Create SMPLXMotion instance from smplx_data. @@ -736,7 +684,7 @@ def from_amass_data(cls, amass_data, insert_rest_pose: bool, flat_hand_mean: boo global_orient = (amass2humandata_adj * spRotation.from_rotvec(global_orient)).as_rotvec() # type: ignore # transl_0 = transl[0, :] # transl = amass2humandata_adj.apply(transl - transl_0) + transl_0 - transl = Converter.vec_amass2humandata(transl) + transl = ConverterMotion.vec_amass2humandata(transl) # TODO: all axis offset height_offset = transl[0, 1] diff --git a/xrfeitoria/utils/converter.py b/xrfeitoria/utils/converter.py new file mode 100644 index 00000000..a39905f1 --- /dev/null +++ b/xrfeitoria/utils/converter.py @@ -0,0 +1,218 @@ +"""Convertor for different spaces""" + +import numpy as np + +from ..data_structure.constants import Vector + + +def rotation_matrix(angles: Vector, order='xyz', degrees: bool = True) -> np.ndarray: + """ + Args: + angles (Tuple[float, float, float]): Rotation angles in degrees or radians. + order (str, optional): Rotation order. Defaults to 'xyz'. + degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. + Returns: + ndarray: Rotation matrix 3x3. + + Examples: + >>> rotation_matrix((0, 0, 0), order='xyz') + + References: + https://programming-surgeon.com/en/euler-angle-python-en/ + """ + if degrees: + angles = np.deg2rad(angles) + + theta1, theta2, theta3 = angles + c1 = np.cos(theta1) + s1 = np.sin(theta1) + c2 = np.cos(theta2) + s2 = np.sin(theta2) + c3 = np.cos(theta3) + s3 = np.sin(theta3) + + if order == 'xzx': + matrix = np.array( + [ + [c2, -c3 * s2, s2 * s3], + [c1 * s2, c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3], + [s1 * s2, c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3], + ] + ) + elif order == 'xyx': + matrix = np.array( + [ + [c2, s2 * s3, c3 * s2], + [s1 * s2, c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1], + [-c1 * s2, c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3], + ] + ) + elif order == 'yxy': + matrix = np.array( + [ + [c1 * c3 - c2 * s1 * s3, s1 * s2, c1 * s3 + c2 * c3 * s1], + [s2 * s3, c2, -c3 * s2], + [-c3 * s1 - c1 * c2 * s3, c1 * s2, c1 * c2 * c3 - s1 * s3], + ] + ) + elif order == 'yzy': + matrix = np.array( + [ + [c1 * c2 * c3 - s1 * s3, -c1 * s2, c3 * s1 + c1 * c2 * s3], + [c3 * s2, c2, s2 * s3], + [-c1 * s3 - c2 * c3 * s1, s1 * s2, c1 * c3 - c2 * s1 * s3], + ] + ) + elif order == 'zyz': + matrix = np.array( + [ + [c1 * c2 * c3 - s1 * s3, -c3 * s1 - c1 * c2 * s3, c1 * s2], + [c1 * s3 + c2 * c3 * s1, c1 * c3 - c2 * s1 * s3, s1 * s2], + [-c3 * s2, s2 * s3, c2], + ] + ) + elif order == 'zxz': + matrix = np.array( + [ + [c1 * c3 - c2 * s1 * s3, -c1 * s3 - c2 * c3 * s1, s1 * s2], + [c3 * s1 + c1 * c2 * s3, c1 * c2 * c3 - s1 * s3, -c1 * s2], + [s2 * s3, c3 * s2, c2], + ] + ) + elif order == 'xyz': + matrix = np.array( + [ + [c2 * c3, -c2 * s3, s2], + [c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1], + [s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2], + ] + ) + elif order == 'xzy': + matrix = np.array( + [ + [c2 * c3, -s2, c2 * s3], + [s1 * s3 + c1 * c3 * s2, c1 * c2, c1 * s2 * s3 - c3 * s1], + [c3 * s1 * s2 - c1 * s3, c2 * s1, c1 * c3 + s1 * s2 * s3], + ] + ) + elif order == 'yxz': + matrix = np.array( + [ + [c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3, c2 * s1], + [c2 * s3, c2 * c3, -s2], + [c1 * s2 * s3 - c3 * s1, c1 * c3 * s2 + s1 * s3, c1 * c2], + ] + ) + elif order == 'yzx': + matrix = np.array( + [ + [c1 * c2, s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3], + [s2, c2 * c3, -c2 * s3], + [-c2 * s1, c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3], + ] + ) + elif order == 'zyx': + matrix = np.array( + [ + [c1 * c2, c1 * s2 * s3 - c3 * s1, s1 * s3 + c1 * c3 * s2], + [c2 * s1, c1 * c3 + s1 * s2 * s3, c3 * s1 * s2 - c1 * s3], + [-s2, c2 * s3, c2 * c3], + ] + ) + elif order == 'zxy': + matrix = np.array( + [ + [c1 * c3 - s1 * s2 * s3, -c2 * s1, c1 * s3 + c3 * s1 * s2], + [c3 * s1 + c1 * s2 * s3, c1 * c2, s1 * s3 - c1 * c3 * s2], + [-c2 * s3, s2, c2 * c3], + ] + ) + + return matrix + + +class ConverterMotion: + @classmethod + def vec_humandata2smplx(cls, vector: np.ndarray) -> np.ndarray: + """From humandata transl (in **OpenCV space**) to SMPLX armature's **pelvis + local space** in Blender. (The pelvis local space is designed to be the same + with **SMPL space**.) + + [right, front, up]: (-x, -z, -y) ==> (-x, z, y) + + Args: + vector (np.ndarray): of shape (N, 3) or (3,) + + Returns: + np.ndarray: of shape (N, 3) or (3,) + """ + if vector.shape == (3,): + ret = np.array([vector[0], -vector[1], -vector[2]], dtype=vector.dtype) + elif vector.ndim == 2 and vector.shape[1] == 3: + ret = np.array([vector[:, 0], -vector[:, 1], -vector[:, 2]]).T + else: + raise ValueError(f'vector.shape={vector.shape}') + return ret + + @classmethod + def vec_smplx2humandata(cls, vector: np.ndarray) -> np.ndarray: + # vice versa + return cls.vec_humandata2smplx(vector) + + @classmethod + def vec_amass2humandata(cls, vector: np.ndarray) -> np.ndarray: + """From amass transl (pelvis's local space) to humandata transl (in **OpenCV + space**) + + [right, front, up]: (x, y, z) ==> (-x, -z, -y) + + (CAUTION: we can see amass animation actors face back + in blender via the smplx add-on) + + Args: + vector (np.ndarray): of shape (N, 3) or (3,) + + Returns: + np.ndarray: of shape (N, 3) or (3,) + """ + if vector.shape == (3,): + vector = np.array([-vector[0], -vector[2], -vector[1]], dtype=vector.dtype) + elif vector.ndim == 2 and vector.shape[1] == 3: + vector = np.array([-vector[:, 0], -vector[:, 2], -vector[:, 1]]).T + else: + raise ValueError(f'vector.shape={vector.shape}') + return vector + + +class ConverterUnreal: + UNITS_SCALE = 100 # 1 meter = 100 cm + + @classmethod + def rotation_camera_from_ue(cls, euler, degrees=True) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention.""" + # convert to left-handed + x, y, z = -euler[1], -euler[2], -euler[0] + return rotation_matrix([x, y, z], order='xyz', degrees=degrees) + + @classmethod + def rotation_from_ue(cls, euler, degrees=True) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention.""" + return rotation_matrix(euler, 'zxy', degrees=degrees) + + @classmethod + def location_from_ue(cls, vector: np.ndarray) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention. + + [right, front, up]: (x, y, z) ==> (y, -z, x) + + Args: + vector (np.ndarray): of shape (3,) or (N, 3) + + Returns: + np.ndarray: of shape (3,) or (N, 3) + """ + if vector.shape == (3,): + ret = np.array([vector[1], -vector[2], vector[0]]) / cls.UNITS_SCALE + elif vector.ndim == 2 and vector.shape[1] == 3: + ret = np.array([vector[:, 1], -vector[:, 2], vector[:, 0]]) / cls.UNITS_SCALE + return ret diff --git a/xrfeitoria/utils/functions/unreal_functions.py b/xrfeitoria/utils/functions/unreal_functions.py index 51aaaf53..3b1b994d 100644 --- a/xrfeitoria/utils/functions/unreal_functions.py +++ b/xrfeitoria/utils/functions/unreal_functions.py @@ -1,8 +1,10 @@ """Remote functions for unreal.""" +import json +from functools import lru_cache from typing import List, Optional, Tuple, Union -from ...data_structure.constants import Vector +from ...data_structure.constants import Vector, color_type from ...rpc import remote_unreal try: @@ -11,19 +13,35 @@ except ImportError: pass +# Constants +mask_colors: List[color_type] = [] + @remote_unreal() +def get_mask_color_file() -> str: + """Returns the path of the mask color file. + + Returns: + str: The path of the mask color file. + """ + return XRFeitoriaUnrealFactory.constants.MASK_COLOR_FILE.as_posix() + + +@lru_cache def get_mask_color(stencil_value: int) -> 'Tuple[int, int, int]': - """Get mask color from stencil value. + """Retrieves the RGB color value associated with the given stencil value. Args: - stencil_value (int): stencil value + stencil_value (int): The stencil value for which to retrieve the color. Returns: - Tuple[int, int, int]: mask color. (r, g, b) in [0, 255] + Tuple[int, int, int]: The RGB color value associated with the stencil value. """ - # TODO: move this to local, not remote - return XRFeitoriaUnrealFactory.utils_actor.get_mask_color(stencil_value) + global mask_colors + if len(mask_colors) == 0: + with open(get_mask_color_file(), 'r') as f: + mask_colors = json.load(f) + return mask_colors[stencil_value]['rgb'] @remote_unreal() From 9e957169bd2896c55201dedc45551f4453561bfb Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 16:02:17 +0800 Subject: [PATCH 18/27] Update ue convention convert --- xrfeitoria/camera/camera_parameter.py | 7 ++-- xrfeitoria/data_structure/constants.py | 2 +- xrfeitoria/renderer/renderer_unreal.py | 45 +++++++++++++++----------- xrfeitoria/utils/converter.py | 45 +++++++++++++++++++------- 4 files changed, 63 insertions(+), 36 deletions(-) diff --git a/xrfeitoria/camera/camera_parameter.py b/xrfeitoria/camera/camera_parameter.py index e137329e..64a1f708 100644 --- a/xrfeitoria/camera/camera_parameter.py +++ b/xrfeitoria/camera/camera_parameter.py @@ -9,7 +9,7 @@ from xrprimer.transform.convention.camera import convert_camera_parameter from ..data_structure.constants import PathLike, Vector -from ..utils.converter import rotation_matrix +from ..utils.converter import ConverterUnreal class CameraParameter(PinholeCameraParameter): @@ -209,9 +209,8 @@ def from_unreal_convention( ) # extrinsic matrix RT - x, y, z = -rotation[1], -rotation[2], -rotation[0] - R = rotation_matrix([x, y, z], order='xyz', degrees=True) - _T = np.array([location[1], -location[2], location[0]]) / 100.0 # unit: meter + R = ConverterUnreal.rotation_camera_from_ue(rotation, degrees=True) + _T = ConverterUnreal.location_from_ue(location) T = -R @ _T # construct camera parameter diff --git a/xrfeitoria/data_structure/constants.py b/xrfeitoria/data_structure/constants.py index 707fad99..2bdaefca 100644 --- a/xrfeitoria/data_structure/constants.py +++ b/xrfeitoria/data_structure/constants.py @@ -25,7 +25,7 @@ plugin_name_blender = 'XRFeitoriaBpy' plugin_name_unreal = 'XRFeitoriaUnreal' plugin_name_pattern = '{plugin_name}-{plugin_version}-{engine_version}-{platform}' -xf_obj_name = '[XF]{obj_type}-{obj_idx:03d}' +xf_obj_name = '(XF){obj_type}-{obj_idx:03d}' ##### Path Constants ##### diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 1fe0e882..9f6eef5e 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -1,4 +1,3 @@ -import json import shutil import socket from pathlib import Path @@ -6,8 +5,9 @@ from loguru import logger -from ..data_structure.constants import PathLike, RenderOutputEnumUnreal, actor_info_type +from ..data_structure.constants import PathLike, RenderOutputEnumUnreal from ..rpc import remote_unreal +from ..utils.converter import ConverterUnreal from ..utils.functions import unreal_functions from .renderer_base import RendererBase, render_status @@ -206,17 +206,15 @@ def convert_vertices(folder: Path) -> None: vertices_files = sorted(folder.glob('*.dat')) if not vertices_files: return - # Read all vertices files into a list - vertices = [ - np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) for vertices_file in vertices_files - ] - - # Stack all vertices into one array with shape (frame, verts, 3) - vertices = np.stack(vertices) - # Convert convention from unreal to opencv, [x, y, z] -> [y, -z, x] - vertices = np.stack([vertices[:, :, 1], -vertices[:, :, 2], vertices[:, :, 0]], axis=-1) - vertices /= 100.0 # convert from cm to m - + # Read all vertices files into an ndarray, shape: (frame, vertex, 3) + vertices = np.stack( + [ + np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) + for vertices_file in vertices_files + ] + ) + # Convert from ue camera space to opencv camera space convention + vertices = ConverterUnreal.location_from_ue(vertices) # Save the vertices in a compressed `.npz` file np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) # Remove the folder @@ -236,20 +234,29 @@ def convert_actor_infos(folder: Path) -> None: # Read all actor info files into a list location = [] rotation = [] + stencil_value = [] mask_color = [] for actor_info_file in actor_info_files: with open(actor_info_file, 'rb') as f: dat = np.frombuffer(f.read(), np.float32).reshape(7) - location.append(dat[:3]) - rotation.append(dat[3:6]) + location.append(ConverterUnreal.location_from_ue(dat[:3])) + rotation.append(ConverterUnreal.rotation_from_ue(dat[3:6])) + stencil_value.append(int(dat[6])) mask_color.append(unreal_functions.get_mask_color(int(dat[6]))) - location = np.array(location) / 100.0 # convert from cm to m - rotation = np.array(rotation) - mask_color = np.array(mask_color) + location = np.stack(location) # shape: (frame, 3) + rotation = np.stack(rotation) # shape: (frame, 3, 3) + stencil_value = np.array(stencil_value) # shape: (frame,) + mask_color = np.array(mask_color) # shape: (frame, 3) # Save the actor infos in a compressed `.npz` file - np.savez_compressed(folder.with_suffix('.npz'), location=location, rotation=rotation, mask_color=mask_color) + np.savez_compressed( + file=folder.with_suffix('.npz'), + location=location, + rotation=rotation, + stencil_value=stencil_value, + mask_color=mask_color, + ) # Remove the folder shutil.rmtree(folder) diff --git a/xrfeitoria/utils/converter.py b/xrfeitoria/utils/converter.py index a39905f1..03db09bc 100644 --- a/xrfeitoria/utils/converter.py +++ b/xrfeitoria/utils/converter.py @@ -1,11 +1,13 @@ -"""Convertor for different spaces""" +"""Converter for different spaces.""" + +from typing import Union import numpy as np from ..data_structure.constants import Vector -def rotation_matrix(angles: Vector, order='xyz', degrees: bool = True) -> np.ndarray: +def rotation_matrix(angles: Union[Vector, np.ndarray], order='xyz', degrees: bool = True) -> np.ndarray: """ Args: angles (Tuple[float, float, float]): Rotation angles in degrees or radians. @@ -185,19 +187,38 @@ def vec_amass2humandata(cls, vector: np.ndarray) -> np.ndarray: class ConverterUnreal: - UNITS_SCALE = 100 # 1 meter = 100 cm + UNITS_SCALE = 100.0 # 1 meter = 100 cm + ROTATION_OFFSET = [0, 0, -90.0] # (x, y, z) in degrees, around z-axis (left-handed) @classmethod def rotation_camera_from_ue(cls, euler, degrees=True) -> np.ndarray: - """Convert from ue camera space to opencv camera space convention.""" - # convert to left-handed + """Convert from ue camera space to opencv camera space convention. + Note: convert to left-handed + + Args: + euler (np.ndarray): of shape (3,) + degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. + + Returns: + np.ndarray: Rotation matrix 3x3. + """ x, y, z = -euler[1], -euler[2], -euler[0] return rotation_matrix([x, y, z], order='xyz', degrees=degrees) @classmethod - def rotation_from_ue(cls, euler, degrees=True) -> np.ndarray: - """Convert from ue camera space to opencv camera space convention.""" - return rotation_matrix(euler, 'zxy', degrees=degrees) + def rotation_from_ue(cls, euler, offset=ROTATION_OFFSET, degrees=True) -> np.ndarray: + """Convert from ue camera space to opencv camera space convention. + + Args: + euler (np.ndarray): of shape (3,) + offset (np.ndarray, optional): of shape (3,). Defaults to ROTATION_OFFSET [0, 0, -90.0]. + degrees (bool, optional): Whether the input angles are in degrees. Defaults to True. + + Returns: + np.ndarray: Rotation matrix 3x3. + """ + _euler = np.array(euler) + np.array(offset) + return rotation_matrix(_euler, 'zxy', degrees=degrees) @classmethod def location_from_ue(cls, vector: np.ndarray) -> np.ndarray: @@ -206,13 +227,13 @@ def location_from_ue(cls, vector: np.ndarray) -> np.ndarray: [right, front, up]: (x, y, z) ==> (y, -z, x) Args: - vector (np.ndarray): of shape (3,) or (N, 3) + vector (np.ndarray): of shape (3,) or (... , 3) Returns: - np.ndarray: of shape (3,) or (N, 3) + np.ndarray: of shape (3,) or (... , 3) """ if vector.shape == (3,): ret = np.array([vector[1], -vector[2], vector[0]]) / cls.UNITS_SCALE - elif vector.ndim == 2 and vector.shape[1] == 3: - ret = np.array([vector[:, 1], -vector[:, 2], vector[:, 0]]) / cls.UNITS_SCALE + elif vector.ndim >= 2 and vector.shape[-1] == 3: + ret = np.stack([vector[..., 1], -vector[..., 2], vector[..., 0]], axis=-1) / cls.UNITS_SCALE return ret From c4bd6403707203bc124085118ea79059feb1294c Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 17:48:19 +0800 Subject: [PATCH 19/27] Update `refine_smpl_x_from_actor_info` --- samples/unreal/07_amass.py | 46 +++++++++------ xrfeitoria/data_structure/constants.py | 2 +- xrfeitoria/renderer/renderer_unreal.py | 60 +++++++++++-------- xrfeitoria/utils/anim/__init__.py | 8 ++- xrfeitoria/utils/anim/motion.py | 3 +- xrfeitoria/utils/anim/utils.py | 80 ++++++++++++++++++++++++-- 6 files changed, 151 insertions(+), 48 deletions(-) diff --git a/samples/unreal/07_amass.py b/samples/unreal/07_amass.py index e0e8ae22..7d2774f5 100644 --- a/samples/unreal/07_amass.py +++ b/samples/unreal/07_amass.py @@ -8,36 +8,37 @@ SMPL-XL: a parametric human model based on SMPL-X in a layered representation, introduced in https://synbody.github.io/ Amass: a large database of human motion, introduced in https://amass.is.tue.mpg.de/ """ + from pathlib import Path import xrfeitoria as xf from xrfeitoria.data_structure.models import RenderPass from xrfeitoria.utils import setup_logger -from xrfeitoria.utils.anim import dump_humandata, load_amass_motion +from xrfeitoria.utils.anim import dump_humandata, load_amass_motion, refine_smpl_x_from_actor_info from ..config import unreal_exec, unreal_project # prepare the assets #################### -root = Path('.cache/sample-amass').resolve() # modify this to your own path +assets_root = Path('.cache/sample-amass').resolve() # modify this to your own path # 1. Download Amass from https://amass.is.tue.mpg.de/download.php # For example, download ACCAD (SMPL-X N) from https://download.is.tue.mpg.de/download.php?domain=amass&sfile=amass_per_dataset/smplx/neutral/mosh_results/ACCAD.tar.bz2 # and use `ACCAD/s001/EricCamper04_stageii.npz` from the uncompressed folder -amass_file = root / 'EricCamper04_stageii.npz' +amass_file = assets_root / 'EricCamper04_stageii.npz' # 2.1 Download SMPL-XL model from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.fbx # or from https://openxlab.org.cn/datasets/OpenXDLab/SynBody/tree/main/Assets # With downloading this, you are agreeing to CC BY-NC-SA 4.0 License (https://creativecommons.org/licenses/by-nc-sa/4.0/). -smpl_xl_file = root / 'SMPL-XL-001.fbx' +smpl_xl_file = assets_root / 'SMPL-XL-001.fbx' # 2.2 Download the meta information from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.npz -smpl_xl_meta_file = root / 'SMPL-XL-001.npz' +smpl_xl_meta_file = assets_root / 'SMPL-XL-001.npz' # 3. Define the output file path -seq_name = 'seq_amass' -output_path = Path(__file__).resolve().parents[2] / 'output/samples/unreal' / Path(__file__).stem -output_path.mkdir(parents=True, exist_ok=True) -saved_humandata_file = output_path / 'output.npz' +seq_name = Path(__file__).stem +output_path = Path(__file__).resolve().parents[2] / 'output/samples/unreal' +seq_dir = output_path / seq_name +saved_humandata_file = seq_dir / 'smplx' / 'output.npz' def main(background: bool = False): @@ -45,27 +46,31 @@ def main(background: bool = False): motion = load_amass_motion(amass_file) motion.convert_fps(30) # convert the motion from 120fps (amass) to 30fps + motion.cut_motion(end_frame=10) # cut the motion to 10 frames, for demonstration purpose motion_data = motion.get_motion_data() xf_runner = xf.init_unreal(exec_path=unreal_exec, project_path=unreal_project, background=background) # Import SMPL-XL model - actor_path = xf_runner.utils.import_asset(smpl_xl_file) + actor_path = xf_runner.utils.import_asset(smpl_xl_file, replace=False) - with xf_runner.Sequence.new(seq_name=seq_name, level='/Game/Levels/Playground', seq_length=motion.n_frames) as seq: + with xf_runner.sequence( + seq_name=seq_name, level='/Game/Levels/Playground', seq_length=motion.n_frames, replace=True + ) as seq: seq.show() # Spawn the actor, and add motion data as FK animation actor = seq.spawn_actor( actor_asset_path=actor_path, - location=(0, 0, 0), + location=(3, 0, 0), rotation=(0, 0, 0), stencil_value=1, motion_data=motion_data, ) + actor_name = actor.name camera = seq.spawn_camera( - location=(0, 2.5, 0.6), + location=(3, 2.5, 0.6), rotation=(0, 0, -90), ) @@ -73,17 +78,26 @@ def main(background: bool = False): seq.add_to_renderer( output_path=output_path, resolution=(1920, 1080), - render_passes=[RenderPass('img', 'png')], + render_passes=[RenderPass('img', 'jpg')], + export_skeleton=True, + export_vertices=True, ) # Save the motion data as annotation in humandata format defined in https://github.com/open-mmlab/mmhuman3d/blob/main/docs/human_data.md - dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file) + dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file, actor_name=actor_name) # render xf_runner.render() + # refine smplx parameters + refine_smpl_x_from_actor_info( + smpl_x_file=saved_humandata_file, + actor_info_file=seq_dir / 'actor_infos' / f'{actor_name}.npz', + replace_smpl_x_file=True, + ) + logger.info('🎉 [bold green]Success!') - output_img = output_path / seq_name / 'img' / camera.name / '0000.png' + output_img = seq_dir / 'img' / camera.name / '0000.png' if output_img.exists(): logger.info(f'Check the output in "{output_img.as_posix()}"') if not background: diff --git a/xrfeitoria/data_structure/constants.py b/xrfeitoria/data_structure/constants.py index 2bdaefca..ed5b7359 100644 --- a/xrfeitoria/data_structure/constants.py +++ b/xrfeitoria/data_structure/constants.py @@ -25,7 +25,7 @@ plugin_name_blender = 'XRFeitoriaBpy' plugin_name_unreal = 'XRFeitoriaUnreal' plugin_name_pattern = '{plugin_name}-{plugin_version}-{engine_version}-{platform}' -xf_obj_name = '(XF){obj_type}-{obj_idx:03d}' +xf_obj_name = 'XF-{obj_type}-{obj_idx:03d}' ##### Path Constants ##### diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 9f6eef5e..4242d189 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -195,35 +195,20 @@ def convert_camera(camera_file: Path) -> None: cam_param.dump(camera_file.with_suffix('.json').as_posix()) camera_file.unlink() - def convert_vertices(folder: Path) -> None: - """Convert vertices from `.dat` to `.npz`. Merge all vertices files into one - `.npz` file with structure of: {'verts': np.ndarray, 'faces': None} - - Args: - folder (Path): Path to the folder containing vertices files. - """ - # Get all vertices files in the folder and sort them - vertices_files = sorted(folder.glob('*.dat')) - if not vertices_files: - return - # Read all vertices files into an ndarray, shape: (frame, vertex, 3) - vertices = np.stack( - [ - np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) - for vertices_file in vertices_files - ] - ) - # Convert from ue camera space to opencv camera space convention - vertices = ConverterUnreal.location_from_ue(vertices) - # Save the vertices in a compressed `.npz` file - np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) - # Remove the folder - shutil.rmtree(folder) - def convert_actor_infos(folder: Path) -> None: """Convert stencil value from `.dat` to `.npz`. Merge all actor info files into one. + actor_info files are in the format of: + ``` + { + 'location': np.ndarray, # shape: (frame, 3) + 'rotation': np.ndarray, # shape: (frame, 3, 3) + 'stencil_value': np.ndarray, # shape: (frame,) + 'mask_color': np.ndarray, # shape: (frame, 3) + } + ``` + Args: folder (Path): Path to the folder containing actor info files. """ @@ -260,6 +245,31 @@ def convert_actor_infos(folder: Path) -> None: # Remove the folder shutil.rmtree(folder) + def convert_vertices(folder: Path) -> None: + """Convert vertices from `.dat` to `.npz`. Merge all vertices files into one + `.npz` file with structure of: {'verts': np.ndarray, 'faces': None} + + Args: + folder (Path): Path to the folder containing vertices files. + """ + # Get all vertices files in the folder and sort them + vertices_files = sorted(folder.glob('*.dat')) + if not vertices_files: + return + # Read all vertices files into an ndarray, shape: (frame, vertex, 3) + vertices = np.stack( + [ + np.frombuffer(vertices_file.read_bytes(), np.float32).reshape(-1, 3) + for vertices_file in vertices_files + ] + ) + # Convert from ue camera space to opencv camera space convention + vertices = ConverterUnreal.location_from_ue(vertices) + # Save the vertices in a compressed `.npz` file + np.savez_compressed(folder.with_suffix('.npz'), verts=vertices, faces=None) + # Remove the folder + shutil.rmtree(folder) + console = get_console() try: spinner: Spinner = console._live.renderable diff --git a/xrfeitoria/utils/anim/__init__.py b/xrfeitoria/utils/anim/__init__.py index 9e9e61a3..94332d75 100644 --- a/xrfeitoria/utils/anim/__init__.py +++ b/xrfeitoria/utils/anim/__init__.py @@ -1 +1,7 @@ -from .utils import dump_humandata, load_amass_motion, load_humandata_motion +from .utils import ( + dump_humandata, + load_amass_motion, + load_humandata_motion, + refine_smpl_x, + refine_smpl_x_from_actor_info, +) diff --git a/xrfeitoria/utils/anim/motion.py b/xrfeitoria/utils/anim/motion.py index 46443d05..daccd2c1 100644 --- a/xrfeitoria/utils/anim/motion.py +++ b/xrfeitoria/utils/anim/motion.py @@ -194,7 +194,8 @@ def sample_motion(self, n_frames: int): self.insert_rest_pose() def cut_motion(self, start_frame: Optional[int] = None, end_frame: Optional[int] = None): - """Cut the motion sequence to a given number of frames. + """Cut the motion sequence to a given number of frames (to [start_frame, + end_frame]) Args: start_frame (Optional[int], optional): The start frame to cut to. Defaults to None. diff --git a/xrfeitoria/utils/anim/utils.py b/xrfeitoria/utils/anim/utils.py index 0e5b4afa..b3dc548b 100644 --- a/xrfeitoria/utils/anim/utils.py +++ b/xrfeitoria/utils/anim/utils.py @@ -1,10 +1,12 @@ """Utilities for animation data loading and dumping.""" from pathlib import Path -from typing import Union +from typing import Optional, Union import numpy as np +from scipy.spatial.transform import Rotation as spRotation +from ...actor.actor_base import ActorBase from ...data_structure.constants import PathLike from .motion import SMPLMotion, SMPLXMotion @@ -58,7 +60,12 @@ def load_humandata_motion(input_humandata_path: PathLike) -> Union[SMPLMotion, S return src_motion -def dump_humandata(motion: Union[SMPLMotion, SMPLXMotion], save_filepath: PathLike, meta_filepath: PathLike) -> None: +def dump_humandata( + motion: Union[SMPLMotion, SMPLXMotion], + save_filepath: PathLike, + meta_filepath: PathLike, + actor_name: Optional[str] = None, +) -> None: """Dump human data to a file. This function must be associate with a meta file provided by SMPL-XL. @@ -66,6 +73,7 @@ def dump_humandata(motion: Union[SMPLMotion, SMPLXMotion], save_filepath: PathLi motion (Union[SMPLMotion, SMPLXMotion]): The motion data to be dumped. save_filepath (PathLike): The file path to save the dumped data. meta_filepath (PathLike): The file path to the meta information, storing the parameters of the SMPL-XL model. + actor_name (Optional[str], optional): The name of the actor. Defaults to None. Note: HumanData is a structure of smpl/smplx data defined in https://github.com/open-mmlab/mmhuman3d/blob/main/docs/human_data.md @@ -88,7 +96,7 @@ def dump_humandata(motion: Union[SMPLMotion, SMPLXMotion], save_filepath: PathLi 'right_hand_pose': right_hand_pose, # (n_frames, 45) 'expression': expression, # (n_frames, 10) }, - 'meta': {'gender': 'neutral'}, # optional + 'meta': {'gender': 'neutral', 'actor_name': '(XF)actor-001'}, # optional } """ meta_info = np.load(meta_filepath, allow_pickle=True) @@ -96,10 +104,15 @@ def dump_humandata(motion: Union[SMPLMotion, SMPLXMotion], save_filepath: PathLi smpl_x = meta_info['smplx'].item() elif 'smpl' in meta_info.keys(): smpl_x = meta_info['smpl'].item() + + _meta_ = meta_info['meta'].item() + if actor_name: + _meta_['actor_name'] = actor_name + motion.dump_humandata( filepath=save_filepath, betas=smpl_x['betas'], - meta=meta_info['meta'].item(), + meta=_meta_, global_orient_offset=smpl_x['global_orient'], transl_offset=smpl_x['transl'], root_location_t0=smpl_x['root_location_t0'], @@ -107,6 +120,65 @@ def dump_humandata(motion: Union[SMPLMotion, SMPLXMotion], save_filepath: PathLi ) +def refine_smpl_x( + smpl_x_file: Path, + replace_smpl_x_file: bool = False, + offset_location: np.ndarray = np.zeros(3), + offset_rotation: np.ndarray = np.eye(3), +) -> None: + """Refine translation and rotation of SMPL-X parameters.""" + + # Load SMPL-X data + smpl_x_file = Path(smpl_x_file) + data = dict(np.load(smpl_x_file, allow_pickle=True)) + if 'smplx' in data.keys(): + smpl_x_type = 'smplx' + elif 'smpl' in data.keys(): + smpl_x_type = 'smpl' + else: + raise ValueError(f'Unknown keys in {smpl_x_file}: {data.keys()}') + + # Convert offset_rotation + if offset_rotation.shape == (3, 3): + offset_rotation = spRotation.from_matrix(offset_rotation) + elif offset_rotation.shape == (4,): + offset_rotation = spRotation.from_quat(offset_rotation) + else: + raise ValueError('Please convert offset_rotation to 3x3 matrix or 4-dim quaternion.') + + smpl_x_data = data[smpl_x_type].item() + global_orient = smpl_x_data['global_orient'] + global_orient = spRotation.from_rotvec(global_orient) + global_orient = (offset_rotation * global_orient).as_rotvec() + transl = smpl_x_data['transl'] + transl = offset_rotation.apply(transl - transl[0]) + transl[0] + transl += offset_location + + smpl_x_data['global_orient'] = global_orient.astype(np.float32) + smpl_x_data['transl'] = transl.astype(np.float32) + data[smpl_x_type] = smpl_x_data + + if replace_smpl_x_file: + np.savez(smpl_x_file, **data) + else: + np.savez(smpl_x_file.parent / f'{smpl_x_file.stem}_refined.npz', **data) + + +def refine_smpl_x_from_actor_info(smpl_x_file: Path, actor_info_file: Path, replace_smpl_x_file: bool = False): + """Refine translation and rotation of SMPL-X parameters from actor info file.""" + actor_info = np.load(actor_info_file, allow_pickle=True) + location = actor_info['location'] + rotation = actor_info['rotation'] + assert np.all(location == location[0]) and np.all(rotation == rotation[0]) + + refine_smpl_x( + smpl_x_file=smpl_x_file, + replace_smpl_x_file=replace_smpl_x_file, + offset_location=location[0], + offset_rotation=rotation[0], + ) + + if __name__ == '__main__': """Python -m xrfeitoria.utils.anim.utils.""" motion = load_amass_motion('.cache/ACCAD/s001/EricCamper04_stageii.npz') From bc7ee9c113c82cf683a8993cadb31d834dbc8048 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 17:49:45 +0800 Subject: [PATCH 20/27] `@deprecated` --- xrfeitoria/factory.py | 24 ++++++++++++++++++++---- xrfeitoria/sequence/sequence_wrapper.py | 4 ++-- xrfeitoria/sequence/sequence_wrapper.pyi | 8 ++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/xrfeitoria/factory.py b/xrfeitoria/factory.py index b778ace5..faf97adb 100644 --- a/xrfeitoria/factory.py +++ b/xrfeitoria/factory.py @@ -1,5 +1,7 @@ from typing import Optional +from typing_extensions import deprecated + from . import _tls from .data_structure.constants import EngineEnum, PathLike, default_level_blender from .utils import setup_logger @@ -51,7 +53,7 @@ def __init__( from .actor.actor_blender import ActorBlender, ShapeBlenderWrapper # isort:skip from .material.material_blender import MaterialBlender # isort:skip from .renderer.renderer_blender import RendererBlender # isort:skip - from .sequence.sequence_wrapper import SequenceWrapperBlender, sequence_wrapper_blender # isort:skip + from .sequence.sequence_wrapper import sequence_wrapper_blender # isort:skip from .utils.runner import BlenderRPCRunner # isort:skip from .utils.functions import blender_functions # isort:skip @@ -63,8 +65,8 @@ def __init__( self.Shape = ShapeBlenderWrapper self.Renderer = RendererBlender self.render = self.Renderer.render_jobs + # self.Sequence = SequenceWrapperBlender self.sequence = sequence_wrapper_blender - self.Sequence = SequenceWrapperBlender self.utils = blender_functions self._rpc_runner = BlenderRPCRunner( engine_exec=engine_exec, @@ -76,6 +78,13 @@ def __init__( new_process=new_process, ) + @property + @deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) + def Sequence(self): + from .sequence.sequence_wrapper import SequenceWrapperBlender + + return SequenceWrapperBlender + class XRFeitoriaUnreal: """Factory class contains all the classes and functions for Unreal. @@ -119,7 +128,7 @@ def __init__( from .camera.camera_unreal import CameraUnreal # isort:skip from .actor.actor_unreal import ActorUnreal, ShapeUnrealWrapper # isort:skip from .renderer.renderer_unreal import RendererUnreal # isort:skip - from .sequence.sequence_wrapper import SequenceWrapperUnreal, sequence_wrapper_unreal # isort:skip + from .sequence.sequence_wrapper import sequence_wrapper_unreal # isort:skip from .utils.runner import UnrealRPCRunner # isort:skip from .utils.functions import unreal_functions # isort:skip @@ -130,8 +139,8 @@ def __init__( self.Shape = ShapeUnrealWrapper self.Renderer = RendererUnreal self.render = self.Renderer.render_jobs + # self.Sequence = SequenceWrapperUnreal self.sequence = sequence_wrapper_unreal - self.Sequence = SequenceWrapperUnreal self.utils = unreal_functions self._rpc_runner = UnrealRPCRunner( engine_exec=engine_exec, @@ -143,6 +152,13 @@ def __init__( new_process=new_process, ) + @property + @deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) + def Sequence(self): + from .sequence.sequence_wrapper import SequenceWrapperUnreal + + return SequenceWrapperUnreal + class init_blender(XRFeitoriaBlender): """Initialize Blender with ``XRFeitoria``, which would start Blender as RPC server, diff --git a/xrfeitoria/sequence/sequence_wrapper.py b/xrfeitoria/sequence/sequence_wrapper.py index b5104e14..6f44c5de 100644 --- a/xrfeitoria/sequence/sequence_wrapper.py +++ b/xrfeitoria/sequence/sequence_wrapper.py @@ -15,7 +15,7 @@ __all__ = ['sequence_wrapper_blender', 'sequence_wrapper_unreal'] -@deprecated('Use `sequence` function instead.', category=DeprecationWarning) +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperBlender: """Sequence utils class.""" @@ -75,7 +75,7 @@ def open(cls, seq_name: str) -> ContextManager[SequenceBase]: cls._seq.close() -@deprecated('Use `sequence` function instead.', category=DeprecationWarning) +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperUnreal: """Sequence utils class for Unreal.""" diff --git a/xrfeitoria/sequence/sequence_wrapper.pyi b/xrfeitoria/sequence/sequence_wrapper.pyi index d4a6075d..5827a0a4 100644 --- a/xrfeitoria/sequence/sequence_wrapper.pyi +++ b/xrfeitoria/sequence/sequence_wrapper.pyi @@ -1,18 +1,25 @@ from typing import ContextManager, List, Optional, Union +from typing_extensions import deprecated + from ..data_structure.constants import default_level_blender from .sequence_blender import SequenceBlender as SequenceBlender from .sequence_unreal import SequenceUnreal as SequenceUnreal +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperBlender: + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def new( cls, seq_name: str, level: str = ..., seq_fps: int = ..., seq_length: int = ..., replace: bool = ... ) -> ContextManager[SequenceBlender]: ... + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def open(cls, seq_name: str) -> ContextManager[SequenceBlender]: ... +@deprecated('Use `xf_runner.sequence` function instead.', category=DeprecationWarning) class SequenceWrapperUnreal: + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def new( cls, @@ -23,6 +30,7 @@ class SequenceWrapperUnreal: replace: bool = ..., seq_dir: Optional[str] = ..., ) -> ContextManager[SequenceUnreal]: ... + @deprecated('This class is deprecated, use `xf_runner.sequence` function instead.', category=DeprecationWarning) @classmethod def open(cls, seq_name: str, seq_dir: Optional[str] = ...) -> ContextManager[SequenceUnreal]: ... From 50c0f010c2e1c7a56086b6ab91a2336f5c3536da Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 18:07:19 +0800 Subject: [PATCH 21/27] Update file_name_format in custom_movie_pipeline.py and add warning message in renderer_unreal.py --- src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py | 2 +- xrfeitoria/renderer/renderer_unreal.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py b/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py index 7fddcdb7..35f2c8cd 100644 --- a/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py +++ b/src/XRFeitoriaUnreal/Content/Python/custom_movie_pipeline.py @@ -522,7 +522,7 @@ def main(): sequence_path='/Game/Sequences/NewSequence', resolution=[1920, 1080], output_path='E:/Datasets/tmp', - file_name_format='{sequence_name}/{render_pass}/{frame_number}', + file_name_format='{sequence_name}/{render_pass}/{camera_name}/{frame_number}', console_variables={'r.MotionBlurQuality': 0.0}, anti_alias={'enable': False}, ) diff --git a/xrfeitoria/renderer/renderer_unreal.py b/xrfeitoria/renderer/renderer_unreal.py index 4242d189..64d15d79 100644 --- a/xrfeitoria/renderer/renderer_unreal.py +++ b/xrfeitoria/renderer/renderer_unreal.py @@ -282,6 +282,10 @@ def convert_vertices(folder: Path) -> None: seq_name = job.sequence_path.split('/')[-1] seq_path = Path(job.output_path).resolve() / seq_name file_name_format = job.file_name_format # TODO: use this to rename the files + if file_name_format != '{sequence_name}/{render_pass}/{camera_name}/{frame_number}': # XXX: hard-coded + logger.warning( + 'The `file_name_format` in renderer is not the default value, which may cause some issues in post-processing. ' + ) text = f'job {idx + 1}/{len(cls.render_queue)}: seq_name="{seq_name}", post-processing...' spinner.update(text=text) From 5426e20cb5e744cdc5873da91a112ac238ac525e Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 19:22:08 +0800 Subject: [PATCH 22/27] Refactor factory.py to use TYPE_CHECKING for deprecated import --- xrfeitoria/factory.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/xrfeitoria/factory.py b/xrfeitoria/factory.py index faf97adb..2ad0a1cc 100644 --- a/xrfeitoria/factory.py +++ b/xrfeitoria/factory.py @@ -1,11 +1,14 @@ -from typing import Optional - -from typing_extensions import deprecated +from typing import TYPE_CHECKING, Optional from . import _tls from .data_structure.constants import EngineEnum, PathLike, default_level_blender from .utils import setup_logger +if TYPE_CHECKING: + from typing_extensions import deprecated +else: + deprecated = lambda *args, **kwargs: (lambda func: func) + __all__ = ['init_blender', 'init_unreal'] From d168cae6c86687f065330f20bec87b9f0c077c16 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 19:56:41 +0800 Subject: [PATCH 23/27] bug fix --- samples/utils.py | 5 ++++- xrfeitoria/object/object_utils.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/samples/utils.py b/samples/utils.py index 915b4357..be275212 100644 --- a/samples/utils.py +++ b/samples/utils.py @@ -38,7 +38,10 @@ def visualize_vertices(camera_name, actor_names: List[str], seq_output_path: Pat logger.info('Visualizing vertices') # fixed file structure img_path = seq_output_path / 'img' / camera_name / f'{frame_idx:04d}.png' - camera_param_json = seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json' + if (seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json').exists(): + camera_param_json = seq_output_path / 'camera_params' / camera_name / f'{frame_idx:04d}.json' + else: + camera_param_json = seq_output_path / 'camera_params' / f'{camera_name}.json' # load img and camera parameters img = np.array(Image.open(img_path.as_posix())) diff --git a/xrfeitoria/object/object_utils.py b/xrfeitoria/object/object_utils.py index 7ee5eae5..21e407e4 100644 --- a/xrfeitoria/object/object_utils.py +++ b/xrfeitoria/object/object_utils.py @@ -411,7 +411,7 @@ def _generate_obj_name_in_engine(obj_type: 'Literal["camera", "actor"]') -> str: Returns: str: Name of the new object. """ - objs = [obj for obj in bpy.data.objects if obj_type in obj.name and obj.name.startswith(xf_obj_name[:4])] + objs = [obj for obj in bpy.data.objects if obj_type in obj.name and obj.name.startswith(xf_obj_name[:3])] # return f'[XF]{obj_type}-{collection.name}-{(len(objs)+1):03}' return xf_obj_name.format(obj_type=obj_type, obj_idx=(len(objs) + 1)) @@ -670,7 +670,7 @@ def _generate_obj_name_in_engine(obj_type: 'Literal["camera", "actor"]') -> str: actors = [ actor for actor in actors - if obj_type in actor.get_actor_label() and actor.get_actor_label().startswith(xf_obj_name[:4]) + if obj_type in actor.get_actor_label() and actor.get_actor_label().startswith(xf_obj_name[:3]) ] return xf_obj_name.format(obj_type=obj_type, obj_idx=(len(actors) + 1)) From be73ca330e5d7ff1ea46d77183c88143a46f9c36 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 22:17:53 +0800 Subject: [PATCH 24/27] Update depth material path in constants.py --- src/XRFeitoriaUnreal/Content/Python/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XRFeitoriaUnreal/Content/Python/constants.py b/src/XRFeitoriaUnreal/Content/Python/constants.py index 97e0c662..2616a5a3 100644 --- a/src/XRFeitoriaUnreal/Content/Python/constants.py +++ b/src/XRFeitoriaUnreal/Content/Python/constants.py @@ -20,7 +20,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]: PLUGIN_NAME = 'XRFeitoriaUnreal' MATERIAL_PATHS = { - 'depth': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_depth_EXR', + 'depth': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_depth', 'mask': f'/{PLUGIN_NAME}/Materials/MRQ/PPM_mask_MRQ', 'flow': f'/{PLUGIN_NAME}/Materials/PPM_velocity', 'diffuse': f'/{PLUGIN_NAME}/Materials/PPM_diffusecolor', From da56afa191234c07b75d05cd4667262535c3d5a0 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Mon, 18 Mar 2024 23:54:47 +0800 Subject: [PATCH 25/27] Add `inverse` option to get_depth method --- xrfeitoria/utils/viewer.py | 47 +++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/xrfeitoria/utils/viewer.py b/xrfeitoria/utils/viewer.py index 31fc8363..b6f2b65a 100644 --- a/xrfeitoria/utils/viewer.py +++ b/xrfeitoria/utils/viewer.py @@ -1,5 +1,7 @@ """Utils for loading images and annotations.""" + import os +from functools import lru_cache from pathlib import Path from typing import List, Tuple, Union @@ -72,20 +74,27 @@ def get_flow(self) -> np.ndarray: img = flow_vis.flow_to_color(flow, convert_to_bgr=False) return img - def get_depth(self, depth_rescale: float = 1.0) -> np.ndarray: + def get_depth(self, inverse: bool = False, depth_rescale: float = 1.0) -> np.ndarray: """Get depth in `.exr` format. Args: - depth_rescale (float, optional): scaling the depth - to map it into (0, 255). Depth values great than - `depth_rescale` will be clipped. Defaults to 1.0. + inverse (bool, optional): whether to inverse the depth. + If True, white (255) represents the farthest, and black (0) represents the nearest. + if False, white (255) represents the nearest, and black (0) represents the farthest. + Defaults to False. + depth_rescale (float, optional): scaling the depth to map it into (0, 255). + ``depth = depth / depth_rescale``. + Depth values greater than `depth_rescale` will be clipped. Defaults to 1.0. Returns: np.ndarray: depth data of shape (H, W, 3) """ depth = self.exr_mat img = self.float2int(depth / depth_rescale) - img[img == 0] = 255 + if inverse: + img = 255 - img + else: + img[img == 0] = 255 return img @@ -120,6 +129,17 @@ def __init__(self, sequence_dir: PathLike) -> None: """ self.sequence_dir = Path(sequence_dir) + @property + @lru_cache + def camera_names(self) -> List[str]: + camera_folders = list(self.sequence_dir.glob(f'{self.IMG}/*')) + return [camera_folder.name for camera_folder in camera_folders] + + @property + @lru_cache + def frame_num(self) -> int: + return len(list(self.sequence_dir.glob(f'{self.IMG}/{self.camera_names[0]}/*'))) + def get_img(self, camera_name: str, frame: int) -> np.ndarray: """Get rgb image of the given frame ('img/{frame:04d}.*') @@ -189,15 +209,19 @@ def get_mask(self, camera_name: str, frame: int) -> np.ndarray: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) return img - def get_depth(self, camera_name: str, frame: int, depth_rescale=1.0) -> np.ndarray: + def get_depth(self, camera_name: str, frame: int, inverse: bool = False, depth_rescale: float = 1.0) -> np.ndarray: """Get depth of the given frame ('depth/{frame:04d}.*') Args: camera_name (str): the camera name frame (int): the frame number - depth_rescale (float, optional): scaling the depth - to map it into (0, 255). Depth values great than - `depth_rescale` will be clipped. Defaults to 1.0. + inverse (bool, optional): whether to inverse the depth. + If True, white (255) represents the farthest, and black (0) represents the nearest. + if False, white (255) represents the nearest, and black (0) represents the farthest. + Defaults to False. + depth_rescale (float, optional): scaling the depth to map it into (0, 255). + ``depth = depth / depth_rescale``. + Depth values greater than `depth_rescale` will be clipped. Defaults to 1.0. Returns: np.ndarray: depth of shape (H, W, 3) @@ -209,7 +233,7 @@ def get_depth(self, camera_name: str, frame: int, depth_rescale=1.0) -> np.ndarr if not file_path.exists(): raise ValueError(f'Depth of {frame}-frame not found: {file_path}') if file_path.suffix == '.exr': - return ExrReader(file_path).get_depth(depth_rescale=depth_rescale) + return ExrReader(file_path).get_depth(inverse=inverse, depth_rescale=depth_rescale) else: img = cv2.imread(str(file_path)) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) @@ -221,9 +245,6 @@ def get_flow(self, camera_name: str, frame: int) -> np.ndarray: Args: camera_name (str): the camera name frame (int): the frame number - depth_rescale (float, optional): scaling the depth - to map it into (0, 255). Depth values great than - `depth_rescale` will be clipped. Defaults to 1.0. Returns: np.ndarray: optical flow of shape (H, W, 3) From 33e905da973cfe09809deabe8fc3cca02773ca81 Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Tue, 19 Mar 2024 00:12:28 +0800 Subject: [PATCH 26/27] Refactor file paths and update motion duration --- samples/blender/07_amass.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/samples/blender/07_amass.py b/samples/blender/07_amass.py index 602fd8ad..01f111b6 100644 --- a/samples/blender/07_amass.py +++ b/samples/blender/07_amass.py @@ -21,26 +21,26 @@ # prepare the assets #################### -root = Path('.cache/sample-amass').resolve() # modify this to your own path +asset_root = Path('.cache/sample-amass').resolve() # modify this to your own path # 1. Download Amass from https://amass.is.tue.mpg.de/download.php # For example, download ACCAD (SMPL-X N) from https://download.is.tue.mpg.de/download.php?domain=amass&sfile=amass_per_dataset/smplx/neutral/mosh_results/ACCAD.tar.bz2 # and use `ACCAD/s001/EricCamper04_stageii.npz` from the uncompressed folder -amass_file = root / 'EricCamper04_stageii.npz' +amass_file = asset_root / 'EricCamper04_stageii.npz' # 2.1 Download SMPL-XL model from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.fbx # or from https://openxlab.org.cn/datasets/OpenXDLab/SynBody/tree/main/Assets # With downloading this, you are agreeing to CC BY-NC-SA 4.0 License (https://creativecommons.org/licenses/by-nc-sa/4.0/). -smpl_xl_file = root / 'SMPL-XL-001.fbx' +smpl_xl_file = asset_root / 'SMPL-XL-001.fbx' # 2.2 Download the meta information from https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/assets/SMPL-XL-001.npz -smpl_xl_meta_file = root / 'SMPL-XL-001.npz' +smpl_xl_meta_file = asset_root / 'SMPL-XL-001.npz' # 3. Define the output file path -seq_name = 'seq_amass' -output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender' / Path(__file__).stem -output_path.mkdir(parents=True, exist_ok=True) -saved_humandata_file = output_path / seq_name / 'output.npz' -saved_blend_file = output_path / seq_name / 'output.blend' +seq_name = Path(__file__).stem +output_path = Path(__file__).resolve().parents[2] / 'output/samples/blender' +seq_dir = output_path / seq_name +saved_humandata_file = seq_dir / 'smplx' / 'output.npz' +saved_blend_file = seq_dir / 'output.blend' @remote_blender() @@ -57,6 +57,7 @@ def main(background: bool = False): motion = load_amass_motion(amass_file) motion.convert_fps(30) # convert the motion from 120fps (amass) to 30fps + motion.cut_motion(end_frame=10) # cut the motion to 10 frames, for demonstration purpose motion_data = motion.get_motion_data() # modify this to your blender executable path @@ -64,7 +65,7 @@ def main(background: bool = False): exec_path='C:/Program Files/Blender Foundation/Blender 3.6/blender.exe', background=background ) - with xf_runner.Sequence.new(seq_name=seq_name, seq_length=motion.n_frames) as seq: + with xf_runner.sequence(seq_name=seq_name, seq_length=motion.n_frames) as seq: # Import SMPL-XL model actor = xf_runner.Actor.import_from_file(smpl_xl_file) apply_scale(actor.name) # SMPL-XL model is imported with scale, we need to apply scale to it From ae7a2f1afb593abec765a579d45353a6a1f0c4af Mon Sep 17 00:00:00 2001 From: HaiyiMei Date: Tue, 19 Mar 2024 17:55:55 +0800 Subject: [PATCH 27/27] get rid of `os` --- xrfeitoria/utils/functions/blender_functions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xrfeitoria/utils/functions/blender_functions.py b/xrfeitoria/utils/functions/blender_functions.py index a7621c16..49d2fc6a 100644 --- a/xrfeitoria/utils/functions/blender_functions.py +++ b/xrfeitoria/utils/functions/blender_functions.py @@ -1,6 +1,5 @@ """Remote functions for blender.""" -import os from pathlib import Path from typing import Dict, List, Literal, Optional, Tuple @@ -141,7 +140,7 @@ def save_blend(save_path: 'PathLike' = None, pack: bool = False): # path.resolve() would do Network Drive Handling like: # X:/path/to/file.blend -> //xxx.xxx.xxx.xxx/Drive/path/to/file.blend # which made Blender failed to save (Windows) - save_path = Path(os.path.abspath(save_path)) + save_path = Path(save_path).absolute() # set suffix to .blend if save_path.suffix != '.blend': save_path = save_path.with_suffix('.blend')