From ff13f7ce728276f13f4062859007311b1b89b580 Mon Sep 17 00:00:00 2001 From: Zeyu Ma Date: Tue, 22 Aug 2023 19:26:17 -0400 Subject: [PATCH 1/3] optimize_terrain_diskusage_flag --- worldgen/core.py | 48 ++++++++++++++++++++-------------------- worldgen/terrain/core.py | 5 ++++- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/worldgen/core.py b/worldgen/core.py index 8b9dc3422..188534994 100644 --- a/worldgen/core.py +++ b/worldgen/core.py @@ -267,6 +267,7 @@ def execute_tasks( reset_assets=True, focal_length=None, dryrun=False, + optimize_terrain_diskusage=True, ): if input_folder != output_folder: if reset_assets: @@ -334,38 +335,37 @@ def execute_tasks( if Task.FineTerrain in task: terrain = Terrain(scene_seed, surface.registry, task=task, on_the_fly_asset_folder=output_folder/"assets") terrain.fine_terrain(output_folder) - + if not optimize_terrain_diskusage: + terrain.load_glb(output_folder, delete=True) + group_collections() - if task == [Task.FineTerrain]: - os.symlink(input_folder / output_blend_name, output_folder / output_blend_name) - os.symlink(input_folder / "MaskTag.json", output_folder / "MaskTag.json") - os.symlink(input_folder / "version.txt", output_folder / "version.txt") - os.symlink(input_folder / 'polycounts.txt', output_folder / 'polycounts.txt') - else: - if input_folder is not None: - for mesh in os.listdir(input_folder): - if (mesh.endswith(".glb") or mesh.endswith(".b_displacement.npy")) and not os.path.islink(output_folder / mesh): - os.symlink(input_folder / mesh, output_folder / mesh) - if Task.Coarse in task or Task.Populate in task: - - with Timer(f'Writing output blendfile'): - logging.info(f'Writing output blendfile to {output_folder / output_blend_name}') - bpy.ops.wm.save_mainfile(filepath=str(output_folder / output_blend_name)) - tag_system.save_tag(path=str(output_folder / "MaskTag.json")) - - with (output_folder/ "version.txt").open('w') as f: - f.write(f"{VERSION}\n") - - with (output_folder/'polycounts.txt').open('w') as f: - save_polycounts(f) + if input_folder is not None: + for mesh in os.listdir(input_folder): + if (mesh.endswith(".glb") or mesh.endswith(".b_displacement.npy")) and not os.path.islink(output_folder / mesh): + os.symlink(input_folder / mesh, output_folder / mesh) + if Task.Coarse in task or Task.Populate in task or Task.FineTerrain in task: + + with Timer(f'Writing output blendfile'): + logging.info(f'Writing output blendfile to {output_folder / output_blend_name}') + if optimize_terrain_diskusage and task == [Task.FineTerrain]: os.symlink(input_folder / output_blend_name, output_folder / output_blend_name) + else: bpy.ops.wm.save_mainfile(filepath=str(output_folder / output_blend_name)) + + tag_system.save_tag(path=str(output_folder / "MaskTag.json")) + + with (output_folder/ "version.txt").open('w') as f: + f.write(f"{VERSION}\n") + + with (output_folder/'polycounts.txt').open('w') as f: + save_polycounts(f) for col in bpy.data.collections['unique_assets'].children: col.hide_viewport = False if Task.Render in task or Task.GroundTruth in task or Task.MeshSave in task: terrain = Terrain(scene_seed, surface.registry, task=task, on_the_fly_asset_folder=output_folder/"assets") - terrain.load_glb(output_folder) + if optimize_terrain_diskusage: + terrain.load_glb(output_folder) if Task.Render in task or Task.GroundTruth in task: render(scene_seed, output_folder=output_folder, camera_id=camera_id, resample_idx=resample_idx) diff --git a/worldgen/terrain/core.py b/worldgen/terrain/core.py index 396afcb44..26d2b2a57 100644 --- a/worldgen/terrain/core.py +++ b/worldgen/terrain/core.py @@ -271,7 +271,7 @@ def fine_terrain(self, output_folder): np.save(output_folder / f"{mesh_name}.b_displacement", fine_meshes[mesh_name].blender_displacements) delete(obj) - def load_glb(self, output_folder): + def load_glb(self, output_folder, delete=False): for mesh_name in os.listdir(output_folder): if not mesh_name.endswith(".glb"): continue mesh_name = mesh_name[:-4] @@ -289,6 +289,9 @@ def load_glb(self, output_folder): object_to_copy_from.hide_viewport = True if mesh_name in hidden_in_viewport: object_to_copy_to.hide_viewport = True + if delete: + os.unlink(output_folder/f"{mesh_name}.glb") + os.unlink(output_folder/f"{mesh_name}.b_displacement.npy") def compute_camera_space_sdf(self, XYZ): sdf = np.ones(len(XYZ), dtype=np.float32) * 1e9 From d23862303ff71879254fe740b69141e006804461 Mon Sep 17 00:00:00 2001 From: Zeyu Ma Date: Tue, 22 Aug 2023 19:46:26 -0400 Subject: [PATCH 2/3] no redundant glb saving --- worldgen/core.py | 4 +--- worldgen/terrain/core.py | 44 ++++++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/worldgen/core.py b/worldgen/core.py index 188534994..c8e091b19 100644 --- a/worldgen/core.py +++ b/worldgen/core.py @@ -334,9 +334,7 @@ def execute_tasks( if Task.FineTerrain in task: terrain = Terrain(scene_seed, surface.registry, task=task, on_the_fly_asset_folder=output_folder/"assets") - terrain.fine_terrain(output_folder) - if not optimize_terrain_diskusage: - terrain.load_glb(output_folder, delete=True) + terrain.fine_terrain(output_folder, optimize_terrain_diskusage=optimize_terrain_diskusage) group_collections() diff --git a/worldgen/terrain/core.py b/worldgen/terrain/core.py index 26d2b2a57..830f30329 100644 --- a/worldgen/terrain/core.py +++ b/worldgen/terrain/core.py @@ -256,7 +256,7 @@ def coarse_terrain(self): self.tag_terrain(self.terrain_objs[name]) return main_obj - def fine_terrain(self, output_folder): + def fine_terrain(self, output_folder, optimize_terrain_diskusage=True): # redo sampling to achieve attribute -> surface correspondance self.sample_surface_templates() if (self.on_the_fly_asset_folder / Assets.Ocean).exists(): @@ -267,31 +267,35 @@ def fine_terrain(self, output_folder): for mesh_name in fine_meshes: obj = fine_meshes[mesh_name].export_blender(mesh_name + "_fine") if mesh_name not in hidden_in_viewport: self.tag_terrain(obj) - Mesh(obj=obj).save(output_folder / f"{mesh_name}.glb") - np.save(output_folder / f"{mesh_name}.b_displacement", fine_meshes[mesh_name].blender_displacements) - delete(obj) + if not optimize_terrain_diskusage: + object_to_copy_from = bpy.data.objects[mesh_name] + self.copy_materials_and_displacements(mesh_name, obj, object_to_copy_from, fine_meshes[mesh_name].blender_displacements) + else: + Mesh(obj=obj).save(output_folder / f"{mesh_name}.glb") + np.save(output_folder / f"{mesh_name}.b_displacement", fine_meshes[mesh_name].blender_displacements) + delete(obj) - def load_glb(self, output_folder, delete=False): + def copy_materials_and_displacements(self, mesh_name, object_to_copy_to, object_to_copy_from, displacements): + mat = object_to_copy_from.data.materials[0] + object_to_copy_to.data.materials.append(mat) + mesh_name_unapplied = mesh_name + if mesh_name + "_unapplied" in bpy.data.objects.keys(): + mesh_name_unapplied = mesh_name + "_unapplied" + for mod_name in displacements: + move_modifier(object_to_copy_to, bpy.data.objects[mesh_name_unapplied].modifiers[mod_name]) + object_to_copy_from.hide_render = True + object_to_copy_from.hide_viewport = True + if mesh_name in hidden_in_viewport: + object_to_copy_to.hide_viewport = True + + def load_glb(self, output_folder): for mesh_name in os.listdir(output_folder): if not mesh_name.endswith(".glb"): continue mesh_name = mesh_name[:-4] object_to_copy_to = Mesh(path=output_folder/f"{mesh_name}.glb").export_blender(mesh_name + "_fine") - object_to_copy_from = bpy.data.objects[mesh_name] - mat = object_to_copy_from.data.materials[0] - object_to_copy_to.data.materials.append(mat) - mesh_name_unapplied = mesh_name - if mesh_name + "_unapplied" in bpy.data.objects.keys(): - mesh_name_unapplied = mesh_name + "_unapplied" - for mod_name in np.load(output_folder / f"{mesh_name}.b_displacement.npy"): - move_modifier(object_to_copy_to, bpy.data.objects[mesh_name_unapplied].modifiers[mod_name]) - object_to_copy_from.hide_render = True - object_to_copy_from.hide_viewport = True - if mesh_name in hidden_in_viewport: - object_to_copy_to.hide_viewport = True - if delete: - os.unlink(output_folder/f"{mesh_name}.glb") - os.unlink(output_folder/f"{mesh_name}.b_displacement.npy") + displacements = np.load(output_folder / f"{mesh_name}.b_displacement.npy") + self.copy_materials_and_displacements(mesh_name, object_to_copy_to, object_to_copy_from, displacements) def compute_camera_space_sdf(self, XYZ): sdf = np.ones(len(XYZ), dtype=np.float32) * 1e9 From ed81a1066f067b0898c11deb61fe09967616bea7 Mon Sep 17 00:00:00 2001 From: pvl-bot Date: Sun, 27 Aug 2023 17:27:12 -0400 Subject: [PATCH 3/3] Disable optimimze_terrain_diskusage unless using high_quality_terrain --- worldgen/config/performance/high_quality_terrain.gin | 4 +++- worldgen/core.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/worldgen/config/performance/high_quality_terrain.gin b/worldgen/config/performance/high_quality_terrain.gin index d3ca82cd6..892667666 100644 --- a/worldgen/config/performance/high_quality_terrain.gin +++ b/worldgen/config/performance/high_quality_terrain.gin @@ -2,4 +2,6 @@ OpaqueSphericalMesher.pixels_per_cube = 0.92 TransparentSphericalMesher.pixels_per_cube = 1.38 get_surface_type.degrade_sdf_to_displacement = 0 -voronoi_grains_chance = 0.5 \ No newline at end of file +voronoi_grains_chance = 0.5 + +execute_tasks.optimize_terrain_diskusage=True \ No newline at end of file diff --git a/worldgen/core.py b/worldgen/core.py index c8e091b19..8b48d2571 100644 --- a/worldgen/core.py +++ b/worldgen/core.py @@ -267,7 +267,7 @@ def execute_tasks( reset_assets=True, focal_length=None, dryrun=False, - optimize_terrain_diskusage=True, + optimize_terrain_diskusage=False, ): if input_folder != output_folder: if reset_assets: