Integrate OcMesher

.gitmodules

@@ -31,3 +31,6 @@
 [submodule "infinigen/datagen/customgt/dependencies/glfw"]
 	path = infinigen/datagen/customgt/dependencies/glfw
 	url = https://github.com/glfw/glfw.git
+[submodule "infinigen/OcMesher"]
+	path = infinigen/OcMesher
+	url = [email protected]:princeton-vl/OcMesher.git

docs/CHANGELOG.md

@@ -20,3 +20,6 @@ v1.1.0
 - Update to blender 3.6, install blender either via pip or standalone
 - Restructure project into an `infinigen` python package and `infinigen_examples` directory
 - Add unit tests
+
+v1.2.0
+- Integrate OcMesher terrain option - see https://github.com/princeton-vl/OcMesher

docs/ConfiguringInfinigen.md

@@ -103,7 +103,6 @@ If you have more than one GPU and are using a `local_*.gin` compute config, each
 
 Generating a video, stereo or other dataset typically requires more render jobs, so we must instruct `manage_jobs.py` to run those jobs. `datagen/configs/data_schema/` provides many options for you to use in your `--pipeline_configs`, including `monocular_video.gin` and `stereo.gin`. <br> These configs are typically mutually exclusive, and you must include at least one </br>
 
-:exclamation: Our terrain system resolves its signed distance function (SDF) to view-specific meshes, which must be updated as the camera moves. For video rendering, we strongly recommend using the `high_quality_terrain` config to avoid perceptible flickering and temporal aliasing. This config meshes the SDF at very high detail, to create seamless video. However, it has high compute costs, so we recommend also using `--pipeline_config cuda_terrain` on a machine with an NVIDIA GPU. For applications with fast moving cameras, you may need to update the terrain mesh more frequently by decreasing `iterate_scene_tasks.view_block_size = 16`.
 
 To create longer videos, modify `iterate_scene_tasks.frame_range` in `monocular_video.gin` (note: we use 24fps video by default). `iterate_scene_tasks.view_block_size` controls how many frames will be grouped into each `fine_terrain` and render / ground-truth task.
 
@@ -166,14 +165,14 @@ If you have any issues with these commands, or wish to customize them to your ne
 
 All commands below are shown with using `local_256GB` config, but you can attempt to swap this for any compute config as discussed in [Configuring available computing resources](#configuring-available-computing-resources).
 
-#### Creating videos similar to the intro video
+#### Creating high quality videos
 
-Most videos in the "Introducing Infinigen" launch video were made using commands similar to the following:
+We recommend this command as a starting point for generating high quality videos. Generating multi-view consistent terrain is not computationally tractible without CUDA accelleration, so make sure to follow the CUDA Terrain instructions in Installation.md, and we recommend not to remove the `cuda_terrain` flag below.
 
 ````
 python -m infinigen.datagen.manage_jobs --output_folder outputs/my_videos --num_scenes 500 \
     --pipeline_config slurm monocular_video cuda_terrain opengl_gt \
-    --cleanup big_files --warmup_sec 60000 --config high_quality_terrain
+    --cleanup big_files --warmup_sec 60000 --config video high_quality_terrain
 ````
 
 #### Creating large-scale stereo datasets
@@ -219,7 +218,7 @@ python -m infinigen.datagen.manage_jobs --output_folder outputs/my_videos --num_
 ```
 python -m infinigen.datagen.manage_jobs --output_folder outputs/my_videos --num_scenes 500 \
     --pipeline_config slurm monocular_video cuda_terrain opengl_gt \
-    --cleanup big_files --warmup_sec 30000 --config high_quality_terrain \
+    --cleanup big_files --warmup_sec 30000 --config video high_quality_terrain \
     --overrides camera.camera_pose_proposal.altitude=["uniform", 20, 30]
 ```
 
@@ -229,7 +228,7 @@ python -m infinigen.datagen.manage_jobs --output_folder outputs/my_videos --num_
 ```
 python -m infinigen.datagen.manage_jobs --output_folder outputs/my_videos --num_scenes 500 \
     --pipeline_config slurm monocular_video cuda_terrain opengl_gt \
-    --cleanup big_files --warmup_sec 30000 --config high_quality_terrain \
+    --cleanup big_files --warmup_sec 30000 --config video high_quality_terrain \
     --pipeline_overrides iterate_scene_tasks.frame_range=[1,25]
 ```
 

infinigen/__init__.py

@@ -1,3 +1,3 @@
 import logging
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"

infinigen/core/execute_tasks.py

@@ -342,7 +342,7 @@ def execute_tasks(
 
     group_collections()
 
-    if input_folder is not None:
+    if input_folder is not None and input_folder != output_folder:
         for mesh in os.listdir(input_folder):
             if (mesh.endswith(".glb") or mesh.endswith(".b_displacement.npy")) and not os.path.islink(output_folder / mesh):
                 os.symlink(input_folder / mesh, output_folder / mesh)

infinigen/core/nodes/node_info.py

@@ -42,8 +42,10 @@ class Nodes:
     CombineHSV = 'ShaderNodeCombineHSV'
     SeparateRGB = 'ShaderNodeSeparateRGB'
     SeparateColor = 'ShaderNodeSeparateColor'
+    CompSeparateColor = 'CompositorNodeSeparateColor'
     CombineRGB = 'ShaderNodeCombineRGB'
     CombineColor = 'ShaderNodeCombineColor'
+    CompCombineColor = 'CompositorNodeCombineColor'
 
     #bl3.5 additions
     SeparateComponents = 'GeometryNodeSeparateComponents'

infinigen/core/rendering/render.py

@@ -138,7 +138,12 @@ def configure_compositor_output(nw, frames_folder, image_denoised, image_noisy,
             setattr(viewlayer.cycles, f"use_pass_{viewlayer_pass}", True)
         slot_input = file_output_node.file_slots.new(socket_name)
         render_socket = render_layers.outputs[socket_name]
-        nw.links.new(render_socket, slot_input)
+        if viewlayer_pass == "vector":
+            separate_color = nw.new_node(Nodes.CompSeparateColor, [render_socket])
+            comnbine_color = nw.new_node(Nodes.CompCombineColor, [0, (separate_color, 3), (separate_color, 2), 0])
+            nw.links.new(comnbine_color.outputs[0], slot_input)
+        else:
+            nw.links.new(render_socket, slot_input)
         file_slot_list.append(file_output_node.file_slots[slot_input.name])
 
     slot_input = file_output_node.file_slots['Image']

infinigen/core/util/blender.py

@@ -206,10 +206,11 @@ def __exit__(self, *_):
 
 
 def select_none():
-    if bpy.context.active_object is not None:
+    if hasattr(bpy.context, "active_object") and bpy.context.active_object is not None:
         bpy.context.active_object.select_set(False)
-    for obj in bpy.context.selected_objects:
-        obj.select_set(False)
+    if hasattr(bpy.context, "selected_objects"):
+        for obj in bpy.context.selected_objects:
+            obj.select_set(False)
 
 
 def select(objs):

infinigen/datagen/configs/data_schema/block_terrain_experiment.gin

@@ -0,0 +1,10 @@
+iterate_scene_tasks.global_tasks = [
+    {'name': 'coarse', 'func': @queue_coarse}, 
+    {'name': "populate", 'func': @queue_populate},
+    {'name': 'backuppopulate', 'func': @renderbackup/queue_populate, 'condition': 'prev_failed'},
+]
+iterate_scene_tasks.view_dependent_tasks = [
+    {'name': "fineterrain", 'func': @queue_fine_terrain},
+]
+
+queue_populate.input_prefix = "coarse"

infinigen/datagen/configs/data_schema/monocular_video.gin

@@ -1,16 +1,15 @@
 iterate_scene_tasks.frame_range = [1, 192]
-iterate_scene_tasks.view_block_size = 8
+iterate_scene_tasks.view_block_size = 192
+iterate_scene_tasks.cam_block_size = 8
 iterate_scene_tasks.cam_id_ranges = [1, 1]
 
 iterate_scene_tasks.global_tasks = [
     {'name': 'coarse', 'func': @queue_coarse}, 
+    {'name': "fineterrain", 'func': @queue_fine_terrain},
     {'name': "populate", 'func': @queue_populate},
     {'name': 'backuppopulate', 'func': @renderbackup/queue_populate, 'condition': 'prev_failed'},
-    {'name': "fineterrain", 'func': @queue_fine_terrain},
-]
-iterate_scene_tasks.view_dependent_tasks = [
-    {'name': "fineterrain", 'func': @queue_fine_terrain},
 ]
+iterate_scene_tasks.view_dependent_tasks = []
 iterate_scene_tasks.camera_dependent_tasks = [
     {'name': 'shortrender', 'func': @rendershort/queue_render}, 
     {'name': 'backuprender', 'func': @renderbackup/queue_render, 'condition': 'prev_failed'}, 

infinigen/datagen/job_funcs.py

@@ -121,6 +121,7 @@ def queue_populate(
     seed,
     configs,
     taskname=None,
+    input_prefix="fine",
     overrides=[],
     input_indices=None, output_indices=None,
     **kwargs,
@@ -132,7 +133,7 @@ def queue_populate(
     input_suffix = get_suffix(input_indices)
     output_suffix = get_suffix(output_indices)
 
-    input_folder = folder/f'coarse{input_suffix}'
+    input_folder = folder/f'{input_prefix}{input_suffix}'
     output_folder = input_folder
 
     cmd = get_cmd(seed, 'populate', configs, taskname, 

infinigen/datagen/manage_jobs.py

@@ -187,7 +187,9 @@ def init_db_from_existing(output_folder: Path):
     existing_db = pd.read_csv(db_path, converters={"configs": literal_eval})
 
     def init_scene(seed_folder):
-        if not seed_folder.is_dir():
+        if not seed_folder.is_symlink() and not seed_folder.is_dir():
+            return None
+        if seed_folder.is_symlink() and not seed_folder.readlink().is_dir():
             return None
         if not (seed_folder/'logs').exists():
             logger.warning(f'Skipping {seed_folder=} due to missing "logs" subdirectory')

infinigen/terrain/assets/landtiles/ant_landscape.py

@@ -13,7 +13,7 @@
 
 from infinigen.terrain.land_process.erosion import run_erosion
 from infinigen.terrain.land_process.snowfall import run_snowfall
-from infinigen.terrain.utils import smooth, random_int
+from infinigen.terrain.utils import smooth, random_nat
 from infinigen.core.util.organization import AssetFile, LandTile
 
 
@@ -23,7 +23,7 @@ def create(
     subdivision_y,
 ):
     def presets(**kwargs):
-        bpy.ops.mesh.landscape_add(ant_terrain_name="Landscape", land_material="", water_material="", texture_block="", at_cursor=True, smooth_mesh=True, tri_face=False, sphere_mesh=False, subdivision_x=subdivision_x, subdivision_y=subdivision_y, mesh_size=2, mesh_size_x=2, mesh_size_y=2, random_seed=max(0, random_int()), water_plane=False, water_level=0.01, remove_double=False, show_main_settings=True, show_noise_settings=True, show_displace_settings=True, refresh=True, auto_refresh=True, **kwargs)
+        bpy.ops.mesh.landscape_add(ant_terrain_name="Landscape", land_material="", water_material="", texture_block="", at_cursor=True, smooth_mesh=True, tri_face=False, sphere_mesh=False, subdivision_x=subdivision_x, subdivision_y=subdivision_y, mesh_size=2, mesh_size_x=2, mesh_size_y=2, random_seed=random_nat(), water_plane=False, water_level=0.01, remove_double=False, show_main_settings=True, show_noise_settings=True, show_displace_settings=True, refresh=True, auto_refresh=True, **kwargs)
 
     if preset_name == LandTile.Canyon:
         strata = np.random.randint(6, 12)

infinigen/terrain/assets/landtiles/custom.py

@@ -141,6 +141,7 @@ def multi_mountains_asset(
         coverage=params["coverage"],
         slope_freq=params["slope_freq"],
         slope_height=params["slope_height"],
+        is_asset=True,
     )
     heightmap = mountains.get_heightmap(X, Y)
     mountains.cleanup()
@@ -175,6 +176,7 @@ def coast_asset(
         coverage=params1["coverage"],
         slope_freq=params1["slope_freq"],
         slope_height=params1["slope_height"],
+        is_asset=True,
     )
     heightmap = mountains.get_heightmap(X, Y)
     mountains.cleanup()

infinigen/terrain/core.py

@@ -12,16 +12,19 @@
 import gin
 import numpy as np
 from mathutils.bvhtree import BVHTree
+
+from infinigen.OcMesher.ocmesher import OcMesher as UntexturedOcMesher
 from infinigen.terrain.mesher import OpaqueSphericalMesher, TransparentSphericalMesher, UniformMesher
 from infinigen.terrain.scene import scene, transfer_scene_info
 from infinigen.terrain.surface_kernel.core import SurfaceKernel
-from infinigen.terrain.utils import Mesh, move_modifier, Vars, AttributeType, FieldsType
+from infinigen.terrain.utils import Mesh, move_modifier, Vars, AttributeType, FieldsType, get_caminfo, write_attributes
 from infinigen.terrain.assets.ocean import ocean_asset
 from infinigen.core.util.blender import SelectObjects, delete
 from infinigen.core.util.logging import Timer
 from infinigen.core.util.math import FixedSeed, int_hash
 from infinigen.core.util.organization import SurfaceTypes, Attributes, Task, TerrainNames, ElementNames, Transparency, Materials, Assets, ElementTag, Tags, SelectionCriterions
 from infinigen.assets.utils.tag import tag_object, tag_system
+
 from numpy import ascontiguousarray as AC
 
 logger = logging.getLogger(__name__)
@@ -39,6 +42,36 @@ def get_surface_type(surface, degrade_sdf_to_displacement=True):
             return SurfaceTypes.Displacement
         return surface.type
 
+
+class OcMesher(UntexturedOcMesher):
+    def __init__(self, cameras, **kwargs):
+        UntexturedOcMesher.__init__(self, get_caminfo(cameras)[0], **kwargs)
+
+    def __call__(self, kernels):
+        sdf_kernels = [(lambda x, k0=k: k0(x)[Vars.SDF]) for k in kernels]
+        meshes, in_view_tags = UntexturedOcMesher.__call__(self, sdf_kernels)
+        with Timer("compute attributes"):
+            write_attributes(kernels, None, meshes)
+            for mesh, tag in zip(meshes, in_view_tags):
+                mesh.vertex_attributes[Tags.OutOfView] = (~tag).astype(np.int32)
+        with Timer("concat meshes"):
+            mesh = Mesh.cat(meshes)
+        return mesh
+
+class CollectiveOcMesher(UntexturedOcMesher):
+    def __init__(self, cameras, **kwargs):
+        UntexturedOcMesher.__init__(self, get_caminfo(cameras)[0], **kwargs)
+
+    def __call__(self, kernels):
+        sdf_kernels = [lambda x: np.stack([k(x)[Vars.SDF] for k in kernels], -1).min(axis=-1)]
+        mesh, in_view_tag = UntexturedOcMesher.__call__(self, sdf_kernels)
+        mesh = mesh[0]
+        with Timer("compute attributes"):
+            write_attributes(kernels, mesh, [])
+            mesh.vertex_attributes[Tags.OutOfView] = (~in_view_tag[0]).astype(np.int32)
+        mesh = Mesh(mesh=mesh)
+        return mesh
+
 @gin.configurable
 class Terrain:
     def __init__(
@@ -99,11 +132,10 @@ def cleanup(self):
 
     @gin.configurable("export")
     def export(self,
-        spherical=False,
+        dynamic=False,
+        spherical=True, # false for OcMesher
         cameras=None,
         main_terrain_only=False,
-        collective_transparent_overrides={},
-        coarse_hidden=True,
         remove_redundant_attrs=True,
     ):
         meshes_dict = {}
@@ -112,11 +144,9 @@ def export(self,
             opaque_elements = [element for element in self.elements_list if element.transparency == Transparency.Opaque]
             if opaque_elements != []:
                 attributes_dict[TerrainNames.OpaqueTerrain] = set()
-                if spherical:
-                    if coarse_hidden:
-                        mesher = OpaqueSphericalMesher(cameras=cameras)
-                    else:
-                        mesher = TransparentSphericalMesher(cameras=cameras)
+                if dynamic:
+                    if spherical: mesher = OpaqueSphericalMesher(cameras=cameras)
+                    else: mesher = OcMesher(cameras=cameras)
                 else:
                     mesher = UniformMesher()
                 with Timer(f"meshing {TerrainNames.OpaqueTerrain}"):
@@ -128,12 +158,13 @@ def export(self,
         individual_transparent_elements = [element for element in self.elements_list if element.transparency == Transparency.IndividualTransparent]
         for element in individual_transparent_elements:
             if not main_terrain_only or element.__class__.name == self.main_terrain:
-                if spherical:
+                if dynamic:
                     special_args = {}
                     if element.__class__.name == ElementNames.Atmosphere:
-                        special_args["coarse_multiplier"] = 64
-                        special_args["upscale"] = 1
-                    mesher = TransparentSphericalMesher(cameras=cameras, **special_args)
+                        special_args["pixels_per_cube"] = 100
+                        special_args["inv_scale"] = 1
+                    if spherical: mesher = TransparentSphericalMesher(cameras=cameras, **special_args)
+                    else: mesher = OcMesher(cameras=cameras, simplify_occluded=False, **special_args)
                 else: mesher = UniformMesher(enclosed=True)
                 with Timer(f"meshing {element.__class__.name}"):
                     mesh = mesher([element])
@@ -144,8 +175,9 @@ def export(self,
             collective_transparent_elements = [element for element in self.elements_list if element.transparency == Transparency.CollectiveTransparent]
             if collective_transparent_elements != []:
                 attributes_dict[TerrainNames.CollectiveTransparentTerrain] = set()
-                if spherical:
-                    mesher = TransparentSphericalMesher(cameras=cameras, **collective_transparent_overrides)
+                if dynamic:
+                    if spherical: mesher = TransparentSphericalMesher(cameras=cameras)
+                    else: mesher = CollectiveOcMesher(cameras=cameras, simplify_occluded=False)
                 else:
                     mesher = UniformMesher()
                 with Timer(f"meshing {TerrainNames.CollectiveTransparentTerrain}"):
@@ -154,7 +186,7 @@ def export(self,
                 for element in collective_transparent_elements:
                     attributes_dict[TerrainNames.CollectiveTransparentTerrain].update(element.attributes)
 
-        if main_terrain_only or spherical:
+        if main_terrain_only or dynamic:
             for mesh_name in meshes_dict:
                 mesh_name_unapplied = mesh_name
                 if mesh_name + "_unapplied" in bpy.data.objects.keys():
@@ -173,7 +205,7 @@ def export(self,
                     if get_surface_type(surface) == SurfaceTypes.BlenderDisplacement:
                         meshes_dict[mesh_name].blender_displacements.append(surface.mod_name)
 
-        if spherical:
+        if dynamic:
             if remove_redundant_attrs:
                 for mesh_name in meshes_dict:
                     if len(attributes_dict[mesh_name]) == 1:
@@ -263,7 +295,7 @@ def fine_terrain(self, output_folder, optimize_terrain_diskusage=True):
             with FixedSeed(int_hash(["Ocean", self.seed])):
                 ocean_asset(output_folder / Assets.Ocean, bpy.context.scene.frame_start, bpy.context.scene.frame_end, link_folder=self.on_the_fly_asset_folder / Assets.Ocean)
         self.surfaces_into_sdf()
-        fine_meshes, _ = self.export(spherical=True, cameras=[bpy.context.scene.camera])
+        fine_meshes, _ = self.export(dynamic=True, cameras=[bpy.context.scene.camera])
         for mesh_name in fine_meshes:
             obj = fine_meshes[mesh_name].export_blender(mesh_name + "_fine")
             if mesh_name not in hidden_in_viewport: self.tag_terrain(obj)

infinigen/terrain/elements/core.py

@@ -6,15 +6,18 @@
 
 from ctypes import POINTER, c_float, c_int32, c_size_t
 
+import gin
 import numpy as np
 from numpy import ascontiguousarray as AC
 from infinigen.terrain.utils import ASFLOAT, ASINT, Vars, load_cdll, register_func
 from infinigen.core.util.organization import Materials
 
-
+@gin.configurable
 class Element:
     called_time = {}
-    def __init__(self, lib_name, material, transparency):
+    def __init__(self, lib_name, material, transparency, bounding_shape=None, is_asset=False):
+        self.bounding_shape = bounding_shape
+        self.is_asset = is_asset
         if lib_name in Element.called_time:
             lib_name_X = f"{lib_name}_{Element.called_time[lib_name]}"
             print(f"{lib_name} already loaded, loading {lib_name_X} instead")
@@ -76,6 +79,14 @@ def __call__(self, positions, sdf_only=False):
             else:
                 auxs.append(None)
         self.call(N, ASFLOAT(AC(positions.astype(np.float32))), ASFLOAT(sdf), *[POINTER(c_float)() if x is None else ASFLOAT(x) for x in auxs])
+        bounding_shape = self.bounding_shape
+        if not self.is_asset and bounding_shape is not None:
+            if bounding_shape[0] == "cube":
+                x_min, x_max, y_min, y_max, z_min, z_max = bounding_shape[1:]
+                out_bound = (positions[:, 0] < x_min) | (positions[:, 0] > x_max) \
+                    | (positions[:, 1] < y_min) | (positions[:, 1] > y_max) \
+                    | (positions[:, 2] < z_min) | (positions[:, 2] > z_max)
+                sdf[out_bound] = 1e6
         ret = {}
         ret[Vars.SDF] = sdf