Render tiled-camera heightfields via the mesh path

CHANGELOG.md

@@ -82,6 +82,7 @@
 
 ### Fixed
 
+- Fix `SensorTiledCamera` not rendering heightfield (`HFIELD`) shapes, which were missing from the render BVH. Heightfields are now rendered through the existing mesh path (they are triangulated `wp.Mesh` shapes), which also resolves a tiled-camera render-performance regression caused by the unused heightfield branch lowering the render kernel's GPU occupancy.
 - Fix `eval_fk()` overwriting VBD-simulated `JointType.CABLE` body poses.
 - Fix hydroelastic SDF contact surfaces dropping the central region under deep interpenetration. The broadphase used to skip subgrids whose centers were deeper than the SDF narrow band, leaving a hole in the contact patch when overlap exceeded the narrow-band thickness. Broadphase now visits every subgrid in the SDF coarse grid (block coordinates are derived arithmetically from per-shape SDF coarse-texture dimensions); sampling at far-inside locations is correct because the coarse SDF is dense and accurate everywhere. On-disk SDF caches written by earlier versions are transparently re-cooked on first load (`_sdf_cache.CACHE_FORMAT_VERSION` bumped to `2`)
 - Fix `SolverXPBD` `body_parent_f` reporting to include `Control.joint_f` contributions and accumulate multiple inbound joint contributions, matching the `SolverMuJoCo` and `SolverFeatherstone` convention.

newton/_src/geometry/bvh.py

@@ -156,6 +156,8 @@ def is_supported_shape_type(shape_type: wp.int32) -> wp.bool:
         return True
     if shape_type == GeoType.MESH:
         return True
+    if shape_type == GeoType.HFIELD:
+        return True
     if shape_type == GeoType.GAUSSIAN:
         return True
     return False
@@ -205,7 +207,9 @@ def compute_shape_local_bounds(
     min_point = wp.vec3(MAXVAL)
     max_point = wp.vec3(-MAXVAL)
 
-    if in_shape_type[tid] == GeoType.MESH:
+    if in_shape_type[tid] == GeoType.MESH or in_shape_type[tid] == GeoType.HFIELD:
+        # Heightfields store their triangulated terrain as a wp.Mesh in
+        # shape_source_ptr, so their local AABB is computed the same way.
         mesh = wp.mesh_get(in_shape_ptr[tid])
         for i in range(mesh.points.shape[0]):
             min_point = wp.min(min_point, mesh.points[i])
@@ -288,7 +292,7 @@ def compute_shape_bvh_bounds(
         lower, upper = compute_ellipsoid_bounds(transform, size)
     elif geom_type == GeoType.BOX:
         lower, upper = compute_box_bounds(transform, size)
-    elif geom_type == GeoType.MESH or geom_type == GeoType.GAUSSIAN:
+    elif geom_type == GeoType.MESH or geom_type == GeoType.GAUSSIAN or geom_type == GeoType.HFIELD:
         min_bounds = shape_bounds[shape_index, 0]
         max_bounds = shape_bounds[shape_index, 1]
         lower, upper = compute_shape_bounds(transform, size, min_bounds, max_bounds)

newton/_src/sensors/warp_raytrace/raytrace.py

@@ -147,6 +147,8 @@ def closest_hit_shape(
                     hit_color = wp.vec3f(0.0)
 
                     shape_type = shape_types[si]
+                    # Heightfields are triangulated meshes; RenderContext remaps
+                    # HFIELD -> MESH, so this branch renders them too.
                     if shape_type == GeoType.MESH:
                         hit_distance, hit_normal, hit_u, hit_v, hit_face_id = _ray_intersect_mesh_smooth(
                             shape_transforms[si],
@@ -159,18 +161,6 @@ def closest_hit_shape(
                             wp.static(config.enable_backface_culling),
                             closest_hit.distance,
                         )
-                    elif shape_type == GeoType.HFIELD:
-                        hit_distance, hit_normal, hit_u, hit_v, hit_face_id = _ray_intersect_mesh_smooth(
-                            shape_transforms[si],
-                            shape_sizes[si],
-                            ray_origin_world,
-                            ray_dir_world,
-                            shape_source_ptr[si],
-                            wp.int32(-1),
-                            mesh_data,
-                            wp.static(config.enable_backface_culling),
-                            closest_hit.distance,
-                        )
                     elif shape_type == GeoType.PLANE:
                         hit_distance, hit_normal = _plane_hit_with_culling(
                             shape_transforms[si],
@@ -447,6 +437,8 @@ def closest_hit_shape_depth_only(
                     hit_dist = -1.0
 
                     shape_type = shape_types[si]
+                    # Heightfields are triangulated meshes; RenderContext remaps
+                    # HFIELD -> MESH, so this branch renders them too.
                     if shape_type == GeoType.MESH:
                         hit_dist, _normal, _u, _v, _face = raycast.ray_intersect_mesh(
                             shape_transforms[si],
@@ -457,16 +449,6 @@ def closest_hit_shape_depth_only(
                             wp.static(config.enable_backface_culling),
                             closest_hit.distance,
                         )
-                    elif shape_type == GeoType.HFIELD:
-                        hit_dist, _normal, _u, _v, _face = raycast.ray_intersect_mesh(
-                            shape_transforms[si],
-                            ray_origin_world,
-                            ray_dir_world,
-                            shape_sizes[si],
-                            shape_source_ptr[si],
-                            wp.static(config.enable_backface_culling),
-                            closest_hit.distance,
-                        )
                     elif shape_type == GeoType.PLANE:
                         hit_dist, _plane_normal = _plane_hit_with_culling(
                             shape_transforms[si],
@@ -698,6 +680,8 @@ def first_hit_shape(
                     hit_dist = wp.float32(-1)
 
                     shape_type = shape_types[si]
+                    # Heightfields are triangulated meshes; RenderContext remaps
+                    # HFIELD -> MESH, so this branch renders them too.
                     if shape_type == GeoType.MESH:
                         hit_dist, _normal, _u, _v, _face = raycast.ray_intersect_mesh(
                             shape_transforms[si],
@@ -708,16 +692,6 @@ def first_hit_shape(
                             False,
                             max_dist,
                         )
-                    elif shape_type == GeoType.HFIELD:
-                        hit_dist, _normal, _u, _v, _face = raycast.ray_intersect_mesh(
-                            shape_transforms[si],
-                            ray_origin_world,
-                            ray_dir_world,
-                            shape_sizes[si],
-                            shape_source_ptr[si],
-                            False,
-                            max_dist,
-                        )
                     elif shape_type == GeoType.PLANE:
                         hit_dist, _plane_normal = _plane_hit_with_culling(
                             shape_transforms[si],

newton/_src/sensors/warp_raytrace/render_context.py

@@ -9,7 +9,7 @@
 import numpy as np
 import warp as wp
 
-from ...geometry import Gaussian, Mesh
+from ...geometry import Gaussian, GeoType, Mesh
 from ...sim import Model, State
 from ...utils import load_texture, normalize_texture
 from .render import create_kernel
@@ -68,6 +68,7 @@ def __init__(self, world_count: int = 1, config: Config | None = None, device: s
         self.shape_source_ptr: wp.array[wp.uint64] | None = None
         self.shape_texture_ids: wp.array[wp.int32] | None = None
         self.shape_mesh_data_ids: wp.array[wp.int32] | None = None
+        self.shape_render_type: wp.array[wp.int32] | None = None
 
         self.mesh_data: wp.array[MeshData] | None = None
         self.texture_data: wp.array[TextureData] | None = None
@@ -106,6 +107,21 @@ def init_from_model(self, model: Model, load_textures: bool = True):
         self.shape_world_index = model.shape_world
         self.shape_source_ptr = model.shape_source_ptr
 
+        # Heightfields are triangulated meshes (their wp.Mesh lives in
+        # shape_source_ptr), so the renderer treats them as meshes: it reuses
+        # the MESH ray-intersection path, which keeps heightfield handling out
+        # of the render kernels entirely (no extra shape-type branch, so no
+        # register/occupancy cost). The remapped type array is what the render
+        # kernel dispatches on; model.shape_type (HFIELD) is left untouched for
+        # collision and BVH bounds.
+        self.shape_render_type = model.shape_type
+        if model.shape_type is not None:
+            shape_type_np = model.shape_type.numpy()
+            if np.any(shape_type_np == int(GeoType.HFIELD)):
+                shape_type_np = shape_type_np.copy()
+                shape_type_np[shape_type_np == int(GeoType.HFIELD)] = int(GeoType.MESH)
+                self.shape_render_type = wp.array(shape_type_np, dtype=wp.int32, device=model.shape_type.device)
+
         if model.particle_q is not None and model.particle_q.shape[0]:
             self.__has_particles = True
             if model.tri_indices is not None and model.tri_indices.shape[0]:
@@ -301,7 +317,7 @@ def render(
                     model.bvh_shapes_group_roots,
                     # Shapes
                     model.bvh_shape_enabled,
-                    model.shape_type,
+                    self.shape_render_type,  # HFIELD remapped to MESH; renderer treats heightfields as meshes
                     model.shape_scale,
                     self.shape_colors,
                     model.bvh_shape_world_transforms,

newton/tests/test_sensor_tiled_camera_heightfield.py

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 The Newton Developers
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+import unittest
+
+import numpy as np
+import warp as wp
+
+import newton
+from newton import Heightfield
+from newton.sensors import SensorTiledCamera
+
+
+class TestSensorTiledCameraHeightfield(unittest.TestCase):
+    """The tiled camera must render heightfield (HFIELD) shapes.
+
+    SensorRaycast (removed in favor of SensorTiledCamera) supported
+    heightfields, so the replacement must too.
+    """
+
+    @unittest.skipUnless(wp.is_cuda_available(), "Requires CUDA")
+    def test_renders_flat_heightfield_from_above(self):
+        # Flat heightfield at z=1 spanning [-2, 2]^2.
+        data = np.full((3, 3), 1.0, dtype=np.float32)
+        hf = Heightfield(data=data, nrow=3, ncol=3, hx=2.0, hy=2.0, min_z=1.0, max_z=1.0)
+        builder = newton.ModelBuilder()
+        builder.add_shape_heightfield(heightfield=hf)
+        model = builder.finalize()
+        state = model.state()
+
+        res = 16
+        sensor = SensorTiledCamera(model=model)
+        sensor.utils.create_default_light(enable_shadows=False)
+        sensor.utils.assign_checkerboard_material_to_all_shapes()
+        # 30-deg fov: footprint half-extent at depth 4 is 4*tan(15)=1.07 < 2,
+        # so the terrain robustly fills the whole frame.
+        rays = sensor.utils.compute_pinhole_camera_rays(res, res, math.radians(30.0))
+        depth = sensor.utils.create_depth_image_output(res, res)
+        newton.geometry.build_bvh_shape(model, state)
+        newton.geometry.build_bvh_particle(model, state)
+        sensor.sync_transforms(state)
+
+        # Camera 5m above origin, identity orientation => looks straight down (-z).
+        # At depth 4 the 45-deg footprint half-extent is 4*tan(22.5)=1.66 < 2,
+        # so every ray hits the terrain.
+        cam = wp.array(
+            [[wp.transformf(wp.vec3f(0.0, 0.0, 5.0), wp.quatf(0.0, 0.0, 0.0, 1.0))]],
+            dtype=wp.transformf,
+        )
+        sensor.render_config.render_order = SensorTiledCamera.RenderOrder.PIXEL_PRIORITY
+        sensor.update(state, cam, rays, depth_image=depth)
+
+        d = depth.numpy()[0, 0]  # .numpy() syncs the device-to-host copy
+        hit = int(np.count_nonzero(d > 0.0))
+        # The terrain covers the whole frame, but ~10-15% of rays miss along
+        # triangle edges (non-watertight mesh_query_ray); measured stable across
+        # resolution and camera offset, so require "most" pixels rather than all.
+        self.assertGreaterEqual(
+            hit,
+            int(res * res * 0.8),
+            msg=f"heightfield should fill most of the view; only {hit}/{res * res} pixels hit",
+        )
+        # Every ray that hits sees the flat surface at z=1 from z=5: depth ~4,
+        # up to ~4.25 toward the frame edges (ray-angle cosine).
+        hit_depths = d[d > 0.0]
+        self.assertGreater(float(hit_depths.min()), 3.9)
+        self.assertLess(float(hit_depths.max()), 4.4)
+        center = float(d[res // 2, res // 2])
+        self.assertAlmostEqual(center, 4.0, delta=0.05, msg=f"center depth {center}, expected ~4.0 (5 - 1)")
+
+
+if __name__ == "__main__":
+    unittest.main()