OMPE-96088: Fix cartpole camera anomalies for newton + rtx variants

source/isaaclab/changelog.d/jmart-cartpole-rtx.minor.rst

@@ -0,0 +1,8 @@
+Added
+^^^^^
+
+* Added the :meth:`~isaaclab.physics.physics_manager_cfg.PhysicsCfg.provides_implicit_damping` and
+  :meth:`~isaaclab.renderers.renderer_cfg.RendererCfg.provides_temporal_camera_data` capability
+  methods, so physics and renderer backends can declare whether a camera observation carries the
+  temporal information a policy needs to infer velocity (used to decide frame stacking). Base
+  defaults: physics has implicit damping (``True``); a renderer provides no temporal data (``False``).

source/isaaclab/isaaclab/physics/physics_manager_cfg.py

@@ -30,3 +30,14 @@ class PhysicsCfg:
 
     class_type: type[PhysicsManager] | Any = MISSING
     """The physics manager class to use. Must be set by subclasses."""
+
+    def provides_implicit_damping(self) -> bool:
+        """Whether this backend's integrator has implicit numerical damping.
+
+        With implicit damping (PhysX, OV-PhysX) a camera policy can infer velocity from a
+        single frame. Without it (Newton's symplectic integrator) the policy needs a temporal
+        cue in the observation (e.g. frame stacking).
+
+        The base default is ``True``; backends without implicit damping override to ``False``.
+        """
+        return True

source/isaaclab/isaaclab/renderers/renderer_cfg.py

@@ -13,3 +13,15 @@ class RendererCfg:
     """Configuration for a renderer."""
 
     renderer_type: str = "default"
+
+    def provides_temporal_camera_data(self, data_type: str) -> bool:
+        """Whether this renderer's ``data_type`` output carries temporal information.
+
+        Under a physics backend without implicit damping (e.g. Newton), a camera policy
+        needs a temporal cue to infer velocity. Renderers that accumulate frames over time
+        (temporal AA / DLSS) supply it; pure rasterizers and non-beauty AOVs do not.
+
+        The base default is ``False`` (assume no temporal information); renderer subclasses
+        override per output type.
+        """
+        return False

source/isaaclab_newton/changelog.d/jmart-cartpole-rtx.minor.rst

@@ -0,0 +1,7 @@
+Added
+^^^^^
+
+* Overrode :meth:`provides_implicit_damping` on :class:`NewtonCfg` to return ``False`` (its
+  symplectic integrator has no implicit damping) and :meth:`provides_temporal_camera_data` on
+  :class:`NewtonWarpRendererCfg` to return ``False`` (the rasterizer accumulates no temporal data),
+  so camera tasks can auto-enable frame stacking for the Newton combos that need it.

source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py

@@ -164,3 +164,7 @@ def __post_init__(self):
                 self.collision_decimation,
                 self.num_substeps,
             )
+
+    def provides_implicit_damping(self) -> bool:
+        # Newton's symplectic integrator has no implicit damping.
+        return False

source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py

@@ -16,6 +16,10 @@ class NewtonWarpRendererCfg(RendererCfg):
     renderer_type: str = "newton_warp"
     """Type identifier for Newton Warp renderer."""
 
+    def provides_temporal_camera_data(self, data_type: str) -> bool:
+        # Pure rasterizer: no temporal accumulation on any output.
+        return False
+
     enable_textures: bool = True
     """Enable texture-mapped rendering for meshes."""
 

source/isaaclab_ov/changelog.d/jmart-cartpole-rtx.minor.rst

@@ -0,0 +1,6 @@
+Added
+^^^^^
+
+* Overrode :meth:`provides_temporal_camera_data` on :class:`OVRTXRendererCfg` to return ``True``
+  only for the ``rgb``/``rgba`` beauty buffer (temporally accumulated by DLSS), matching Isaac RTX;
+  other AOVs return ``False``.

source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py

@@ -26,6 +26,11 @@ class OVRTXRendererCfg(RendererCfg):
     renderer_type: str = "ovrtx"
     """Type identifier for OVRTX renderer."""
 
+    def provides_temporal_camera_data(self, data_type: str) -> bool:
+        # OV-RTX, like Isaac RTX, temporally accumulates only the rgb/rgba beauty buffer
+        # (DLSS); the other AOVs bypass it.
+        return data_type in ("rgb", "rgba")
+
     temp_usd_dir: str | None = None
     """Directory for temporary combined USD files (scene + injected cameras).
     Used by the OVRTX renderer when building the render scope; must be writable.

source/isaaclab_physx/changelog.d/jmart-cartpole-rtx.minor.rst

@@ -0,0 +1,6 @@
+Added
+^^^^^
+
+* Overrode :meth:`provides_temporal_camera_data` on :class:`IsaacRtxRendererCfg` to return ``True``
+  only for the ``rgb``/``rgba`` beauty buffer (temporally accumulated by DLSS); the depth, albedo,
+  simple_shading, and segmentation AOVs return ``False`` as they bypass DLSS.

source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py

@@ -23,6 +23,11 @@ class IsaacRtxRendererCfg(RendererCfg):
     renderer_type: str = "isaac_rtx"
     """Type identifier for Isaac RTX renderer."""
 
+    def provides_temporal_camera_data(self, data_type: str) -> bool:
+        # Only the rgb/rgba beauty buffer is temporally accumulated by DLSS; the depth,
+        # albedo, simple_shading, and segmentation AOVs bypass it.
+        return data_type in ("rgb", "rgba")
+
     semantic_filter: str | list[str] = "*:*"
     """A string or a list specifying a semantic filter predicate. Defaults to ``"*:*"``.
 

source/isaaclab_tasks/changelog.d/jmart-cartpole-rtx.rst

@@ -0,0 +1,11 @@
+Fixed
+^^^^^
+
+* Fixed the camera-based Cartpole task failing to converge under Newton physics with the RTX
+  ``depth``, ``albedo``, and ``simple_shading`` AOV observations. These AOVs bypass DLSS temporal
+  accumulation, so the observation carried no temporal cue for the policy to infer velocity from
+  (Newton's symplectic integrator has no implicit damping). The ``frame_stack`` default resolver
+  now enables 2-frame stacking for these Newton + RTX AOVs, matching the existing Newton + Warp
+  behavior; Newton + RTX ``rgb`` keeps single-frame observations as DLSS already supplies the cue.
+  The resolver now reads the backend capability flags (``PhysicsCfg.provides_implicit_damping``,
+  ``RendererCfg.provides_temporal_camera_data``) instead of hard-coding backend types.

source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py

@@ -31,23 +31,30 @@
 class CartpoleCameraEnv(CartpoleEnv):
     """Cartpole environment driven by camera observations.
 
-    Uses temporal observations for the Newton + Warp combo as it does not have the same implicit benefit
-    as the RTX renderer (implicit temporal anti-aliasing).
+    Stacks frames to supply the temporal cue Newton needs when the render lacks one; see
+    :meth:`_resolve_frame_stack_default`.
     """
 
     cfg: CartpoleCameraEnvCfg
 
     @staticmethod
     def _resolve_frame_stack_default(camera_cfg, physics_cfg) -> int:
-        """Return ``2`` for the Newton + Warp combo (no implicit damping, no temporal AA),
-        ``1`` otherwise."""
-        from isaaclab_newton.physics import NewtonCfg
-        from isaaclab_newton.renderers import NewtonWarpRendererCfg
-
-        is_newton_warp = isinstance(physics_cfg, NewtonCfg) and isinstance(
-            getattr(camera_cfg, "renderer_cfg", None), NewtonWarpRendererCfg
-        )
-        return 2 if is_newton_warp else 1
+        """Default frame-stack size from the backend capability flags.
+
+        Stack ``2`` frames when the policy needs a temporal cue to infer velocity but the
+        observation carries none -- the physics backend has no implicit damping AND the
+        renderer provides no temporal data for this data type. Otherwise ``1``. The capability
+        lives on the backend configs, not here:
+        :meth:`~isaaclab.physics.physics_manager_cfg.PhysicsCfg.provides_implicit_damping` and
+        :meth:`~isaaclab.renderers.renderer_cfg.RendererCfg.provides_temporal_camera_data`.
+        """
+        if physics_cfg is None or physics_cfg.provides_implicit_damping():
+            return 1
+        renderer_cfg = getattr(camera_cfg, "renderer_cfg", None)
+        data_types = getattr(camera_cfg, "data_types", None) or []
+        data_type = data_types[0] if data_types else ""
+        has_temporal = renderer_cfg is not None and renderer_cfg.provides_temporal_camera_data(data_type)
+        return 1 if has_temporal else 2
 
     def __init__(self, cfg: CartpoleCameraEnvCfg, render_mode: str | None = None, **kwargs):
         # Flatten preset wrappers so the frame-stack resolution below sees concrete types.

source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env_cfg.py

@@ -55,7 +55,9 @@ class BaseCartpoleCameraEnvCfg(CartpoleEnvCfg):
         frame_stack: int = -1
         """Number of frames to stack along the channel dim.
 
-        ``-1`` (default) auto-resolves to ``2`` for the Newton + Warp combo and ``1`` otherwise.
+        ``-1`` (default) auto-resolves to ``2`` when the physics lacks damping and the render
+        carries no temporal cue, else ``1``; see
+        :meth:`~isaaclab_tasks.core.cartpole.cartpole_direct_camera_env.CartpoleCameraEnv._resolve_frame_stack_default`.
         Set to ``1`` to force single-frame; set to ``N > 1`` to force an explicit stack size.
         """
 

source/isaaclab_tasks/test/core/test_cartpole_camera_presets_frame_stack.py

@@ -84,6 +84,49 @@ def test_newton_with_warp_renderer_stacks(self):
         assert isinstance(cfg.tiled_camera.renderer_cfg, NewtonWarpRendererCfg)
         assert _policy_default(cfg) == 2
 
+    # Under Newton + RTX the default splits by data type: rgb gets a DLSS temporal cue
+    # (stack=1); depth/albedo/simple_shading do not (stack=2).
+
+    def test_newton_rtx_depth_stacks(self):
+        """Newton + RTX + depth — depth bypasses DLSS, so it needs explicit stacking."""
+        cfg = _resolve("newton_mjwarp", "depth")
+        assert isinstance(cfg.sim.physics, NewtonCfg)
+        assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg)
+        assert cfg.tiled_camera.data_types == ["depth"]
+        assert _policy_default(cfg) == 2
+
+    def test_newton_rtx_albedo_stacks(self):
+        """Newton + RTX + albedo — albedo bypasses DLSS, so it needs explicit stacking."""
+        cfg = _resolve("newton_mjwarp", "albedo")
+        assert isinstance(cfg.sim.physics, NewtonCfg)
+        assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg)
+        assert cfg.tiled_camera.data_types == ["albedo"]
+        assert _policy_default(cfg) == 2
+
+    def test_newton_rtx_simple_shading_stacks(self):
+        """Newton + RTX + simple_shading — bypasses DLSS, so it needs explicit stacking."""
+        cfg = _resolve("newton_mjwarp", "simple_shading_diffuse_mdl")
+        assert isinstance(cfg.sim.physics, NewtonCfg)
+        assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg)
+        assert cfg.tiled_camera.data_types == ["simple_shading_diffuse_mdl"]
+        assert _policy_default(cfg) == 2
+
+    def test_newton_rtx_rgb_does_not_stack(self):
+        """Newton + RTX + rgb — DLSS supplies the temporal cue, so rgb stays single-frame
+        even though the other RTX AOVs stack."""
+        cfg = _resolve("newton_mjwarp", "rgb")
+        assert isinstance(cfg.sim.physics, NewtonCfg)
+        assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg)
+        assert cfg.tiled_camera.data_types == ["rgb"]
+        assert _policy_default(cfg) == 1
+
+    def test_physx_rtx_depth_does_not_stack(self):
+        """PhysX + RTX + depth — implicit damping means even a non-temporal AOV stays at 1."""
+        cfg = _resolve("physx", "depth")
+        assert isinstance(cfg.sim.physics, PhysxCfg)
+        assert cfg.tiled_camera.data_types == ["depth"]
+        assert _policy_default(cfg) == 1
+
 
 class TestObsSpaceBumpArithmetic:
     """The env class bumps ``observation_space[0] *= frame_stack`` (channel-first) when stacking —