diff --git a/source/isaaclab/changelog.d/jmart-cartpole-rtx.minor.rst b/source/isaaclab/changelog.d/jmart-cartpole-rtx.minor.rst new file mode 100644 index 000000000000..4c6fbe40a3f2 --- /dev/null +++ b/source/isaaclab/changelog.d/jmart-cartpole-rtx.minor.rst @@ -0,0 +1,8 @@ +Added +^^^^^ + +* Added the :meth:`~isaaclab.physics.physics_manager_cfg.PhysicsCfg.provides_implicit_damping` and + :meth:`~isaaclab.renderers.renderer_cfg.RendererCfg.provides_temporal_camera_data` capability + methods, so physics and renderer backends can declare whether a camera observation carries the + temporal information a policy needs to infer velocity (used to decide frame stacking). Base + defaults: physics has implicit damping (``True``); a renderer provides no temporal data (``False``). diff --git a/source/isaaclab/isaaclab/physics/physics_manager_cfg.py b/source/isaaclab/isaaclab/physics/physics_manager_cfg.py index d8e68f0b41a2..c8dfd63c3152 100644 --- a/source/isaaclab/isaaclab/physics/physics_manager_cfg.py +++ b/source/isaaclab/isaaclab/physics/physics_manager_cfg.py @@ -30,3 +30,14 @@ class PhysicsCfg: class_type: type[PhysicsManager] | Any = MISSING """The physics manager class to use. Must be set by subclasses.""" + + def provides_implicit_damping(self) -> bool: + """Whether this backend's integrator has implicit numerical damping. + + With implicit damping (PhysX, OV-PhysX) a camera policy can infer velocity from a + single frame. Without it (Newton's symplectic integrator) the policy needs a temporal + cue in the observation (e.g. frame stacking). + + The base default is ``True``; backends without implicit damping override to ``False``. + """ + return True diff --git a/source/isaaclab/isaaclab/renderers/renderer_cfg.py b/source/isaaclab/isaaclab/renderers/renderer_cfg.py index 276add3a0727..36dcef92c807 100644 --- a/source/isaaclab/isaaclab/renderers/renderer_cfg.py +++ b/source/isaaclab/isaaclab/renderers/renderer_cfg.py @@ -13,3 +13,15 @@ class RendererCfg: """Configuration for a renderer.""" renderer_type: str = "default" + + def provides_temporal_camera_data(self, data_type: str) -> bool: + """Whether this renderer's ``data_type`` output carries temporal information. + + Under a physics backend without implicit damping (e.g. Newton), a camera policy + needs a temporal cue to infer velocity. Renderers that accumulate frames over time + (temporal AA / DLSS) supply it; pure rasterizers and non-beauty AOVs do not. + + The base default is ``False`` (assume no temporal information); renderer subclasses + override per output type. + """ + return False diff --git a/source/isaaclab_newton/changelog.d/jmart-cartpole-rtx.minor.rst b/source/isaaclab_newton/changelog.d/jmart-cartpole-rtx.minor.rst new file mode 100644 index 000000000000..2d71aa957552 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jmart-cartpole-rtx.minor.rst @@ -0,0 +1,7 @@ +Added +^^^^^ + +* Overrode :meth:`provides_implicit_damping` on :class:`NewtonCfg` to return ``False`` (its + symplectic integrator has no implicit damping) and :meth:`provides_temporal_camera_data` on + :class:`NewtonWarpRendererCfg` to return ``False`` (the rasterizer accumulates no temporal data), + so camera tasks can auto-enable frame stacking for the Newton combos that need it. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py index feb9b28db386..ddb3f1719d7e 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py @@ -164,3 +164,7 @@ def __post_init__(self): self.collision_decimation, self.num_substeps, ) + + def provides_implicit_damping(self) -> bool: + # Newton's symplectic integrator has no implicit damping. + return False diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py index 9249fbf4ee71..82183a42ec45 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py @@ -16,6 +16,10 @@ class NewtonWarpRendererCfg(RendererCfg): renderer_type: str = "newton_warp" """Type identifier for Newton Warp renderer.""" + def provides_temporal_camera_data(self, data_type: str) -> bool: + # Pure rasterizer: no temporal accumulation on any output. + return False + enable_textures: bool = True """Enable texture-mapped rendering for meshes.""" diff --git a/source/isaaclab_ov/changelog.d/jmart-cartpole-rtx.minor.rst b/source/isaaclab_ov/changelog.d/jmart-cartpole-rtx.minor.rst new file mode 100644 index 000000000000..cbe11a8128db --- /dev/null +++ b/source/isaaclab_ov/changelog.d/jmart-cartpole-rtx.minor.rst @@ -0,0 +1,6 @@ +Added +^^^^^ + +* Overrode :meth:`provides_temporal_camera_data` on :class:`OVRTXRendererCfg` to return ``True`` + only for the ``rgb``/``rgba`` beauty buffer (temporally accumulated by DLSS), matching Isaac RTX; + other AOVs return ``False``. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py index f60affcdc779..54d4fe7fab5c 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py @@ -26,6 +26,11 @@ class OVRTXRendererCfg(RendererCfg): renderer_type: str = "ovrtx" """Type identifier for OVRTX renderer.""" + def provides_temporal_camera_data(self, data_type: str) -> bool: + # OV-RTX, like Isaac RTX, temporally accumulates only the rgb/rgba beauty buffer + # (DLSS); the other AOVs bypass it. + return data_type in ("rgb", "rgba") + temp_usd_dir: str | None = None """Directory for temporary combined USD files (scene + injected cameras). Used by the OVRTX renderer when building the render scope; must be writable. diff --git a/source/isaaclab_physx/changelog.d/jmart-cartpole-rtx.minor.rst b/source/isaaclab_physx/changelog.d/jmart-cartpole-rtx.minor.rst new file mode 100644 index 000000000000..bc9e66bae132 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/jmart-cartpole-rtx.minor.rst @@ -0,0 +1,6 @@ +Added +^^^^^ + +* Overrode :meth:`provides_temporal_camera_data` on :class:`IsaacRtxRendererCfg` to return ``True`` + only for the ``rgb``/``rgba`` beauty buffer (temporally accumulated by DLSS); the depth, albedo, + simple_shading, and segmentation AOVs return ``False`` as they bypass DLSS. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 2f765546bafe..0e2c8266ea2a 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -23,6 +23,11 @@ class IsaacRtxRendererCfg(RendererCfg): renderer_type: str = "isaac_rtx" """Type identifier for Isaac RTX renderer.""" + def provides_temporal_camera_data(self, data_type: str) -> bool: + # Only the rgb/rgba beauty buffer is temporally accumulated by DLSS; the depth, + # albedo, simple_shading, and segmentation AOVs bypass it. + return data_type in ("rgb", "rgba") + semantic_filter: str | list[str] = "*:*" """A string or a list specifying a semantic filter predicate. Defaults to ``"*:*"``. diff --git a/source/isaaclab_tasks/changelog.d/jmart-cartpole-rtx.rst b/source/isaaclab_tasks/changelog.d/jmart-cartpole-rtx.rst new file mode 100644 index 000000000000..eb55cf739d14 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jmart-cartpole-rtx.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed the camera-based Cartpole task failing to converge under Newton physics with the RTX + ``depth``, ``albedo``, and ``simple_shading`` AOV observations. These AOVs bypass DLSS temporal + accumulation, so the observation carried no temporal cue for the policy to infer velocity from + (Newton's symplectic integrator has no implicit damping). The ``frame_stack`` default resolver + now enables 2-frame stacking for these Newton + RTX AOVs, matching the existing Newton + Warp + behavior; Newton + RTX ``rgb`` keeps single-frame observations as DLSS already supplies the cue. + The resolver now reads the backend capability flags (``PhysicsCfg.provides_implicit_damping``, + ``RendererCfg.provides_temporal_camera_data``) instead of hard-coding backend types. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py index c0ccb26a4dda..e38323f12681 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py @@ -31,23 +31,30 @@ class CartpoleCameraEnv(CartpoleEnv): """Cartpole environment driven by camera observations. - Uses temporal observations for the Newton + Warp combo as it does not have the same implicit benefit - as the RTX renderer (implicit temporal anti-aliasing). + Stacks frames to supply the temporal cue Newton needs when the render lacks one; see + :meth:`_resolve_frame_stack_default`. """ cfg: CartpoleCameraEnvCfg @staticmethod def _resolve_frame_stack_default(camera_cfg, physics_cfg) -> int: - """Return ``2`` for the Newton + Warp combo (no implicit damping, no temporal AA), - ``1`` otherwise.""" - from isaaclab_newton.physics import NewtonCfg - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - is_newton_warp = isinstance(physics_cfg, NewtonCfg) and isinstance( - getattr(camera_cfg, "renderer_cfg", None), NewtonWarpRendererCfg - ) - return 2 if is_newton_warp else 1 + """Default frame-stack size from the backend capability flags. + + Stack ``2`` frames when the policy needs a temporal cue to infer velocity but the + observation carries none -- the physics backend has no implicit damping AND the + renderer provides no temporal data for this data type. Otherwise ``1``. The capability + lives on the backend configs, not here: + :meth:`~isaaclab.physics.physics_manager_cfg.PhysicsCfg.provides_implicit_damping` and + :meth:`~isaaclab.renderers.renderer_cfg.RendererCfg.provides_temporal_camera_data`. + """ + if physics_cfg is None or physics_cfg.provides_implicit_damping(): + return 1 + renderer_cfg = getattr(camera_cfg, "renderer_cfg", None) + data_types = getattr(camera_cfg, "data_types", None) or [] + data_type = data_types[0] if data_types else "" + has_temporal = renderer_cfg is not None and renderer_cfg.provides_temporal_camera_data(data_type) + return 1 if has_temporal else 2 def __init__(self, cfg: CartpoleCameraEnvCfg, render_mode: str | None = None, **kwargs): # Flatten preset wrappers so the frame-stack resolution below sees concrete types. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env_cfg.py index 89cb2db922d2..a1afbd5d094b 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env_cfg.py @@ -55,7 +55,9 @@ class BaseCartpoleCameraEnvCfg(CartpoleEnvCfg): frame_stack: int = -1 """Number of frames to stack along the channel dim. - ``-1`` (default) auto-resolves to ``2`` for the Newton + Warp combo and ``1`` otherwise. + ``-1`` (default) auto-resolves to ``2`` when the physics lacks damping and the render + carries no temporal cue, else ``1``; see + :meth:`~isaaclab_tasks.core.cartpole.cartpole_direct_camera_env.CartpoleCameraEnv._resolve_frame_stack_default`. Set to ``1`` to force single-frame; set to ``N > 1`` to force an explicit stack size. """ diff --git a/source/isaaclab_tasks/test/core/test_cartpole_camera_presets_frame_stack.py b/source/isaaclab_tasks/test/core/test_cartpole_camera_presets_frame_stack.py index 2540c4563877..45def73a7a07 100644 --- a/source/isaaclab_tasks/test/core/test_cartpole_camera_presets_frame_stack.py +++ b/source/isaaclab_tasks/test/core/test_cartpole_camera_presets_frame_stack.py @@ -84,6 +84,49 @@ def test_newton_with_warp_renderer_stacks(self): assert isinstance(cfg.tiled_camera.renderer_cfg, NewtonWarpRendererCfg) assert _policy_default(cfg) == 2 + # Under Newton + RTX the default splits by data type: rgb gets a DLSS temporal cue + # (stack=1); depth/albedo/simple_shading do not (stack=2). + + def test_newton_rtx_depth_stacks(self): + """Newton + RTX + depth — depth bypasses DLSS, so it needs explicit stacking.""" + cfg = _resolve("newton_mjwarp", "depth") + assert isinstance(cfg.sim.physics, NewtonCfg) + assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg) + assert cfg.tiled_camera.data_types == ["depth"] + assert _policy_default(cfg) == 2 + + def test_newton_rtx_albedo_stacks(self): + """Newton + RTX + albedo — albedo bypasses DLSS, so it needs explicit stacking.""" + cfg = _resolve("newton_mjwarp", "albedo") + assert isinstance(cfg.sim.physics, NewtonCfg) + assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg) + assert cfg.tiled_camera.data_types == ["albedo"] + assert _policy_default(cfg) == 2 + + def test_newton_rtx_simple_shading_stacks(self): + """Newton + RTX + simple_shading — bypasses DLSS, so it needs explicit stacking.""" + cfg = _resolve("newton_mjwarp", "simple_shading_diffuse_mdl") + assert isinstance(cfg.sim.physics, NewtonCfg) + assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg) + assert cfg.tiled_camera.data_types == ["simple_shading_diffuse_mdl"] + assert _policy_default(cfg) == 2 + + def test_newton_rtx_rgb_does_not_stack(self): + """Newton + RTX + rgb — DLSS supplies the temporal cue, so rgb stays single-frame + even though the other RTX AOVs stack.""" + cfg = _resolve("newton_mjwarp", "rgb") + assert isinstance(cfg.sim.physics, NewtonCfg) + assert isinstance(cfg.tiled_camera.renderer_cfg, IsaacRtxRendererCfg) + assert cfg.tiled_camera.data_types == ["rgb"] + assert _policy_default(cfg) == 1 + + def test_physx_rtx_depth_does_not_stack(self): + """PhysX + RTX + depth — implicit damping means even a non-temporal AOV stays at 1.""" + cfg = _resolve("physx", "depth") + assert isinstance(cfg.sim.physics, PhysxCfg) + assert cfg.tiled_camera.data_types == ["depth"] + assert _policy_default(cfg) == 1 + class TestObsSpaceBumpArithmetic: """The env class bumps ``observation_space[0] *= frame_stack`` (channel-first) when stacking —