From 83e396a77cc0f050fdc57293aa1b9c32194e1649 Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:14:33 +0800 Subject: [PATCH 1/4] fix(sim): publish render state only for visual consumers --- .../topics/simulation-system/rendering.md | 12 +++--- embodichain/lab/gym/envs/base_env.py | 5 ++- embodichain/lab/sim/sim_manager.py | 11 ++++- tests/gym/envs/test_env_seed.py | 17 ++++++++ tests/sim/test_sim_manager.py | 41 +++++++++++++++++++ 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/agent_context/topics/simulation-system/rendering.md b/agent_context/topics/simulation-system/rendering.md index f91065b30..0aff2f0aa 100644 --- a/agent_context/topics/simulation-system/rendering.md +++ b/agent_context/topics/simulation-system/rendering.md @@ -6,11 +6,13 @@ and readiness reporting. Return to the [simulation overview](simulation-system.m ## Rendering does not advance physics `NewtonPhysicsCfg.sync_to_renderer=None` preserves DexSim's consumer-aware -per-step policy. Camera rendering and reset observations explicitly synchronize -on demand; correctness must not depend on continuous render sync in headless -training. `SimulationManager.sync_render_state()` and backend render hooks -publish state without advancing physics time. Marker publication while paused -uses `capture_visualization(force=True)`. +per-step policy. Nonempty camera groups, Viser captures and simulation-time +recording frames synchronize when consumed. Empty camera groups must not publish +render transforms. Reset publishes immediately only for an open native window; +headless state-only observations need no render synchronization. +`SimulationManager.sync_render_state()` remains an explicit publication API and +backend render hooks publish without advancing physics time. Marker publication +while paused uses `capture_visualization(force=True)`. Viser forces headless mode and is mutually exclusive with the native window. An empty server may start before assets are declared; it must not finalize diff --git a/embodichain/lab/gym/envs/base_env.py b/embodichain/lab/gym/envs/base_env.py index a8d65ab5a..a4ff36980 100644 --- a/embodichain/lab/gym/envs/base_env.py +++ b/embodichain/lab/gym/envs/base_env.py @@ -930,7 +930,10 @@ def reset( self._initialize_episode(reset_ids, **options) self._elapsed_steps[reset_ids] = 0 - self.sim.sync_render_state() + # Cameras and Viser synchronize when they capture. Only the native + # window needs an immediate publication after this direct reset. + if self.sim.is_window_opened: + self.sim.sync_render_state() self.sim.capture_visualization_safely(force=True) with self._profiler.section("get_obs"): diff --git a/embodichain/lab/sim/sim_manager.py b/embodichain/lab/sim/sim_manager.py index 659756bdc..1eaeab9c9 100644 --- a/embodichain/lab/sim/sim_manager.py +++ b/embodichain/lab/sim/sim_manager.py @@ -1662,14 +1662,18 @@ def finalize_newton_physics(self) -> None: def render_camera_group(self, group_ids: list[int]) -> None: """Synchronize physics state and render camera groups. + An empty group list skips synchronization and rendering. Startup + readiness reporting still runs for state-only observations. + Args: group_ids (list[int]): The list of camera group ids to render. Note: This interface is only valid when Ray Tracing rendering backend is enabled. """ - self.sync_render_state() - self._world.render_camera_group(group_ids) + if group_ids: + self.sync_render_state() + self._world.render_camera_group(group_ids) self._log_scene_summary() def update( @@ -3982,6 +3986,9 @@ def _step_window_record_from_sim_update( state.accumulated_sim_time = max( 0.0, state.accumulated_sim_time - state.time_step ) + # Publish from the simulation thread only when a frame is due. The + # render-thread recorder must not wait on its own render publication. + self.sync_render_state() return self._capture_window_record_frame(state) def _save_window_record_worker( diff --git a/tests/gym/envs/test_env_seed.py b/tests/gym/envs/test_env_seed.py index 15c78fe08..420fe8886 100644 --- a/tests/gym/envs/test_env_seed.py +++ b/tests/gym/envs/test_env_seed.py @@ -44,6 +44,7 @@ def __init__(self) -> None: self._num_envs = 1 self.sim = SimpleNamespace( device=torch.device("cpu"), + is_window_opened=False, reset_objects_state=MagicMock( side_effect=lambda **_: self.reset_events.append("reset_objects") ), @@ -107,6 +108,7 @@ def _set_seed(seed: int) -> int: def test_reset_publishes_episode_state_before_visual_observation() -> None: """Reset state must reach render consumers before capture and observations.""" env = _ResetEnv() + env.sim.is_window_opened = True env.reset() @@ -119,6 +121,21 @@ def test_reset_publishes_episode_state_before_visual_observation() -> None: ] +def test_headless_reset_leaves_publication_to_visual_consumers() -> None: + """State-only training resets must not publish the entire render scene.""" + env = _ResetEnv() + + env.reset() + + env.sim.sync_render_state.assert_not_called() + assert env.reset_events == [ + "reset_objects", + "initialize_episode", + "capture", + "get_obs", + ] + + def test_named_component_stream_rewinds_on_explicit_seed_and_is_independent(): env = _ResetEnv() command = env.get_generator("commands") diff --git a/tests/sim/test_sim_manager.py b/tests/sim/test_sim_manager.py index 35fc62e49..700a58c33 100644 --- a/tests/sim/test_sim_manager.py +++ b/tests/sim/test_sim_manager.py @@ -1248,6 +1248,20 @@ def test_render_camera_group_syncs_state_before_rendering() -> None: assert lifecycle == ["sync", "render", "summary"] +def test_empty_camera_group_does_not_publish_render_state() -> None: + """State-only observations must not cross the physics-to-render bridge.""" + sim = object.__new__(SimulationManager) + sim.sync_render_state = MagicMock() + sim._world = SimpleNamespace(render_camera_group=MagicMock()) + sim._log_scene_summary = MagicMock() + + sim.render_camera_group([]) + + sim.sync_render_state.assert_not_called() + sim._world.render_camera_group.assert_not_called() + sim._log_scene_summary.assert_called_once_with() + + def test_register_kinematic_joint_trajectory_expands_each_arena() -> None: sim, builder = _make_runtime_control_sim_manager() frame_count = 3 @@ -2017,6 +2031,15 @@ def test_start_window_record_rejects_concurrent_sessions() -> None: def test_headless_recording_uses_sim_time_and_captures_frames() -> None: sim = _make_sim_manager() + lifecycle: list[str] = [] + sim.sync_render_state = MagicMock(side_effect=lambda: lifecycle.append("sync")) + original_capture = sim._capture_window_record_frame + + def capture(state: _WindowRecordState) -> int: + lifecycle.append("capture") + return original_capture(state) + + sim._capture_window_record_frame = capture assert sim.start_window_record(look_at=DEFAULT_LOOK_AT, fps=5, max_memory=1) state = sim._window_record_state @@ -2030,9 +2053,11 @@ def test_headless_recording_uses_sim_time_and_captures_frames() -> None: sim._step_window_record_from_sim_update(state, physics_dt=0.1) assert len(state.frames) == 0 + assert lifecycle == [] sim._step_window_record_from_sim_update(state, physics_dt=0.1) assert len(state.frames) == 1 + assert lifecycle == ["sync", "capture"] assert sim._window_record_camera.render_count == 1 np.testing.assert_allclose( sim._window_record_camera.last_pose, @@ -2040,6 +2065,22 @@ def test_headless_recording_uses_sim_time_and_captures_frames() -> None: ) +def test_render_thread_recording_does_not_republish_render_state() -> None: + """Native recording consumes state already published by simulation calls.""" + sim = _make_sim_manager(window=object()) + sim.sync_render_state = MagicMock() + assert sim.start_window_record(look_at=DEFAULT_LOOK_AT, fps=5, max_memory=1) + state = sim._window_record_state + assert state is not None + assert state.capture_from_sim_update is False + state.last_capture_time = 0.0 + + sim._step_window_record(state) + + assert len(state.frames) == 1 + sim.sync_render_state.assert_not_called() + + def test_stop_window_record_waits_for_background_export(monkeypatch) -> None: sim = _make_sim_manager() assert sim.start_window_record(look_at=DEFAULT_LOOK_AT, fps=5, max_memory=1) From 2033b8043daefed29ad97e5981ba1eafa5bf1146 Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:59:20 +0800 Subject: [PATCH 2/4] fix(rendering): share publication within explicit render frames --- .../topics/simulation-system/rendering.md | 32 +++- .../embodichain/embodichain.lab.sim.cfg.rst | 24 +++ .../embodichain.lab.sim.sim_manager.rst | 25 +++ embodichain/lab/gym/envs/base_env.py | 23 +-- embodichain/lab/sim/cfg/simulation.py | 26 ++- embodichain/lab/sim/physics/base.py | 9 + embodichain/lab/sim/physics/newton.py | 8 + embodichain/lab/sim/sim_manager.py | 115 ++++++++++-- embodichain/lab/visualization/runtime.py | 7 +- tests/gym/envs/test_env_seed.py | 12 +- tests/gym/envs/test_substep_sensors.py | 3 +- tests/sim/test_sim_manager.py | 177 ++++++++++++++++++ tests/visualization/test_runtime.py | 61 ++++++ 13 files changed, 474 insertions(+), 48 deletions(-) diff --git a/agent_context/topics/simulation-system/rendering.md b/agent_context/topics/simulation-system/rendering.md index 0aff2f0aa..c10e36a84 100644 --- a/agent_context/topics/simulation-system/rendering.md +++ b/agent_context/topics/simulation-system/rendering.md @@ -5,14 +5,30 @@ and readiness reporting. Return to the [simulation overview](simulation-system.m ## Rendering does not advance physics -`NewtonPhysicsCfg.sync_to_renderer=None` preserves DexSim's consumer-aware -per-step policy. Nonempty camera groups, Viser captures and simulation-time -recording frames synchronize when consumed. Empty camera groups must not publish -render transforms. Reset publishes immediately only for an open native window; -headless state-only observations need no render synchronization. -`SimulationManager.sync_render_state()` remains an explicit publication API and -backend render hooks publish without advancing physics time. Marker publication -while paused uses `capture_visualization(force=True)`. +`SimulationManager.render_frame()` owns a read-only consumption phase after +physics, interval events or reset writes. An open native window, nonempty camera +groups, due simulation-time recording and due Viser captures share one state +publication in that phase. Empty camera groups alone are not consumers; an open +window remains a consumer even with `NewtonPhysicsCfg.sync_to_renderer=False`. +Viser decides whether a frame is due before invoking its publication callback. + +`update()` renders after each substep by default. Gym uses +`update(render_final_step=False)` and enters `render_frame()` around observations +after interval events, so the final camera, recording and Viser reads share the +post-event state. Standalone callers combining those consumers can use the same +sequence. Reset completes all writes before a new frame with forced Viser capture. +An independent camera/capture call still publishes fresh state. No publication +cache survives a frame, so paused edits and direct writes between frames do not +need mutation counters. State edits inside a read-only frame require an explicit +`sync_render_state()` before further reads; that API always publishes. + +The Newton backend step hook suppresses automatic DexSim publication during +manager-owned updates to avoid publishing twice. The manager honors explicit +`sync_to_renderer=True` even without consumers; `None`/`False` publish only for +consumers. Raw DexSim `World.update()` retains its configured automatic policy. +Render-thread recording only consumes already-published state and must never +call the blocking physics-to-render bridge. Marker publication while paused uses +`capture_visualization(force=True)` without advancing physics. Viser forces headless mode and is mutually exclusive with the native window. An empty server may start before assets are declared; it must not finalize diff --git a/docs/source/api_reference/embodichain/embodichain.lab.sim.cfg.rst b/docs/source/api_reference/embodichain/embodichain.lab.sim.cfg.rst index aa5ff9025..bfe6e24e4 100644 --- a/docs/source/api_reference/embodichain/embodichain.lab.sim.cfg.rst +++ b/docs/source/api_reference/embodichain/embodichain.lab.sim.cfg.rst @@ -104,3 +104,27 @@ Constructors and ``from_dict()`` accept only the current fields. Surface-element properties are grouped under ``attrs.surface_props``. Volume elasticity uses ``youngs`` and ``poissons``. Configuration dictionaries serialize this same schema; no legacy aliases or field migration are provided. + +World-level configuration module +-------------------------------- + +The same world-level configuration types and backend selectors are available +from ``embodichain.lab.sim.cfg.simulation``. For manager-owned Newton steps, +``sync_to_renderer=None`` or ``False`` publishes only for visual consumers; +``True`` additionally requests publication on steps without consumers. An open +window remains a consumer regardless of this automatic policy. + +.. currentmodule:: embodichain.lab.sim.cfg.simulation + +.. autosummary:: + + DLSSCfg + RenderCfg + GPUMemoryCfg + PhysicsBackendCfg + DefaultPhysicsCfg + NewtonCollisionPipelineCfg + NewtonPhysicsCfg + physics_cfg_for_backend + physics_backend_from_cfg + validate_physics_cfg diff --git a/docs/source/api_reference/embodichain/embodichain.lab.sim.sim_manager.rst b/docs/source/api_reference/embodichain/embodichain.lab.sim.sim_manager.rst index c1b99b2a5..882493ee8 100644 --- a/docs/source/api_reference/embodichain/embodichain.lab.sim.sim_manager.rst +++ b/docs/source/api_reference/embodichain/embodichain.lab.sim.sim_manager.rst @@ -29,6 +29,31 @@ not write drive targets. Set this field to ``None`` to opt out or select prevents automatic recreation. ``GizmoCfg(ik_start_enabled=True)`` activates native IK on the first update with an open window, as used by the robot tutorial. +Physics and visual consumption +------------------------------ + +State-only headless stepping does not publish Newton state to the renderer. +An open native window, camera groups, due recording and due Viser captures +request publication when they consume state. ``render_frame()`` shares that +publication within one read-only consumption phase. Gym arranges this phase +after physics and interval events automatically. Standalone callers combining +camera observations with recording or Viser can use: + +.. code-block:: python + + sim.update(physics_dt, step=1, render_final_step=False) + # Complete direct state edits before entering the read-only frame. + with sim.render_frame(): + sim.render_camera_group(camera_group_ids) + # Due recording and Viser captures run on context exit. + +The phase never advances physics and never caches publication across frames. +Independent camera/capture calls request fresh publication. If state changes +inside a frame, call ``sync_render_state()`` explicitly before further reads. +Opening a window and rendering while paused also request publication. Native +render-thread recording consumes already-published state without invoking the +blocking physics-to-render bridge from that thread. + .. rubric:: Classes .. autosummary:: diff --git a/embodichain/lab/gym/envs/base_env.py b/embodichain/lab/gym/envs/base_env.py index a4ff36980..673828e2c 100644 --- a/embodichain/lab/gym/envs/base_env.py +++ b/embodichain/lab/gym/envs/base_env.py @@ -602,7 +602,11 @@ def _advance_physics(self) -> None: sensor for sensor in self.sensors.values() if sensor.requires_substep_update ] if not sensors: - self.sim.update(self.physics_dt, self.cfg.sim_steps_per_control) + self.sim.update( + self.physics_dt, + self.cfg.sim_steps_per_control, + render_final_step=False, + ) return for sensor in sensors: sensor.begin_control_step() @@ -615,6 +619,7 @@ def sample_substep(dt: float) -> None: self.physics_dt, self.cfg.sim_steps_per_control, after_substep=sample_substep, + render_final_step=False, ) def _update_sim_state(self, **kwargs): @@ -930,14 +935,9 @@ def reset( self._initialize_episode(reset_ids, **options) self._elapsed_steps[reset_ids] = 0 - # Cameras and Viser synchronize when they capture. Only the native - # window needs an immediate publication after this direct reset. - if self.sim.is_window_opened: - self.sim.sync_render_state() - self.sim.capture_visualization_safely(force=True) - - with self._profiler.section("get_obs"): - obs = self.get_obs(**options) + with self.sim.render_frame(force_visualization=True): + with self._profiler.section("get_obs"): + obs = self.get_obs(**options) with self._profiler.section("get_info"): info = self.get_info(**options) @@ -1012,8 +1012,9 @@ def step( with self._profiler.section("update_sim_state"): self._update_sim_state(**kwargs) - with self._profiler.section("get_obs"): - obs = self.get_obs(**kwargs) + with self.sim.render_frame(): + with self._profiler.section("get_obs"): + obs = self.get_obs(**kwargs) with self._profiler.section("get_info"): info = self.get_info(**kwargs) with self._profiler.section("reward"): diff --git a/embodichain/lab/sim/cfg/simulation.py b/embodichain/lab/sim/cfg/simulation.py index 1e1f68372..9451364a8 100644 --- a/embodichain/lab/sim/cfg/simulation.py +++ b/embodichain/lab/sim/cfg/simulation.py @@ -32,6 +32,19 @@ from embodichain.utils import configclass, logger +__all__ = [ + "DLSSCfg", + "RenderCfg", + "GPUMemoryCfg", + "PhysicsBackendCfg", + "DefaultPhysicsCfg", + "NewtonCollisionPipelineCfg", + "NewtonPhysicsCfg", + "physics_cfg_for_backend", + "physics_backend_from_cfg", + "validate_physics_cfg", +] + if TYPE_CHECKING: from dexsim.engine.newton_physics import NewtonCfg from dexsim.engine.newton_physics.solvers_cfg import NewtonSolverCfg @@ -521,11 +534,14 @@ class NewtonPhysicsCfg(PhysicsBackendCfg): """ sync_to_renderer: bool | None = None - """Automatic Newton-to-render sync policy for each physics update. - - ``None`` preserves DexSim's consumer-aware default, ``True`` always syncs, - and ``False`` disables per-step sync. Camera rendering still performs an - explicit on-demand sync through :class:`SimulationManager`. + """Newton-to-render publication policy. + + ``None`` and ``False`` let :class:`SimulationManager` publish on demand + for windows, cameras, recording and browser visualization. ``True`` also + requests publication on steps without a visual consumer. The manager batches + consumers within each render frame. Direct DexSim ``World.update`` callers + retain the underlying automatic policy: window-aware for ``None``, always + for ``True``, and disabled for ``False``. """ debug_mode: bool = False diff --git a/embodichain/lab/sim/physics/base.py b/embodichain/lab/sim/physics/base.py index b4c27f76f..50606d8ce 100644 --- a/embodichain/lab/sim/physics/base.py +++ b/embodichain/lab/sim/physics/base.py @@ -119,6 +119,15 @@ def sync_render_state(self, result: "dexsim.scene.Scene") -> None: """ del result + def _step(self, physics_dt: float) -> None: + """Advance physics; the manager owns subsequent visual consumption.""" + self._manager._world.update(physics_dt) + + @property + def _requires_step_render_sync(self) -> bool: + """Whether the caller explicitly requested publication on every step.""" + return False + def prepare_for_teardown(self) -> None: """Release backend-owned views before Spawn releases their parents. diff --git a/embodichain/lab/sim/physics/newton.py b/embodichain/lab/sim/physics/newton.py index 786ce98fb..8a684447a 100644 --- a/embodichain/lab/sim/physics/newton.py +++ b/embodichain/lab/sim/physics/newton.py @@ -111,6 +111,14 @@ def sync_render_state(self, result: "dexsim.scene.Scene") -> None: ) backend.sync_to_dexsim(result.world) + def _step(self, physics_dt: float) -> None: + """Let the manager batch window and offscreen consumers after stepping.""" + self._manager._world.update(physics_dt, sync_to_dexsim=False) + + @property + def _requires_step_render_sync(self) -> bool: + return self._manager.sim_config.physics_cfg.sync_to_renderer is True + def prepare_for_teardown(self) -> None: """Release Newton render views while Spawn still owns their parents.""" if self._runtime_device is not None and self._runtime_device.startswith("cuda"): diff --git a/embodichain/lab/sim/sim_manager.py b/embodichain/lab/sim/sim_manager.py index 1eaeab9c9..a180e0124 100644 --- a/embodichain/lab/sim/sim_manager.py +++ b/embodichain/lab/sim/sim_manager.py @@ -666,6 +666,8 @@ def __init__( self._ready_spawn_topology_revision = -1 self._synced_spawn_render_topology_revision = -1 self._camera_attachment_topology_revision = -1 + self._render_state_published: bool | None = None + self._pending_record_dt = 0.0 self._visualization_runtime = None self._visualization_overlays: SceneOverlays | None = None @@ -1016,16 +1018,18 @@ def start_visualization(self) -> VisualizationRuntime | None: ) self._visualization_error_reported = False logger.log_info(f"Viser visualization ready at {runtime.endpoint}") + before_capture = None if ( getattr(self, "_spawn_scene", None) is not None and self.spawn_result is not None ): - self.sync_render_state() + before_capture = self._ensure_render_state runtime.capture( sim_step=self._visualization_sim_step, sim_time=self._visualization_sim_time, overlays=self._visualization_overlays, force=True, + before_capture=before_capture, ) return runtime @@ -1067,13 +1071,13 @@ def capture_visualization( != self._visualization_topology_revision ): self.refresh_visualization() - self.sync_render_state() return runtime.capture( sim_step=self._visualization_sim_step, sim_time=self._visualization_sim_time, overlays=self._visualization_overlays, force=force, capture_camera_images=capture_camera_images, + before_capture=self._ensure_render_state, ) def capture_visualization_safely( @@ -1618,6 +1622,76 @@ def _sync_spawn_render_state(self, result: Scene) -> None: return self.physics.sync_render_state(result) self._synced_spawn_render_topology_revision = topology_revision + if getattr(self, "_render_state_published", None) is not None: + self._render_state_published = True + + def _ensure_render_state(self) -> None: + """Reuse publication only inside a caller-owned, read-only frame.""" + if getattr(self, "_render_state_published", None) is True: + return + self.sync_render_state() + if getattr(self, "_render_state_published", None) is not None: + self._render_state_published = True + + @contextmanager + def render_frame( + self, + *, + force_visualization: bool = False, + capture_camera_images: bool = True, + ) -> Iterator[None]: + """Share one state publication across the frame's visual consumers. + + Render camera observations inside this context. On exit, due recording + and Viser frames consume the same state. An open native window is a + consumer even when automatic Newton synchronization is disabled. + + .. attention:: + Complete physics steps, resets and direct state writes before + entering. If state must change inside the context, explicitly call + :meth:`sync_render_state` afterwards. Publication is never cached + across contexts or independent camera/visualization calls. + + Args: + force_visualization: Bypass the Viser frame-rate limiter. + capture_camera_images: Allow Viser to capture camera images. + + Yields: + Control to the caller rendering this frame's camera observations. + """ + previous = getattr(self, "_render_state_published", None) + self._render_state_published = False + record_dt = self._pending_record_dt + self._pending_record_dt = 0.0 + observations_complete = False + try: + if self.is_window_opened or ( + record_dt > 0 and self.physics._requires_step_render_sync + ): + self._ensure_render_state() + yield + observations_complete = True + if ( + self._window_record_state is not None + and self._window_record_state.capture_from_sim_update + and record_dt > 0 + ): + with self.profiler.section("window_record_capture"): + self._step_window_record_from_sim_update( + self._window_record_state, record_dt + ) + if self.sim_config.visualization.backend == "viser": + with self.profiler.section("visualization_capture"): + self.capture_visualization_safely( + force=force_visualization, + capture_camera_images=capture_camera_images, + ) + finally: + if not observations_complete: + self._pending_record_dt += record_dt + # A nested frame may contain an explicit publication after a state + # edit. Never restore an outer frame's obsolete publication flag. + self._render_state_published = None if previous is None else False def sync_render_state(self) -> None: """Publish current physics state to render resources without stepping. @@ -1634,6 +1708,8 @@ def sync_render_state(self) -> None: "Render-state synchronization requires a prepared Spawn scene." ) self.physics.sync_render_state(result) + if getattr(self, "_render_state_published", None) is not None: + self._render_state_published = True def enable_physics(self, enable: bool) -> None: """Enable or disable physics simulation. @@ -1662,8 +1738,9 @@ def finalize_newton_physics(self) -> None: def render_camera_group(self, group_ids: list[int]) -> None: """Synchronize physics state and render camera groups. - An empty group list skips synchronization and rendering. Startup - readiness reporting still runs for state-only observations. + An empty group list skips camera rendering. An open native window still + consumes current state. Use :meth:`render_frame` to share publication + with recording and browser visualization. Args: group_ids (list[int]): The list of camera group ids to render. @@ -1671,8 +1748,9 @@ def render_camera_group(self, group_ids: list[int]) -> None: Note: This interface is only valid when Ray Tracing rendering backend is enabled. """ + if group_ids or self.is_window_opened: + self._ensure_render_state() if group_ids: - self.sync_render_state() self._world.render_camera_group(group_ids) self._log_scene_summary() @@ -1682,6 +1760,7 @@ def update( step: int = 10, *, after_substep: Callable[[float], None] | None = None, + render_final_step: bool = True, ) -> None: """Advance physics explicitly and publish the resulting simulation state. @@ -1695,6 +1774,9 @@ def update( after_substep: Observer called with the elapsed physics time after each world update, before recording and visualization. It runs inside this update call and is not retained by the manager. + render_final_step: Render the final substep immediately. Gym defers + it until after interval events, then enters :meth:`render_frame` + around observations to share publication with other consumers. """ with self.profiler.section("sim_update", is_root=True): with self.profiler.section("gpu_physics_check"): @@ -1711,25 +1793,16 @@ def update( with _temporary_warp_kernel_log_suppression( self.sim_config.physics_cfg ): - self._world.update(physics_dt) + self.physics._step(physics_dt) self._visualization_sim_step += 1 self._visualization_sim_time += physics_dt if after_substep is not None: with self.profiler.section("after_substep"): after_substep(physics_dt) - if ( - self._window_record_state is not None - and self._window_record_state.capture_from_sim_update - ): - with self.profiler.section("window_record_capture"): - self._step_window_record_from_sim_update( - self._window_record_state, physics_dt - ) - if self.sim_config.visualization.backend == "viser": - with self.profiler.section("visualization_capture"): - self.capture_visualization_safely( - capture_camera_images=i == step - 1 - ) + self._pending_record_dt += physics_dt + if render_final_step or i < step - 1: + with self.render_frame(capture_camera_images=i == step - 1): + pass if step > 0: self._log_scene_summary() @@ -1889,6 +1962,8 @@ def open_window(self) -> bool: ): self.enable_window_camera_pose_hotkey(**self._window_camera_pose_hotkey_cfg) self.is_window_opened = True + if self.spawn_result is not None: + self.sync_render_state() self._log_scene_summary() return True @@ -3988,7 +4063,7 @@ def _step_window_record_from_sim_update( ) # Publish from the simulation thread only when a frame is due. The # render-thread recorder must not wait on its own render publication. - self.sync_render_state() + self._ensure_render_state() return self._capture_window_record_frame(state) def _save_window_record_worker( diff --git a/embodichain/lab/visualization/runtime.py b/embodichain/lab/visualization/runtime.py index 9fa020e98..8bf02824a 100644 --- a/embodichain/lab/visualization/runtime.py +++ b/embodichain/lab/visualization/runtime.py @@ -21,7 +21,7 @@ from collections import deque from dataclasses import dataclass, replace from time import perf_counter -from typing import Generic, TypeVar +from typing import Callable, Generic, TypeVar from .backends.base import VisualizationBackend from .cfg import VisualizationCfg @@ -487,6 +487,7 @@ def capture( overlays: SceneOverlays | None = None, force: bool = False, capture_camera_images: bool = True, + before_capture: Callable[[], None] | None = None, ) -> bool: """Capture a due frame and enqueue it without waiting for Viser. @@ -498,6 +499,8 @@ def capture( capture_camera_images: Whether camera images may be captured in this call. Simulation batches disable this for intermediate physics substeps. + before_capture: Publish source state only after a frame is due, + immediately before reading scene or camera data. Returns: ``True`` when a frame was captured, otherwise ``False`` when limited. @@ -522,6 +525,8 @@ def capture( ) if not scene_due and not image_due: return False + if before_capture is not None: + before_capture() if scene_due: if pose_due: self._next_capture_time = now + 1.0 / self.cfg.scene_fps diff --git a/tests/gym/envs/test_env_seed.py b/tests/gym/envs/test_env_seed.py index 420fe8886..b2a8db6e9 100644 --- a/tests/gym/envs/test_env_seed.py +++ b/tests/gym/envs/test_env_seed.py @@ -55,6 +55,7 @@ def __init__(self) -> None: side_effect=lambda **_: self.reset_events.append("capture") ), ) + self.sim.render_frame = self._render_frame self._profiler = _ProfilerStub() self._task_success = torch.zeros(1, dtype=torch.bool) self._detached_uids_for_reset: list[str] = [] @@ -62,6 +63,13 @@ def __init__(self) -> None: self.event_manager = MagicMock() self.initial_random_value = 0.0 + @contextmanager + def _render_frame(self, *, force_visualization: bool = False): + if self.sim.is_window_opened: + self.sim.sync_render_state() + yield + self.sim.capture_visualization_safely(force=force_visualization) + def is_task_success(self, **kwargs) -> torch.Tensor: del kwargs return torch.zeros(1, dtype=torch.bool) @@ -116,8 +124,8 @@ def test_reset_publishes_episode_state_before_visual_observation() -> None: "reset_objects", "initialize_episode", "sync_render", - "capture", "get_obs", + "capture", ] @@ -131,8 +139,8 @@ def test_headless_reset_leaves_publication_to_visual_consumers() -> None: assert env.reset_events == [ "reset_objects", "initialize_episode", - "capture", "get_obs", + "capture", ] diff --git a/tests/gym/envs/test_substep_sensors.py b/tests/gym/envs/test_substep_sensors.py index 84cf50f04..4593222f4 100644 --- a/tests/gym/envs/test_substep_sensors.py +++ b/tests/gym/envs/test_substep_sensors.py @@ -28,7 +28,8 @@ def test_standard_step_schedules_sensor_after_each_physics_substep(enabled): env.cfg = SimpleNamespace(sim_steps_per_control=3) env.sim_cfg = SimpleNamespace(physics_dt=0.01) - def update(dt, n, *, after_substep=None): + def update(dt, n, *, after_substep=None, render_final_step=True): + assert render_final_step is False events.append(("manager", dt, n)) for _ in range(n): events.append(("physics", dt)) diff --git a/tests/sim/test_sim_manager.py b/tests/sim/test_sim_manager.py index 700a58c33..08a122521 100644 --- a/tests/sim/test_sim_manager.py +++ b/tests/sim/test_sim_manager.py @@ -35,6 +35,7 @@ DefaultPhysicsCfg, DLSSCfg, MarkerCfg, + NewtonPhysicsCfg, RenderCfg, RobotCfg, RobotPresetCfg, @@ -258,6 +259,9 @@ def __init__(self) -> None: self.stopped = False def capture(self, **kwargs: object) -> bool: + before_capture = kwargs.pop("before_capture", None) + if before_capture is not None: + before_capture() self.capture_calls.append(kwargs) return True @@ -319,6 +323,8 @@ def _make_sim_manager( sim._window_camera_pose_input_control = None sim._env = FakeEnv() sim._world = FakeWorld() + sim.physics = DefaultPhysicsBackend(sim) + sim._pending_record_dt = 0.0 sim._native_default_plane = object() sim._default_plane = SimpleNamespace(native=lambda: sim._native_default_plane) sim._visualization_runtime = None @@ -341,6 +347,9 @@ def _make_visualization_sim_manager() -> ( sim.profiler = Profiler(None, torch.device("cpu")) sim._is_initialized_gpu_physics = False sim._world = FakeWorld() + sim.physics = DefaultPhysicsBackend(sim) + sim._pending_record_dt = 0.0 + sim.is_window_opened = False sim.prepare = MagicMock() sim.sync_render_state = MagicMock() sim._window_record_state = None @@ -1237,6 +1246,7 @@ def start_visualization(sim: SimulationManager) -> None: def test_render_camera_group_syncs_state_before_rendering() -> None: lifecycle: list[str] = [] sim = object.__new__(SimulationManager) + sim.is_window_opened = False sim.sync_render_state = MagicMock(side_effect=lambda: lifecycle.append("sync")) sim._world = SimpleNamespace( render_camera_group=lambda _group_ids: lifecycle.append("render") @@ -1251,6 +1261,7 @@ def test_render_camera_group_syncs_state_before_rendering() -> None: def test_empty_camera_group_does_not_publish_render_state() -> None: """State-only observations must not cross the physics-to-render bridge.""" sim = object.__new__(SimulationManager) + sim.is_window_opened = False sim.sync_render_state = MagicMock() sim._world = SimpleNamespace(render_camera_group=MagicMock()) sim._log_scene_summary = MagicMock() @@ -1262,6 +1273,172 @@ def test_empty_camera_group_does_not_publish_render_state() -> None: sim._log_scene_summary.assert_called_once_with() +def test_empty_camera_group_still_publishes_for_an_open_window() -> None: + sim = object.__new__(SimulationManager) + sim.is_window_opened = True + sim.sync_render_state = MagicMock() + sim._world = SimpleNamespace(render_camera_group=MagicMock()) + sim._log_scene_summary = MagicMock() + + sim.render_camera_group([]) + + sim.sync_render_state.assert_called_once_with() + sim._world.render_camera_group.assert_not_called() + + +def test_recording_and_viser_share_substep_publication() -> None: + sim, runtime = _make_visualization_sim_manager() + sim._window_record_state = _WindowRecordState( + time_step=0.01, + max_memory_bytes=1024, + output_dir="unused", + video_name="unused", + save_kwargs={}, + capture_from_sim_update=True, + ) + sim._capture_window_record_frame = MagicMock() + + sim.update(0.01, 1) + + sim.sync_render_state.assert_called_once_with() + sim._capture_window_record_frame.assert_called_once() + assert len(runtime.capture_calls) == 1 + + +def _make_render_frame_sim(*, window=False, viser=False, automatic_sync=False): + """Track published state through real manager/backend scheduling methods.""" + sim, runtime = _make_visualization_sim_manager() + sim.sim_config.physics_cfg = NewtonPhysicsCfg(sync_to_renderer=automatic_sync) + sim.sim_config.visualization.backend = "viser" if viser else "none" + sim.physics = NewtonPhysicsBackend(sim) + sim.is_window_opened = window + sim._log_scene_summary = lambda: None + sim._visualization_manifest_topology_revision = sim._visualization_topology_revision + state = {"value": 0, "published": None} + publications = [] + rendered = [] + sim._spawn_scene = SimpleNamespace( + builder=SimpleNamespace(is_finalized=True, result=object()) + ) + + def advance(_dt, *, sync_to_dexsim): + # Automatic publication must not duplicate the manager's frame. + assert sync_to_dexsim is False + state["value"] += 1 + + def publish(_result): + state["published"] = state["value"] + publications.append(state["value"]) + + sim._world.update = advance + sim._world.render_camera_group = lambda ids: rendered.append(state["published"]) + sim.physics.sync_render_state = publish + sim.sync_render_state = SimulationManager.sync_render_state.__get__(sim) + return sim, runtime, state, publications, rendered + + +@pytest.mark.parametrize("automatic_sync", [None, False, True]) +@pytest.mark.parametrize("window", [False, True]) +def test_window_publication_respects_consumers_with_all_newton_policies( + automatic_sync, window +): + sim, _, _, publications, rendered = _make_render_frame_sim( + window=window, automatic_sync=automatic_sync + ) + for _ in range(3): + sim.update(0.01, 1, render_final_step=False) + with sim.render_frame(): + sim.render_camera_group([]) + assert publications == ([1, 2, 3] if window or automatic_sync is True else []) + assert rendered == [] + + +@pytest.mark.parametrize("camera", [False, True]) +@pytest.mark.parametrize("record", [False, True]) +@pytest.mark.parametrize("viser", [False, True]) +def test_final_substep_consumers_share_publication(camera, record, viser): + sim, runtime, state, publications, rendered = _make_render_frame_sim(viser=viser) + recorded = [] + if record: + sim._window_record_state = _WindowRecordState( + time_step=0.01, + max_memory_bytes=1024, + output_dir="unused", + video_name="unused", + save_kwargs={}, + capture_from_sim_update=True, + ) + sim._capture_window_record_frame = lambda _: recorded.append(state["published"]) + sim.update(0.01, 3, render_final_step=False) + # Interval events may directly edit state after physics. All final-frame + # consumers must see this edit, not a cached final-substep publication. + state["value"] = 30 + with sim.render_frame(): + sim.render_camera_group([3] if camera else []) + expected = [1, 2] if record or viser else [] + if camera or record or viser: + expected += [30] + assert publications == expected + assert rendered == ([30] if camera else []) + assert recorded == ([1, 2, 30] if record else []) + assert len(runtime.capture_calls) == (3 if viser else 0) + + +def test_render_frames_and_independent_reads_refresh_direct_state_writes(): + sim, _, state, publications, rendered = _make_render_frame_sim() + for value in (4, 7): + state["value"] = value # A reset/pose write without a physics step. + with sim.render_frame(): + sim.render_camera_group([1]) + sim.render_camera_group([2]) + state["value"] = 9 + sim.render_camera_group([1]) + state["value"] = 12 + sim.render_camera_group([1]) + assert publications == [4, 7, 9, 12] + assert rendered == [4, 4, 7, 7, 9, 12] + + +def test_explicit_publication_inside_frame_refreshes_after_state_edit(): + sim, _, state, publications, rendered = _make_render_frame_sim() + with sim.render_frame(): + sim.render_camera_group([1]) + state["value"] = 5 + sim.sync_render_state() + sim.render_camera_group([1]) + assert publications == [0, 5] + assert rendered == [0, 5] + + +def test_failed_frame_discards_publication_and_preserves_record_cadence(): + sim, _, state, publications, _ = _make_render_frame_sim() + sim.update(0.01, 1, render_final_step=False) + with pytest.raises(ValueError, match="observation failed"): + with sim.render_frame(): + sim.render_camera_group([1]) + raise ValueError("observation failed") + assert sim._render_state_published is None + assert sim._pending_record_dt == pytest.approx(0.01) + state["value"] = 8 + with sim.render_frame(): + sim.render_camera_group([1]) + assert publications == [1, 8] + assert sim._pending_record_dt == 0 + + +def test_failed_publication_retries_within_the_same_frame(): + sim, _, _, _, _ = _make_render_frame_sim() + sim.physics.sync_render_state = MagicMock( + side_effect=[RuntimeError("bridge"), None] + ) + with sim.render_frame(): + with pytest.raises(RuntimeError, match="bridge"): + sim.render_camera_group([1]) + sim.render_camera_group([1]) + sim.render_camera_group([2]) + assert sim.physics.sync_render_state.call_count == 2 + + def test_register_kinematic_joint_trajectory_expands_each_arena() -> None: sim, builder = _make_runtime_control_sim_manager() frame_count = 3 diff --git a/tests/visualization/test_runtime.py b/tests/visualization/test_runtime.py index 2505d902a..8d009a64d 100644 --- a/tests/visualization/test_runtime.py +++ b/tests/visualization/test_runtime.py @@ -21,6 +21,7 @@ from time import monotonic, sleep import numpy as np +import pytest from embodichain.lab.visualization import ( CameraImage, @@ -325,6 +326,66 @@ def test_runtime_keeps_backend_lifecycle_on_update_thread() -> None: assert runtime.stats.published_image_frames == 1 +@pytest.mark.parametrize("due", ["none", "pose", "deformable", "image", "forced"]) +def test_capture_publishes_source_once_and_only_when_a_consumer_is_due(due): + published = threading.Event() + exporter = _Exporter(published) + runtime = VisualizationRuntime( + exporter, + VisualizationCfg(backend="viser"), + backend=_Backend(published, threading.Event()), + ) + runtime.start() + try: + runtime._next_capture_time = 0.0 if due == "pose" else float("inf") + runtime._next_deformable_capture_time = ( + 0.0 if due == "deformable" else float("inf") + ) + runtime._next_image_capture_time = 0.0 if due == "image" else float("inf") + calls = [] + + def publish(): + # Publication precedes both scene reads and camera rendering. + assert exporter.dynamic_capture_flags == [] + assert exporter.image_capture_count == 0 + calls.append("publish") + + captured = runtime.capture( + sim_step=1, + sim_time=0.01, + force=due == "forced", + before_capture=publish, + ) + assert captured is (due != "none") + assert calls == ([] if due == "none" else ["publish"]) + finally: + runtime.stop() + + +def test_failed_source_publication_does_not_consume_capture_deadlines(): + published = threading.Event() + runtime = VisualizationRuntime( + _Exporter(published), + VisualizationCfg(backend="viser"), + backend=_Backend(published, threading.Event()), + ) + runtime.start() + try: + runtime._next_capture_time = 0.0 + runtime._next_deformable_capture_time = float("inf") + runtime._next_image_capture_time = float("inf") + + def fail(): + raise RuntimeError("publication failed") + + with pytest.raises(RuntimeError, match="publication failed"): + runtime.capture(sim_step=1, sim_time=0.01, before_capture=fail) + assert runtime._next_capture_time == 0.0 + assert runtime.capture(sim_step=1, sim_time=0.01) + finally: + runtime.stop() + + def test_runtime_publishes_latest_replay_state_on_update_thread() -> None: published = threading.Event() backend = _Backend(published, threading.Event()) From 02afa2356580221a0b31d23da5d50b3454463500 Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:09:49 +0800 Subject: [PATCH 3/4] test(rendering): align profiler probes with backend stepping --- tests/sim/test_sim_profiler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/sim/test_sim_profiler.py b/tests/sim/test_sim_profiler.py index 3acb0ec04..6cb2a2c14 100644 --- a/tests/sim/test_sim_profiler.py +++ b/tests/sim/test_sim_profiler.py @@ -23,6 +23,7 @@ from embodichain.lab.sim import Profiler, ProfilerCfg, SimulationManager from embodichain.lab.sim.cfg import DefaultPhysicsCfg +from embodichain.lab.sim.physics.default import DefaultPhysicsBackend pytestmark = pytest.mark.no_sim @@ -46,6 +47,9 @@ def _make_sim_update_probe(profiler: Profiler) -> SimulationManager: sim.device = torch.device("cpu") sim._is_initialized_gpu_physics = False sim._world = _WorldUpdateProbe() + sim.physics = DefaultPhysicsBackend(sim) + sim.is_window_opened = False + sim._pending_record_dt = 0.0 sim._window_record_state = None sim._visualization_runtime = None sim._visualization_sim_step = 0 @@ -67,6 +71,7 @@ def test_standalone_sim_update_is_profile_root() -> None: sim.update(step=2) + assert sim._world.update_calls == 2 assert "sim_update" in profiler._stats assert "sim_update.gpu_physics_check" in profiler._stats assert "sim_update.physics_steps" in profiler._stats @@ -98,7 +103,7 @@ def test_visualization_capture_is_profiled_per_sim_step() -> None: sim = _make_sim_update_probe(profiler) sim.sim_config.visualization.backend = "viser" camera_capture_flags: list[bool] = [] - sim.capture_visualization_safely = lambda *, capture_camera_images: ( + sim.capture_visualization_safely = lambda *, capture_camera_images, force=False: ( camera_capture_flags.append(capture_camera_images) ) From 4d9a9eecb6f9174f4723f27b33f854820da7a333 Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:37:24 +0800 Subject: [PATCH 4/4] test(rendering): complete backend step fixture for Warp log checks --- tests/sim/test_sim_manager_cfg.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/sim/test_sim_manager_cfg.py b/tests/sim/test_sim_manager_cfg.py index a96f8e93c..47abd7f7f 100644 --- a/tests/sim/test_sim_manager_cfg.py +++ b/tests/sim/test_sim_manager_cfg.py @@ -234,6 +234,7 @@ def fake_init() -> None: sim_manager.wp.config.log_level = previous_log_level +@pytest.mark.no_sim def test_newton_warp_log_suppression_covers_world_update() -> None: previous_log_level = sim_manager.wp.config.log_level observed_log_levels = [] @@ -259,9 +260,12 @@ def update(self, _physics_dt: float) -> None: _world=World(), _visualization_sim_step=0, _visualization_sim_time=0.0, + _pending_record_dt=0.0, _window_record_state=None, + render_frame=lambda **_kwargs: nullcontext(), _log_scene_summary=lambda: None, ) + manager.physics = DefaultPhysicsBackend(manager) try: SimulationManager.update(manager, physics_dt=0.01, step=1) assert observed_log_levels == [sim_manager.wp.LOG_WARNING]