diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d30646ad1..a108a93ad 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,8 +7,8 @@ /scripts/train_*.py @TATP-233 @caozx1110 # Task and environment ownership -/src/unilab/envs/motion_tracking/ @caozx1110 -/src/unilab/envs/manipulation/ @Mingrui-Yu +/src/unilab/tasks/motion_tracking/ @caozx1110 +/src/unilab/tasks/manipulation/ @Mingrui-Yu /scripts/motion/ @caozx1110 # Project process and docs diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index cc7bacc49..630a89483 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -37,7 +37,7 @@ body: label: Reproduction description: Exact command, config, or sequence that reproduces the issue. placeholder: | - uv run scripts/train_offpolicy.py algo=sac task=sac/g1_walk_flat/mujoco ... + uv run scripts/train_sac.py task=g1_walk_flat/mujoco ... validations: required: true diff --git a/.gitignore b/.gitignore index d1de5a693..e4c38a1cc 100644 --- a/.gitignore +++ b/.gitignore @@ -58,7 +58,7 @@ run_summary.json src/unilab/assets/checkpoints/ scripts/benchmark/outputs/ -src/unilab/algos/torch/rsl_rl +src/unilab/algos/rsl_rl third-party temp/ @@ -81,6 +81,12 @@ src/unilab/assets/motions/x2/*.csv # Robot mesh assets (downloaded from HF at runtime) src/unilab/assets/robots/x2/meshes/*.STL +src/unilab/assets/robots/a2arm/meshes/* +!src/unilab/assets/robots/a2arm/meshes/.gitkeep +src/unilab/assets/robots/t800/assets/* +!src/unilab/assets/robots/t800/assets/.gitkeep +src/unilab/assets/robots/t800/textures/* +!src/unilab/assets/robots/t800/textures/.gitkeep # Grasp cache assets (downloaded from HF at runtime) src/unilab/assets/caches/*.npy diff --git a/AGENTS.md b/AGENTS.md index 8b02a7549..9baf79cbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,19 +39,19 @@ UniLab 是一个 **高性能、模块化、contract 驱动** 的 RL infrastructu ## Sim2Sim 跨后端配置契约 -`src/unilab/training/sim2sim.py` 按 dotted path 维护三类字段： +`src/unilab/utils/sim2sim.py` 按 dotted path 维护三类字段： - **DENYLIST**（差异即 `CrossBackendIncompatibleError`）：`algo.obs_groups`、`env.control_config.action_scale`、`algo.policy.actor_hidden_dims` / `critic_hidden_dims`、`algo.empirical_normalization` / `algo.obs_normalization`、`env.sampling_mode`。`env.*` 子集对**任一方向**的不对称出现也 fail-closed；`algo` 专属字段目标缺省时按设计跳过（跨算法合法）。 - **WARNING_LIST**：`reward.*`、`env.control_config.simulate_action_latency`、`env.ctrl_dt`。 - **ALLOWLIST**（自由覆盖）：`training.sim_backend`、`env.scene`、`training.play_steps`、`env.domain_rand`、`env.noise_config`、`env.commands.vel_limit`。 -训练时 `ExperimentTracker.start()` 把上述字段写入 `run_config.json` 的 `contract_snapshot`（不改 checkpoint 格式，旧 run 无 snapshot 时 fallback + warning）；五个 play 入口在建 env 前调用 `resolve_sim2sim_config` 校验，并用 `policy_load_dim_guard` 包裹 checkpoint 加载以把维度不匹配的隐晦报错重抛为显式诊断。设 `training.sim2sim_strict=false` 可把 DENYLIST 差异降级为 warning（默认 `true`）。DENYLIST 字段在每个后端 owner 配置中显式声明并保持跨后端一致（范例：`conf/ppo/task/g1_walk_flat/{mujoco,motrix}.yaml`）；跨后端契约审计见 `scripts/audit_sim2sim_contracts.py`。 +训练时 `ExperimentTracker.start()` 把上述字段写入 `run_config.json` 的 `contract_snapshot`（不改 checkpoint 格式，旧 run 无 snapshot 时 fallback + warning）；五个 play 入口在建 env 前调用 `resolve_sim2sim_config` 校验，并用 `policy_load_dim_guard` 包裹 checkpoint 加载以把维度不匹配的隐晦报错重抛为显式诊断。设 `training.sim2sim_strict=false` 可把 DENYLIST 差异降级为 warning（默认 `true`）。DENYLIST 字段在共享 base owner 与后端 owner 配置中显式声明并保持跨后端一致（范例：`conf/ppo/task/g1_walk_flat/{base,mujoco,motrix}.yaml`）；跨后端契约审计见 `scripts/audit_sim2sim_contracts.py`。 ## Pointers - PPO: `scripts/train_rsl_rl.py` - APPO: `scripts/train_appo.py` -- SAC / TD3: `scripts/train_offpolicy.py` +- SAC / TD3 / FlashSAC: `scripts/train_sac.py` / `scripts/train_td3.py` / `scripts/train_flashsac.py` - env contract: `src/unilab/base/np_env.py` - backend contract: `src/unilab/base/backend/base.py` - training run helpers: `src/unilab/training/run.py` @@ -59,7 +59,7 @@ UniLab 是一个 **高性能、模块化、contract 驱动** 的 RL infrastructu - shared numeric helpers: `src/unilab/utils/rotation.py`, `src/unilab/utils/geometry.py` - config schema: `src/unilab/structured_configs.py` - async runner: `src/unilab/ipc/async_runner.py` -- sim2sim 跨后端契约: `src/unilab/training/sim2sim.py` +- sim2sim 跨后端契约: `src/unilab/utils/sim2sim.py` ## GitHub CLI (gh) 速查 diff --git a/Makefile b/Makefile index b09ce494e..b09dd8b20 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ clean: find . -type d -name ".ruff_cache" -exec rm -rf {} + find . -type d -name "htmlcov" -exec rm -rf {} + find . -type f -name ".coverage" -delete - rm -f train_appo.log train_offpolicy.log train_rsl_rl.log MUJOCO_LOG.TXT + rm -f train_appo.log train_sac.log train_flashsac.log train_rsl_rl.log MUJOCO_LOG.TXT find src/unilab/assets/.cache -type f ! -name '.gitkeep' -delete 2>/dev/null || true find src/unilab/assets/caches -type f ! -name '.gitkeep' -delete 2>/dev/null || true find src/unilab/assets/checkpoints -type f ! -name '.gitkeep' -delete 2>/dev/null || true diff --git a/conf/appo/task/allegro_inhand/base.yaml b/conf/appo/task/allegro_inhand/base.yaml new file mode 100644 index 000000000..a36f24c9f --- /dev/null +++ b/conf/appo/task/allegro_inhand/base.yaml @@ -0,0 +1,138 @@ +# @package _global_ +# Canonical Allegro rotation Manager-Based task declaration. Backend leaves own +# only backend identity and algorithm/runtime tuning. +env: + scene: + model_file: src/unilab/assets/robots/allegro_hand/scene.xml + default_keyframe_name: home + entities: + robot: + root_body_name: ball + joint_names: + - ffj0 + - ffj1 + - ffj2 + - ffj3 + - mfj0 + - mfj1 + - mfj2 + - mfj3 + - rfj0 + - rfj1 + - rfj2 + - rfj3 + - thj0 + - thj1 + - thj2 + - thj3 + body_names: [ball, ff_tip, mf_tip, rf_tip, th_tip] + actuator_names: + - ffa0 + - ffa1 + - ffa2 + - ffa3 + - mfa0 + - mfa1 + - mfa2 + - mfa3 + - rfa0 + - rfa1 + - rfa2 + - rfa3 + - tha0 + - tha1 + - tha2 + - tha3 + sim_dt: 0.005 + ctrl_dt: 0.05 + max_episode_seconds: 20.0 + observations: + policy: + history_length: 3 + flatten_history_dim: true + terms: + rotation: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroRotationObservation + params: + entity_name: robot + action_name: hand + joint_noise: 0.02 + torque_estimate_kp: 1.0 + torque_estimate_kd: 0.1 + actions: + hand: + _target_: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroIncrementalPositionActionCfg + entity_name: robot + actuator_names: [".*"] + action_scale: 0.041666666666666664 + raw_action_clip: [-1.0, 1.0] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_hand_ball: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroHandBallReset + mode: reset + params: + entity_name: robot + # null explicitly selects the model home pose. A configured path is + # fail-closed when missing or malformed. + grasp_cache_path: null + joint_noise: 0.0 + ball_velocity_noise: 0.0 + ball_z_offset: 0.0 + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [1.0, 1.0] + kd_range: [0.1, 0.1] + operation: abs + terminations: + dropped: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroDropTermination + params: + observation_group: policy + observation_term: rotation + minimum_ball_height: 0.125 + time_out: + func: unilab.envs.mdp.time_out + time_out: true + scale_rewards_by_dt: true + policy_observation_group: policy + critic_observation_group: null + +reward: + rotate: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroRotateReward + weight: 1.25 + params: + state_term_name: dropped + rotation_axis: [0.0, 0.0, 1.0] + clip_min: -0.5 + clip_max: 0.5 + obj_linvel: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.object_linear_velocity_l1 + weight: -0.3 + params: + state_term_name: dropped + pose_diff: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.hand_pose_deviation_l2 + weight: -0.3 + params: + state_term_name: dropped + torque: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.estimated_torque_l2 + weight: -0.1 + params: + state_term_name: dropped + work: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.estimated_work_l2 + weight: -2.0 + params: + state_term_name: dropped + drop: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.dropped + weight: 0.0 + params: + state_term_name: dropped diff --git a/conf/appo/task/allegro_inhand/drake.yaml b/conf/appo/task/allegro_inhand/drake.yaml new file mode 100644 index 000000000..edbb75562 --- /dev/null +++ b/conf/appo/task/allegro_inhand/drake.yaml @@ -0,0 +1,50 @@ +# @package _global_ +defaults: + - /task/allegro_inhand/base + - _self_ + +training: + task_name: AllegroInhandRotation + sim_backend: drake + play_steps: 200 + render_spacing: 0.5 + cam_distance: 1.5 + cam_lookat: [0.75, 0.75, 0] + cam_elevation: -20.0 + replay_queue_size: 4 +algo: + num_envs: 1024 + steps_per_env: 8 + max_iterations: 3000 + save_interval: 500 + algorithm: + value_loss_coef: 4.0 + entropy_coef: 0.01 + learning_rate: 0.001 + desired_kl: 0.025 + adaptive_kl_factor: 2.0 + adaptive_lr_factor: 1.5 + num_learning_epochs: 5 + num_mini_batches: 4 + clip_param: 0.2 + gamma: 0.99 + lam: 0.95 + max_grad_norm: 1.0 + use_clipped_value_loss: true + schedule: adaptive + actor: + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + distribution_cfg: + class_name: rsl_rl.modules.distribution.GaussianDistribution + init_std: 1.0 + std_type: scalar + critic: + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true +env: + drake_backend_mode: batch + events: + pd_gains: null diff --git a/conf/appo/task/allegro_inhand/motrix.yaml b/conf/appo/task/allegro_inhand/motrix.yaml index c027729a8..1a89e3b37 100644 --- a/conf/appo/task/allegro_inhand/motrix.yaml +++ b/conf/appo/task/allegro_inhand/motrix.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/allegro_inhand/base + - _self_ + training: task_name: AllegroInhandRotation sim_backend: motrix @@ -7,6 +11,9 @@ training: cam_distance: 1.5 cam_lookat: [0.75, 0.75, 0] cam_elevation: -20.0 +env: + events: + pd_gains: null algo: num_envs: 16384 steps_per_env: 8 @@ -39,27 +46,3 @@ algo: hidden_dims: [512, 256, 128] activation: elu obs_normalization: true -reward: - scales: - rotate: 1.25 - obj_linvel: -0.3 - pose_diff: -0.3 - torque: -0.1 - work: -2.0 - drop: 0.0 - angvel_clip_min: -0.5 - angvel_clip_max: 0.5 - reset_z_threshold: 0.125 -env: - gen_grasp: false - max_episode_seconds: 20.0 - grasp_cache_path: caches/allegro_grasp_50k.npy - # Keep only grasp/pose reset variation. All online DR terms stay disabled. - domain_rand: - randomize_base_mass: false - random_com: false - randomize_gravity: false - push_robots: false - joint_noise: 0.0 - ball_vel_noise: 0.0 - ball_z_offset: 0.0 diff --git a/conf/appo/task/allegro_inhand/mujoco.yaml b/conf/appo/task/allegro_inhand/mujoco.yaml index c24a442bc..7d4bcfbd0 100644 --- a/conf/appo/task/allegro_inhand/mujoco.yaml +++ b/conf/appo/task/allegro_inhand/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/allegro_inhand/base + - _self_ + training: task_name: AllegroInhandRotation sim_backend: mujoco @@ -41,27 +45,3 @@ algo: hidden_dims: [512, 256, 128] activation: elu obs_normalization: true -reward: - scales: - rotate: 1.25 - obj_linvel: -0.3 - pose_diff: -0.3 - torque: -0.1 - work: -2.0 - drop: 0.0 - angvel_clip_min: -0.5 - angvel_clip_max: 0.5 - reset_z_threshold: 0.125 -env: - gen_grasp: false - max_episode_seconds: 20.0 - grasp_cache_path: caches/allegro_grasp_50k.npy - # Keep only grasp/pose reset variation. All online DR terms stay disabled. - domain_rand: - randomize_base_mass: false - random_com: false - randomize_gravity: false - push_robots: false - joint_noise: 0.0 - ball_vel_noise: 0.0 - ball_z_offset: 0.0 diff --git a/conf/appo/task/g1_23dof_climb_tracking/motrix.yaml b/conf/appo/task/g1_23dof_climb_tracking/motrix.yaml index 7844addb7..ae708bf1d 100644 --- a/conf/appo/task/g1_23dof_climb_tracking/motrix.yaml +++ b/conf/appo/task/g1_23dof_climb_tracking/motrix.yaml @@ -1,65 +1,8 @@ # @package _global_ +defaults: + - /task/g1_23dof_climb_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking23Dof sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 20000 - save_interval: 500 -env: - sampling_mode: adaptive - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_23dof_climb_tracking/mujoco.yaml b/conf/appo/task/g1_23dof_climb_tracking/mujoco.yaml index 21380ecb4..3586affd7 100644 --- a/conf/appo/task/g1_23dof_climb_tracking/mujoco.yaml +++ b/conf/appo/task/g1_23dof_climb_tracking/mujoco.yaml @@ -1,65 +1,85 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking23Dof sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 20000 save_interval: 500 + algorithm: + adaptive_kl_factor: 1.2 + adaptive_lr_factor: 1.1 + env: - sampling_mode: adaptive - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/g1/scene_climb_20_z_scale_1_23dof.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + max_episode_seconds: 15.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + commands: + motion: + params: + motion_file: motions/g1/climb_20_z_scale_1.0_23dof.npz + sampling_mode: adaptive + truncate_on_clip_end: false + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.3} + ee_body_pos: + params: + command_name: motion + threshold: 0.3 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/appo/task/g1_23dof_flip_tracking/motrix.yaml b/conf/appo/task/g1_23dof_flip_tracking/motrix.yaml index 05098dac7..a600df907 100644 --- a/conf/appo/task/g1_23dof_flip_tracking/motrix.yaml +++ b/conf/appo/task/g1_23dof_flip_tracking/motrix.yaml @@ -1,86 +1,14 @@ # @package _global_ +defaults: + - /task/g1_23dof_flip_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking23Dof sim_backend: motrix - play_steps: 1000 + algo: - num_envs: 1024 - steps_per_env: 24 - max_iterations: 3500 - save_interval: 500 algorithm: - num_learning_epochs: 10 - num_mini_batches: 8 - clip_param: 0.2 - gamma: 0.99 - lam: 0.95 - value_loss_coef: 1.0 - entropy_coef: 0.005 - learning_rate: 1.0e-3 - max_grad_norm: 1.0 - use_clipped_value_loss: true - schedule: adaptive - desired_kl: 0.01 adaptive_kl_factor: 2.0 adaptive_lr_factor: 1.5 - optimizer: adam - tau: 1.0 - target_update_freq: 1 - vtrace_clip_rho: 1.0 - vtrace_clip_c: 1.0 enable_compile: true -env: - sampling_mode: start - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_23dof_flip_tracking/mujoco.yaml b/conf/appo/task/g1_23dof_flip_tracking/mujoco.yaml index 6676d9360..6798916ca 100644 --- a/conf/appo/task/g1_23dof_flip_tracking/mujoco.yaml +++ b/conf/appo/task/g1_23dof_flip_tracking/mujoco.yaml @@ -1,8 +1,13 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking23Dof sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 steps_per_env: 24 @@ -13,58 +18,84 @@ algo: num_learning_epochs: 10 num_mini_batches: 8 desired_kl: 0.01 + adaptive_kl_factor: 1.2 + adaptive_lr_factor: 1.1 + env: - sampling_mode: start - truncate_on_clip_end: true + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + commands: + motion: + params: + motion_file: motions/g1/flip_360_001__A304_23dof.npz + sampling_mode: start + truncate_on_clip_end: true + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + motion_clip_end: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_clip_end + time_out: true + params: {command_name: motion} + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + action_rate_l2: + weight: -0.005 diff --git a/conf/appo/task/g1_23dof_motion_tracking/motrix.yaml b/conf/appo/task/g1_23dof_motion_tracking/motrix.yaml index aa74c1a66..9d8ecf00d 100644 --- a/conf/appo/task/g1_23dof_motion_tracking/motrix.yaml +++ b/conf/appo/task/g1_23dof_motion_tracking/motrix.yaml @@ -1,29 +1,8 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTracking23Dof sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 5000 - save_interval: 500 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_23dof_motion_tracking/mujoco.yaml b/conf/appo/task/g1_23dof_motion_tracking/mujoco.yaml index 3b31907a5..8605feb92 100644 --- a/conf/appo/task/g1_23dof_motion_tracking/mujoco.yaml +++ b/conf/appo/task/g1_23dof_motion_tracking/mujoco.yaml @@ -1,32 +1,67 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTracking23Dof sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 5000 - save_interval: 500 - algorithm: - adaptive_kl_factor: 2.0 - adaptive_lr_factor: 1.5 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: &g1_23dof_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: *g1_23dof_joints + body_names: &tracked_bodies_23dof + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_roll_rubber_hand + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_roll_rubber_hand + commands: + motion: + params: + motion_file: motions/g1/dance1_subject2_part_23dof.npz + body_names: *tracked_bodies_23dof + terminations: + ee_body_pos: + params: + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand diff --git a/conf/appo/task/g1_23dof_walk_flat/base.yaml b/conf/appo/task/g1_23dof_walk_flat/base.yaml new file mode 100644 index 000000000..66c3da41e --- /dev/null +++ b/conf/appo/task/g1_23dof_walk_flat/base.yaml @@ -0,0 +1,66 @@ +# @package _global_ +# Canonical G1 23-DoF walk Manager-Based task declaration (APPO owners; mirrors the PPO base). +# Inherits the 29-DoF flat contract and swaps the scene to the 23-DoF model +# (no waist roll/pitch, no wrist pitch/yaw) with the 23-entry pose weights. +defaults: + - /task/g1_walk_flat/base + - _self_ + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + +reward: + pose: + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/appo/task/g1_23dof_walk_flat/mujoco.yaml b/conf/appo/task/g1_23dof_walk_flat/mujoco.yaml index e68cd0af1..aa34c3339 100644 --- a/conf/appo/task/g1_23dof_walk_flat/mujoco.yaml +++ b/conf/appo/task/g1_23dof_walk_flat/mujoco.yaml @@ -1,36 +1,13 @@ # @package _global_ +# MuJoCo APPO owner: inherits the shared 23-DoF flat Manager-Based contract +# from base.yaml and only carries backend/algo identity. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + training: task_name: G1Walk23DofFlat sim_backend: mujoco algo: max_iterations: 500 save_interval: 100 -env: - control_config: - action_scale: 0.25 - curriculum: - enabled: false - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.2 - feet_phase: 1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.25 - base_height: -500.0 - orientation: -5.0 - action_rate: -0.01 - pose: -0.1 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.55 - max_tilt_deg: 25.0 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/appo/task/g1_23dof_wall_flip_tracking/motrix.yaml b/conf/appo/task/g1_23dof_wall_flip_tracking/motrix.yaml index 3b543c9c4..3a43778c1 100644 --- a/conf/appo/task/g1_23dof_wall_flip_tracking/motrix.yaml +++ b/conf/appo/task/g1_23dof_wall_flip_tracking/motrix.yaml @@ -1,50 +1,33 @@ # @package _global_ +defaults: + - /task/g1_23dof_wall_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking23Dof sim_backend: motrix - play_steps: 1000 + replay_queue_size: null + algo: - num_envs: 1024 + steps_per_env: 24 max_iterations: 5000 - save_interval: 500 algorithm: num_learning_epochs: 5 num_mini_batches: 4 - clip_param: 0.2 - gamma: 0.99 - lam: 0.95 - value_loss_coef: 1.0 entropy_coef: 0.01 - learning_rate: 1.0e-3 - max_grad_norm: 1.0 - use_clipped_value_loss: true - schedule: adaptive desired_kl: 0.01 - adaptive_kl_factor: 2.0 - adaptive_lr_factor: 1.5 - optimizer: adam - tau: 1.0 - target_update_freq: 1 - vtrace_clip_rho: 1.0 - vtrace_clip_c: 1.0 enable_compile: true + +env: + actions: + joint_pos: + scale: 0.25 + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 1.0 + motion_body_ori: + weight: 1.0 + motion_ee_body_pos_z: null + action_rate_l2: + weight: -0.1 diff --git a/conf/appo/task/g1_23dof_wall_flip_tracking/mujoco.yaml b/conf/appo/task/g1_23dof_wall_flip_tracking/mujoco.yaml index 0decec32d..a140546c1 100644 --- a/conf/appo/task/g1_23dof_wall_flip_tracking/mujoco.yaml +++ b/conf/appo/task/g1_23dof_wall_flip_tracking/mujoco.yaml @@ -1,87 +1,27 @@ # @package _global_ +defaults: + - /task/g1_23dof_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking23Dof sim_backend: mujoco - play_steps: 1000 replay_queue_size: 5 + algo: - num_envs: 1024 steps_per_env: 20 max_iterations: 7000 - save_interval: 500 algorithm: num_learning_epochs: 6 - num_mini_batches: 8 - clip_param: 0.2 - gamma: 0.99 - lam: 0.95 - value_loss_coef: 1.0 - entropy_coef: 0.005 - learning_rate: 1.0e-3 - max_grad_norm: 1.0 - use_clipped_value_loss: true - schedule: adaptive desired_kl: 0.008 adaptive_kl_factor: 2.0 adaptive_lr_factor: 1.5 - optimizer: adam - tau: 1.0 - target_update_freq: 1 - vtrace_clip_rho: 1.0 - vtrace_clip_c: 1.0 enable_compile: true + env: - sampling_mode: start - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof_with_wall.xml + commands: + motion: + params: + motion_file: motions/g1/flip_from_wall_104__A304_23dof.npz diff --git a/conf/appo/task/g1_climb_tracking/motrix.yaml b/conf/appo/task/g1_climb_tracking/motrix.yaml index ec5684fe2..01a98604a 100644 --- a/conf/appo/task/g1_climb_tracking/motrix.yaml +++ b/conf/appo/task/g1_climb_tracking/motrix.yaml @@ -1,71 +1,8 @@ # @package _global_ +defaults: + - /task/g1_climb_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 20000 - save_interval: 500 -env: - sampling_mode: adaptive - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_climb_tracking/mujoco.yaml b/conf/appo/task/g1_climb_tracking/mujoco.yaml index bff7b13c8..8d15fff57 100644 --- a/conf/appo/task/g1_climb_tracking/mujoco.yaml +++ b/conf/appo/task/g1_climb_tracking/mujoco.yaml @@ -1,71 +1,87 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 20000 save_interval: 500 + algorithm: + adaptive_kl_factor: 1.2 + adaptive_lr_factor: 1.1 + env: - sampling_mode: adaptive - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/g1/scene_climb_20_z_scale_1.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + max_episode_seconds: 15.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + "waist_(roll|pitch)_joint": 0.43857731392336724 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + ".*_wrist_(pitch|yaw)_joint": 0.07450087032950714 + commands: + motion: + params: + motion_file: motions/g1/climb_20_z_scale_1.0.npz + sampling_mode: adaptive + truncate_on_clip_end: false + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.3} + ee_body_pos: + params: + command_name: motion + threshold: 0.3 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/appo/task/g1_flip_tracking/motrix.yaml b/conf/appo/task/g1_flip_tracking/motrix.yaml index 9b2fd0c57..d5f5e8b76 100644 --- a/conf/appo/task/g1_flip_tracking/motrix.yaml +++ b/conf/appo/task/g1_flip_tracking/motrix.yaml @@ -1,92 +1,8 @@ # @package _global_ +defaults: + - /task/g1_flip_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - steps_per_env: 24 - max_iterations: 3500 - save_interval: 500 - algorithm: - num_learning_epochs: 10 - num_mini_batches: 8 - clip_param: 0.2 - gamma: 0.99 - lam: 0.95 - value_loss_coef: 1.0 - entropy_coef: 0.005 - learning_rate: 1.0e-3 - max_grad_norm: 1.0 - use_clipped_value_loss: true - schedule: adaptive - desired_kl: 0.01 - adaptive_kl_factor: 2.0 - adaptive_lr_factor: 1.5 - optimizer: adam - tau: 1.0 - target_update_freq: 1 - vtrace_clip_rho: 1.0 - vtrace_clip_c: 1.0 - enable_compile: true -env: - sampling_mode: start - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_flip_tracking/mujoco.yaml b/conf/appo/task/g1_flip_tracking/mujoco.yaml index 628bdbb12..b8f05ad61 100644 --- a/conf/appo/task/g1_flip_tracking/mujoco.yaml +++ b/conf/appo/task/g1_flip_tracking/mujoco.yaml @@ -1,8 +1,13 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 steps_per_env: 24 @@ -29,64 +34,84 @@ algo: vtrace_clip_rho: 1.0 vtrace_clip_c: 1.0 enable_compile: true + env: - sampling_mode: start - truncate_on_clip_end: true + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + "waist_(roll|pitch)_joint": 0.43857731392336724 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + ".*_wrist_(pitch|yaw)_joint": 0.07450087032950714 + commands: + motion: + params: + motion_file: motions/g1/flip_360_001__A304.npz + sampling_mode: start + truncate_on_clip_end: true + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + motion_clip_end: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_clip_end + time_out: true + params: {command_name: motion} + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 \ No newline at end of file + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + action_rate_l2: + weight: -0.005 diff --git a/conf/appo/task/g1_motion_tracking/motrix.yaml b/conf/appo/task/g1_motion_tracking/motrix.yaml index eabea1ad6..22bbdadfa 100644 --- a/conf/appo/task/g1_motion_tracking/motrix.yaml +++ b/conf/appo/task/g1_motion_tracking/motrix.yaml @@ -1,30 +1,8 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTracking sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 5000 - save_interval: 500 -env: -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/appo/task/g1_motion_tracking/mujoco.yaml b/conf/appo/task/g1_motion_tracking/mujoco.yaml index ec978a19c..b1898ef6d 100644 --- a/conf/appo/task/g1_motion_tracking/mujoco.yaml +++ b/conf/appo/task/g1_motion_tracking/mujoco.yaml @@ -3,6 +3,7 @@ training: task_name: G1MotionTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 5000 @@ -10,24 +11,224 @@ algo: algorithm: adaptive_kl_factor: 2.0 adaptive_lr_factor: 1.5 + env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: &g1_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: *g1_joints + body_names: &tracked_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_yaw_link + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_yaw_link + geom_names: + - left_foot1_collision + - left_foot2_collision + - left_foot3_collision + - left_foot4_collision + - left_foot5_collision + - left_foot6_collision + - left_foot7_collision + - right_foot1_collision + - right_foot2_collision + - right_foot3_collision + - right_foot4_collision + - right_foot5_collision + - right_foot6_collision + - right_foot7_collision + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 10.0 + observations: + actor: + terms: + command: &command_obs + func: unilab.envs.mdp.generated_commands + params: {command_name: motion} + motion_anchor_pos_b: &anchor_pos_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_pos_b + params: {command_name: motion} + motion_anchor_ori_b: &anchor_ori_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_ori_b + params: {command_name: motion} + base_lin_vel: &base_lin_vel_obs + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + base_ang_vel: &base_ang_vel_obs + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + joint_pos: &joint_pos_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_pos_rel + params: {command_name: motion} + joint_vel: &joint_vel_obs + func: unilab.envs.mdp.joint_vel_rel + actions: &actions_obs + func: unilab.envs.mdp.last_action + critic: + terms: + command: *command_obs + motion_anchor_pos_b: *anchor_pos_obs + motion_anchor_ori_b: *anchor_ori_obs + base_lin_vel: *base_lin_vel_obs + base_ang_vel: *base_ang_vel_obs + joint_pos: *joint_pos_obs + joint_vel: *joint_vel_obs + actions: *actions_obs + body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.robot_body_pos_b + params: {command_name: motion} + body_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.robot_body_ori_b + params: {command_name: motion} + actions: + joint_pos: + _target_: unilab.tasks.motion_tracking.common.manager_terms.MotionJointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + command_name: motion + commands: + motion: + _target_: unilab.tasks.motion_tracking.common.manager_terms.MotionCommandCfg + entity_name: robot + resampling_time_range: [1.0e9, 1.0e9] + params: + motion_file: motions/g1/dance1_subject2_part.npz + anchor_body_name: torso_link + body_names: *tracked_bodies + sampling_mode: adaptive + sampling_start_ratio: 0.0 + truncate_on_clip_end: false + pose_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.01, 0.01] + roll: [-0.1, 0.1] + pitch: [-0.1, 0.1] + yaw: [-0.2, 0.2] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.2, 0.2] + roll: [-0.52, 0.52] + pitch: [-0.52, 0.52] + yaw: [-0.78, 0.78] + joint_position_range: [-0.1, 0.1] + joint_default_position_range: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + anchor_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_anchor_pos_z_only + params: {command_name: motion, threshold: 0.25} + anchor_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_anchor_ori + params: + command_name: motion + threshold: 0.8 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_motion_body_pos_z_only + params: + command_name: motion + threshold: 0.25 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + policy_observation_group: actor + critic_observation_group: critic + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_global_root_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_anchor_position_error_exp + weight: 0.5 + params: {command_name: motion, std: 0.3} + motion_global_root_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_anchor_orientation_error_exp + weight: 0.5 + params: {command_name: motion, std: 0.4} + motion_body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_error_exp + weight: 1.0 + params: {command_name: motion, std: 0.3} + motion_body_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_orientation_error_exp + weight: 1.0 + params: {command_name: motion, std: 0.4} + motion_body_lin_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_body_linear_velocity_error_exp + weight: 1.0 + params: {command_name: motion, std: 1.0} + motion_body_ang_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_body_angular_velocity_error_exp + weight: 1.0 + params: {command_name: motion, std: 3.14} + motion_joint_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_position_error_exp + weight: 0.0 + params: {command_name: motion, std: 0.2} + motion_joint_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_velocity_error_exp + weight: 0.0 + params: {command_name: motion, std: 1.0} + action_rate_l2: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.1 + joint_limit: + func: unilab.tasks.motion_tracking.common.manager_terms.joint_pos_limits + weight: -10.0 + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: ".*" diff --git a/conf/appo/task/g1_walk_flat/base.yaml b/conf/appo/task/g1_walk_flat/base.yaml new file mode 100644 index 000000000..8b35fdd94 --- /dev/null +++ b/conf/appo/task/g1_walk_flat/base.yaml @@ -0,0 +1,262 @@ +# @package _global_ +# Canonical G1 29-DoF walk Manager-Based task declaration (APPO owners; mirrors the PPO base). +# Backend owner leaves inherit this file and only override backend/algo tuning +# or explicitly disabled terms. Observation scaling follows the legacy profile +# (unit scales); the walk profile lives in conf/sac/task/g1_walk_flat/base.yaml. +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + body_names: [pelvis] + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + # Observation noise matches the legacy noise_config (level=1.0, actor-only + # since the critic reads clean observations): gyro +/-0.2, gravity +/-0.05, + # joint pos +/-0.01, joint vel +/-1.5, applied before term scaling. + enable_corruption: true + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.2 + n_max: 0.2 + operation: add + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.05 + n_max: 0.05 + operation: add + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + operation: add + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -1.5 + n_max: 1.5 + operation: add + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + tilt: + func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded + params: + max_tilt_deg: 25.0 + base_height: + func: unilab.envs.mdp.root_height_below_minimum + params: + minimum_height: 0.55 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel + weight: 2.0 + params: + tracking_sigma: 0.25 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel + weight: 0.2 + params: + tracking_sigma: 0.25 + command_name: twist + feet_phase: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase + weight: 1.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.0 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.g1.manager_terms.lin_vel_z + weight: -1.0 + ang_vel_xy: + func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy + weight: -0.25 + base_height: + func: unilab.tasks.locomotion.g1.manager_terms.base_height + weight: -500.0 + params: + target_height: 0.754 + orientation: + func: unilab.tasks.locomotion.g1.manager_terms.orientation + weight: -5.0 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.01 + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.1 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/appo/task/g1_walk_flat/mujoco.yaml b/conf/appo/task/g1_walk_flat/mujoco.yaml index 0fce8bf11..6ee7b66ed 100644 --- a/conf/appo/task/g1_walk_flat/mujoco.yaml +++ b/conf/appo/task/g1_walk_flat/mujoco.yaml @@ -1,36 +1,13 @@ # @package _global_ +# MuJoCo APPO owner: inherits the shared 29-DoF flat Manager-Based contract +# from base.yaml and only carries backend/algo identity. +defaults: + - /task/g1_walk_flat/base + - _self_ + training: task_name: G1WalkFlat sim_backend: mujoco algo: max_iterations: 500 save_interval: 100 -env: - control_config: - action_scale: 0.25 - curriculum: - enabled: false - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.2 - feet_phase: 1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.25 - base_height: -500.0 - orientation: -5.0 - action_rate: -0.01 - pose: -0.1 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.55 - max_tilt_deg: 25.0 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/appo/task/g1_wall_flip_tracking/motrix.yaml b/conf/appo/task/g1_wall_flip_tracking/motrix.yaml index 83cca7b39..3ce1d8cc3 100644 --- a/conf/appo/task/g1_wall_flip_tracking/motrix.yaml +++ b/conf/appo/task/g1_wall_flip_tracking/motrix.yaml @@ -1,50 +1,32 @@ # @package _global_ +defaults: + - /task/g1_wall_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking sim_backend: motrix - play_steps: 1000 + replay_queue_size: null + algo: - num_envs: 1024 + steps_per_env: 24 max_iterations: 5000 - save_interval: 500 algorithm: num_learning_epochs: 5 num_mini_batches: 4 - clip_param: 0.2 - gamma: 0.99 - lam: 0.95 - value_loss_coef: 1.0 entropy_coef: 0.01 - learning_rate: 1.0e-3 - max_grad_norm: 1.0 - use_clipped_value_loss: true - schedule: adaptive desired_kl: 0.01 - adaptive_kl_factor: 2.0 - adaptive_lr_factor: 1.5 - optimizer: adam - tau: 1.0 - target_update_freq: 1 - vtrace_clip_rho: 1.0 - vtrace_clip_c: 1.0 - enable_compile: true + +env: + actions: + joint_pos: + scale: 0.25 + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 1.0 + motion_body_ori: + weight: 1.0 + motion_ee_body_pos_z: null + action_rate_l2: + weight: -0.1 diff --git a/conf/appo/task/g1_wall_flip_tracking/mujoco.yaml b/conf/appo/task/g1_wall_flip_tracking/mujoco.yaml index 0083729c0..d177ef8b4 100644 --- a/conf/appo/task/g1_wall_flip_tracking/mujoco.yaml +++ b/conf/appo/task/g1_wall_flip_tracking/mujoco.yaml @@ -1,93 +1,24 @@ # @package _global_ +defaults: + - /task/g1_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking sim_backend: mujoco - play_steps: 1000 replay_queue_size: 5 + algo: - num_envs: 1024 steps_per_env: 20 max_iterations: 7000 - save_interval: 500 algorithm: num_learning_epochs: 6 - num_mini_batches: 8 - clip_param: 0.2 - gamma: 0.99 - lam: 0.95 - value_loss_coef: 1.0 - entropy_coef: 0.005 - learning_rate: 1.0e-3 - max_grad_norm: 1.0 - use_clipped_value_loss: true - schedule: adaptive desired_kl: 0.008 - adaptive_kl_factor: 2.0 - adaptive_lr_factor: 1.5 - optimizer: adam - tau: 1.0 - target_update_freq: 1 - vtrace_clip_rho: 1.0 - vtrace_clip_c: 1.0 - enable_compile: true + env: - sampling_mode: start - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 \ No newline at end of file + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_with_wall.xml + commands: + motion: + params: + motion_file: motions/g1/flip_from_wall_104__A304.npz diff --git a/conf/appo/task/go1_joystick_flat/base.yaml b/conf/appo/task/go1_joystick_flat/base.yaml new file mode 100644 index 000000000..8a2f3bf57 --- /dev/null +++ b/conf/appo/task/go1_joystick_flat/base.yaml @@ -0,0 +1,244 @@ +# @package _global_ +# Canonical Go1 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go1/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: trunk + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + body_names: [trunk] + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: trunk + mass_distribution_params: [-1.5, 1.5] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: trunk + com_range: + x: [-0.05, 0.05] + y: [0.0, 0.0] + z: [0.0, 0.0] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [35.0, 35.0] + kd_range: [0.5, 0.5] + operation: abs + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [15.0, 15.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + # Legacy Go1 sums four matching feet while this community term returns their mean. + weight: 0.96 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/appo/task/go1_joystick_flat/motrix.yaml b/conf/appo/task/go1_joystick_flat/motrix.yaml index f7343fd5d..d4bb53521 100644 --- a/conf/appo/task/go1_joystick_flat/motrix.yaml +++ b/conf/appo/task/go1_joystick_flat/motrix.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go1_joystick_flat/base + - _self_ + training: task_name: Go1JoystickFlat sim_backend: motrix @@ -14,21 +18,22 @@ algo: entropy_coef: 1.0e-3 desired_kl: 0.008 env: - sim_dt: 0.01 commands: - vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] + twist: + ranges: + lin_vel_x: [0.5, 0.5] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + push_robot: null reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.015 - action_smooth: -0.01 - similar_to_default: -0.15 - swing_feet_z: 2.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + action_rate: + weight: -0.015 + action_smooth: + func: unilab.envs.mdp.action_acc_l2 + weight: -0.01 + similar_to_default: + weight: -0.15 + contact: null + swing_feet_z: + weight: 2.0 diff --git a/conf/appo/task/go1_joystick_flat/mujoco.yaml b/conf/appo/task/go1_joystick_flat/mujoco.yaml index 8646817a5..5443809b6 100644 --- a/conf/appo/task/go1_joystick_flat/mujoco.yaml +++ b/conf/appo/task/go1_joystick_flat/mujoco.yaml @@ -1,18 +1,12 @@ # @package _global_ +defaults: + - /task/go1_joystick_flat/base + - _self_ + training: task_name: Go1JoystickFlat sim_backend: mujoco algo: max_iterations: 150 reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - tracking_sigma: 0.25 - base_height_target: 0.3 + swing_feet_z: null diff --git a/conf/appo/task/go2_joystick_flat/base.yaml b/conf/appo/task/go2_joystick_flat/base.yaml new file mode 100644 index 000000000..be7364f78 --- /dev/null +++ b/conf/appo/task/go2_joystick_flat/base.yaml @@ -0,0 +1,209 @@ +# @package _global_ +# Canonical Go2 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [31.5, 38.5] + kd_range: [0.45, 0.55] + operation: abs + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + alive: + func: unilab.envs.mdp.is_alive + weight: 0.0 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + weight: 0.24 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/appo/task/go2_joystick_flat/motrix.yaml b/conf/appo/task/go2_joystick_flat/motrix.yaml index fc4597bbd..b46281fe6 100644 --- a/conf/appo/task/go2_joystick_flat/motrix.yaml +++ b/conf/appo/task/go2_joystick_flat/motrix.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + training: task_name: Go2JoystickFlat sim_backend: motrix @@ -7,18 +11,7 @@ algo: steps_per_env: 24 max_iterations: 180 env: - sim_dt: 0.015 -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - alive: 0.0 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + # The former 0.015/0.02 ratio was not an integer number of physics substeps. + # MBA keeps the shared 0.01/0.02 contract and fails on fractional ratios. + events: + pd_gains: null diff --git a/conf/appo/task/go2_joystick_flat/mujoco.yaml b/conf/appo/task/go2_joystick_flat/mujoco.yaml index d25c356ad..aec4a0437 100644 --- a/conf/appo/task/go2_joystick_flat/mujoco.yaml +++ b/conf/appo/task/go2_joystick_flat/mujoco.yaml @@ -1,20 +1,10 @@ # @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + training: task_name: Go2JoystickFlat sim_backend: mujoco algo: max_iterations: 150 -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - alive: 0.0 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 diff --git a/conf/appo/task/sharpa_inhand/mujoco_hora.yaml b/conf/appo/task/sharpa_inhand/mujoco_hora.yaml index 47aa3947a..5bdcab1b5 100644 --- a/conf/appo/task/sharpa_inhand/mujoco_hora.yaml +++ b/conf/appo/task/sharpa_inhand/mujoco_hora.yaml @@ -23,7 +23,7 @@ interactive: algo: algo_log_name: hora_appo runtime_impl: hora_appo - runtime_resolver: unilab.algos.torch.hora.appo:resolve_hora_appo_runtime + runtime_resolver: unilab.algos.hora.appo:resolve_hora_appo_runtime num_envs: 2048 steps_per_env: 8 max_iterations: 305 @@ -38,11 +38,11 @@ algo: actor: 0 priv_info: 0 actor: - class_name: unilab.algos.torch.hora:HoraActorModel + class_name: unilab.algos.hora:HoraActorModel priv_info_embed_dim: 9 priv_mlp_hidden_dims: [256, 128, 9] critic: - class_name: unilab.algos.torch.hora:HoraCriticModel + class_name: unilab.algos.hora:HoraCriticModel priv_info_embed_dim: 9 priv_mlp_hidden_dims: [256, 128, 9] algorithm: diff --git a/conf/flashsac/config.yaml b/conf/flashsac/config.yaml new file mode 100644 index 000000000..27b3064cb --- /dev/null +++ b/conf/flashsac/config.yaml @@ -0,0 +1,152 @@ +defaults: + - _self_ + - task: g1_walk_flat/mujoco + +algo: + algo: flashsac + algo_log_name: flash_sac + load_run: "-1" + seed: 1 + num_envs: 1024 + batch_size: 2048 + replay_buffer_n: 512 + updates_per_step: 2 + learning_starts: 98 + policy_frequency: 2 + max_iterations: 5000 + save_interval: 1000 + gamma: 0.97 + tau: 0.01 + actor_lr: 3.0e-4 + critic_lr: 3.0e-4 + actor_hidden_dim: 128 + critic_hidden_dim: 256 + num_atoms: 101 + obs_normalization: false + use_layer_norm: false + algo_params: + normalize_reward: true + normalized_g_max: 5.0 + actor_num_blocks: 2 + critic_num_blocks: 2 + actor_bc_alpha: 0.0 + actor_noise_zeta_mu: 2.0 + actor_noise_zeta_max: 16 + critic_min_v: -5.0 + critic_max_v: 5.0 + temp_initial_value: 0.01 + temp_target_sigma: 0.15 + temp_target_entropy: null + learning_rate_init: 3.0e-4 + learning_rate_peak: 3.0e-4 + learning_rate_end: 1.5e-4 + learning_rate_warmup_steps: 0 + learning_rate_decay_steps: 500000 + n_step: 1 + amp_dtype: auto + use_compile: true + use_cuda_graph_critic: false + use_cuda_graph_actor: false + use_cuda_graph_critic_packed_staging: false + use_cuda_graph_actor_packed_staging: false + +training: + task_name: G1WalkFlat + # list[int] | null; null/[] = auto-selected single-device behavior; + # [d0] = explicit single CUDA device; [d0..dN-1] = N-way data parallel, + # rank i trains on cuda:devices[i]. + devices: null + # list[list[int]] | null; one CPU-id segment per rank for the collector's + # MuJoCo pool; null = auto partition of cpu_count // world_size per rank. + dp_collector_cpu_ids: null + logger: tensorboard + wandb_project: unilab + wandb_entity: null + wandb_group: null + wandb_job_type: null + wandb_name: null + wandb_tags: [] + wandb_notes: null + wandb_mode: null + sim_backend: mujoco + nan_guard: + enabled: true + buffer_size: 100 + max_envs_to_dump: 5 + output_dir: null + use_amp: true + play_only: false + no_play: false + sim2sim_strict: true + play_render_mode: auto + export_onnx: true + play_env_num: 16 + play_steps: 800 + cam_distance: 6.0 + cam_elevation: -20.0 + cam_azimuth: 90.0 + log_root: null + log_dir: null + env_steps_per_sync: 1 + trace_enabled: false + trace_output_dir: null + trace_thread_time: false + trace_cuda_events: true + nvtx_profile_ranges: false + replay_prefetch_mode: one_tick + torch_threads: + enabled: true + # "auto" resolves per process role from host CPU count with conservative caps. + # Override these from Hydra when benchmarking a specific machine. + learner_num_threads: auto + collector_num_threads: auto + learner_num_interop_threads: 1 + collector_num_interop_threads: 1 + compile_threads: auto + set_env_vars: true + +interactive: + action_mode: zero + policy_obs_mode: auto + show_target_bodies: false + show_reward_debug: false + target_show_axes: false + target_body_names: "" + target_max_bodies: 0 + target_marker_radius: 0.02 + target_axis_length: 0.08 + target_marker_alpha: 0.75 + reward_debug_show_velocity: false + reward_debug_lin_vel_scale: 0.08 + reward_debug_ang_vel_scale: 0.05 + reward_debug_show_connectors: false + reward_debug_show_global_anchor: false + camera_follow_body: true + camera_focus_body_name: "" + camera_height_offset: 0.15 + camera_distance: null + camera_elevation: null + camera_azimuth: null + use_env_visual_model: true + speed: 1.0 + start_paused: false + keyboard: false + keyboard_step_lin: 0.1 + keyboard_step_ang: 0.2 + +env: + post_step_forward_sensor: false + # adaptive_chunk_size: auto-tune the MuJoCo BatchEnvPool chunk_size at materialize + # (cache-backed). chunk_size (int) manually overrides and wins; null => use default. + adaptive_chunk_size: true + chunk_size: null + +hydra: + run: + dir: . + output_subdir: null + job: + chdir: false + job_logging: + root: + handlers: [console] diff --git a/conf/flashsac/task/g1_23dof_walk_flat/base.yaml b/conf/flashsac/task/g1_23dof_walk_flat/base.yaml new file mode 100644 index 000000000..c7571348f --- /dev/null +++ b/conf/flashsac/task/g1_23dof_walk_flat/base.yaml @@ -0,0 +1,66 @@ +# @package _global_ +# Canonical G1 23-DoF walk Manager-Based task declaration (off-policy owners). +# Inherits the 29-DoF off-policy contract and swaps the scene to the 23-DoF +# model (no waist roll/pitch, no wrist pitch/yaw) with 23-entry pose weights. +defaults: + - /task/g1_walk_flat/base + - _self_ + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + +reward: + pose: + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/flashsac/task/g1_23dof_walk_flat/motrix.yaml b/conf/flashsac/task/g1_23dof_walk_flat/motrix.yaml new file mode 100644 index 000000000..487f03787 --- /dev/null +++ b/conf/flashsac/task/g1_23dof_walk_flat/motrix.yaml @@ -0,0 +1,45 @@ +# @package _global_ +# FlashSAC Motrix 23-DoF owner: mirrors flashsac/g1_walk_flat/motrix.yaml with +# the 23-DoF contract (algo identity kept, Motrix-direction retuning). +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofFlat + sim_backend: motrix +algo: + num_envs: 4096 + learning_starts: 49 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + replay_buffer_n: 256 + tau: 0.05 +env: + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null +reward: + tracking_lin_vel: + weight: 2.2 + tracking_ang_vel: + weight: 1.8 + penalty_ang_vel_xy: + weight: -1.2 + penalty_orientation: + weight: -12.0 + penalty_action_rate: + weight: -2.5 + pose: + weight: -0.6 + params: + pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + weight: -5.0 + feet_phase: + weight: 6.0 + params: + tracking_sigma: 0.008 + alive: + weight: 12.0 diff --git a/conf/flashsac/task/g1_23dof_walk_flat/mujoco.yaml b/conf/flashsac/task/g1_23dof_walk_flat/mujoco.yaml new file mode 100644 index 000000000..77e8071ab --- /dev/null +++ b/conf/flashsac/task/g1_23dof_walk_flat/mujoco.yaml @@ -0,0 +1,29 @@ +# @package _global_ +# FlashSAC MuJoCo 23-DoF owner: 23-DoF off-policy contract plus the FlashSAC +# algo identity and its reward retuning. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofFlat + sim_backend: mujoco +algo: + num_envs: 4096 + learning_starts: 49 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + replay_buffer_n: 256 + tau: 0.05 +reward: + penalty_action_rate: + weight: -5.0 + penalty_feet_ori: + weight: -25.0 + feet_phase: + params: + tracking_sigma: 0.005 + pose: + params: + pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/flashsac/task/g1_walk_flat/base.yaml b/conf/flashsac/task/g1_walk_flat/base.yaml new file mode 100644 index 000000000..a058d94f5 --- /dev/null +++ b/conf/flashsac/task/g1_walk_flat/base.yaml @@ -0,0 +1,272 @@ +# @package _global_ +# Canonical G1 29-DoF walk Manager-Based task declaration (off-policy owners). +# Backend owner leaves inherit this file and only override backend/algo tuning +# or explicitly disabled terms. Observation scaling follows the walk profile +# (gyro x0.25, joint velocity x0.05, critic linear velocity x2.0); every +# off-policy owner carries the penalty curriculum. +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + body_names: [pelvis] + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + # Observation noise matches the legacy off-policy noise_config (level=1.0, + # actor-only; gyro/gravity/linvel scales were 0.0 there): joint pos + # +/-0.01 and joint vel +/-0.1, applied before term scaling. + enable_corruption: true + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + operation: add + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.1 + n_max: 0.1 + operation: add + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + scale: 2.0 + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 1.0 + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + tilt: + func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded + params: + max_tilt_deg: 65.0 + base_height: + func: unilab.envs.mdp.root_height_below_minimum + params: + minimum_height: 0.3 + curriculum: + penalty_scaling: + func: unilab.tasks.locomotion.g1.manager_terms.G1PenaltyCurriculum + # Effective schedule matches the tuned legacy baseline: the legacy env + # halved the shared override dict once per env construction (two probe + # envs + the collector in every off-policy runner), so collectors actually + # trained at 1/8 initial / 1/4 cap of these YAML weights. The manager + # runtime isolates each env, so the tuned effective range is declared + # explicitly here. + params: + initial_scale: 0.125 + min_scale: 0.125 + max_scale: 0.25 + level_down_threshold: 150.0 + level_up_threshold: 750.0 + degree: 0.001 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel + weight: 2.0 + params: + tracking_sigma: 0.25 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel + weight: 1.5 + params: + tracking_sigma: 0.25 + command_name: twist + penalty_ang_vel_xy: + func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy + weight: -1.0 + penalty_orientation: + func: unilab.tasks.locomotion.g1.manager_terms.orientation + weight: -10.0 + penalty_action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -4.0 + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.5 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori + weight: -20.0 + feet_phase: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase + weight: 5.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.04 + min_forward_speed: 0.0 + command_name: twist + alive: + func: unilab.tasks.locomotion.g1.manager_terms.alive + weight: 10.0 diff --git a/conf/flashsac/task/g1_walk_flat/mjwarp.yaml b/conf/flashsac/task/g1_walk_flat/mjwarp.yaml new file mode 100644 index 000000000..bd3fb1705 --- /dev/null +++ b/conf/flashsac/task/g1_walk_flat/mjwarp.yaml @@ -0,0 +1,39 @@ +# @package _global_ +# Configured-only FlashSAC mjwarp owner. Mirrors the MuJoCo owner's algo / +# env / reward identity; mjwarp-specific host-adapter settings follow +# conf/sac/task/g1_walk_flat/mjwarp.yaml. Offline record reuses +# MuJoCo rendering; native playback and device-resident runtime are absent. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: mjwarp + play_render_mode: record +algo: + num_envs: 4096 + learning_starts: 49 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + replay_buffer_n: 256 + tau: 0.05 +env: + mjwarp_nconmax: 128 + mjwarp_njmax: 256 + render_spacing: 2.0 + events: + # Legacy mjwarp owners disable kp/kd and armature randomization. + pd_gains: null +reward: + penalty_action_rate: + weight: -5.0 + penalty_feet_ori: + weight: -25.0 + feet_phase: + params: + tracking_sigma: 0.005 + pose: + params: + pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/flashsac/task/g1_walk_flat/motrix.yaml b/conf/flashsac/task/g1_walk_flat/motrix.yaml new file mode 100644 index 000000000..c8d8e6b8d --- /dev/null +++ b/conf/flashsac/task/g1_walk_flat/motrix.yaml @@ -0,0 +1,45 @@ +# @package _global_ +# FlashSAC Motrix owner: keeps the MuJoCo owner's FlashSAC algo identity while +# adopting the Motrix-direction reward retuning; kp/kd randomization disabled. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: motrix +algo: + num_envs: 4096 + learning_starts: 49 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + replay_buffer_n: 256 + tau: 0.05 +env: + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null +reward: + tracking_lin_vel: + weight: 2.2 + tracking_ang_vel: + weight: 1.8 + penalty_ang_vel_xy: + weight: -1.2 + penalty_orientation: + weight: -12.0 + penalty_action_rate: + weight: -2.5 + pose: + weight: -0.6 + params: + pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + weight: -5.0 + feet_phase: + weight: 6.0 + params: + tracking_sigma: 0.008 + alive: + weight: 12.0 diff --git a/conf/flashsac/task/g1_walk_flat/mujoco.yaml b/conf/flashsac/task/g1_walk_flat/mujoco.yaml new file mode 100644 index 000000000..a82b90883 --- /dev/null +++ b/conf/flashsac/task/g1_walk_flat/mujoco.yaml @@ -0,0 +1,30 @@ +# @package _global_ +# FlashSAC MuJoCo owner: inherits the 29-DoF off-policy Manager-Based contract +# and carries the FlashSAC algo identity plus its reward retuning +# (stiffer action-rate / feet-orientation penalties, tighter feet-phase sigma). +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: mujoco +algo: + num_envs: 4096 + learning_starts: 49 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + replay_buffer_n: 256 + tau: 0.05 +reward: + penalty_action_rate: + weight: -5.0 + penalty_feet_ori: + weight: -25.0 + feet_phase: + params: + tracking_sigma: 0.005 + pose: + params: + pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/flashsac/task/go2_joystick_flat/base.yaml b/conf/flashsac/task/go2_joystick_flat/base.yaml new file mode 100644 index 000000000..409129c16 --- /dev/null +++ b/conf/flashsac/task/go2_joystick_flat/base.yaml @@ -0,0 +1,206 @@ +# @package _global_ +# Canonical Go2 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [31.5, 38.5] + kd_range: [0.45, 0.55] + operation: abs + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + weight: 0.24 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/flashsac/task/go2_joystick_flat/mujoco.yaml b/conf/flashsac/task/go2_joystick_flat/mujoco.yaml new file mode 100644 index 000000000..7fbfdec78 --- /dev/null +++ b/conf/flashsac/task/go2_joystick_flat/mujoco.yaml @@ -0,0 +1,118 @@ +# @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + +training: + task_name: Go2JoystickFlat + sim_backend: mujoco +algo: + num_envs: 1024 + learning_starts: 50 + max_iterations: 4000 + save_interval: 1000 + updates_per_step: 2 + batch_size: 2048 + replay_buffer_n: 4096 + tau: 0.05 +env: + scene: + entities: + robot: + body_names: [base] + actions: + joint_pos: + scale: 0.4 + observations: + policy: + enable_corruption: true + terms: + joint_pos: + noise: + _target_: UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + joint_vel: + noise: + _target_: UniformNoiseCfg + n_min: -0.1 + n_max: 0.1 + critic: + enable_corruption: true + terms: + joint_pos: + noise: + _target_: UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + joint_vel: + noise: + _target_: UniformNoiseCfg + n_min: -0.1 + n_max: 0.1 + events: + randomize_rigid_body_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: base + mass_distribution_params: [-1.5, 1.5] + operation: add + recompute_inertia: false + randomize_rigid_body_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: base + com_range: + x: [-0.05, 0.05] + randomize_physics_scene_gravity: + func: unilab.envs.mdp.randomize_physics_scene_gravity + mode: reset + params: + gravity_distribution_params: + - [0.0, 0.0, -9.81] + - [0.0, 0.0, -9.81] + operation: abs + push_by_setting_velocity: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [15.0, 15.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] +reward: + tracking_lin_vel: + weight: 1.0 + params: + std: 0.6324555320336759 + tracking_ang_vel: + weight: 0.2 + params: + std: 0.6324555320336759 + lin_vel_z: + weight: -5.0 + ang_vel_xy: + weight: -0.1 + base_height: + weight: -20.0 + action_rate: + weight: -0.02 + similar_to_default: + weight: -0.4 + contact: + weight: 1.5 + swing_feet_z: + weight: 4.0 diff --git a/conf/hora_distill/student_model/hora_actor.yaml b/conf/hora_distill/student_model/hora_actor.yaml index 4aa117957..05c17bc83 100644 --- a/conf/hora_distill/student_model/hora_actor.yaml +++ b/conf/hora_distill/student_model/hora_actor.yaml @@ -1,6 +1,6 @@ # Teacher -> student `algo.model` mapping for HoraActorModel teachers (PPO/APPO). # -# Loaded by unilab.algos.torch.hora.distill_config and merged next to the +# Loaded by unilab.algos.hora.distill_config and merged next to the # Hydra-composed teacher owner config mounted at `teacher_owner`, so every # student field interpolates directly from the teacher owner YAML. The teacher # owner config stays the single source of truth for these hyperparameters; diff --git a/conf/hora_distill/student_model/hora_sac.yaml b/conf/hora_distill/student_model/hora_sac.yaml index a462608f0..8b72b5ff2 100644 --- a/conf/hora_distill/student_model/hora_sac.yaml +++ b/conf/hora_distill/student_model/hora_sac.yaml @@ -1,6 +1,6 @@ -# Teacher -> student `algo.model` mapping for hora_sac teachers (offpolicy SAC). +# Teacher -> student `algo.model` mapping for hora_sac teachers (off-policy SAC). # -# Loaded by unilab.algos.torch.hora.distill_config and merged next to the +# Loaded by unilab.algos.hora.distill_config and merged next to the # Hydra-composed teacher owner config mounted at `teacher_owner`. Values come # from the teacher owner YAML; the `oc.select` fallback after the comma only # applies when the teacher owner config does not define the field at all. diff --git a/conf/offpolicy/algo/flashsac.yaml b/conf/offpolicy/algo/flashsac.yaml deleted file mode 100644 index 05f4d05cc..000000000 --- a/conf/offpolicy/algo/flashsac.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# @package _global_ -algo: - algo: flashsac - algo_log_name: flash_sac - load_run: "-1" - seed: 1 - num_envs: 1024 - batch_size: 2048 - replay_buffer_n: 512 - updates_per_step: 2 - learning_starts: 98 - policy_frequency: 2 - max_iterations: 5000 - save_interval: 1000 - gamma: 0.97 - tau: 0.01 - actor_lr: 3.0e-4 - critic_lr: 3.0e-4 - actor_hidden_dim: 128 - critic_hidden_dim: 256 - num_atoms: 101 - obs_normalization: false - use_layer_norm: false - algo_params: - normalize_reward: true - normalized_g_max: 5.0 - actor_num_blocks: 2 - critic_num_blocks: 2 - actor_bc_alpha: 0.0 - actor_noise_zeta_mu: 2.0 - actor_noise_zeta_max: 16 - critic_min_v: -5.0 - critic_max_v: 5.0 - temp_initial_value: 0.01 - temp_target_sigma: 0.15 - temp_target_entropy: null - learning_rate_init: 3.0e-4 - learning_rate_peak: 3.0e-4 - learning_rate_end: 1.5e-4 - learning_rate_warmup_steps: 0 - learning_rate_decay_steps: 500000 - n_step: 1 - amp_dtype: auto - use_compile: true - use_cuda_graph_critic: false - use_cuda_graph_actor: false - use_cuda_graph_critic_packed_staging: false - use_cuda_graph_actor_packed_staging: false diff --git a/conf/offpolicy/algo/sac.yaml b/conf/offpolicy/algo/sac.yaml deleted file mode 100644 index 4e6efeeb0..000000000 --- a/conf/offpolicy/algo/sac.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# @package _global_ -algo: - algo: sac - algo_log_name: fast_sac - runtime_impl: null - runtime_resolver: null - load_run: "-1" - seed: 1 - num_envs: 4096 - # Learner batch size for one SAC update. Symmetry may sample fewer replay rows - # and expand them inside the learner. - batch_size: 8192 - replay_buffer_n: 512 - updates_per_step: 4 - learning_starts: 1 - policy_frequency: 4 - max_iterations: 500 - save_interval: 500 - gamma: 0.97 - tau: 0.125 - actor_lr: 3.0e-4 - critic_lr: 3.0e-4 - actor_hidden_dim: 512 - critic_hidden_dim: 768 - num_atoms: 101 - obs_normalization: false - use_layer_norm: true - use_symmetry: false - actor: {} - algo_params: - alpha_lr: 3.0e-4 - alpha_init: 0.01 - target_entropy_ratio: 0.0 - max_grad_norm: 0.0 - amp_dtype: auto - use_compile: true - use_cuda_graph_critic: false - use_cuda_graph_actor: false - use_cuda_graph_critic_packed_staging: false - use_cuda_graph_actor_packed_staging: false diff --git a/conf/offpolicy/algo/td3.yaml b/conf/offpolicy/algo/td3.yaml deleted file mode 100644 index 3d217abc7..000000000 --- a/conf/offpolicy/algo/td3.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# @package algo -algo: td3 -algo_log_name: fast_td3 -load_run: "-1" -seed: 1 -num_envs: 4096 -batch_size: 8192 -replay_buffer_n: 1000 -updates_per_step: 4 -learning_starts: 1 -policy_frequency: 2 -max_iterations: 5000 -save_interval: 500 -gamma: 0.97 -tau: 0.1 -actor_lr: 3.0e-4 -critic_lr: 3.0e-4 -actor_hidden_dim: 512 -critic_hidden_dim: 1024 -num_atoms: 101 -obs_normalization: true -use_layer_norm: false -algo_params: - weight_decay: 0.1 - v_min: -10.0 - v_max: 10.0 - init_scale: 0.01 - log_std_min: -1.6 - log_std_max: -0.22 - policy_noise: 0.2 - noise_clip: 0.5 - use_cdq: true diff --git a/conf/offpolicy/task/flashsac/g1_23dof_walk_flat/motrix.yaml b/conf/offpolicy/task/flashsac/g1_23dof_walk_flat/motrix.yaml deleted file mode 100644 index d1698fb4c..000000000 --- a/conf/offpolicy/task/flashsac/g1_23dof_walk_flat/motrix.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# @package _global_ -# Motrix owner for FlashSAC G1 23-DoF walk flat. -# Mirrors 29-DoF flashsac/g1_walk_flat/motrix.yaml: -# - Keeps the mujoco owner's FlashSAC algo identity -# - Adopts the Motrix-direction env + reward tuning (kp/kd rand off, retuned shaping) -training: - task_name: G1Walk23DofFlat - sim_backend: motrix -algo: - num_envs: 4096 - learning_starts: 49 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - replay_buffer_n: 256 - tau: 0.05 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 2.2 - tracking_ang_vel: 1.8 - penalty_ang_vel_xy: -1.2 - penalty_orientation: -12.0 - penalty_action_rate: -2.5 - pose: -0.6 - penalty_feet_ori: -5.0 - feet_phase: 6.0 - alive: 12.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - close_feet_threshold: 0.15 - pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/flashsac/g1_23dof_walk_flat/mujoco.yaml b/conf/offpolicy/task/flashsac/g1_23dof_walk_flat/mujoco.yaml deleted file mode 100644 index 34e63d24d..000000000 --- a/conf/offpolicy/task/flashsac/g1_23dof_walk_flat/mujoco.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# @package _global_ -training: - task_name: G1Walk23DofFlat - sim_backend: mujoco -algo: - num_envs: 4096 - learning_starts: 49 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - #use_symmetry: true - replay_buffer_n: 256 - tau: 0.05 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -5.0 - pose: -0.5 - penalty_feet_ori: -25.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.005 - close_feet_threshold: 0.15 - pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/flashsac/g1_walk_flat/mjwarp.yaml b/conf/offpolicy/task/flashsac/g1_walk_flat/mjwarp.yaml deleted file mode 100644 index 4a7602acb..000000000 --- a/conf/offpolicy/task/flashsac/g1_walk_flat/mjwarp.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# @package _global_ -# Configured-only mjwarp owner for FlashSAC G1 walk flat. Mirrors the mujoco -# owner's algo / env / reward identity; mjwarp-specific host-adapter settings -# follow conf/offpolicy/task/sac/g1_walk_flat/mjwarp.yaml. Offline record -# reuses MuJoCo rendering; native playback and device-resident runtime are absent. -training: - task_name: G1WalkFlat - sim_backend: mjwarp - play_render_mode: record -algo: - num_envs: 4096 - learning_starts: 49 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - #use_symmetry: true - replay_buffer_n: 256 - tau: 0.05 -env: - mjwarp_nconmax: 128 - mjwarp_njmax: 256 - render_spacing: 2.0 - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - randomize_dof_armature: false - randomize_body_gravity_compensation: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -5.0 - pose: -0.5 - penalty_feet_ori: -25.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.005 - close_feet_threshold: 0.15 - pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/flashsac/g1_walk_flat/motrix.yaml b/conf/offpolicy/task/flashsac/g1_walk_flat/motrix.yaml deleted file mode 100644 index 5f61c51b6..000000000 --- a/conf/offpolicy/task/flashsac/g1_walk_flat/motrix.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# @package _global_ -# Motrix owner for FlashSAC G1 walk flat. -# Keeps the mujoco owner's FlashSAC algo identity (num_envs/updates_per_step/ -# replay_buffer_n/tau plus the distributional critic from conf/offpolicy/algo/flashsac.yaml) -# while adopting the Motrix-direction env + reward tuning used by the SAC motrix owner -# (kp/kd randomization off, retuned reward shaping, tighter feet-phase sigma). -# control_config.action_scale stays 1.0 to keep sim2sim DENYLIST parity with mujoco. -training: - task_name: G1WalkFlat - sim_backend: motrix -algo: - num_envs: 4096 - learning_starts: 49 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - replay_buffer_n: 256 - tau: 0.05 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.2 - tracking_ang_vel: 1.8 - penalty_ang_vel_xy: -1.2 - penalty_orientation: -12.0 - penalty_action_rate: -2.5 - pose: -0.6 - penalty_feet_ori: -5.0 - feet_phase: 6.0 - alive: 12.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - close_feet_threshold: 0.15 - pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/flashsac/g1_walk_flat/mujoco.yaml b/conf/offpolicy/task/flashsac/g1_walk_flat/mujoco.yaml deleted file mode 100644 index 35624c3b6..000000000 --- a/conf/offpolicy/task/flashsac/g1_walk_flat/mujoco.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# @package _global_ -training: - task_name: G1WalkFlat - sim_backend: mujoco -algo: - num_envs: 4096 - learning_starts: 49 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - #use_symmetry: true - replay_buffer_n: 256 - tau: 0.05 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -5.0 - pose: -0.5 - penalty_feet_ori: -25.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.005 - close_feet_threshold: 0.15 - pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/flashsac/go2_joystick_flat/mujoco.yaml b/conf/offpolicy/task/flashsac/go2_joystick_flat/mujoco.yaml deleted file mode 100644 index 031d8e2b7..000000000 --- a/conf/offpolicy/task/flashsac/go2_joystick_flat/mujoco.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# @package _global_ -training: - task_name: Go2JoystickFlat - sim_backend: mujoco -algo: - num_envs: 1024 - learning_starts: 50 - max_iterations: 4000 - save_interval: 1000 - updates_per_step: 2 - batch_size: 2048 - replay_buffer_n: 4096 - tau: 0.05 -env: - control_config: - action_scale: 0.4 - domain_rand: - randomize_kp: true - randomize_kd: true - randomize_base_mass: true - random_com: true - randomize_gravity: true - push_robots: true - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -20.0 - action_rate: -0.02 - similar_to_default: -0.4 - contact: 1.5 - swing_feet_z: 4.0 - tracking_sigma: 0.4 - base_height_target: 0.3 \ No newline at end of file diff --git a/conf/offpolicy/task/sac/g1_23dof_flip_tracking/mujoco.yaml b/conf/offpolicy/task/sac/g1_23dof_flip_tracking/mujoco.yaml deleted file mode 100644 index f48e6108a..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_flip_tracking/mujoco.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# @package _global_ -training: - task_name: G1FlipTrackingSAC23Dof - sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 4096 - max_iterations: 25000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.005 - target_entropy_ratio: 0.05 - max_grad_norm: 10.0 -env: - sampling_mode: mixed - sampling_start_ratio: 0.1 - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/offpolicy/task/sac/g1_23dof_motion_tracking/motrix.yaml b/conf/offpolicy/task/sac/g1_23dof_motion_tracking/motrix.yaml deleted file mode 100644 index e042ee799..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_motion_tracking/motrix.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# @package _global_ -# G1 23-DoF Motion Tracking SAC — Motrix variant for sim2sim eval. -# Inherits the mujoco training config in full and only switches the rendering -# backend so checkpoints trained on mujoco can be replayed via motrix's native -# renderer (`eval --sim motrix`). Training on motrix is not the intended path. -defaults: - - /task/sac/g1_23dof_motion_tracking/mujoco - - _self_ - -training: - task_name: G1MotionTrackingSAC23Dof - sim_backend: motrix -env: - # motrix backend's kp/kd override path is broken on column slices, and DR - # is not desirable during deterministic sim2sim eval anyway. Match the - # `g1_walk_flat/motrix.yaml` convention by switching them off. - domain_rand: - randomize_kp: false - randomize_kd: false diff --git a/conf/offpolicy/task/sac/g1_23dof_motion_tracking/mujoco.yaml b/conf/offpolicy/task/sac/g1_23dof_motion_tracking/mujoco.yaml deleted file mode 100644 index a68d07fae..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_motion_tracking/mujoco.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# @package _global_ -training: - task_name: G1MotionTrackingSAC23Dof - sim_backend: mujoco -algo: - num_envs: 2048 - max_iterations: 25000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.1 - target_entropy_ratio: 0.5 - max_grad_norm: 10.0 -env: - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - truncate_on_clip_end: true - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -2.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/offpolicy/task/sac/g1_23dof_walk_flat/motrix.yaml b/conf/offpolicy/task/sac/g1_23dof_walk_flat/motrix.yaml deleted file mode 100644 index d67013177..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_walk_flat/motrix.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# @package _global_ -training: - task_name: G1Walk23DofFlat - sim_backend: motrix -algo: - num_envs: 2048 - learning_starts: 1 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: false - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 -reward: - scales: - tracking_lin_vel: 2.2 - tracking_ang_vel: 1.8 - penalty_ang_vel_xy: -1.2 - penalty_orientation: -12.0 - penalty_action_rate: -2.5 - pose: -0.6 - penalty_feet_ori: -5.0 - feet_phase: 6.0 - alive: 12.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_23dof_walk_flat/mujoco.yaml b/conf/offpolicy/task/sac/g1_23dof_walk_flat/mujoco.yaml deleted file mode 100644 index 1f2ca273c..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_walk_flat/mujoco.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# @package _global_ -training: - task_name: G1Walk23DofFlat - sim_backend: mujoco -algo: - num_envs: 2048 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: true - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_23dof_walk_rough/motrix.yaml b/conf/offpolicy/task/sac/g1_23dof_walk_rough/motrix.yaml deleted file mode 100644 index 844fb2f2a..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_walk_rough/motrix.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# @package _global_ -training: - task_name: G1Walk23DofRough - sim_backend: motrix -algo: - num_envs: 2048 - learning_starts: 1 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: false - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - sim_dt: 0.01 - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 -reward: - scales: - tracking_lin_vel: 2.2 - tracking_ang_vel: 1.8 - penalty_ang_vel_xy: -1.2 - penalty_orientation: -12.0 - penalty_action_rate: -2.5 - pose: -0.6 - penalty_feet_ori: -5.0 - feet_phase: 6.0 - alive: 12.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_23dof_walk_rough/mujoco.yaml b/conf/offpolicy/task/sac/g1_23dof_walk_rough/mujoco.yaml deleted file mode 100644 index 16689327b..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_walk_rough/mujoco.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# @package _global_ -training: - task_name: G1Walk23DofRough - sim_backend: mujoco -algo: - num_envs: 2048 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: true - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_23dof_wall_flip_tracking/mujoco.yaml b/conf/offpolicy/task/sac/g1_23dof_wall_flip_tracking/mujoco.yaml deleted file mode 100644 index f67d34d2e..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_wall_flip_tracking/mujoco.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# @package _global_ -training: - task_name: G1WallFlipTrackingSAC23Dof - sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 4096 - max_iterations: 25000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.005 - target_entropy_ratio: 0.0 - max_grad_norm: 10.0 -env: - sampling_mode: uniform - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 1000000000.0 - ee_body_pos_z_threshold: 1000000000.0 - terminate_on_undesired_contacts: false - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/offpolicy/task/sac/g1_23dof_wbt_obs/mujoco.yaml b/conf/offpolicy/task/sac/g1_23dof_wbt_obs/mujoco.yaml deleted file mode 100644 index d0623bad2..000000000 --- a/conf/offpolicy/task/sac/g1_23dof_wbt_obs/mujoco.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# @package _global_ -training: - task_name: G1WBTObs23Dof - sim_backend: mujoco -algo: - num_envs: 4096 - max_iterations: 140000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.1 - target_entropy_ratio: 0.5 - max_grad_norm: 10.0 -env: - sim_dt: 0.005 - sensor: - local_linvel: pelvis_local_linvel - gyro: pelvis_gyro - upvector: pelvis_upvector - control_config: - action_scale: 2.0 - simulate_action_latency: true - anchor_pos_z_threshold: 0.40 - ee_body_pos_z_threshold: 0.5 - truncate_on_clip_end: true - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.5 - scale_gyro: 0.2 - enable_zero_linvel: true - enable_zero_anchor_pos: true - enable_anchor_ori_noise: true - scale_anchor_ori: 0.05 - obs_history_length: 5 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 1.0] - random_com: true - com_offset_x: [-0.05, 0.05] - randomize_com_y: true - com_offset_y: [-0.05, 0.05] - randomize_com_z: true - com_offset_z: [-0.05, 0.05] - randomize_gravity: false - gravity_range: [[0.0, 0.0, -9.81], [0.0, 0.0, -9.81]] - push_robots: true - push_interval: 200 - max_force: [300.0, 300.0, 120.0] - push_body_name: null - randomize_kp: true - kp_multiplier_range: [0.9, 1.1] - randomize_kd: true - kd_multiplier_range: [0.85, 1.15] - randomize_geom_friction: true - friction_range: [0.3, 1.2] - friction_geom_pattern: "^(left|right)_foot[1-7]_collision$" - enable_encoder_bias: true - encoder_bias_range: [-0.01, 0.01] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 1.0 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -5.0 - undesired_contacts: -0.1 - joint_acc_l2: -2.5e-7 - joint_torque_l2: -1e-5 diff --git a/conf/offpolicy/task/sac/g1_flip_tracking/mujoco.yaml b/conf/offpolicy/task/sac/g1_flip_tracking/mujoco.yaml deleted file mode 100644 index 48a956b4f..000000000 --- a/conf/offpolicy/task/sac/g1_flip_tracking/mujoco.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# @package _global_ -training: - task_name: G1FlipTrackingSAC - sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 4096 - max_iterations: 25000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.005 - target_entropy_ratio: 0.05 - max_grad_norm: 10.0 -env: - sampling_mode: mixed - sampling_start_ratio: 0.1 - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/offpolicy/task/sac/g1_motion_tracking/motrix.yaml b/conf/offpolicy/task/sac/g1_motion_tracking/motrix.yaml deleted file mode 100644 index eddb1839f..000000000 --- a/conf/offpolicy/task/sac/g1_motion_tracking/motrix.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# @package _global_ -# G1 Whole-Body Tracking (WBT) FastSAC — Motrix variant for sim2sim eval. -# Inherits the mujoco training config in full and only switches the rendering -# backend so checkpoints trained on mujoco can be replayed via motrix's native -# renderer (`eval --sim motrix`). Training on motrix is not the intended path. -defaults: - - /task/sac/g1_motion_tracking/mujoco - - _self_ - -training: - task_name: G1MotionTrackingSAC - sim_backend: motrix -env: - # motrix backend's kp/kd override path is broken on column slices, and DR - # is not desirable during deterministic sim2sim eval anyway. Match the - # `g1_walk_flat/motrix.yaml` convention by switching them off. - domain_rand: - randomize_kp: false - randomize_kd: false diff --git a/conf/offpolicy/task/sac/g1_motion_tracking/mujoco.yaml b/conf/offpolicy/task/sac/g1_motion_tracking/mujoco.yaml deleted file mode 100644 index 598feccbb..000000000 --- a/conf/offpolicy/task/sac/g1_motion_tracking/mujoco.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# @package _global_ -# G1 Whole-Body Tracking (WBT) with FastSAC on MuJoCo. -# Hyperparameters aligned with holosoma g1-29dof-wbt-fast-sac. -training: - task_name: G1MotionTrackingSAC - sim_backend: mujoco -algo: - num_envs: 2048 - max_iterations: 25000 - save_interval: 1000 - # --- holosoma WBT-specific overrides (vs sac.yaml defaults) --- - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.1 - target_entropy_ratio: 0.5 - max_grad_norm: 10.0 -env: - control_config: - action_scale: 2.0 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - truncate_on_clip_end: true - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 - seed: null -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -2.0 - undesired_contacts: -0.1 diff --git a/conf/offpolicy/task/sac/g1_walk_flat/mjwarp.yaml b/conf/offpolicy/task/sac/g1_walk_flat/mjwarp.yaml deleted file mode 100644 index dc9e23014..000000000 --- a/conf/offpolicy/task/sac/g1_walk_flat/mjwarp.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# @package _global_ -# Configured-only SAC owner for the mjwarp host adapter. Offline record reuses -# MuJoCo rendering; native playback and device-resident runtime are absent. -training: - task_name: G1WalkFlat - sim_backend: mjwarp - play_render_mode: record -algo: - num_envs: 2048 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: true - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - mjwarp_nconmax: 128 - mjwarp_njmax: 256 - render_spacing: 2.0 - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - randomize_dof_armature: false - randomize_body_gravity_compensation: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_walk_flat/motrix.yaml b/conf/offpolicy/task/sac/g1_walk_flat/motrix.yaml deleted file mode 100644 index a193eeb51..000000000 --- a/conf/offpolicy/task/sac/g1_walk_flat/motrix.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# @package _global_ -training: - task_name: G1WalkFlat - sim_backend: motrix -algo: - num_envs: 2048 - learning_starts: 1 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: false - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 0.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.2 - tracking_ang_vel: 1.8 - penalty_ang_vel_xy: -1.2 - penalty_orientation: -12.0 - penalty_action_rate: -2.5 - pose: -0.6 - penalty_feet_ori: -5.0 - feet_phase: 6.0 - alive: 12.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_walk_flat/mujoco.yaml b/conf/offpolicy/task/sac/g1_walk_flat/mujoco.yaml deleted file mode 100644 index 0b0e2d4bd..000000000 --- a/conf/offpolicy/task/sac/g1_walk_flat/mujoco.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# @package _global_ -training: - task_name: G1WalkFlat - sim_backend: mujoco -algo: - num_envs: 2048 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: true - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_walk_rough/motrix.yaml b/conf/offpolicy/task/sac/g1_walk_rough/motrix.yaml deleted file mode 100644 index 4144b1713..000000000 --- a/conf/offpolicy/task/sac/g1_walk_rough/motrix.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# @package _global_ -training: - task_name: G1WalkRough - sim_backend: motrix -algo: - num_envs: 2048 - learning_starts: 1 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: false - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - sim_dt: 0.01 - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 -reward: - scales: - tracking_lin_vel: 2.2 - tracking_ang_vel: 1.8 - penalty_ang_vel_xy: -1.2 - penalty_orientation: -12.0 - penalty_action_rate: -2.5 - pose: -0.6 - penalty_feet_ori: -5.0 - feet_phase: 6.0 - alive: 12.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_walk_rough/mujoco.yaml b/conf/offpolicy/task/sac/g1_walk_rough/mujoco.yaml deleted file mode 100644 index bd9a1282a..000000000 --- a/conf/offpolicy/task/sac/g1_walk_rough/mujoco.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# @package _global_ -training: - task_name: G1WalkRough - sim_backend: mujoco -algo: - num_envs: 2048 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - use_symmetry: true - algo_params: - alpha_init: 0.001 - target_entropy_ratio: 0.0 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/sac/g1_wall_flip_tracking/mujoco.yaml b/conf/offpolicy/task/sac/g1_wall_flip_tracking/mujoco.yaml deleted file mode 100644 index da48802f1..000000000 --- a/conf/offpolicy/task/sac/g1_wall_flip_tracking/mujoco.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# @package _global_ -training: - task_name: G1WallFlipTrackingSAC - sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 4096 - max_iterations: 25000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.005 - target_entropy_ratio: 0.0 - max_grad_norm: 10.0 -env: - sampling_mode: uniform - truncate_on_clip_end: true - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 1000000000.0 - ee_body_pos_z_threshold: 1000000000.0 - terminate_on_undesired_contacts: false - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml b/conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml deleted file mode 100644 index 5fd91fc07..000000000 --- a/conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# @package _global_ -training: - task_name: G1WBTObs - sim_backend: mujoco -algo: - num_envs: 4096 - max_iterations: 140000 - save_interval: 1000 - gamma: 0.99 - tau: 0.05 - num_atoms: 501 - updates_per_step: 4 - policy_frequency: 2 - use_symmetry: false - algo_params: - alpha_init: 0.1 - target_entropy_ratio: 0.5 - max_grad_norm: 10.0 -env: - sim_dt: 0.005 - sensor: - local_linvel: pelvis_local_linvel - gyro: pelvis_gyro - upvector: pelvis_upvector - control_config: - action_scale: 2.0 - simulate_action_latency: true - anchor_pos_z_threshold: 0.40 - ee_body_pos_z_threshold: 0.5 - truncate_on_clip_end: true - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.5 - scale_gyro: 0.2 - enable_zero_linvel: true - enable_zero_anchor_pos: true - enable_anchor_ori_noise: true - scale_anchor_ori: 0.05 - obs_history_length: 5 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 1.0] - random_com: true - com_offset_x: [-0.05, 0.05] - randomize_com_y: true - com_offset_y: [-0.05, 0.05] - randomize_com_z: true - com_offset_z: [-0.05, 0.05] - randomize_gravity: false - gravity_range: [[0.0, 0.0, -9.81], [0.0, 0.0, -9.81]] - push_robots: true - push_interval: 200 - max_force: [300.0, 300.0, 120.0] - push_body_name: null - randomize_kp: true - kp_multiplier_range: [0.9, 1.1] - randomize_kd: true - kd_multiplier_range: [0.85, 1.15] - randomize_geom_friction: true - friction_range: [0.3, 1.2] - friction_geom_pattern: "^(left|right)_foot[1-7]_collision$" - enable_encoder_bias: true - encoder_bias_range: [-0.01, 0.01] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 1.0 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -5.0 - undesired_contacts: -0.1 - joint_acc_l2: -2.5e-7 - joint_torque_l2: -1e-5 diff --git a/conf/offpolicy/task/sac/go2_footstand/drake.yaml b/conf/offpolicy/task/sac/go2_footstand/drake.yaml deleted file mode 100644 index 31035486b..000000000 --- a/conf/offpolicy/task/sac/go2_footstand/drake.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# @package _global_ -training: - task_name: Go2FootStand - sim_backend: drake - no_play: true - play_steps: 400 - play_env_num: 1 - play_render_mode: record - -algo: - algo_log_name: fast_sac_drake - num_envs: 512 - batch_size: 1024 - replay_buffer_n: 512 - updates_per_step: 2 - learning_starts: 2 - max_iterations: 300 - save_interval: 100 - actor_hidden_dim: 256 - critic_hidden_dim: 512 - obs_normalization: true - use_layer_norm: true - algo_params: - alpha_init: 0.005 - target_entropy_ratio: 0.0 - use_compile: false - -env: - sim_dt: 0.004 - drake_backend_mode: batch - drake_nthread: 20 - add_body_sensors: true - obs_history_len: 15 - energy_termination_threshold: 200.0 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 - scale_gravity: 0.05 - scale_linvel: 0.1 - control_config: - action_scale: 0.3 - domain_rand: - randomize_floor_friction: false - randomize_link_mass: false - torso_added_mass_range: null - randomize_torso_com: false - randomize_dof_armature: false - randomize_reset_joint_qpos: true - reset_joint_qpos_range: [-0.05, 0.05] - -reward: - scales: - height: 2.0 - orientation: 2.0 - contact: -1.0 - action_rate: -0.01 - termination: -2.0 - dof_pos_limits: -0.5 - torques: 0.0 - pose: -0.1 - penalty_contact: -0.2 - tar: 0.8 - rear_feet_contact: 0.5 - rear_leg_symmetry: -0.2 - front_leg_motion: -0.05 - upright_stability: -0.2 - knee_clearance: -0.5 - stay_still: -0.1 - energy: -0.003 - dof_acc: -2.5e-7 - tracking_sigma: 0.25 - base_height_target: 0.3 - knee_height_target: 0.08 diff --git a/conf/offpolicy/task/sac/go2w_joystick_flat/drake.yaml b/conf/offpolicy/task/sac/go2w_joystick_flat/drake.yaml deleted file mode 100644 index fa05273e4..000000000 --- a/conf/offpolicy/task/sac/go2w_joystick_flat/drake.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _global_ -training: - task_name: Go2WJoystickFlat - sim_backend: drake - no_play: true - play_steps: 400 - play_env_num: 1 - play_render_mode: record - -algo: - algo_log_name: fast_sac_drake - num_envs: 512 - batch_size: 1024 - replay_buffer_n: 512 - updates_per_step: 2 - learning_starts: 2 - max_iterations: 300 - save_interval: 100 - actor_hidden_dim: 256 - critic_hidden_dim: 512 - obs_normalization: true - use_layer_norm: true - algo_params: - alpha_init: 0.005 - target_entropy_ratio: 0.0 - use_compile: false - -env: - drake_backend_mode: batch - drake_nthread: 20 - scene: - model_file: src/unilab/assets/robots/go2w/scene_flat.xml - commands: - vel_limit: - - [0.0, 0.0, -1.0] - - [1.0, 0.0, 1.0] - control_config: - action_scale: 0.5 - wheel_action_scale: 10.0 - Kp: 50.0 - Kd: 1.5 - wheel_Kd: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - push_robots: false - -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.75 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - orientation: -2.0 - action_rate: -0.005 - similar_to_default: -0.5 - torques: -0.0002 - wheel_vel: 0.0 - alive: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.4 diff --git a/conf/offpolicy/task/td3/g1_23dof_walk_flat/mujoco.yaml b/conf/offpolicy/task/td3/g1_23dof_walk_flat/mujoco.yaml deleted file mode 100644 index 850f155e8..000000000 --- a/conf/offpolicy/task/td3/g1_23dof_walk_flat/mujoco.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# @package _global_ -training: - task_name: G1Walk23DofFlat - sim_backend: mujoco -algo: - max_iterations: 100000 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/td3/g1_walk_flat/mujoco.yaml b/conf/offpolicy/task/td3/g1_walk_flat/mujoco.yaml deleted file mode 100644 index 54f87699c..000000000 --- a/conf/offpolicy/task/td3/g1_walk_flat/mujoco.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# @package _global_ -training: - task_name: G1WalkFlat - sim_backend: mujoco -algo: - max_iterations: 100000 -env: - control_config: - action_scale: 1.0 - gait_phase_init_mode: "offset_phase" - reset_base_qvel_limit: 0.5 - curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_gyro: 0.0 - scale_gravity: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 0.1 - scale_linvel: 0.0 - seed: null -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 1.5 - penalty_ang_vel_xy: -1.0 - penalty_orientation: -10.0 - penalty_action_rate: -4.0 - pose: -0.5 - penalty_feet_ori: -20.0 - feet_phase: 5.0 - alive: 10.0 - tracking_sigma: 0.25 - base_height_target: 0.754 - min_base_height: 0.3 - max_tilt_deg: 65.0 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.04 - close_feet_threshold: 0.15 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/offpolicy/task/td3/go1_joystick_flat/motrix.yaml b/conf/offpolicy/task/td3/go1_joystick_flat/motrix.yaml deleted file mode 100644 index 40c9f28b7..000000000 --- a/conf/offpolicy/task/td3/go1_joystick_flat/motrix.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# @package _global_ -training: - task_name: Go1JoystickFlat - sim_backend: motrix -algo: - num_envs: 1024 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - batch_size: 8192 - replay_buffer_n: 1024 -env: - commands: - vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 diff --git a/conf/offpolicy/task/td3/go2_joystick_flat/motrix.yaml b/conf/offpolicy/task/td3/go2_joystick_flat/motrix.yaml deleted file mode 100644 index 8b32bd78a..000000000 --- a/conf/offpolicy/task/td3/go2_joystick_flat/motrix.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# @package _global_ -training: - task_name: Go2JoystickFlat - sim_backend: motrix -algo: - num_envs: 1024 - learning_starts: 10 - max_iterations: 5000 - save_interval: 1000 - updates_per_step: 8 - batch_size: 8192 - replay_buffer_n: 1024 -env: - commands: - vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] - domain_rand: - randomize_kp: false - randomize_kd: false -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 diff --git a/conf/ppo/config.yaml b/conf/ppo/config.yaml index a5210ce18..2954e01ae 100644 --- a/conf/ppo/config.yaml +++ b/conf/ppo/config.yaml @@ -28,7 +28,7 @@ algo: activation: elu class_name: ActorCritic algorithm: - class_name: unilab.algos.torch.rsl_rl_ppo:FinalObservationAwarePPO + class_name: unilab.algos.rsl_rl_ppo:FinalObservationAwarePPO value_loss_coef: 1.0 use_clipped_value_loss: true clip_param: 0.2 diff --git a/conf/ppo/task/a2_joystick_flat/base.yaml b/conf/ppo/task/a2_joystick_flat/base.yaml new file mode 100644 index 000000000..87203212a --- /dev/null +++ b/conf/ppo/task/a2_joystick_flat/base.yaml @@ -0,0 +1,296 @@ +# @package _global_ +# Canonical A2 flat Manager-Based task declaration. The MuJoCo owner leaf adds +# only training identity/tuning; all task behavior is declared here for Hydra. +env: + scene: + model_file: src/unilab/assets/robots/a2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base_link + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + body_names: [base_link] + geom_names: [floor] + actuator_names: + - FL_hip + - FL_thigh + - FL_calf + - FR_hip + - FR_thigh + - FR_calf + - RL_hip + - RL_thigh + - RL_calf + - RR_hip + - RR_thigh + - RR_calf + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + command_name: twist + command_threshold: 0.1 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + command_name: twist + command_threshold: 0.1 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [5.0, 5.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.1 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: base_link + mass_distribution_params: [0.0, 8.0] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: base_link + com_range: + x: [-0.08, 0.08] + y: [-0.08, 0.08] + z: [-0.08, 0.08] + foot_friction: + func: unilab.envs.mdp.geom_friction + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + geom_names: floor + ranges: [0.3, 1.6] + operation: scale + axes: [0] + shared_random: true + joint_armature: + func: unilab.envs.mdp.joint_armature + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*" + ranges: [0.9, 1.1] + operation: scale + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + actuator_names: ".*" + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [8.0, 8.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.4 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.4 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.02 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.25 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + weight: 0.5 + params: + frequency: 2.0 + command_name: twist + command_threshold: 0.1 + sensor_names: [FL_foot_contact, FR_foot_contact, RL_foot_contact, RR_foot_contact] + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + command_name: twist + command_threshold: 0.1 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 + stand_still: + func: unilab.tasks.locomotion.common.manager_terms.stand_still_l1 + weight: -4.0 + params: + command_name: twist + command_threshold: 0.1 + hip_deviation: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -1.0 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_hip_joint" + stand_feet_air: + func: unilab.tasks.locomotion.common.manager_terms.feet_air_while_standing + weight: -1.0 + params: + command_name: twist + command_threshold: 0.1 + sensor_names: [FL_foot_contact, FR_foot_contact, RL_foot_contact, RR_foot_contact] + contact_threshold: 0.1 diff --git a/conf/ppo/task/a2_joystick_flat/mujoco.yaml b/conf/ppo/task/a2_joystick_flat/mujoco.yaml index b0863e9db..8a5f336ca 100644 --- a/conf/ppo/task/a2_joystick_flat/mujoco.yaml +++ b/conf/ppo/task/a2_joystick_flat/mujoco.yaml @@ -1,84 +1,22 @@ # @package _global_ -# A2 (leg-only Unitree A2) joystick flat task. Same isomorphic task as -# Go2JoystickFlat: 12-DOF velocity tracking with a gait phase. Robot identity -# (asset path, standing height 0.465, A2 leg PD gains) lives in A2JoystickCfg; -# this YAML carries training + reward only, mirroring the Go2 task. +defaults: + - /task/a2_joystick_flat/base + - _self_ + training: task_name: A2JoystickFlat sim_backend: mujoco algo: num_envs: 1024 - max_iterations: 500 # A2 (19.6 kg, ~2.8x Go2) + full DR needs more budget than Go2's flat task + max_iterations: 500 empirical_normalization: true obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: learning_rate: 3.0e-4 entropy_coef: 1.0e-3 -env: - # Domain randomization for sim2real deployment. A2JoystickDomainRandomizationProvider - # caches the dof-armature + geom-friction baselines, so randomize_dof_armature and - # randomize_ground_friction are ON. Ground friction is effective because the floor - # geom is the priority geom (scene_flat.xml). Ranges reference unitree_rl_mjlab A2 - # events: joint_armature scale [0.9,1.1], foot friction [0.3,1.6]. randomize_body_mass - # stays off (base_body_mass not cached); gravity OFF (constant on flat ground). - # env.domain_rand is sim2sim ALLOWLIST (free override). - domain_rand: - randomize_base_mass: true - added_mass_range: [0.0, 8.0] - - randomize_body_mass: false # provider does not cache base_body_mass - body_mass_multiplier_range: [0.9, 1.1] - - random_com: true - com_offset_x: [-0.08, 0.08] - com_offset_y: [-0.08, 0.08] - com_offset_z: [-0.08, 0.08] - - randomize_gravity: false # flat ground: gravity constant on the real robot; randomizing only slows training. - - randomize_ground_friction: true # floor is the priority geom, so this moves the foot-ground friction - ground_friction_multiplier_range: [0.3, 1.6] # mjlab foot_friction range - - randomize_dof_armature: true - dof_armature_multiplier_range: [0.9, 1.1] # mjlab joint_armature scale - - randomize_kp: true - kp_multiplier_range: [0.9, 1.1] - - randomize_kd: true - kd_multiplier_range: [0.9, 1.1] - - push_robots: true - push_interval: 400 # control steps between base velocity pushes - max_force: [1.0, 1.0, 0.5] - push_body_name: base_link # A2 base body (Go2 uses "base"); required or push has no target. - # Standing-aware commands so the policy trains on genuine zero-command samples - # (rel_standing_envs fraction forced to stand) and resamples mid-episode every 5s. env.commands is - # a sim2sim ALLOWLIST subset (vel_limit) / free fields; rel_standing_envs and - # resampling_time are declared fields on Commands so Hydra struct mode accepts them. - commands: - rel_standing_envs: 0.1 - resampling_time: 5.0 -reward: - # command_threshold gates the phase-driven gait rewards (swing_feet_z / contact) - # so the A2 stands still at zero command instead of marching in place. - command_threshold: 0.1 - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.4 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.02 - similar_to_default: -0.25 - contact: 0.5 - swing_feet_z: 4.0 - stand_still: -4.0 - hip_deviation: -1.0 - stand_feet_air: -1.0 # penalize feet leaving the ground at zero command (gated off during locomotion) - tracking_sigma: 0.25 - base_height_target: 0.40 diff --git a/conf/ppo/task/allegro_inhand/base.yaml b/conf/ppo/task/allegro_inhand/base.yaml new file mode 100644 index 000000000..afafae38c --- /dev/null +++ b/conf/ppo/task/allegro_inhand/base.yaml @@ -0,0 +1,139 @@ +# @package _global_ +# Canonical Allegro rotation Manager-Based task declaration. Backend leaves own +# only backend identity and algorithm/runtime tuning. +env: + scene: + model_file: src/unilab/assets/robots/allegro_hand/scene.xml + default_keyframe_name: home + entities: + robot: + # The hand is fixed; the free ball is the task's single root-state entity. + root_body_name: ball + joint_names: + - ffj0 + - ffj1 + - ffj2 + - ffj3 + - mfj0 + - mfj1 + - mfj2 + - mfj3 + - rfj0 + - rfj1 + - rfj2 + - rfj3 + - thj0 + - thj1 + - thj2 + - thj3 + body_names: [ball, ff_tip, mf_tip, rf_tip, th_tip] + actuator_names: + - ffa0 + - ffa1 + - ffa2 + - ffa3 + - mfa0 + - mfa1 + - mfa2 + - mfa3 + - rfa0 + - rfa1 + - rfa2 + - rfa3 + - tha0 + - tha1 + - tha2 + - tha3 + sim_dt: 0.005 + ctrl_dt: 0.05 + max_episode_seconds: 20.0 + observations: + policy: + history_length: 3 + flatten_history_dim: true + terms: + rotation: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroRotationObservation + params: + entity_name: robot + action_name: hand + joint_noise: 0.02 + torque_estimate_kp: 1.0 + torque_estimate_kd: 0.1 + actions: + hand: + _target_: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroIncrementalPositionActionCfg + entity_name: robot + actuator_names: [".*"] + action_scale: 0.041666666666666664 + raw_action_clip: [-1.0, 1.0] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_hand_ball: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroHandBallReset + mode: reset + params: + entity_name: robot + # null explicitly selects the model home pose. A configured path is + # fail-closed when missing or malformed. + grasp_cache_path: null + joint_noise: 0.0 + ball_velocity_noise: 0.0 + ball_z_offset: 0.0 + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [1.0, 1.0] + kd_range: [0.1, 0.1] + operation: abs + terminations: + dropped: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroDropTermination + params: + observation_group: policy + observation_term: rotation + minimum_ball_height: 0.125 + time_out: + func: unilab.envs.mdp.time_out + time_out: true + scale_rewards_by_dt: true + policy_observation_group: policy + critic_observation_group: null + +reward: + rotate: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.AllegroRotateReward + weight: 1.25 + params: + state_term_name: dropped + rotation_axis: [0.0, 0.0, 1.0] + clip_min: -0.5 + clip_max: 0.5 + obj_linvel: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.object_linear_velocity_l1 + weight: -0.3 + params: + state_term_name: dropped + pose_diff: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.hand_pose_deviation_l2 + weight: -0.3 + params: + state_term_name: dropped + torque: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.estimated_torque_l2 + weight: -0.1 + params: + state_term_name: dropped + work: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.estimated_work_l2 + weight: -2.0 + params: + state_term_name: dropped + drop: + func: unilab.tasks.manipulation.allegro_inhand.manager_terms.dropped + weight: 0.0 + params: + state_term_name: dropped diff --git a/conf/ppo/task/allegro_inhand/drake.yaml b/conf/ppo/task/allegro_inhand/drake.yaml new file mode 100644 index 000000000..dc7f0e09c --- /dev/null +++ b/conf/ppo/task/allegro_inhand/drake.yaml @@ -0,0 +1,46 @@ +# @package _global_ +defaults: + - /task/allegro_inhand/base + - _self_ + +training: + task_name: AllegroInhandRotation + sim_backend: drake + render_spacing: 0.5 + cam_distance: 1.5 + cam_lookat: [0.75, 0.75, 0] + cam_elevation: -20.0 +algo: + num_envs: 16384 + num_steps_per_env: 8 + max_iterations: 201 + obs_groups: + actor: [policy] + critic: [policy] + actor: + class_name: rsl_rl.models.MLPModel + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + distribution_cfg: + class_name: rsl_rl.modules.distribution.GaussianDistribution + init_std: 1.0 + std_type: scalar + critic: + class_name: rsl_rl.models.MLPModel + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + algorithm: + value_loss_coef: 4.0 + desired_kl: 0.02 +env: + drake_backend_mode: batch + events: + # Drake consumes the fixed MJCF actuator gains and does not expose reset + # gain mutation in the installed production adapter. + pd_gains: null +play_profile: + enabled: true + env: + render_spacing: 2.0 diff --git a/conf/ppo/task/allegro_inhand/motrix.yaml b/conf/ppo/task/allegro_inhand/motrix.yaml index 3608d9743..8df1fe13b 100644 --- a/conf/ppo/task/allegro_inhand/motrix.yaml +++ b/conf/ppo/task/allegro_inhand/motrix.yaml @@ -1,7 +1,16 @@ # @package _global_ +defaults: + - /task/allegro_inhand/base + - _self_ + training: task_name: AllegroInhandRotation sim_backend: motrix +env: + # Legacy Motrix consumed the MJCF-native gains; only MuJoCo used the explicit + # host-side position-gain override. + events: + pd_gains: null algo: num_envs: 16384 num_steps_per_env: 8 @@ -26,30 +35,6 @@ algo: algorithm: value_loss_coef: 4.0 desired_kl: 0.02 -reward: - scales: - rotate: 1.25 - obj_linvel: -0.3 - pose_diff: -0.3 - torque: -0.1 - work: -2.0 - drop: 0.0 - angvel_clip_min: -0.5 - angvel_clip_max: 0.5 - reset_z_threshold: 0.125 -env: - gen_grasp: false - max_episode_seconds: 20.0 - grasp_cache_path: caches/allegro_grasp_50k.npy - # Keep only grasp/pose reset variation. All online DR terms stay disabled. - domain_rand: - randomize_base_mass: false - random_com: false - randomize_gravity: false - push_robots: false - joint_noise: 0.0 - ball_vel_noise: 0.0 - ball_z_offset: 0.0 play_profile: enabled: true env: diff --git a/conf/ppo/task/allegro_inhand/mujoco.yaml b/conf/ppo/task/allegro_inhand/mujoco.yaml index 17d639913..3538defdc 100644 --- a/conf/ppo/task/allegro_inhand/mujoco.yaml +++ b/conf/ppo/task/allegro_inhand/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/allegro_inhand/base + - _self_ + training: task_name: AllegroInhandRotation sim_backend: mujoco @@ -30,30 +34,6 @@ algo: algorithm: value_loss_coef: 4.0 desired_kl: 0.02 -reward: - scales: - rotate: 1.25 - obj_linvel: -0.3 - pose_diff: -0.3 - torque: -0.1 - work: -2.0 - drop: 0.0 - angvel_clip_min: -0.5 - angvel_clip_max: 0.5 - reset_z_threshold: 0.125 -env: - gen_grasp: false - max_episode_seconds: 20.0 - grasp_cache_path: caches/allegro_grasp_50k.npy - # Keep only grasp/pose reset variation. All online DR terms stay disabled. - domain_rand: - randomize_base_mass: false - random_com: false - randomize_gravity: false - push_robots: false - joint_noise: 0.0 - ball_vel_noise: 0.0 - ball_z_offset: 0.0 play_profile: enabled: true env: diff --git a/conf/ppo/task/allegro_inhand_grasp/motrix.yaml b/conf/ppo/task/allegro_inhand_grasp/motrix.yaml index 6b5cdca31..e75de1bd9 100644 --- a/conf/ppo/task/allegro_inhand_grasp/motrix.yaml +++ b/conf/ppo/task/allegro_inhand_grasp/motrix.yaml @@ -1,36 +1,94 @@ # @package _global_ defaults: - - /task/allegro_inhand/motrix + - /task/allegro_inhand/base - _self_ training: task_name: AllegroInhandRotationGrasp sim_backend: motrix no_play: true + algo: - max_iterations: 1000 # infinite rollout -reward: - scales: - rotate: 0.0 - obj_linvel: 0.0 - pose_diff: 0.0 - torque: 0.0 - work: 0.0 - drop: 0.0 + num_envs: 16384 + num_steps_per_env: 8 + max_iterations: 1000 # infinite rollout until the recorder raises RunComplete + obs_groups: + actor: [policy] + critic: [policy] + actor: + class_name: rsl_rl.models.MLPModel + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + distribution_cfg: + class_name: rsl_rl.modules.distribution.GaussianDistribution + init_std: 1.0 + std_type: scalar + critic: + class_name: rsl_rl.models.MLPModel + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + algorithm: + value_loss_coef: 4.0 + desired_kl: 0.02 + env: - gen_grasp: true max_episode_seconds: 3.0 - grasp_cache_path: caches/allegro_grasp_50k.npy - grasp_collection_target: 50000 - grasp_auto_save: true - grasp_quality_check: true - grasp_min_contacts: 2 - domain_rand: - randomize_base_mass: false - random_com: false - push_robots: false - ball_vel_noise: 0.0 - joint_noise: 0.25 # random sampling of the grasp poses + actions: + hand: + action_scale: 0.0 + events: + pd_gains: null + reset_hand_ball: + params: + grasp_cache_path: null + joint_noise: 0.25 + ball_velocity_noise: 0.0 + ball_z_offset: 0.0 + terminations: + invalid_grasp: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityTermination + params: + entity_name: robot + observation_group: policy + observation_term: rotation + fingertip_body_names: [ff_tip, mf_tip, rf_tip, th_tip] + contact_sensor_names: [ff_contact, mf_contact, rf_contact, th_contact] + max_fingertip_distance: 0.1 + minimum_contacts: 2 + minimum_ball_height: 0.125 + enabled: true + metrics: + fingertips_close: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: fingertips_close} + enough_contacts: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: enough_contacts} + ball_held: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: ball_held} + valid: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: valid} + recorders: + grasp_cache: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspRecorder + params: + quality_term_name: invalid_grasp + output_path: caches/allegro_grasp_50k.npy + collection_target: 50000 + auto_save: true + +reward: + rotate: {weight: 0.0} + obj_linvel: {weight: 0.0} + pose_diff: {weight: 0.0} + torque: {weight: 0.0} + work: {weight: 0.0} + drop: {weight: 0.0} + play_profile: enabled: true env: diff --git a/conf/ppo/task/allegro_inhand_grasp/mujoco.yaml b/conf/ppo/task/allegro_inhand_grasp/mujoco.yaml index e6e62f2f6..693937e3d 100644 --- a/conf/ppo/task/allegro_inhand_grasp/mujoco.yaml +++ b/conf/ppo/task/allegro_inhand_grasp/mujoco.yaml @@ -1,36 +1,98 @@ # @package _global_ defaults: - - /task/allegro_inhand/mujoco + - /task/allegro_inhand/base - _self_ training: task_name: AllegroInhandRotationGrasp sim_backend: mujoco no_play: true + render_spacing: 0.5 + cam_distance: 1.5 + cam_lookat: [0.75, 0.75, 0] + cam_elevation: -20.0 + algo: - max_iterations: 1000 # infinite rollout -reward: - scales: - rotate: 0.0 - obj_linvel: 0.0 - pose_diff: 0.0 - torque: 0.0 - work: 0.0 - drop: 0.0 + num_envs: 16384 + num_steps_per_env: 8 + max_iterations: 1000 # infinite rollout until the recorder raises RunComplete + obs_groups: + actor: [policy] + critic: [policy] + actor: + class_name: rsl_rl.models.MLPModel + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + distribution_cfg: + class_name: rsl_rl.modules.distribution.GaussianDistribution + init_std: 1.0 + std_type: scalar + critic: + class_name: rsl_rl.models.MLPModel + hidden_dims: [512, 256, 128] + activation: elu + obs_normalization: true + algorithm: + value_loss_coef: 4.0 + desired_kl: 0.02 + env: - gen_grasp: true max_episode_seconds: 3.0 - grasp_cache_path: caches/allegro_grasp_50k.npy - grasp_collection_target: 50000 - grasp_auto_save: true - grasp_quality_check: true - grasp_min_contacts: 2 - domain_rand: - randomize_base_mass: false - random_com: false - push_robots: false - ball_vel_noise: 0.0 - joint_noise: 0.25 # random sampling of the grasp poses + actions: + # The collector holds each sampled reset pose; policy output is intentionally ignored. + hand: + action_scale: 0.0 + events: + reset_hand_ball: + params: + grasp_cache_path: null + joint_noise: 0.25 + ball_velocity_noise: 0.0 + ball_z_offset: 0.0 + terminations: + invalid_grasp: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityTermination + params: + entity_name: robot + observation_group: policy + observation_term: rotation + fingertip_body_names: [ff_tip, mf_tip, rf_tip, th_tip] + contact_sensor_names: [ff_contact, mf_contact, rf_contact, th_contact] + max_fingertip_distance: 0.1 + minimum_contacts: 2 + minimum_ball_height: 0.125 + enabled: true + metrics: + fingertips_close: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: fingertips_close} + enough_contacts: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: enough_contacts} + ball_held: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: ball_held} + valid: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspQualityMetric + params: {quality_term_name: invalid_grasp, condition: valid} + recorders: + grasp_cache: + func: unilab.tasks.manipulation.allegro_inhand.grasp_gen.AllegroGraspRecorder + params: + quality_term_name: invalid_grasp + output_path: caches/allegro_grasp_50k.npy + collection_target: 50000 + auto_save: true + +reward: + rotate: {weight: 0.0} + obj_linvel: {weight: 0.0} + pose_diff: {weight: 0.0} + torque: {weight: 0.0} + work: {weight: 0.0} + drop: {weight: 0.0} + play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_23dof_box_tracking/motrix.yaml b/conf/ppo/task/g1_23dof_box_tracking/motrix.yaml index e453b0721..c9456f1e4 100644 --- a/conf/ppo/task/g1_23dof_box_tracking/motrix.yaml +++ b/conf/ppo/task/g1_23dof_box_tracking/motrix.yaml @@ -1,27 +1,37 @@ # @package _global_ +defaults: + - /task/g1_23dof_box_tracking/mujoco + - _self_ + training: task_name: G1BoxTracking23Dof sim_backend: motrix play_env_num: 16 - play_steps: 1000 + algo: - num_envs: 1024 max_iterations: 40000 - save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.002 desired_kl: 0.01 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + +reward: + motion_global_root_pos: + weight: 1.0 + motion_body_ori: + weight: 1.5 + motion_body_ang_vel: + weight: 1.5 + object_global_ref_position_error_exp: + weight: 4.0 + params: {command_name: motion, std: 0.12} + object_global_ref_orientation_error_exp: + weight: 3.0 + params: {command_name: motion, std: 0.2} + play_profile: enabled: true env: @@ -33,28 +43,3 @@ play_profile: skybox_rgb1: [0.90, 0.90, 0.91] skybox_rgb2: [0.68, 0.68, 0.70] ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.5 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - undesired_contacts: -0.1 - object_global_ref_position_error_exp: 4.0 - object_global_ref_orientation_error_exp: 3.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 - std_object_pos: 0.12 - std_object_ori: 0.2 diff --git a/conf/ppo/task/g1_23dof_box_tracking/mujoco.yaml b/conf/ppo/task/g1_23dof_box_tracking/mujoco.yaml index 4c7b186f3..d835ffe06 100644 --- a/conf/ppo/task/g1_23dof_box_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_box_tracking/mujoco.yaml @@ -1,44 +1,84 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1BoxTracking23Dof sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 30000 save_interval: 500 obs_groups: - actor: - - actor + actor: [actor] algorithm: entropy_coef: 0.005 + +play_profile: + enabled: false + env: null + env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof_with_largebox.xml + entities: + object: + root_body_name: largebox sim_dt: 0.005 - sensor: - gyro: pelvis_gyro - upvector: pelvis_upvector + observations: + actor: + terms: + motion_anchor_pos_b: null + base_lin_vel: null + base_ang_vel: + params: {sensor_name: pelvis_gyro} + critic: + terms: + base_ang_vel: + params: {sensor_name: pelvis_gyro} + object_state: + func: unilab.tasks.motion_tracking.g1.manager_terms.object_state_b + params: {command_name: motion} + commands: + motion: + _target_: unilab.tasks.motion_tracking.g1.manager_terms.BoxMotionCommandCfg + object_entity_name: object + params: + motion_file: motions/g1/sub3_largebox_003_boxconverted_23dof.npz + terminations: + object_pos: + func: unilab.tasks.motion_tracking.g1.manager_terms.bad_object_position + params: {command_name: motion, threshold: 0.25} + object_ori: + func: unilab.tasks.motion_tracking.g1.manager_terms.bad_object_orientation + params: {command_name: motion, threshold: 0.8} + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - undesired_contacts: -0.1 - object_global_ref_position_error_exp: 2.0 - object_global_ref_orientation_error_exp: 2.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 - std_object_pos: 0.2 - std_object_ori: 0.3 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + object_global_ref_position_error_exp: + func: unilab.tasks.motion_tracking.g1.manager_terms.object_global_position_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.2} + object_global_ref_orientation_error_exp: + func: unilab.tasks.motion_tracking.g1.manager_terms.object_global_orientation_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3} diff --git a/conf/ppo/task/g1_23dof_climb_tracking/motrix.yaml b/conf/ppo/task/g1_23dof_climb_tracking/motrix.yaml index f554a13d7..ae708bf1d 100644 --- a/conf/ppo/task/g1_23dof_climb_tracking/motrix.yaml +++ b/conf/ppo/task/g1_23dof_climb_tracking/motrix.yaml @@ -1,74 +1,8 @@ # @package _global_ +defaults: + - /task/g1_23dof_climb_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking23Dof sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 20000 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 -env: - sampling_mode: adaptive - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_23dof_climb_tracking/mujoco.yaml b/conf/ppo/task/g1_23dof_climb_tracking/mujoco.yaml index 34fc4ae59..99706fa98 100644 --- a/conf/ppo/task/g1_23dof_climb_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_climb_tracking/mujoco.yaml @@ -1,74 +1,93 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking23Dof sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 20000 save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.005 desired_kl: 0.01 + +play_profile: + enabled: false + env: null + env: - sampling_mode: adaptive - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/g1/scene_climb_20_z_scale_1_23dof.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + max_episode_seconds: 15.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + commands: + motion: + params: + motion_file: motions/g1/climb_20_z_scale_1.0_23dof.npz + sampling_mode: adaptive + truncate_on_clip_end: false + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.3} + ee_body_pos: + params: + command_name: motion + threshold: 0.3 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/ppo/task/g1_23dof_flip_tracking/motrix.yaml b/conf/ppo/task/g1_23dof_flip_tracking/motrix.yaml index fff96e018..fe5afd7e6 100644 --- a/conf/ppo/task/g1_23dof_flip_tracking/motrix.yaml +++ b/conf/ppo/task/g1_23dof_flip_tracking/motrix.yaml @@ -1,34 +1,29 @@ # @package _global_ +defaults: + - /task/g1_23dof_flip_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking23Dof sim_backend: motrix - play_steps: 1000 + algo: - num_envs: 1024 max_iterations: 30000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 + empirical_normalization: false + +env: + actions: + joint_pos: + scale: 0.25 + reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.05 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_global_root_pos: + weight: 1.0 + motion_body_pos: + weight: 1.0 + motion_body_ori: + weight: 1.0 + motion_ee_body_pos_z: null + action_rate_l2: + weight: -0.05 + undesired_contacts: null diff --git a/conf/ppo/task/g1_23dof_flip_tracking/mujoco.yaml b/conf/ppo/task/g1_23dof_flip_tracking/mujoco.yaml index 8e62a80ff..5cec48e23 100644 --- a/conf/ppo/task/g1_23dof_flip_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_flip_tracking/mujoco.yaml @@ -1,74 +1,104 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking23Dof sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 20000 save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.005 desired_kl: 0.01 + +play_profile: + enabled: false + env: null + env: - sampling_mode: start - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + commands: + motion: + params: + motion_file: motions/g1/flip_360_001__A304_23dof.npz + sampling_mode: start + truncate_on_clip_end: false + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/ppo/task/g1_23dof_motion_tracking/motrix.yaml b/conf/ppo/task/g1_23dof_motion_tracking/motrix.yaml index 6bcc0a74f..2141c58cf 100644 --- a/conf/ppo/task/g1_23dof_motion_tracking/motrix.yaml +++ b/conf/ppo/task/g1_23dof_motion_tracking/motrix.yaml @@ -1,52 +1,37 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTracking23Dof sim_backend: motrix play_env_num: 16 - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + +reward: + motion_global_root_pos: + weight: 1.0 + action_rate_l2: + weight: -0.05 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + play_profile: enabled: true env: render_spacing: 2.5 - scene: - enabled: true - source_model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml - ground_texture_file: src/unilab/assets/robots/g1/textures/floor.png - skybox_rgb1: [0.90, 0.90, 0.91] - skybox_rgb2: [0.68, 0.68, 0.70] - ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.05 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 \ No newline at end of file diff --git a/conf/ppo/task/g1_23dof_motion_tracking/mujoco.yaml b/conf/ppo/task/g1_23dof_motion_tracking/mujoco.yaml index 3a411cd18..8605feb92 100644 --- a/conf/ppo/task/g1_23dof_motion_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_motion_tracking/mujoco.yaml @@ -1,34 +1,67 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTracking23Dof sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 \ No newline at end of file + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: &g1_23dof_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: *g1_23dof_joints + body_names: &tracked_bodies_23dof + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_roll_rubber_hand + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_roll_rubber_hand + commands: + motion: + params: + motion_file: motions/g1/dance1_subject2_part_23dof.npz + body_names: *tracked_bodies_23dof + terminations: + ee_body_pos: + params: + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand diff --git a/conf/ppo/task/g1_23dof_motion_tracking_deploy/motrix.yaml b/conf/ppo/task/g1_23dof_motion_tracking_deploy/motrix.yaml index a0359647e..46d82769f 100644 --- a/conf/ppo/task/g1_23dof_motion_tracking_deploy/motrix.yaml +++ b/conf/ppo/task/g1_23dof_motion_tracking_deploy/motrix.yaml @@ -1,101 +1,42 @@ # @package _global_ +defaults: + - /task/g1_23dof_motion_tracking_deploy/mujoco + - _self_ + training: task_name: G1MotionTracking23DofDeploy sim_backend: motrix play_env_num: 16 - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + env: - sim_dt: 0.005 - sensor: - local_linvel: pelvis_local_linvel - gyro: pelvis_gyro - upvector: pelvis_upvector - domain_rand: - random_com: true - com_offset_x: [-0.025, 0.025] - com_offset_y: [-0.05, 0.05] - com_offset_z: [-0.05, 0.05] - randomize_base_mass: true - added_mass_range: [-1.5, 1.5] - push_robots: true - push_interval: 750 - max_force: [1.0, 1.0, 0.5] - randomize_joint_default_pos: true - joint_default_pos_range: [-0.01, 0.01] - noise_config: - scale_joint_angle: 0.01 - scale_joint_vel: 0.5 - scale_gyro: 0.2 - scale_linvel: 0.5 - scale_gravity: 0.05 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 + events: + foot_friction: null + push_robot: null + +reward: + motion_global_root_pos: + weight: 1.0 + action_rate_l2: + weight: -0.05 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + play_profile: enabled: true env: render_spacing: 2.5 - scene: - enabled: true - source_model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml - ground_texture_file: src/unilab/assets/robots/g1/textures/floor.png - skybox_rgb1: [0.90, 0.90, 0.91] - skybox_rgb2: [0.68, 0.68, 0.70] - ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.05 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_23dof_motion_tracking_deploy/mujoco.yaml b/conf/ppo/task/g1_23dof_motion_tracking_deploy/mujoco.yaml index 4c725f29d..0e4667f90 100644 --- a/conf/ppo/task/g1_23dof_motion_tracking_deploy/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_motion_tracking_deploy/mujoco.yaml @@ -1,85 +1,77 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking_deploy/mujoco + - _self_ + training: task_name: G1MotionTracking23DofDeploy sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 + +_g1_23dof_deploy_action_scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + env: - sim_dt: 0.005 - sensor: - local_linvel: pelvis_local_linvel - gyro: pelvis_gyro - upvector: pelvis_upvector - domain_rand: - random_com: true - com_offset_x: [-0.025, 0.025] - com_offset_y: [-0.05, 0.05] - com_offset_z: [-0.05, 0.05] - randomize_base_mass: true - added_mass_range: [-1.5, 1.5] - push_robots: true - push_interval: 750 - max_force: [1.0, 1.0, 0.5] - randomize_geom_friction: true - friction_range: [0.3, 1.2] - randomize_joint_default_pos: true - joint_default_pos_range: [-0.01, 0.01] - noise_config: - scale_joint_angle: 0.01 - scale_joint_vel: 0.5 - scale_gyro: 0.2 - scale_linvel: 0.5 - scale_gravity: 0.05 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: &g1_23dof_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: *g1_23dof_joints + body_names: &tracked_bodies_23dof + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_roll_rubber_hand + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_roll_rubber_hand + actions: + joint_pos: + scale: ${_g1_23dof_deploy_action_scale} + commands: + motion: + params: + motion_file: motions/g1/dance1_subject2_part_23dof.npz + body_names: *tracked_bodies_23dof + terminations: + ee_body_pos: + params: + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand diff --git a/conf/ppo/task/g1_23dof_walk_flat/base.yaml b/conf/ppo/task/g1_23dof_walk_flat/base.yaml new file mode 100644 index 000000000..7489083e8 --- /dev/null +++ b/conf/ppo/task/g1_23dof_walk_flat/base.yaml @@ -0,0 +1,66 @@ +# @package _global_ +# Canonical G1 23-DoF walk Manager-Based task declaration (PPO/APPO owners). +# Inherits the 29-DoF flat contract and swaps the scene to the 23-DoF model +# (no waist roll/pitch, no wrist pitch/yaw) with the 23-entry pose weights. +defaults: + - /task/g1_walk_flat/base + - _self_ + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + +reward: + pose: + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/ppo/task/g1_23dof_walk_flat/motrix.yaml b/conf/ppo/task/g1_23dof_walk_flat/motrix.yaml index ab1c8a94f..d8fd852d0 100644 --- a/conf/ppo/task/g1_23dof_walk_flat/motrix.yaml +++ b/conf/ppo/task/g1_23dof_walk_flat/motrix.yaml @@ -1,4 +1,11 @@ # @package _global_ +# Motrix owner: inherits the shared 23-DoF flat Manager-Based contract from +# base.yaml, then overrides contract fields for Motrix-specific tuning +# (intentionally non-transferable from MuJoCo; drop overrides to restore parity). +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + training: task_name: G1Walk23DofFlat sim_backend: motrix @@ -17,48 +24,95 @@ algo: learning_rate: 3.0e-4 entropy_coef: 5.0e-3 env: - domain_rand: - randomize_kp: false - randomize_kd: false - control_config: - action_scale: 0.5 + actions: + joint_pos: + scale: 0.5 commands: - vel_limit: - - [0.4, 0.0, 0.0] - - [0.7, 0.0, 0.0] - gait_phase_init_mode: offset_phase - reset_base_qvel_limit: 0.05 - curriculum: - enabled: false - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + twist: + ranges: + lin_vel_x: [0.4, 0.7] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null + reset_root_state_uniform: + params: + velocity_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.05, 0.05] + roll: [-0.05, 0.05] + pitch: [-0.05, 0.05] + yaw: [-0.05, 0.05] + terminations: + tilt: + params: + max_tilt_deg: 35.0 + base_height: + params: + minimum_height: 0.5 reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.25 - forward_progress: 0.0 - under_speed: -0.2 - upper_body_pose: -0.05 - penalty_feet_ori: 0.0 - feet_phase: 1.2 - feet_phase_contrast: 1.5 - feet_phase_contact: 1.0 - feet_double_stance: -1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.2 - base_height: -120.0 - orientation: -2.5 - action_rate: -0.005 - pose: -0.05 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.765 - min_forward_speed_for_gait_reward: 0.05 - min_base_height: 0.5 - max_tilt_deg: 35.0 + tracking_ang_vel: + weight: 0.25 + forward_progress: + func: unilab.tasks.locomotion.g1.manager_terms.forward_progress + weight: 0.0 + params: + command_name: twist + under_speed: + func: unilab.tasks.locomotion.g1.manager_terms.under_speed + weight: -0.2 + params: + command_name: twist + upper_body_pose: + func: unilab.tasks.locomotion.g1.manager_terms.upper_body_pose + weight: -0.05 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori + weight: 0.0 + feet_phase: + weight: 1.2 + params: + min_forward_speed: 0.05 + feet_phase_contrast: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase_contrast + weight: 1.5 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.05 + command_name: twist + feet_phase_contact: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase_contact + weight: 1.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.05 + command_name: twist + feet_double_stance: + func: unilab.tasks.locomotion.g1.manager_terms.feet_double_stance + weight: -1.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.05 + command_name: twist + ang_vel_xy: + weight: -0.2 + base_height: + weight: -120.0 + params: + target_height: 0.765 + orientation: + weight: -2.5 + action_rate: + weight: -0.005 + pose: + weight: -0.05 diff --git a/conf/ppo/task/g1_23dof_walk_flat/mujoco.yaml b/conf/ppo/task/g1_23dof_walk_flat/mujoco.yaml index 0397b885d..e8e5776ed 100644 --- a/conf/ppo/task/g1_23dof_walk_flat/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_walk_flat/mujoco.yaml @@ -1,4 +1,10 @@ # @package _global_ +# MuJoCo owner: inherits the shared 23-DoF flat Manager-Based contract from +# base.yaml and only carries backend/algo identity. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + training: task_name: G1Walk23DofFlat sim_backend: mujoco @@ -8,32 +14,3 @@ algo: obs_groups: actor: - actor -env: - control_config: - action_scale: 0.25 - curriculum: - enabled: false - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.2 - feet_phase: 1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.25 - base_height: -500.0 - orientation: -5.0 - action_rate: -0.01 - pose: -0.1 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.55 - max_tilt_deg: 25.0 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/ppo/task/g1_23dof_walk_rough/mujoco.yaml b/conf/ppo/task/g1_23dof_walk_rough/mujoco.yaml index ea82c6792..cd7700207 100644 --- a/conf/ppo/task/g1_23dof_walk_rough/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_walk_rough/mujoco.yaml @@ -1,4 +1,11 @@ # @package _global_ +# MuJoCo 23-DoF rough owner: inherits the 23-DoF flat Manager-Based contract, +# swaps the scene to the static-hfield rough XML, and enables the penalty +# curriculum (the only PPO walk owner that carries one). +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + training: task_name: G1Walk23DofRough sim_backend: mujoco @@ -9,37 +16,15 @@ algo: actor: - actor env: - control_config: - action_scale: 0.25 + scene: + model_file: src/unilab/assets/robots/g1/scene_rough_23dof.xml curriculum: - enabled: true - initial_scale: 0.5 - min_scale: 0.5 - max_scale: 1.0 - level_down_threshold: 150.0 - level_up_threshold: 750.0 - degree: 0.001 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.2 - feet_phase: 1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.25 - base_height: -500.0 - orientation: -5.0 - action_rate: -0.01 - pose: -0.1 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.55 - max_tilt_deg: 25.0 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_scaling: + func: unilab.tasks.locomotion.g1.manager_terms.G1PenaltyCurriculum + params: + initial_scale: 0.5 + min_scale: 0.5 + max_scale: 1.0 + level_down_threshold: 150.0 + level_up_threshold: 750.0 + degree: 0.001 diff --git a/conf/ppo/task/g1_23dof_wall_flip_tracking/motrix.yaml b/conf/ppo/task/g1_23dof_wall_flip_tracking/motrix.yaml index 04c69a001..ca2f0e7be 100644 --- a/conf/ppo/task/g1_23dof_wall_flip_tracking/motrix.yaml +++ b/conf/ppo/task/g1_23dof_wall_flip_tracking/motrix.yaml @@ -1,58 +1,20 @@ # @package _global_ +defaults: + - /task/g1_23dof_wall_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking23Dof sim_backend: motrix play_env_num: 16 - play_steps: 1000 render_spacing: 3.0 + algo: - num_envs: 1024 max_iterations: 12000 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 + env: motrix_max_iterations: 3 - sampling_mode: start - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + play_profile: enabled: true env: @@ -64,25 +26,3 @@ play_profile: skybox_rgb1: [0.90, 0.90, 0.91] skybox_rgb2: [0.68, 0.68, 0.70] ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_23dof_wall_flip_tracking/mujoco.yaml b/conf/ppo/task/g1_23dof_wall_flip_tracking/mujoco.yaml index 7a49d091d..3f2c7ec81 100644 --- a/conf/ppo/task/g1_23dof_wall_flip_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_23dof_wall_flip_tracking/mujoco.yaml @@ -1,74 +1,26 @@ # @package _global_ +defaults: + - /task/g1_23dof_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking23Dof sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 20000 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 + +play_profile: + enabled: false + env: null + env: - sampling_mode: start - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof_with_wall.xml + commands: + motion: + params: + motion_file: motions/g1/flip_from_wall_104__A304_23dof.npz + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 diff --git a/conf/ppo/task/g1_box_tracking/motrix.yaml b/conf/ppo/task/g1_box_tracking/motrix.yaml index 863ac1215..ba9ad2abc 100644 --- a/conf/ppo/task/g1_box_tracking/motrix.yaml +++ b/conf/ppo/task/g1_box_tracking/motrix.yaml @@ -1,27 +1,37 @@ # @package _global_ +defaults: + - /task/g1_box_tracking/mujoco + - _self_ + training: task_name: G1BoxTracking sim_backend: motrix play_env_num: 16 - play_steps: 1000 + algo: - num_envs: 1024 max_iterations: 40000 - save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.002 desired_kl: 0.01 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + +reward: + motion_global_root_pos: + weight: 1.0 + motion_body_ori: + weight: 1.5 + motion_body_ang_vel: + weight: 1.5 + object_global_ref_position_error_exp: + weight: 4.0 + params: {command_name: motion, std: 0.12} + object_global_ref_orientation_error_exp: + weight: 3.0 + params: {command_name: motion, std: 0.2} + play_profile: enabled: true env: @@ -33,28 +43,3 @@ play_profile: skybox_rgb1: [0.90, 0.90, 0.91] skybox_rgb2: [0.68, 0.68, 0.70] ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.5 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - undesired_contacts: -0.1 - object_global_ref_position_error_exp: 4.0 - object_global_ref_orientation_error_exp: 3.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 - std_object_pos: 0.12 - std_object_ori: 0.2 diff --git a/conf/ppo/task/g1_box_tracking/mujoco.yaml b/conf/ppo/task/g1_box_tracking/mujoco.yaml index 278a8b8f3..73d278bf9 100644 --- a/conf/ppo/task/g1_box_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_box_tracking/mujoco.yaml @@ -1,47 +1,84 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1BoxTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 30000 save_interval: 500 obs_groups: - actor: - - actor + actor: [actor] algorithm: entropy_coef: 0.005 + env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_with_largebox.xml + entities: + object: + root_body_name: largebox sim_dt: 0.005 - sensor: - gyro: pelvis_gyro - upvector: pelvis_upvector + observations: + actor: + terms: + motion_anchor_pos_b: null + base_lin_vel: null + base_ang_vel: + params: {sensor_name: pelvis_gyro} + critic: + terms: + base_ang_vel: + params: {sensor_name: pelvis_gyro} + object_state: + func: unilab.tasks.motion_tracking.g1.manager_terms.object_state_b + params: {command_name: motion} + commands: + motion: + _target_: unilab.tasks.motion_tracking.g1.manager_terms.BoxMotionCommandCfg + object_entity_name: object + params: + motion_file: motions/g1/sub3_largebox_003_boxconverted.npz + terminations: + object_pos: + func: unilab.tasks.motion_tracking.g1.manager_terms.bad_object_position + params: {command_name: motion, threshold: 0.25} + object_ori: + func: unilab.tasks.motion_tracking.g1.manager_terms.bad_object_orientation + params: {command_name: motion, threshold: 0.8} + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - undesired_contacts: -0.1 - object_global_ref_position_error_exp: 2.0 - object_global_ref_orientation_error_exp: 2.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 - std_object_pos: 0.2 - std_object_ori: 0.3 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + object_global_ref_position_error_exp: + func: unilab.tasks.motion_tracking.g1.manager_terms.object_global_position_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.2} + object_global_ref_orientation_error_exp: + func: unilab.tasks.motion_tracking.g1.manager_terms.object_global_orientation_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3} + play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_climb_tracking/motrix.yaml b/conf/ppo/task/g1_climb_tracking/motrix.yaml index 0c30db38e..01a98604a 100644 --- a/conf/ppo/task/g1_climb_tracking/motrix.yaml +++ b/conf/ppo/task/g1_climb_tracking/motrix.yaml @@ -1,84 +1,8 @@ # @package _global_ +defaults: + - /task/g1_climb_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking sim_backend: motrix - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 20000 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 -env: - sampling_mode: adaptive - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 -play_profile: - enabled: true - env: - render_spacing: 2.0 diff --git a/conf/ppo/task/g1_climb_tracking/mujoco.yaml b/conf/ppo/task/g1_climb_tracking/mujoco.yaml index b7111e85a..fd7826eb4 100644 --- a/conf/ppo/task/g1_climb_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_climb_tracking/mujoco.yaml @@ -1,83 +1,95 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1ClimbTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 20000 save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.005 desired_kl: 0.01 + env: - sampling_mode: adaptive - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/g1/scene_climb_20_z_scale_1.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.3 - ee_body_pos_z_threshold: 0.3 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + max_episode_seconds: 15.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + "waist_(roll|pitch)_joint": 0.43857731392336724 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + ".*_wrist_(pitch|yaw)_joint": 0.07450087032950714 + commands: + motion: + params: + motion_file: motions/g1/climb_20_z_scale_1.0.npz + sampling_mode: adaptive + truncate_on_clip_end: false + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.3} + ee_body_pos: + params: + command_name: motion + threshold: 0.3 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} + play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_flip_tracking/motrix.yaml b/conf/ppo/task/g1_flip_tracking/motrix.yaml index 3435bd57a..5948e0166 100644 --- a/conf/ppo/task/g1_flip_tracking/motrix.yaml +++ b/conf/ppo/task/g1_flip_tracking/motrix.yaml @@ -1,38 +1,29 @@ # @package _global_ +defaults: + - /task/g1_flip_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking sim_backend: motrix - play_steps: 1000 + algo: - num_envs: 1024 max_iterations: 30000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 + empirical_normalization: false + +env: + actions: + joint_pos: + scale: 0.25 + reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.05 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 -play_profile: - enabled: true - env: - render_spacing: 2.0 + motion_global_root_pos: + weight: 1.0 + motion_body_pos: + weight: 1.0 + motion_body_ori: + weight: 1.0 + motion_ee_body_pos_z: null + action_rate_l2: + weight: -0.05 + undesired_contacts: null diff --git a/conf/ppo/task/g1_flip_tracking/mujoco.yaml b/conf/ppo/task/g1_flip_tracking/mujoco.yaml index 1edd403da..eea70d92f 100644 --- a/conf/ppo/task/g1_flip_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_flip_tracking/mujoco.yaml @@ -1,83 +1,106 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1FlipTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 20000 save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.005 desired_kl: 0.01 + env: - sampling_mode: start - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + "waist_(roll|pitch)_joint": 0.43857731392336724 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + ".*_wrist_(pitch|yaw)_joint": 0.07450087032950714 + commands: + motion: + params: + motion_file: motions/g1/flip_360_001__A304.npz + sampling_mode: start + truncate_on_clip_end: false + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} + play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_motion_tracking/motrix.yaml b/conf/ppo/task/g1_motion_tracking/motrix.yaml index b4e531856..57f4e35be 100644 --- a/conf/ppo/task/g1_motion_tracking/motrix.yaml +++ b/conf/ppo/task/g1_motion_tracking/motrix.yaml @@ -1,53 +1,37 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTracking sim_backend: motrix play_env_num: 16 - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -env: + +reward: + motion_global_root_pos: + weight: 1.0 + action_rate_l2: + weight: -0.05 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + play_profile: enabled: true env: render_spacing: 2.5 - scene: - enabled: true - source_model_file: src/unilab/assets/robots/g1/scene_flat.xml - ground_texture_file: src/unilab/assets/robots/g1/textures/floor.png - skybox_rgb1: [0.90, 0.90, 0.91] - skybox_rgb2: [0.68, 0.68, 0.70] - ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 1.0 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.05 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_motion_tracking/mujoco.yaml b/conf/ppo/task/g1_motion_tracking/mujoco.yaml index 54ffea9fd..343c454e0 100644 --- a/conf/ppo/task/g1_motion_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_motion_tracking/mujoco.yaml @@ -3,36 +3,237 @@ training: task_name: G1MotionTracking sim_backend: mujoco play_steps: 1000 + algo: num_envs: 1024 max_iterations: 15000 save_interval: 500 obs_groups: - actor: - - actor + actor: [actor] algorithm: entropy_coef: 0.005 + env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: &g1_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: *g1_joints + body_names: &tracked_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_yaw_link + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_yaw_link + geom_names: + - left_foot1_collision + - left_foot2_collision + - left_foot3_collision + - left_foot4_collision + - left_foot5_collision + - left_foot6_collision + - left_foot7_collision + - right_foot1_collision + - right_foot2_collision + - right_foot3_collision + - right_foot4_collision + - right_foot5_collision + - right_foot6_collision + - right_foot7_collision + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 10.0 + observations: + actor: + terms: + command: &command_obs + func: unilab.envs.mdp.generated_commands + params: {command_name: motion} + motion_anchor_pos_b: &anchor_pos_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_pos_b + params: {command_name: motion} + motion_anchor_ori_b: &anchor_ori_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_ori_b + params: {command_name: motion} + base_lin_vel: &base_lin_vel_obs + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + base_ang_vel: &base_ang_vel_obs + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + joint_pos: &joint_pos_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_pos_rel + params: {command_name: motion} + joint_vel: &joint_vel_obs + func: unilab.envs.mdp.joint_vel_rel + actions: &actions_obs + func: unilab.envs.mdp.last_action + critic: + terms: + command: *command_obs + motion_anchor_pos_b: *anchor_pos_obs + motion_anchor_ori_b: *anchor_ori_obs + base_lin_vel: *base_lin_vel_obs + base_ang_vel: *base_ang_vel_obs + joint_pos: *joint_pos_obs + joint_vel: *joint_vel_obs + actions: *actions_obs + body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.robot_body_pos_b + params: {command_name: motion} + body_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.robot_body_ori_b + params: {command_name: motion} + actions: + joint_pos: + _target_: unilab.tasks.motion_tracking.common.manager_terms.MotionJointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + command_name: motion + commands: + motion: + _target_: unilab.tasks.motion_tracking.common.manager_terms.MotionCommandCfg + entity_name: robot + resampling_time_range: [1.0e9, 1.0e9] + params: + motion_file: motions/g1/dance1_subject2_part.npz + anchor_body_name: torso_link + body_names: *tracked_bodies + sampling_mode: adaptive + sampling_start_ratio: 0.0 + truncate_on_clip_end: false + pose_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.01, 0.01] + roll: [-0.1, 0.1] + pitch: [-0.1, 0.1] + yaw: [-0.2, 0.2] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.2, 0.2] + roll: [-0.52, 0.52] + pitch: [-0.52, 0.52] + yaw: [-0.78, 0.78] + joint_position_range: [-0.1, 0.1] + joint_default_position_range: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + anchor_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_anchor_pos_z_only + params: {command_name: motion, threshold: 0.25} + anchor_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_anchor_ori + params: + command_name: motion + threshold: 0.8 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_motion_body_pos_z_only + params: + command_name: motion + threshold: 0.25 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + policy_observation_group: actor + critic_observation_group: critic + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_global_root_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_anchor_position_error_exp + weight: 0.5 + params: {command_name: motion, std: 0.3} + motion_global_root_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_anchor_orientation_error_exp + weight: 0.5 + params: {command_name: motion, std: 0.4} + motion_body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_error_exp + weight: 1.0 + params: {command_name: motion, std: 0.3} + motion_body_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_orientation_error_exp + weight: 1.0 + params: {command_name: motion, std: 0.4} + motion_body_lin_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_body_linear_velocity_error_exp + weight: 1.0 + params: {command_name: motion, std: 1.0} + motion_body_ang_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_body_angular_velocity_error_exp + weight: 1.0 + params: {command_name: motion, std: 3.14} + motion_joint_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_position_error_exp + weight: 0.0 + params: {command_name: motion, std: 0.2} + motion_joint_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_velocity_error_exp + weight: 0.0 + params: {command_name: motion, std: 1.0} + action_rate_l2: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.1 + joint_limit: + func: unilab.tasks.motion_tracking.common.manager_terms.joint_pos_limits + weight: -10.0 + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: ".*" + play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_motion_tracking_deploy/motrix.yaml b/conf/ppo/task/g1_motion_tracking_deploy/motrix.yaml index 6ebc34b25..9d2fe0e6e 100644 --- a/conf/ppo/task/g1_motion_tracking_deploy/motrix.yaml +++ b/conf/ppo/task/g1_motion_tracking_deploy/motrix.yaml @@ -1,107 +1,42 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking_deploy/mujoco + - _self_ + training: task_name: G1MotionTrackingDeploy sim_backend: motrix play_env_num: 16 - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + +env: + events: + foot_friction: null + push_robot: null + +reward: + motion_global_root_pos: + weight: 1.0 + action_rate_l2: + weight: -0.05 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + play_profile: enabled: true env: render_spacing: 2.5 - scene: - enabled: true - source_model_file: src/unilab/assets/robots/g1/scene_flat.xml - ground_texture_file: src/unilab/assets/robots/g1/textures/floor.png - skybox_rgb1: [0.90, 0.90, 0.91] - skybox_rgb2: [0.68, 0.68, 0.70] - ground_texrepeat: [0.25, 0.25] -env: - sim_dt: 0.005 - sensor: - local_linvel: pelvis_local_linvel - gyro: pelvis_gyro - upvector: pelvis_upvector - domain_rand: - random_com: true - com_offset_x: [-0.025, 0.025] - com_offset_y: [-0.05, 0.05] - com_offset_z: [-0.05, 0.05] - randomize_base_mass: true - added_mass_range: [-1.5, 1.5] - push_robots: true - push_interval: 750 - max_force: [1.0, 1.0, 0.5] - randomize_joint_default_pos: true - joint_default_pos_range: [-0.01, 0.01] - # randomize_geom_friction omitted: Motrix does not reliably support it - noise_config: - scale_joint_angle: 0.01 - scale_joint_vel: 0.5 - scale_gyro: 0.2 - scale_linvel: 0.5 - scale_gravity: 0.05 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_motion_tracking_deploy/mujoco.yaml b/conf/ppo/task/g1_motion_tracking_deploy/mujoco.yaml index 39a10a9ca..6b8d5ef89 100644 --- a/conf/ppo/task/g1_motion_tracking_deploy/mujoco.yaml +++ b/conf/ppo/task/g1_motion_tracking_deploy/mujoco.yaml @@ -1,91 +1,84 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: G1MotionTrackingDeploy sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 15000 - save_interval: 500 - obs_groups: - actor: - - actor - algorithm: - entropy_coef: 0.005 + env: sim_dt: 0.005 - sensor: - local_linvel: pelvis_local_linvel - gyro: pelvis_gyro - upvector: pelvis_upvector - domain_rand: - random_com: true - com_offset_x: [-0.025, 0.025] - com_offset_y: [-0.05, 0.05] - com_offset_z: [-0.05, 0.05] - randomize_base_mass: true - added_mass_range: [-1.5, 1.5] - push_robots: true - push_interval: 750 - max_force: [1.0, 1.0, 0.5] - randomize_geom_friction: true - friction_range: [0.3, 1.2] - randomize_joint_default_pos: true - joint_default_pos_range: [-0.01, 0.01] - noise_config: - scale_joint_angle: 0.01 - scale_joint_vel: 0.5 - scale_gyro: 0.2 - scale_linvel: 0.5 - scale_gravity: 0.05 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 1.0 - motion_body_ori: 1.0 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_joint_pos: 0.0 - motion_joint_vel: 0.0 - action_rate_l2: -0.1 - joint_limit: -10.0 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + observations: + actor: + terms: + motion_anchor_pos_b: null + base_lin_vel: null + base_ang_vel: + params: {sensor_name: pelvis_gyro} + critic: + terms: + base_ang_vel: + params: {sensor_name: pelvis_gyro} + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + "waist_(roll|pitch)_joint": 0.43857731392336724 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + ".*_wrist_(pitch|yaw)_joint": 0.07450087032950714 + commands: + motion: + params: + joint_default_position_range: [-0.01, 0.01] + events: + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + body_names: pelvis + mass_distribution_params: [-1.5, 1.5] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + body_names: pelvis + com_range: + x: [-0.025, 0.025] + y: [-0.05, 0.05] + z: [-0.05, 0.05] + foot_friction: + func: unilab.envs.mdp.geom_friction + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + geom_names: ".*" + ranges: [0.3, 1.2] + operation: abs + shared_random: true + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [15.0, 15.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] diff --git a/conf/ppo/task/g1_walk_flat/base.yaml b/conf/ppo/task/g1_walk_flat/base.yaml new file mode 100644 index 000000000..a77fbed0c --- /dev/null +++ b/conf/ppo/task/g1_walk_flat/base.yaml @@ -0,0 +1,262 @@ +# @package _global_ +# Canonical G1 29-DoF walk Manager-Based task declaration (PPO/APPO owners). +# Backend owner leaves inherit this file and only override backend/algo tuning +# or explicitly disabled terms. Observation scaling follows the legacy profile +# (unit scales); the walk profile lives in conf/sac/task/g1_walk_flat/base.yaml. +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + body_names: [pelvis] + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + # Observation noise matches the legacy noise_config (level=1.0, actor-only + # since the critic reads clean observations): gyro +/-0.2, gravity +/-0.05, + # joint pos +/-0.01, joint vel +/-1.5, applied before term scaling. + enable_corruption: true + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.2 + n_max: 0.2 + operation: add + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.05 + n_max: 0.05 + operation: add + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + operation: add + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -1.5 + n_max: 1.5 + operation: add + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + tilt: + func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded + params: + max_tilt_deg: 25.0 + base_height: + func: unilab.envs.mdp.root_height_below_minimum + params: + minimum_height: 0.55 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel + weight: 2.0 + params: + tracking_sigma: 0.25 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel + weight: 0.2 + params: + tracking_sigma: 0.25 + command_name: twist + feet_phase: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase + weight: 1.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.0 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.g1.manager_terms.lin_vel_z + weight: -1.0 + ang_vel_xy: + func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy + weight: -0.25 + base_height: + func: unilab.tasks.locomotion.g1.manager_terms.base_height + weight: -500.0 + params: + target_height: 0.754 + orientation: + func: unilab.tasks.locomotion.g1.manager_terms.orientation + weight: -5.0 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.01 + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.1 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/ppo/task/g1_walk_flat/mjwarp.yaml b/conf/ppo/task/g1_walk_flat/mjwarp.yaml index 2d8c3a9d7..647fc6d00 100644 --- a/conf/ppo/task/g1_walk_flat/mjwarp.yaml +++ b/conf/ppo/task/g1_walk_flat/mjwarp.yaml @@ -1,7 +1,12 @@ # @package _global_ -# Configured-only mjwarp owner for the unified host contract adapter. Offline -# record reuses MuJoCo rendering; native playback and device-resident runtime -# routing are intentionally absent. +# Configured-only mjwarp owner for the unified host contract adapter. Keeps +# DENYLIST parity with the MuJoCo owner; legacy kp/kd randomization is disabled. +# Offline record reuses MuJoCo rendering; native playback and device-resident +# runtime routing are intentionally absent. +defaults: + - /task/g1_walk_flat/base + - _self_ + training: task_name: G1WalkFlat sim_backend: mjwarp @@ -19,39 +24,9 @@ algo: env: mjwarp_nconmax: 128 mjwarp_njmax: 256 - domain_rand: - randomize_kp: false - randomize_kd: false - randomize_dof_armature: false - randomize_body_gravity_compensation: false - control_config: - action_scale: 0.25 - curriculum: - enabled: false - noise_config: - level: 0.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.2 - feet_phase: 1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.25 - base_height: -500.0 - orientation: -5.0 - action_rate: -0.01 - pose: -0.1 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.55 - max_tilt_deg: 25.0 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + events: + # Legacy mjwarp owners disable kp/kd and armature randomization. + pd_gains: null play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_walk_flat/motrix.yaml b/conf/ppo/task/g1_walk_flat/motrix.yaml index 88c971936..b93cc9269 100644 --- a/conf/ppo/task/g1_walk_flat/motrix.yaml +++ b/conf/ppo/task/g1_walk_flat/motrix.yaml @@ -1,7 +1,11 @@ # @package _global_ -# Standalone Motrix owner config: carries the shared contract inline, then -# overrides contract fields for Motrix-specific tuning +# Motrix owner: inherits the shared 29-DoF flat Manager-Based contract from +# base.yaml, then overrides contract fields for Motrix-specific tuning # (intentionally non-transferable from MuJoCo; drop overrides to restore parity). +defaults: + - /task/g1_walk_flat/base + - _self_ + training: task_name: G1WalkFlat sim_backend: motrix @@ -22,50 +26,98 @@ algo: learning_rate: 3.0e-4 entropy_coef: 5.0e-3 env: - domain_rand: - randomize_kp: false - randomize_kd: false - control_config: - action_scale: 0.5 + actions: + joint_pos: + scale: 0.5 commands: - vel_limit: - - [0.4, 0.0, 0.0] - - [0.7, 0.0, 0.0] - gait_phase_init_mode: offset_phase - reset_base_qvel_limit: 0.05 - curriculum: - enabled: false - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 + twist: + ranges: + lin_vel_x: [0.4, 0.7] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null + reset_root_state_uniform: + params: + velocity_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.05, 0.05] + roll: [-0.05, 0.05] + pitch: [-0.05, 0.05] + yaw: [-0.05, 0.05] + terminations: + tilt: + params: + max_tilt_deg: 35.0 + base_height: + params: + minimum_height: 0.5 reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.25 - forward_progress: 0.0 - under_speed: -0.2 - upper_body_pose: -0.05 - penalty_feet_ori: 0.0 - feet_phase: 1.2 - feet_phase_contrast: 1.5 - feet_phase_contact: 1.0 - feet_double_stance: -1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.2 - base_height: -120.0 - orientation: -2.5 - action_rate: -0.005 - pose: -0.05 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.765 - min_forward_speed_for_gait_reward: 0.05 - min_base_height: 0.5 - max_tilt_deg: 35.0 + tracking_ang_vel: + weight: 0.25 + forward_progress: + func: unilab.tasks.locomotion.g1.manager_terms.forward_progress + weight: 0.0 + params: + command_name: twist + under_speed: + func: unilab.tasks.locomotion.g1.manager_terms.under_speed + weight: -0.2 + params: + command_name: twist + upper_body_pose: + func: unilab.tasks.locomotion.g1.manager_terms.upper_body_pose + weight: -0.05 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori + weight: 0.0 + feet_phase: + weight: 1.2 + params: + min_forward_speed: 0.05 + feet_phase_contrast: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase_contrast + weight: 1.5 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.05 + command_name: twist + feet_phase_contact: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase_contact + weight: 1.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.05 + command_name: twist + feet_double_stance: + func: unilab.tasks.locomotion.g1.manager_terms.feet_double_stance + weight: -1.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.008 + min_forward_speed: 0.05 + command_name: twist + ang_vel_xy: + weight: -0.2 + base_height: + weight: -120.0 + params: + target_height: 0.765 + orientation: + weight: -2.5 + action_rate: + weight: -0.005 + pose: + weight: -0.05 play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_walk_flat/mujoco.yaml b/conf/ppo/task/g1_walk_flat/mujoco.yaml index 843d7b85f..a900f8d8d 100644 --- a/conf/ppo/task/g1_walk_flat/mujoco.yaml +++ b/conf/ppo/task/g1_walk_flat/mujoco.yaml @@ -1,6 +1,10 @@ # @package _global_ -# Standalone MuJoCo owner config: carries the shared cross-backend contract inline -# (formerly base.yaml), plus backend-specific tuning. +# MuJoCo owner: inherits the shared 29-DoF flat Manager-Based contract from +# base.yaml and only carries backend/algo identity. +defaults: + - /task/g1_walk_flat/base + - _self_ + training: task_name: G1WalkFlat sim_backend: mujoco @@ -14,35 +18,6 @@ algo: policy: actor_hidden_dims: [512, 256, 128] critic_hidden_dims: [512, 256, 128] -env: - control_config: - action_scale: 0.25 - curriculum: - enabled: false - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 -reward: - scales: - tracking_lin_vel: 2.0 - tracking_ang_vel: 0.2 - feet_phase: 1.0 - lin_vel_z: -1.0 - ang_vel_xy: -0.25 - base_height: -500.0 - orientation: -5.0 - action_rate: -0.01 - pose: -0.1 - tracking_sigma: 0.25 - gait_frequency: 1.5 - feet_phase_swing_height: 0.09 - feet_phase_tracking_sigma: 0.008 - base_height_target: 0.754 - min_base_height: 0.55 - max_tilt_deg: 25.0 - pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] play_profile: enabled: true env: diff --git a/conf/ppo/task/g1_wall_flip_tracking/motrix.yaml b/conf/ppo/task/g1_wall_flip_tracking/motrix.yaml index c595203a6..8fd67bfc7 100644 --- a/conf/ppo/task/g1_wall_flip_tracking/motrix.yaml +++ b/conf/ppo/task/g1_wall_flip_tracking/motrix.yaml @@ -1,64 +1,20 @@ # @package _global_ +defaults: + - /task/g1_wall_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking sim_backend: motrix play_env_num: 16 - play_steps: 1000 render_spacing: 3.0 + algo: - num_envs: 1024 max_iterations: 12000 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 + env: motrix_max_iterations: 3 - sampling_mode: start - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + play_profile: enabled: true env: @@ -70,25 +26,3 @@ play_profile: skybox_rgb1: [0.90, 0.90, 0.91] skybox_rgb2: [0.68, 0.68, 0.70] ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/g1_wall_flip_tracking/mujoco.yaml b/conf/ppo/task/g1_wall_flip_tracking/mujoco.yaml index e11d9b05b..082c23fd9 100644 --- a/conf/ppo/task/g1_wall_flip_tracking/mujoco.yaml +++ b/conf/ppo/task/g1_wall_flip_tracking/mujoco.yaml @@ -1,84 +1,22 @@ # @package _global_ +defaults: + - /task/g1_flip_tracking/mujoco + - _self_ + training: task_name: G1WallFlipTracking sim_backend: mujoco - play_steps: 1000 -algo: - num_envs: 1024 - max_iterations: 20000 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 + env: - sampling_mode: start - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.5475464629911068 - - 0.35066146637882434 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.5475464629911068 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.43857731392336724 - - 0.07450087032950714 - - 0.07450087032950714 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_with_wall.xml + commands: + motion: + params: + motion_file: motions/g1/flip_from_wall_104__A304.npz + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 -play_profile: - enabled: true - env: - render_spacing: 2.0 + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 diff --git a/conf/ppo/task/go1_joystick_flat/base.yaml b/conf/ppo/task/go1_joystick_flat/base.yaml new file mode 100644 index 000000000..8a2f3bf57 --- /dev/null +++ b/conf/ppo/task/go1_joystick_flat/base.yaml @@ -0,0 +1,244 @@ +# @package _global_ +# Canonical Go1 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go1/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: trunk + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + body_names: [trunk] + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: trunk + mass_distribution_params: [-1.5, 1.5] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: trunk + com_range: + x: [-0.05, 0.05] + y: [0.0, 0.0] + z: [0.0, 0.0] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [35.0, 35.0] + kd_range: [0.5, 0.5] + operation: abs + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [15.0, 15.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + # Legacy Go1 sums four matching feet while this community term returns their mean. + weight: 0.96 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/ppo/task/go1_joystick_flat/drake.yaml b/conf/ppo/task/go1_joystick_flat/drake.yaml index 3c9f63c80..f1251763b 100644 --- a/conf/ppo/task/go1_joystick_flat/drake.yaml +++ b/conf/ppo/task/go1_joystick_flat/drake.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go1_joystick_flat/base + - _self_ + training: task_name: Go1JoystickFlat sim_backend: drake @@ -23,26 +27,17 @@ algo: obs_groups: actor: - actor + critic: + - critic env: drake_backend_mode: batch drake_nthread: 0 - scene: - model_file: src/unilab/assets/robots/go1/scene_flat.xml - domain_rand: - randomize_base_mass: false - random_com: false - push_robots: false + events: + base_mass: null + base_com: null + pd_gains: null + push_robot: null reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + contact: null diff --git a/conf/ppo/task/go1_joystick_flat/motrix.yaml b/conf/ppo/task/go1_joystick_flat/motrix.yaml index d84ba1b03..e05a46d7b 100644 --- a/conf/ppo/task/go1_joystick_flat/motrix.yaml +++ b/conf/ppo/task/go1_joystick_flat/motrix.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go1_joystick_flat/base + - _self_ + training: task_name: Go1JoystickFlat sim_backend: motrix @@ -20,6 +24,8 @@ algo: obs_groups: actor: - actor + critic: + - critic empirical_normalization: true policy: init_noise_std: 0.5 @@ -28,21 +34,16 @@ algo: entropy_coef: 1.0e-3 env: commands: - vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] + twist: + ranges: + lin_vel_x: [0.5, 0.5] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + # Motrix has no formal root velocity-delta capability; do not fall back to force push. + push_robot: null reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + contact: null play_profile: enabled: true env: diff --git a/conf/ppo/task/go1_joystick_flat/mujoco.yaml b/conf/ppo/task/go1_joystick_flat/mujoco.yaml index 794266ad3..867b319b8 100644 --- a/conf/ppo/task/go1_joystick_flat/mujoco.yaml +++ b/conf/ppo/task/go1_joystick_flat/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go1_joystick_flat/base + - _self_ + training: task_name: Go1JoystickFlat sim_backend: mujoco @@ -21,19 +25,8 @@ algo: obs_groups: actor: - actor -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + critic: + - critic play_profile: enabled: true env: diff --git a/conf/ppo/task/go1_joystick_rough/motrix.yaml b/conf/ppo/task/go1_joystick_rough/motrix.yaml index 21be8340a..0af29a93b 100644 --- a/conf/ppo/task/go1_joystick_rough/motrix.yaml +++ b/conf/ppo/task/go1_joystick_rough/motrix.yaml @@ -1,121 +1,18 @@ # @package _global_ +defaults: + - /task/go1_joystick_rough/mujoco + - _self_ + training: task_name: Go1JoystickRough sim_backend: motrix - play_steps: 500 - play_env_num: 16 - cam_tracking: true - cam_tracking_env_idx: 0 - cam_tracking_extra_envs: 9 - -interactive: - action_mode: policy - policy_obs_mode: auto - camera_follow_body: true - use_env_visual_model: false algo: num_envs: 2048 - num_steps_per_env: 24 - max_iterations: 1000 - empirical_normalization: false - obs_groups: - actor: - - actor - critic: - - critic - policy: - init_noise_std: 1.0 - algorithm: - learning_rate: 1.0e-3 - entropy_coef: 1.0e-2 env: render_offset_mode: zero - control_config: - action_scale: 0.25 - hip_action_scale: 0.125 - non_hip_action_scale: 0.25 - clip_actions: 100.0 - commands: - vel_limit: [[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]] - resampling_time: 10.0 - heading_command: true - heading_range: [-3.141592653589793, 3.141592653589793] - rel_standing_envs: 0.1 - terrain_curriculum: - enabled: false scene: model_file: src/unilab/assets/robots/go1/go1.xml - fragment_files: - - src/unilab/assets/robots/go1/locomotion_task.xml - terrain: - hfield_name: terrain_hfield - geom_name: floor - generator: - seed: 42 - curriculum: false - size: [8.0, 8.0] - num_rows: 6 - num_cols: 6 - border_width: 20.0 - terrain_scan: - enabled: true - geom_name: floor - termination_config: - terrain_out_of_bounds: true - terrain_distance_buffer: 3.0 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 3.0] - random_com: true - randomize_kp: true - kp_multiplier_range: [0.5, 2.0] - randomize_kd: true - kd_multiplier_range: [0.5, 2.0] - push_robots: true - push_interval: 625 - max_force: [1.0, 1.0, 0.5] - -reward: - scales: - lin_vel_z: -2.0 - ang_vel_xy: -0.05 - joint_torques_l2: -2.5e-5 - joint_acc_l2: -2.5e-7 - joint_power: -2.0e-5 - stand_still: -2.0 - hip_pos: -0.5 - joint_pos_penalty: -1.0 - joint_mirror: -0.05 - action_rate: -0.01 - undesired_contacts: -1.0 - contact_forces: -1.5e-4 - tracking_lin_vel: 3.0 - tracking_ang_vel: 1.5 - feet_air_time: 0.5 - feet_air_time_variance: -1.0 - feet_contact_without_cmd: 0.1 - feet_slide: -0.1 - feet_height_body: -5.0 - feet_gait: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.33 - stand_still_command_threshold: 0.1 - joint_pos_penalty_stand_still_scale: 5.0 - joint_pos_penalty_velocity_threshold: 0.5 - joint_pos_penalty_command_threshold: 0.1 - contact_threshold: 1.0 - contact_forces_threshold: 100.0 - feet_air_time_threshold: 0.5 - feet_height_body_target: -0.2 - feet_height_body_tanh_mult: 2.0 - feet_gait_std: 0.7071067811865476 - feet_gait_max_err: 0.2 - feet_gait_velocity_threshold: 0.5 - feet_gait_command_threshold: 0.1 -play_profile: - enabled: true - env: - render_spacing: 2.0 + events: + push_robot: null diff --git a/conf/ppo/task/go1_joystick_rough/mujoco.yaml b/conf/ppo/task/go1_joystick_rough/mujoco.yaml index 04a9dc179..db1c3e920 100644 --- a/conf/ppo/task/go1_joystick_rough/mujoco.yaml +++ b/conf/ppo/task/go1_joystick_rough/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/quadruped_joystick_rough/quadruped + - _self_ + training: task_name: Go1JoystickRough sim_backend: mujoco @@ -20,103 +24,46 @@ algo: num_steps_per_env: 24 max_iterations: 1000 empirical_normalization: false - obs_groups: - actor: - - actor - critic: - - critic - policy: - init_noise_std: 1.0 - algorithm: - learning_rate: 1.0e-3 - entropy_coef: 1.0e-2 + obs_groups: {actor: [actor], critic: [critic]} + policy: {init_noise_std: 1.0} + algorithm: {learning_rate: 1.0e-3, entropy_coef: 1.0e-2} env: sim_dt: 0.005 - control_config: - action_scale: 0.25 - hip_action_scale: 0.125 - non_hip_action_scale: 0.25 - clip_actions: 100.0 - commands: - vel_limit: [[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]] - resampling_time: 10.0 - heading_command: true - heading_range: [-3.141592653589793, 3.141592653589793] - rel_standing_envs: 0.1 - terrain_curriculum: - enabled: false scene: model_file: src/unilab/assets/robots/go1/go1_mujoco.xml - fragment_files: - - src/unilab/assets/robots/go1/locomotion_task.xml - terrain: - hfield_name: terrain_hfield - geom_name: floor - generator: - seed: 42 - curriculum: false - size: [8.0, 8.0] - num_rows: 6 - num_cols: 6 - border_width: 20.0 - terrain_scan: - enabled: true - geom_name: floor - termination_config: - terrain_out_of_bounds: true - terrain_distance_buffer: 3.0 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 3.0] - random_com: true - randomize_kp: true - kp_multiplier_range: [0.5, 2.0] - randomize_kd: true - kd_multiplier_range: [0.5, 2.0] - push_robots: true - push_interval: 625 - max_force: [1.0, 1.0, 0.5] + fragment_files: [src/unilab/assets/robots/go1/locomotion_task.xml] + entities: + robot: + root_body_name: trunk + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + body_names: [trunk] -reward: - scales: - lin_vel_z: -2.0 - ang_vel_xy: -0.05 - joint_torques_l2: -2.5e-5 - joint_acc_l2: -2.5e-7 - joint_power: -2.0e-5 - stand_still: -2.0 - hip_pos: -0.5 - joint_pos_penalty: -1.0 - joint_mirror: -0.05 - action_rate: -0.01 - undesired_contacts: -1.0 - contact_forces: -1.5e-4 - tracking_lin_vel: 3.0 - tracking_ang_vel: 1.5 - feet_air_time: 0.5 - feet_air_time_variance: -1.0 - feet_contact_without_cmd: 0.1 - feet_slide: -0.1 - feet_height_body: -5.0 - feet_gait: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.33 - stand_still_command_threshold: 0.1 - joint_pos_penalty_stand_still_scale: 5.0 - joint_pos_penalty_velocity_threshold: 0.5 - joint_pos_penalty_command_threshold: 0.1 - contact_threshold: 1.0 - contact_forces_threshold: 100.0 - feet_air_time_threshold: 0.5 - feet_height_body_target: -0.2 - feet_height_body_tanh_mult: 2.0 - feet_gait_std: 0.7071067811865476 - feet_gait_max_err: 0.2 - feet_gait_velocity_threshold: 0.5 - feet_gait_command_threshold: 0.1 play_profile: enabled: true - env: - render_spacing: 2.0 + env: {render_spacing: 2.0} diff --git a/conf/ppo/task/go2_footstand/base.yaml b/conf/ppo/task/go2_footstand/base.yaml new file mode 100644 index 000000000..5478fc5b5 --- /dev/null +++ b/conf/ppo/task/go2_footstand/base.yaml @@ -0,0 +1,217 @@ +# @package _global_ +# Canonical Go2 footstand Manager-Based declaration. Hydra owns every task term; +# backend leaves only select identity, tuning, and supported reset capabilities. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + body_names: + - base + - FL_hip + - FL_thigh + - FL_calf + - FR_hip + - FR_thigh + - FR_calf + - RL_hip + - RL_thigh + - RL_calf + - RR_hip + - RR_thigh + - RR_calf + geom_names: [floor] + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.004 + ctrl_dt: 0.02 + max_episode_seconds: 10.0 + adaptive_chunk_size: false + observations: + policy: + enable_corruption: true + terms: + frame: + func: unilab.tasks.locomotion.go2.footstand.frame_observation + params: + action_name: joint_pos + noise: + _target_: UniformNoiseCfg + n_min: [-0.1, -0.1, -0.1, -0.2, -0.2, -0.2, -0.05, -0.05, -0.05, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + n_max: [0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + history_length: 15 + critic: + terms: + frame: + func: unilab.tasks.locomotion.go2.footstand.frame_observation + params: + action_name: joint_pos + history_length: 15 + privileged: + func: unilab.tasks.locomotion.go2.footstand.privileged_observation + params: + action_name: joint_pos + actions: + joint_pos: + _target_: unilab.tasks.locomotion.go2.footstand.FootstandIncrementalActionCfg + entity_name: robot + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + joint_position_limits: + - [-1.0472, 1.0472] + - [-1.5708, 3.4907] + - [-2.7227, -0.83776] + - [-1.0472, 1.0472] + - [-1.5708, 3.4907] + - [-2.7227, -0.83776] + - [-1.0472, 1.0472] + - [-0.5236, 4.5379] + - [-2.7227, -0.83776] + - [-1.0472, 1.0472] + - [-0.5236, 4.5379] + - [-2.7227, -0.83776] + action_scale: 0.3 + clip_actions: 1.0 + kp: 35.0 + kd: 0.5 + simulate_action_latency: false + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + reset_joints: + func: unilab.tasks.locomotion.go2.footstand.FootstandJointReset + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*" + position_offset_range: [-0.05, 0.05] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [35.0, 35.0] + kd_range: [0.5, 0.5] + operation: abs + floor_friction: null + link_mass: null + torso_com: null + joint_armature: null + terminations: + footstand: + func: unilab.tasks.locomotion.go2.footstand.FootstandTermination + params: + action_name: joint_pos + grace_steps: 100 + height_fraction: 0.8 + orientation_threshold: 0.2 + energy_threshold: 200.0 + time_out: + func: unilab.envs.mdp.time_out + time_out: true + policy_observation_group: policy + critic_observation_group: critic + scale_rewards_by_dt: true + +reward: + footstand: + func: unilab.tasks.locomotion.go2.footstand.FootstandReward + weight: 1.0 + params: + state_term_name: footstand + scales: + height: 2.0 + orientation: 2.0 + contact: -1.0 + action_rate: -0.01 + termination: -2.0 + dof_pos_limits: -0.5 + torques: 0.0 + pose: -0.1 + penalty_contact: -0.2 + tar: 0.8 + rear_feet_contact: 0.5 + rear_leg_symmetry: -0.2 + front_leg_motion: -0.05 + upright_stability: -0.2 + knee_clearance: -0.5 + stay_still: -0.1 + energy: -0.003 + dof_acc: -2.5e-7 + soft_joint_pos_limit_factor: 0.9 + knee_height_target: 0.08 + front_feet_min_separation: 0.16 + front_feet_side_margin: 0.04 + rear_hip_abduction_margin: 0.25 + rear_foot_slip_deadband: 0.02 + rear_foot_anchor_radius: 0.03 diff --git a/conf/ppo/task/go2_footstand/drake.yaml b/conf/ppo/task/go2_footstand/drake.yaml index b8c00f148..3d7c02a40 100644 --- a/conf/ppo/task/go2_footstand/drake.yaml +++ b/conf/ppo/task/go2_footstand/drake.yaml @@ -1,36 +1,26 @@ # @package _global_ +defaults: + - /task/go2_footstand/base + - _self_ + training: task_name: Go2FootStand sim_backend: drake env: - sim_dt: 0.004 drake_backend_mode: batch drake_nthread: 0 - add_body_sensors: true - obs_history_len: 15 - energy_termination_threshold: 200.0 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 - scale_gravity: 0.05 - scale_linvel: 0.1 - control_config: - action_scale: 0.3 - domain_rand: - randomize_floor_friction: false - randomize_link_mass: false - torso_added_mass_range: null - randomize_torso_com: false - randomize_dof_armature: false - randomize_reset_joint_qpos: true - reset_joint_qpos_range: [-0.05, 0.05] + events: + # Drake supports the joint-state reset, but not these reset payload fields. + pd_gains: null + floor_friction: null + link_mass: null + torso_com: null + joint_armature: null algo: empirical_normalization: true - num_envs: 1024 + num_envs: 4096 max_iterations: 10000 obs_groups: actor: @@ -41,27 +31,3 @@ algo: init_noise_std: 0.5 algorithm: entropy_coef: 0.005 - -reward: - scales: - height: 2.0 - orientation: 2.0 - contact: -1.0 - action_rate: -0.01 - termination: -2.0 - dof_pos_limits: -0.5 - torques: 0.0 - pose: -0.1 - penalty_contact: -0.2 - tar: 0.8 - rear_feet_contact: 0.5 - rear_leg_symmetry: -0.2 - front_leg_motion: -0.05 - upright_stability: -0.2 - knee_clearance: -0.5 - stay_still: -0.1 - energy: -0.003 - dof_acc: -2.5e-7 - tracking_sigma: 0.25 - base_height_target: 0.3 - knee_height_target: 0.08 diff --git a/conf/ppo/task/go2_footstand/motrix.yaml b/conf/ppo/task/go2_footstand/motrix.yaml index 98d5b3a4e..c53005fbd 100644 --- a/conf/ppo/task/go2_footstand/motrix.yaml +++ b/conf/ppo/task/go2_footstand/motrix.yaml @@ -1,40 +1,27 @@ # @package _global_ +defaults: + - /task/go2_footstand/base + - _self_ + training: task_name: Go2FootStand sim_backend: motrix no_play: true + env: - sim_dt: 0.004 - add_body_sensors: true - obs_history_len: 15 - soft_joint_pos_limit_factor: 0.9 - energy_termination_threshold: 200.0 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 - scale_gravity: 0.05 - scale_linvel: 0.1 - control_config: - action_scale: 0.3 - clip_actions: 1.0 - Kd: 0.5 - domain_rand: - randomize_floor_friction: false - floor_friction_range: [0.6, 1.0] - # Motrix does not implement dof_armature randomization; disabled. - randomize_dof_armature: false - randomize_link_mass: false - link_mass_scale_range: [0.95, 1.05] - torso_added_mass_range: [0.0, 0.0] - randomize_torso_com: false - torso_com_offset_range: [-0.02, 0.02] - randomize_reset_joint_qpos: true - reset_joint_qpos_range: [-0.02, 0.02] + events: + reset_joints: + params: + position_offset_range: [-0.02, 0.02] + # Unsupported model-field payloads remain explicitly disabled for Motrix. + floor_friction: null + link_mass: null + torso_com: null + joint_armature: null + algo: empirical_normalization: true - num_envs: 1024 + num_envs: 4096 max_iterations: 10000 obs_groups: actor: @@ -45,38 +32,38 @@ algo: init_noise_std: 0.5 algorithm: entropy_coef: 0.005 + reward: - scales: - height: 2.0 - orientation: 3.0 - contact: -1.0 - action_rate: -0.01 - termination: -2.0 - dof_pos_limits: -0.5 - torques: 0.0 - pose: -0.1 - penalty_contact: -0.2 - tar: 1.3 - rear_feet_contact: 0.5 - both_rear_feet_contact: 0.25 - rear_foot_slip: -1.0 - rear_foot_anchor: -0.15 - front_feet_air: 0.0 - balanced_footstand: 0.0 - rear_leg_symmetry: -0.2 - rear_leg_splay: -0.25 - front_leg_motion: -0.06 - front_leg_crossing: -2.0 - upright_stability: -0.2 - knee_clearance: -0.5 - stay_still: -0.12 - energy: -0.003 - dof_acc: -2.5e-7 - tracking_sigma: 0.25 - base_height_target: 0.3 - knee_height_target: 0.08 - front_feet_min_separation: 0.18 - front_feet_side_margin: 0.06 - rear_hip_abduction_margin: 0.25 - rear_foot_slip_deadband: 0.012 - rear_foot_anchor_radius: 0.04 + footstand: + params: + scales: + height: 2.0 + orientation: 3.0 + contact: -1.0 + action_rate: -0.01 + termination: -2.0 + dof_pos_limits: -0.5 + torques: 0.0 + pose: -0.1 + penalty_contact: -0.2 + tar: 1.3 + rear_feet_contact: 0.5 + both_rear_feet_contact: 0.25 + rear_foot_slip: -1.0 + rear_foot_anchor: -0.15 + front_feet_air: 0.0 + balanced_footstand: 0.0 + rear_leg_symmetry: -0.2 + rear_leg_splay: -0.25 + front_leg_motion: -0.06 + front_leg_crossing: -2.0 + upright_stability: -0.2 + knee_clearance: -0.5 + stay_still: -0.12 + energy: -0.003 + dof_acc: -2.5e-7 + front_feet_min_separation: 0.18 + front_feet_side_margin: 0.06 + rear_hip_abduction_margin: 0.25 + rear_foot_slip_deadband: 0.012 + rear_foot_anchor_radius: 0.04 diff --git a/conf/ppo/task/go2_footstand/mujoco.yaml b/conf/ppo/task/go2_footstand/mujoco.yaml index cca9fb89d..8a4b428cc 100644 --- a/conf/ppo/task/go2_footstand/mujoco.yaml +++ b/conf/ppo/task/go2_footstand/mujoco.yaml @@ -1,37 +1,61 @@ # @package _global_ +defaults: + - /task/go2_footstand/base + - _self_ + training: task_name: Go2FootStand sim_backend: mujoco + env: - sim_dt: 0.004 - add_body_sensors: true - obs_history_len: 15 - energy_termination_threshold: 200.0 - noise_config: - level: 1.0 - scale_joint_angle: 0.01 - scale_joint_vel: 1.5 - scale_gyro: 0.2 - scale_gravity: 0.05 - scale_linvel: 0.1 - control_config: - action_scale: 0.3 - domain_rand: - randomize_floor_friction: true - floor_friction_range: [0.4, 1.0] - randomize_link_mass: true - link_mass_scale_range: [0.9, 1.1] - torso_added_mass_range: [-1.0, 1.0] - randomize_torso_com: true - torso_com_offset_range: [-0.05, 0.05] - randomize_dof_armature: true - dof_armature_scale_range: [1.0, 1.05] - randomize_reset_joint_qpos: true - reset_joint_qpos_range: [-0.05, 0.05] + events: + floor_friction: + func: unilab.envs.mdp.geom_friction + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + geom_names: floor + ranges: [0.4, 1.0] + operation: abs + link_mass: + func: unilab.tasks.locomotion.go2.footstand.FootstandMassRandomization + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: ".*" + torso_body_name: base + link_mass_scale_range: [0.9, 1.1] + torso_added_mass_range: [-1.0, 1.0] + torso_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: base + com_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.05, 0.05] + joint_armature: + func: unilab.envs.mdp.joint_armature + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*" + ranges: [1.0, 1.05] + operation: scale + algo: empirical_normalization: true - num_envs: 1024 # 4096 # 1024 - # max_iterations: 3000 + num_envs: 4096 max_iterations: 10000 obs_groups: actor: @@ -42,26 +66,3 @@ algo: init_noise_std: 0.5 algorithm: entropy_coef: 0.005 -reward: - scales: - height: 2.0 - orientation: 2.0 - contact: -1.0 - action_rate: -0.01 - termination: -2.0 - dof_pos_limits: -0.5 - torques: 0.0 - pose: -0.1 - penalty_contact: -0.2 - tar: 0.8 - rear_feet_contact: 0.5 - rear_leg_symmetry: -0.2 - front_leg_motion: -0.05 - upright_stability: -0.2 - knee_clearance: -0.5 - stay_still: -0.1 - energy: -0.003 - dof_acc: -2.5e-7 - tracking_sigma: 0.25 - base_height_target: 0.3 - knee_height_target: 0.08 diff --git a/conf/ppo/task/go2_joystick_flat/base.yaml b/conf/ppo/task/go2_joystick_flat/base.yaml new file mode 100644 index 000000000..409129c16 --- /dev/null +++ b/conf/ppo/task/go2_joystick_flat/base.yaml @@ -0,0 +1,206 @@ +# @package _global_ +# Canonical Go2 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [31.5, 38.5] + kd_range: [0.45, 0.55] + operation: abs + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + weight: 0.24 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/ppo/task/go2_joystick_flat/drake.yaml b/conf/ppo/task/go2_joystick_flat/drake.yaml index e246865f7..9f200f570 100644 --- a/conf/ppo/task/go2_joystick_flat/drake.yaml +++ b/conf/ppo/task/go2_joystick_flat/drake.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + training: task_name: Go2JoystickFlat sim_backend: drake @@ -10,6 +14,8 @@ algo: obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: @@ -19,23 +25,5 @@ algo: env: drake_backend_mode: batch drake_nthread: 0 - scene: - model_file: src/unilab/assets/robots/go2/scene_flat.xml - domain_rand: - randomize_kp: false - randomize_kd: false - push_robots: false - -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + events: + pd_gains: null diff --git a/conf/ppo/task/go2_joystick_flat/motrix.yaml b/conf/ppo/task/go2_joystick_flat/motrix.yaml index aca13216d..84282c9cb 100644 --- a/conf/ppo/task/go2_joystick_flat/motrix.yaml +++ b/conf/ppo/task/go2_joystick_flat/motrix.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + training: task_name: Go2JoystickFlat sim_backend: motrix @@ -9,6 +13,8 @@ algo: obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: @@ -16,25 +22,13 @@ algo: entropy_coef: 1.0e-3 env: commands: - vel_limit: - - [0.5, 0.0, 0.0] - - [0.5, 0.0, 0.0] - domain_rand: - randomize_kp: false - randomize_kd: false -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + twist: + ranges: + lin_vel_x: [0.5, 0.5] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + pd_gains: null play_profile: enabled: true env: diff --git a/conf/ppo/task/go2_joystick_flat/mujoco.yaml b/conf/ppo/task/go2_joystick_flat/mujoco.yaml index 35cf09bed..4201cf978 100644 --- a/conf/ppo/task/go2_joystick_flat/mujoco.yaml +++ b/conf/ppo/task/go2_joystick_flat/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + training: task_name: Go2JoystickFlat sim_backend: mujoco @@ -9,24 +13,13 @@ algo: obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: learning_rate: 3.0e-4 entropy_coef: 1.0e-3 -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 play_profile: enabled: true env: diff --git a/conf/ppo/task/go2_joystick_rough/motrix.yaml b/conf/ppo/task/go2_joystick_rough/motrix.yaml index 46992b1f0..a35a5cbc5 100644 --- a/conf/ppo/task/go2_joystick_rough/motrix.yaml +++ b/conf/ppo/task/go2_joystick_rough/motrix.yaml @@ -1,120 +1,18 @@ # @package _global_ +defaults: + - /task/go2_joystick_rough/mujoco + - _self_ + training: task_name: Go2JoystickRough sim_backend: motrix - play_steps: 500 - play_env_num: 16 - cam_tracking: true - cam_tracking_env_idx: 0 - cam_tracking_extra_envs: 9 - -interactive: - action_mode: policy - policy_obs_mode: auto - camera_follow_body: true - use_env_visual_model: false algo: num_envs: 4096 - num_steps_per_env: 24 - max_iterations: 1500 - empirical_normalization: false - obs_groups: - actor: - - actor - critic: - - critic - policy: - init_noise_std: 1.0 - algorithm: - learning_rate: 1.0e-3 - entropy_coef: 1.0e-2 env: render_offset_mode: zero - control_config: - action_scale: 0.25 - hip_action_scale: 0.125 - non_hip_action_scale: 0.25 - clip_actions: 100.0 - commands: - vel_limit: [[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]] - resampling_time: 10.0 - heading_command: true - heading_range: [-3.141592653589793, 3.141592653589793] - rel_standing_envs: 0.1 - terrain_curriculum: - enabled: false scene: model_file: src/unilab/assets/robots/go2/go2.xml - fragment_files: - - src/unilab/assets/robots/go2/locomotion_task.xml - terrain: - hfield_name: terrain_hfield - geom_name: floor - generator: - seed: 42 - curriculum: false - size: [8.0, 8.0] - num_rows: 6 - num_cols: 6 - border_width: 20.0 - terrain_scan: - enabled: true - geom_name: floor - termination_config: - terrain_out_of_bounds: true - terrain_distance_buffer: 3.0 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 3.0] - random_com: true - randomize_kp: true - kp_multiplier_range: [0.5, 2.0] - randomize_kd: true - kd_multiplier_range: [0.5, 2.0] - push_robots: true - push_interval: 625 - max_force: [1.0, 1.0, 0.5] -reward: - scales: - lin_vel_z: -2.0 - ang_vel_xy: -0.05 - joint_torques_l2: -2.5e-5 - joint_acc_l2: -2.5e-7 - joint_power: -2.0e-5 - stand_still: -2.0 - hip_pos: -0.5 - joint_pos_penalty: -1.0 - joint_mirror: -0.05 - action_rate: -0.01 - undesired_contacts: -1.0 - contact_forces: -1.5e-4 - tracking_lin_vel: 3.0 - tracking_ang_vel: 1.5 - feet_air_time: 0.5 - feet_air_time_variance: -1.0 - feet_contact_without_cmd: 0.1 - feet_slide: -0.1 - feet_height_body: -5.0 - feet_gait: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.3 - stand_still_command_threshold: 0.1 - joint_pos_penalty_stand_still_scale: 5.0 - joint_pos_penalty_velocity_threshold: 0.5 - joint_pos_penalty_command_threshold: 0.1 - contact_threshold: 1.0 - contact_forces_threshold: 100.0 - feet_air_time_threshold: 0.5 - feet_height_body_target: -0.2 - feet_height_body_tanh_mult: 2.0 - feet_gait_std: 0.7071067811865476 - feet_gait_max_err: 0.2 - feet_gait_velocity_threshold: 0.5 - feet_gait_command_threshold: 0.1 -play_profile: - enabled: true - env: - render_spacing: 2.0 + events: + push_robot: null diff --git a/conf/ppo/task/go2_joystick_rough/mujoco.yaml b/conf/ppo/task/go2_joystick_rough/mujoco.yaml index ea665a5fa..5d3622fdc 100644 --- a/conf/ppo/task/go2_joystick_rough/mujoco.yaml +++ b/conf/ppo/task/go2_joystick_rough/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/quadruped_joystick_rough/quadruped + - _self_ + training: task_name: Go2JoystickRough sim_backend: mujoco @@ -20,103 +24,46 @@ algo: num_steps_per_env: 24 max_iterations: 1500 empirical_normalization: false - obs_groups: - actor: - - actor - critic: - - critic - policy: - init_noise_std: 1.0 - algorithm: - learning_rate: 1.0e-3 - entropy_coef: 1.0e-2 + obs_groups: {actor: [actor], critic: [critic]} + policy: {init_noise_std: 1.0} + algorithm: {learning_rate: 1.0e-3, entropy_coef: 1.0e-2} env: sim_dt: 0.002 - control_config: - action_scale: 0.25 - hip_action_scale: 0.125 - non_hip_action_scale: 0.25 - clip_actions: 100.0 - commands: - vel_limit: [[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]] - resampling_time: 10.0 - heading_command: true - heading_range: [-3.141592653589793, 3.141592653589793] - rel_standing_envs: 0.1 - terrain_curriculum: - enabled: false scene: model_file: src/unilab/assets/robots/go2/go2_mujoco.xml - fragment_files: - - src/unilab/assets/robots/go2/locomotion_task.xml - terrain: - hfield_name: terrain_hfield - geom_name: floor - generator: - seed: 42 - curriculum: false - size: [8.0, 8.0] - num_rows: 6 - num_cols: 6 - border_width: 20.0 - terrain_scan: - enabled: true - geom_name: floor - termination_config: - terrain_out_of_bounds: true - terrain_distance_buffer: 3.0 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 3.0] - random_com: true - randomize_kp: true - kp_multiplier_range: [0.5, 2.0] - randomize_kd: true - kd_multiplier_range: [0.5, 2.0] - push_robots: true - push_interval: 625 - max_force: [1.0, 1.0, 0.5] + fragment_files: [src/unilab/assets/robots/go2/locomotion_task.xml] + entities: + robot: + root_body_name: base + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + body_names: [base] -reward: - scales: - lin_vel_z: -2.0 - ang_vel_xy: -0.05 - joint_torques_l2: -2.5e-5 - joint_acc_l2: -2.5e-7 - joint_power: -2.0e-5 - stand_still: -2.0 - hip_pos: -0.5 - joint_pos_penalty: -1.0 - joint_mirror: -0.05 - action_rate: -0.01 - undesired_contacts: -1.0 - contact_forces: -1.5e-4 - tracking_lin_vel: 3.0 - tracking_ang_vel: 1.5 - feet_air_time: 0.5 - feet_air_time_variance: -1.0 - feet_contact_without_cmd: 0.1 - feet_slide: -0.1 - feet_height_body: -5.0 - feet_gait: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.3 - stand_still_command_threshold: 0.1 - joint_pos_penalty_stand_still_scale: 5.0 - joint_pos_penalty_velocity_threshold: 0.5 - joint_pos_penalty_command_threshold: 0.1 - contact_threshold: 1.0 - contact_forces_threshold: 100.0 - feet_air_time_threshold: 0.5 - feet_height_body_target: -0.2 - feet_height_body_tanh_mult: 2.0 - feet_gait_std: 0.7071067811865476 - feet_gait_max_err: 0.2 - feet_gait_velocity_threshold: 0.5 - feet_gait_command_threshold: 0.1 play_profile: enabled: true - env: - render_spacing: 2.0 + env: {render_spacing: 2.0} diff --git a/conf/ppo/task/go2w_joystick_flat/base.yaml b/conf/ppo/task/go2w_joystick_flat/base.yaml new file mode 100644 index 000000000..3d057e1ee --- /dev/null +++ b/conf/ppo/task/go2w_joystick_flat/base.yaml @@ -0,0 +1,263 @@ +# @package _global_ +# Canonical Go2W flat Manager-Based task declaration. Backend leaves only own +# backend identity and backend-specific rendering/runtime settings. +env: + scene: + model_file: src/unilab/assets/robots/go2w/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base_link + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - FR_wheel_joint + - FL_wheel_joint + - RR_wheel_joint + - RL_wheel_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + - FR_wheel + - FL_wheel + - RR_wheel + - RL_wheel + body_names: [base_link] + sim_dt: 0.005 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + leg_joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + leg_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + wheel_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_wheel_joint" + actions: + func: unilab.envs.mdp.last_action + params: + action_name: motor + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + leg_joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + leg_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + wheel_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_wheel_joint" + actions: + func: unilab.envs.mdp.last_action + params: + action_name: motor + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + motor_torque: + func: unilab.tasks.locomotion.go2w.manager_terms.motor_torque + params: + action_name: motor + actions: + motor: + _target_: unilab.tasks.locomotion.go2w.manager_terms.Go2WMixedActionCfg + entity_name: robot + actuator_names: [".*"] + leg_action_scale: 0.5 + wheel_action_scale: 10.0 + leg_kp: 50.0 + leg_kd: 1.5 + wheel_kd: 0.5 + clip_actions: 1.0 + simulate_action_latency: false + commands: + twist: + _target_: unilab.tasks.locomotion.go2w.manager_terms.Go2WVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [0.0, 1.0] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [-1.0, 1.0] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + motor_gains: + func: unilab.tasks.locomotion.go2w.manager_terms.randomize_motor_gains + mode: reset + params: + action_name: motor + kp_multiplier_range: [1.0, 1.0] + kd_multiplier_range: [1.0, 1.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.75 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.4 + orientation: + func: unilab.envs.mdp.flat_orientation_l2 + weight: -2.0 + action_rate: + func: unilab.tasks.locomotion.go2w.manager_terms.clipped_action_rate_l2 + weight: -0.005 + params: + action_name: motor + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.5 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + torques: + func: unilab.tasks.locomotion.go2w.manager_terms.motor_torque_l2 + weight: -0.0002 + params: + action_name: motor + wheel_vel: + func: unilab.envs.mdp.joint_vel_l2 + weight: 0.0 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_wheel_joint" + alive: + func: unilab.tasks.locomotion.go2w.manager_terms.constant_alive + weight: 0.5 + upward: + func: unilab.tasks.locomotion.go2w.manager_terms.upward_l2 + weight: 1.0 diff --git a/conf/ppo/task/go2w_joystick_flat/drake.yaml b/conf/ppo/task/go2w_joystick_flat/drake.yaml index 6a5b59ef5..ddbbc1bfa 100644 --- a/conf/ppo/task/go2w_joystick_flat/drake.yaml +++ b/conf/ppo/task/go2w_joystick_flat/drake.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2w_joystick_flat/base + - _self_ + training: task_name: Go2WJoystickFlat sim_backend: drake @@ -10,6 +14,8 @@ algo: obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: @@ -19,36 +25,3 @@ algo: env: drake_backend_mode: batch drake_nthread: 0 - scene: - model_file: src/unilab/assets/robots/go2w/scene_flat.xml - commands: - vel_limit: - - [0.0, 0.0, -1.0] - - [1.0, 0.0, 1.0] - control_config: - action_scale: 0.5 - wheel_action_scale: 10.0 - Kp: 50.0 - Kd: 1.5 - wheel_Kd: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false - push_robots: false - -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.75 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - orientation: -2.0 - action_rate: -0.005 - similar_to_default: -0.5 - torques: -0.0002 - wheel_vel: 0.0 - alive: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.4 diff --git a/conf/ppo/task/go2w_joystick_flat/motrix.yaml b/conf/ppo/task/go2w_joystick_flat/motrix.yaml index 0e86b63b0..01af0050d 100644 --- a/conf/ppo/task/go2w_joystick_flat/motrix.yaml +++ b/conf/ppo/task/go2w_joystick_flat/motrix.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2w_joystick_flat/base + - _self_ + training: task_name: Go2WJoystickFlat sim_backend: motrix @@ -9,6 +13,8 @@ algo: obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: @@ -16,35 +22,6 @@ algo: entropy_coef: 1.0e-3 env: render_offset_mode: zero - commands: - vel_limit: - - [0.0, 0.0, -1.0] - - [1.0, 0.0, 1.0] - control_config: - action_scale: 0.5 - wheel_action_scale: 10.0 - Kp: 50.0 - Kd: 1.5 - wheel_Kd: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.75 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - orientation: -2.0 - action_rate: -0.005 - similar_to_default: -0.5 - torques: -0.0002 - wheel_vel: 0.0 - alive: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.4 play_profile: enabled: true env: diff --git a/conf/ppo/task/go2w_joystick_flat/mujoco.yaml b/conf/ppo/task/go2w_joystick_flat/mujoco.yaml index 602b95a27..3e092334c 100644 --- a/conf/ppo/task/go2w_joystick_flat/mujoco.yaml +++ b/conf/ppo/task/go2w_joystick_flat/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2w_joystick_flat/base + - _self_ + training: task_name: Go2WJoystickFlat sim_backend: mujoco @@ -9,41 +13,13 @@ algo: obs_groups: actor: - actor + critic: + - critic policy: init_noise_std: 0.5 algorithm: learning_rate: 3.0e-4 entropy_coef: 1.0e-3 -env: - commands: - vel_limit: - - [0.0, 0.0, -1.0] - - [1.0, 0.0, 1.0] - control_config: - action_scale: 0.5 - wheel_action_scale: 10.0 - Kp: 50.0 - Kd: 1.5 - wheel_Kd: 0.5 - domain_rand: - randomize_kp: false - randomize_kd: false -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.75 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - orientation: -2.0 - action_rate: -0.005 - similar_to_default: -0.5 - torques: -0.0002 - wheel_vel: 0.0 - alive: 0.5 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.4 play_profile: enabled: true env: diff --git a/conf/ppo/task/go2w_joystick_rough/motrix.yaml b/conf/ppo/task/go2w_joystick_rough/motrix.yaml index d15620ddb..0f4fcbb21 100644 --- a/conf/ppo/task/go2w_joystick_rough/motrix.yaml +++ b/conf/ppo/task/go2w_joystick_rough/motrix.yaml @@ -1,99 +1,19 @@ # @package _global_ +defaults: + - /task/go2w_joystick_rough/mujoco + - _self_ + training: task_name: Go2WJoystickRough sim_backend: motrix - play_steps: 500 - play_env_num: 16 - cam_tracking: true - cam_tracking_env_idx: 0 - cam_tracking_extra_envs: 9 - -interactive: - action_mode: policy - policy_obs_mode: auto - camera_follow_body: true - use_env_visual_model: false -algo: - num_envs: 2048 - num_steps_per_env: 24 - max_iterations: 1200 - empirical_normalization: false - obs_groups: - actor: - - actor - critic: - - critic env: render_offset_mode: zero scene: model_file: src/unilab/assets/robots/go2w/go2w.xml - fragment_files: - - src/unilab/assets/robots/go2w/locomotion_task.xml - terrain: - hfield_name: terrain_hfield - geom_name: floor - generator: - seed: 42 - curriculum: false - size: [8.0, 8.0] - num_rows: 6 - num_cols: 6 - border_width: 20.0 - commands: - vel_limit: [[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]] - resampling_time: 10.0 - heading_command: true - heading_range: [-3.141592653589793, 3.141592653589793] - rel_standing_envs: 0.1 - control_config: - action_scale: 0.25 - hip_action_scale: 0.125 - wheel_action_scale: 5.0 - wheel_Kd: 0.5 - clip_actions: 100.0 - simulate_action_latency: false - terrain_scan: - enabled: true - hfield_name: terrain_hfield - geom_name: floor - termination_config: - terrain_out_of_bounds: true - terrain_distance_buffer: 3.0 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 3.0] - random_com: true - com_offset_x: [-0.05, 0.05] - randomize_kp: true - kp_multiplier_range: [0.5, 1.0] - randomize_kd: true - kd_multiplier_range: [0.5, 1.0] - push_robots: true - push_interval: 625 - max_force: [1.0, 1.0, 0.5] - push_body_name: base_link + events: + push_robot: null + reward: - scales: - tracking_lin_vel: 3.0 - tracking_ang_vel: 1.5 - lin_vel_z: -2.0 - ang_vel_xy: -0.05 - orientation: -2.0 - joint_torques_l2: -2.5e-5 - joint_acc_l2: -2.5e-7 - joint_acc_wheel_l2: -2.5e-9 - joint_power: -2.0e-5 - action_rate: -0.01 - stand_still: -2.0 - hip_pos: -0.5 - joint_pos_penalty: -1.0 - joint_mirror: -0.05 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.4 - only_positive_rewards: false -play_profile: - enabled: true - env: - render_spacing: 2.0 + hip_pos: + weight: -0.5 diff --git a/conf/ppo/task/go2w_joystick_rough/mujoco.yaml b/conf/ppo/task/go2w_joystick_rough/mujoco.yaml index 5bcc31038..f0c77718c 100644 --- a/conf/ppo/task/go2w_joystick_rough/mujoco.yaml +++ b/conf/ppo/task/go2w_joystick_rough/mujoco.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/quadruped_joystick_rough/go2w + - _self_ + training: task_name: Go2WJoystickRough sim_backend: mujoco @@ -20,80 +24,55 @@ algo: num_steps_per_env: 24 max_iterations: 1200 empirical_normalization: false - obs_groups: - actor: - - actor - critic: - - critic + obs_groups: {actor: [actor], critic: [critic]} + env: + sim_dt: 0.005 scene: model_file: src/unilab/assets/robots/go2w/go2w_mujoco.xml - fragment_files: - - src/unilab/assets/robots/go2w/locomotion_task.xml + fragment_files: [src/unilab/assets/robots/go2w/locomotion_task.xml] terrain: - hfield_name: terrain_hfield - geom_name: floor generator: - seed: 42 - curriculum: false - size: [8.0, 8.0] - num_rows: 6 - num_cols: 6 - border_width: 20.0 - commands: - vel_limit: [[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]] - resampling_time: 10.0 - heading_command: true - heading_range: [-3.141592653589793, 3.141592653589793] - rel_standing_envs: 0.1 - control_config: - action_scale: 0.25 - hip_action_scale: 0.125 - wheel_action_scale: 5.0 - wheel_Kd: 0.5 - clip_actions: 100.0 - simulate_action_latency: false - terrain_scan: - enabled: true - hfield_name: terrain_hfield - geom_name: floor - termination_config: - terrain_out_of_bounds: true - terrain_distance_buffer: 3.0 - domain_rand: - randomize_base_mass: true - added_mass_range: [-1.0, 3.0] - random_com: true - com_offset_x: [-0.05, 0.05] - randomize_kp: true - kp_multiplier_range: [0.5, 1.0] - randomize_kd: true - kd_multiplier_range: [0.5, 1.0] - push_robots: true - push_interval: 625 - max_force: [1.0, 1.0, 0.5] - push_body_name: base_link -reward: - scales: - tracking_lin_vel: 3.0 - tracking_ang_vel: 1.5 - lin_vel_z: -2.0 - ang_vel_xy: -0.05 - orientation: -2.0 - joint_torques_l2: -2.5e-5 - joint_acc_l2: -2.5e-7 - joint_acc_wheel_l2: -2.5e-9 - joint_power: -2.0e-5 - action_rate: -0.01 - stand_still: -2.0 - hip_pos: -2.0 - joint_pos_penalty: -1.0 - joint_mirror: -0.05 - upward: 1.0 - tracking_sigma: 0.25 - base_height_target: 0.4 - only_positive_rewards: false + horizontal_scale: 0.1 + entities: + robot: + root_body_name: base_link + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - FR_wheel_joint + - FL_wheel_joint + - RR_wheel_joint + - RL_wheel_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + - FR_wheel + - FL_wheel + - RR_wheel + - RL_wheel + body_names: [base_link] + play_profile: enabled: true - env: - render_spacing: 2.0 + env: {render_spacing: 2.0} diff --git a/conf/ppo/task/quadruped_joystick_rough/base.yaml b/conf/ppo/task/quadruped_joystick_rough/base.yaml new file mode 100644 index 000000000..c916ccd63 --- /dev/null +++ b/conf/ppo/task/quadruped_joystick_rough/base.yaml @@ -0,0 +1,152 @@ +# @package _global_ +# Shared terrain/reset/command owner for all production rough quadrupeds. +env: + scene: + default_keyframe_name: home + terrain: + hfield_name: terrain_hfield + geom_name: floor + generator: + _target_: unilab.tasks.locomotion.common.rough_manager_terms.QuadrupedRoughTerrainCfg + seed: 42 + curriculum: false + size: [8.0, 8.0] + num_rows: 6 + num_cols: 6 + border_width: 20.0 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + commands: + twist: + _target_: unilab.tasks.locomotion.common.rough_manager_terms.RoughVelocityCommandCfg + entity_name: robot + resampling_time_range: [10.0, 10.0] + heading_command: true + heading_control_stiffness: 0.5 + rel_standing_envs: 0.1 + rel_heading_envs: 1.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.08 + ranges: + lin_vel_x: [-1.0, 1.0] + lin_vel_y: [-1.0, 1.0] + ang_vel_z: [-1.0, 1.0] + heading: [-3.141592653589793, 3.141592653589793] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + terrain_root_state: + func: unilab.tasks.locomotion.common.rough_manager_terms.RoughTerrainReset + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.25, 0.5] + roll: [-3.14, 3.14] + pitch: [-3.14, 3.14] + yaw: [-3.14, 3.14] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + promote_frac: 0.5 + demote_frac: 0.25 + cycle_top_frac: 0.5 + spawn_height_margin: 0.05 + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: ".*" + mass_distribution_params: [-1.0, 3.0] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: ".*" + com_range: + x: [-0.05, 0.05] + y: [0.0, 0.0] + z: [0.0, 0.0] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [17.5, 70.0] + kd_range: [0.25, 1.0] + operation: abs + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [12.5, 12.5] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + terrain_out_of_bounds: + func: unilab.tasks.locomotion.common.rough_manager_terms.RoughTerrainOutOfBounds + time_out: true + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + distance_buffer: 3.0 + curriculum: + terrain_levels: + func: unilab.tasks.locomotion.common.rough_manager_terms.RoughTerrainCurriculum + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 3.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 1.5 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -2.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.05 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.01 diff --git a/conf/ppo/task/quadruped_joystick_rough/go2w.yaml b/conf/ppo/task/quadruped_joystick_rough/go2w.yaml new file mode 100644 index 000000000..746943680 --- /dev/null +++ b/conf/ppo/task/quadruped_joystick_rough/go2w.yaml @@ -0,0 +1,127 @@ +# @package _global_ +defaults: + - /task/quadruped_joystick_rough/base + - _self_ + +env: + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: upvector} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + leg_joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + actions: + func: unilab.envs.mdp.last_action + params: {action_name: motor} + critic: + terms: + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: local_linvel} + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: gyro} + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: upvector} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + leg_joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + params: {action_name: motor} + height_scan: + func: unilab.tasks.locomotion.common.rough_manager_terms.RoughHeightScan + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + geom_name: floor + vertical_offset: 0.5 + scale: 5.0 + actions: + motor: + _target_: unilab.tasks.locomotion.go2w.manager_terms.Go2WMixedActionCfg + entity_name: robot + actuator_names: [".*"] + leg_action_scale: 0.25 + hip_action_scale: 0.125 + wheel_action_scale: 5.0 + leg_kp: 35.0 + leg_kd: 0.5 + wheel_kd: 0.5 + clip_actions: 100.0 + simulate_action_latency: false + events: + pd_gains: null + motor_gains: + func: unilab.tasks.locomotion.go2w.manager_terms.randomize_motor_gains + mode: reset + params: + action_name: motor + kp_multiplier_range: [0.5, 1.0] + kd_multiplier_range: [0.5, 1.0] + +reward: + orientation: + func: unilab.envs.mdp.flat_orientation_l2 + weight: -2.0 + motor_torque: + func: unilab.tasks.locomotion.go2w.manager_terms.motor_torque_l2 + weight: -2.5e-5 + params: {action_name: motor} + stand_still: + func: unilab.tasks.locomotion.common.manager_terms.stand_still_l1 + weight: -2.0 + params: + command_name: twist + command_threshold: 0.1 + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + hip_pos: + func: unilab.tasks.locomotion.common.rough_manager_terms.joint_deviation_l2 + weight: -2.0 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_hip_joint" + joint_pos_penalty: + func: unilab.tasks.locomotion.common.rough_manager_terms.joint_deviation_l2 + weight: -1.0 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + upward: + func: unilab.tasks.locomotion.go2w.manager_terms.upward_l2 + weight: 1.0 diff --git a/conf/ppo/task/quadruped_joystick_rough/quadruped.yaml b/conf/ppo/task/quadruped_joystick_rough/quadruped.yaml new file mode 100644 index 000000000..a464a98c8 --- /dev/null +++ b/conf/ppo/task/quadruped_joystick_rough/quadruped.yaml @@ -0,0 +1,89 @@ +# @package _global_ +defaults: + - /task/quadruped_joystick_rough/base + - _self_ + +env: + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: upvector} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + critic: + terms: + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: local_linvel} + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: gyro} + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: upvector} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + height_scan: + func: unilab.tasks.locomotion.common.rough_manager_terms.RoughHeightScan + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + geom_name: floor + vertical_offset: 0.5 + scale: 5.0 + actions: + joint_pos: + _target_: unilab.tasks.locomotion.common.rough_manager_terms.RoughJointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: + ".*_hip_joint": 0.125 + ".*_(thigh|calf)_joint": 0.25 + use_default_offset: true + clip_actions: 100.0 + +reward: + stand_still: + func: unilab.tasks.locomotion.common.manager_terms.stand_still_l1 + weight: -2.0 + params: + command_name: twist + command_threshold: 0.1 + hip_pos: + func: unilab.tasks.locomotion.common.rough_manager_terms.joint_deviation_l2 + weight: -0.5 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_hip_joint" + joint_pos_penalty: + func: unilab.tasks.locomotion.common.rough_manager_terms.joint_deviation_l2 + weight: -1.0 + upward: + func: unilab.tasks.locomotion.go2w.manager_terms.upward_l2 + weight: 1.0 diff --git a/conf/ppo/task/sharpa_inhand/mujoco_hora.yaml b/conf/ppo/task/sharpa_inhand/mujoco_hora.yaml index 7937b66ee..7a7f8c13d 100644 --- a/conf/ppo/task/sharpa_inhand/mujoco_hora.yaml +++ b/conf/ppo/task/sharpa_inhand/mujoco_hora.yaml @@ -14,12 +14,12 @@ interactive: algo: algo_log_name: hora_ppo runtime_impl: hora_ppo - runtime_resolver: unilab.algos.torch.hora.rsl_rl:resolve_hora_ppo_runtime + runtime_resolver: unilab.algos.hora.rsl_rl:resolve_hora_ppo_runtime obs_groups: actor: [actor] critic: [actor] actor: - class_name: unilab.algos.torch.hora:HoraActorModel + class_name: unilab.algos.hora:HoraActorModel hidden_dims: [512, 256, 128] activation: elu obs_normalization: true @@ -30,14 +30,14 @@ algo: init_std: 1.0 std_type: scalar critic: - class_name: unilab.algos.torch.hora:HoraCriticModel + class_name: unilab.algos.hora:HoraCriticModel hidden_dims: [512, 256, 128] activation: elu obs_normalization: true priv_info_embed_dim: 9 priv_mlp_hidden_dims: [256, 128, 9] algorithm: - class_name: unilab.algos.torch.hora:HoraPPO + class_name: unilab.algos.hora:HoraPPO env: obs: diff --git a/conf/ppo/task/stewart_balance/base.yaml b/conf/ppo/task/stewart_balance/base.yaml new file mode 100644 index 000000000..ef8a4c49a --- /dev/null +++ b/conf/ppo/task/stewart_balance/base.yaml @@ -0,0 +1,118 @@ +# @package _global_ +# Canonical Stewart Manager-Based task declaration. Backend leaves own only +# backend identity and algorithm/runtime tuning. +env: + scene: + model_file: src/unilab/assets/robots/stewart/scene.xml + entities: + stewart: + root_body_name: ball + actuator_names: [a0, a1, a2, a3, a4, a5] + body_names: + - ball + - top + - leg00 + - leg10 + - leg01 + - leg11 + - leg02 + - leg12 + - top_connect00 + - top_connect10 + - top_connect01 + - top_connect11 + - top_connect02 + - top_connect12 + # The stiff closed-loop model requires a physics step no larger than ~0.005 s. + sim_dt: 0.004 + ctrl_dt: 0.02 + max_episode_seconds: 24.0 + render_spacing: 4.5 + observations: + policy: + terms: + balance: + func: unilab.tasks.manipulation.stewart.balance.StewartObservation + params: + entity_name: stewart + action_name: tilt + ball_body_name: ball + top_body_name: top + target_rotation_limit_deg: 6.0 + vel_smooth: 0.25 + actions: + tilt: + _target_: unilab.tasks.manipulation.stewart.balance.StewartTiltActionCfg + entity_name: stewart + actuator_names: [a0, a1, a2, a3, a4, a5] + top_body_name: top + ball_body_name: ball + leg_body_names: [leg00, leg10, leg01, leg11, leg02, leg12] + top_connect_body_names: + - top_connect00 + - top_connect10 + - top_connect01 + - top_connect11 + - top_connect02 + - top_connect12 + raw_action_clip: [-1.0, 1.0] + target_rotation_limit_deg: 6.0 + action_smooth: 0.60 + center_control_radius: 0.25 + center_control_min_gain: 0.15 + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_ball: + func: unilab.tasks.manipulation.stewart.balance.StewartBallReset + mode: reset + params: + entity_name: stewart + platform_radius: 0.8 + init_ball_radius_ratio: 0.18 + ball_home_z: 1.2 + terminations: + balance_state: + func: unilab.tasks.manipulation.stewart.balance.StewartBalanceState + params: + observation_group: policy + observation_term: balance + platform_radius: 0.8 + fall_radius: 0.5 + top_center_z: 1.0 + still_xy: 0.12 + still_vel: 0.07 + still_xy_hysteresis: 1.15 + still_vel_hysteresis: 1.20 + zero_vel_thresh: 0.07 + still_steps_needed: 5 + time_out: + func: unilab.envs.mdp.time_out + time_out: true + # Legacy Stewart rewards were discrete per-control-step values, not rates. + scale_rewards_by_dt: false + policy_observation_group: policy + critic_observation_group: null + +reward: + center: + func: unilab.tasks.manipulation.stewart.balance.center_reward + weight: 0.7 + params: + state_term_name: balance_state + progress: + func: unilab.tasks.manipulation.stewart.balance.progress_reward + weight: 0.6 + params: + state_term_name: balance_state + still: + func: unilab.tasks.manipulation.stewart.balance.still_reward + weight: 3.0 + params: + state_term_name: balance_state + fall: + func: unilab.tasks.manipulation.stewart.balance.fall_reward + weight: -6.0 + params: + state_term_name: balance_state diff --git a/conf/ppo/task/stewart_balance/drake.yaml b/conf/ppo/task/stewart_balance/drake.yaml index cdd440832..6be5f7d0d 100644 --- a/conf/ppo/task/stewart_balance/drake.yaml +++ b/conf/ppo/task/stewart_balance/drake.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/stewart_balance/base + - _self_ + training: task_name: StewartBalance sim_backend: drake @@ -42,10 +46,3 @@ algo: gamma: 0.99 lam: 0.95 save_interval: 50 - -reward: - scales: - center: 0.7 - progress: 0.6 - still: 3.0 - fall_penalty: -6.0 diff --git a/conf/ppo/task/stewart_balance/motrix.yaml b/conf/ppo/task/stewart_balance/motrix.yaml index f018225f8..e15d86cf5 100644 --- a/conf/ppo/task/stewart_balance/motrix.yaml +++ b/conf/ppo/task/stewart_balance/motrix.yaml @@ -2,6 +2,10 @@ # Stewart-platform ball-balancing (motrix). A short, runnable PPO baseline, not # tuned for best final performance (raise max_iterations / num_envs for higher # success rates). +defaults: + - /task/stewart_balance/base + - _self_ + training: task_name: StewartBalance sim_backend: motrix @@ -40,12 +44,6 @@ algo: gamma: 0.99 lam: 0.95 save_interval: 50 -reward: - scales: - center: 0.7 - progress: 0.6 - still: 3.0 - fall_penalty: -6.0 play_profile: enabled: true env: diff --git a/conf/ppo/task/t800_walk_flat/base.yaml b/conf/ppo/task/t800_walk_flat/base.yaml new file mode 100644 index 000000000..d27397e86 --- /dev/null +++ b/conf/ppo/task/t800_walk_flat/base.yaml @@ -0,0 +1,287 @@ +# @package _global_ +# Canonical EngineAI T800 25-DoF walk-flat Manager-Based task declaration. +env: + scene: + model_file: src/unilab/assets/robots/t800/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: LINK_BASE + joint_names: + - J00_HIP_PITCH_L + - J01_HIP_ROLL_L + - J02_HIP_YAW_L + - J03_KNEE_PITCH_L + - J04_ANKLE_PITCH_L + - J05_ANKLE_ROLL_L + - J06_HIP_PITCH_R + - J07_HIP_ROLL_R + - J08_HIP_YAW_R + - J09_KNEE_PITCH_R + - J10_ANKLE_PITCH_R + - J11_ANKLE_ROLL_R + - J12_TORSO_YAW + - J13_SHOULDER_PITCH_L + - J14_SHOULDER_ROLL_L + - J15_SHOULDER_YAW_L + - J16_ELBOW_PITCH_L + - J17_ELBOW_YAW_L + - J18_SHOULDER_PITCH_R + - J19_SHOULDER_ROLL_R + - J20_SHOULDER_YAW_R + - J21_ELBOW_PITCH_R + - J22_ELBOW_YAW_R + - J23_HEAD_PITCH + - J24_HEAD_YAW + actuator_names: + - J00_HIP_PITCH_L + - J01_HIP_ROLL_L + - J02_HIP_YAW_L + - J03_KNEE_PITCH_L + - J04_ANKLE_PITCH_L + - J05_ANKLE_ROLL_L + - J06_HIP_PITCH_R + - J07_HIP_ROLL_R + - J08_HIP_YAW_R + - J09_KNEE_PITCH_R + - J10_ANKLE_PITCH_R + - J11_ANKLE_ROLL_R + - J12_TORSO_YAW + - J13_SHOULDER_PITCH_L + - J14_SHOULDER_ROLL_L + - J15_SHOULDER_YAW_L + - J16_ELBOW_PITCH_L + - J17_ELBOW_YAW_L + - J18_SHOULDER_PITCH_R + - J19_SHOULDER_ROLL_R + - J20_SHOULDER_YAW_R + - J21_ELBOW_PITCH_R + - J22_ELBOW_YAW_R + - J23_HEAD_PITCH + - J24_HEAD_YAW + body_names: [LINK_BASE] + sim_dt: 0.002 + ctrl_dt: 0.01 + max_episode_seconds: 20.0 + observations: + policy: + enable_corruption: true + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.2 + n_max: 0.2 + operation: add + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.05 + n_max: 0.05 + operation: add + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + operation: add + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -1.5 + n_max: 1.5 + operation: add + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: {frequency: 1.5, init_mode: offset_phase} + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: {frequency: 1.5, init_mode: offset_phase} + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + policy_observation_group: policy + critic_observation_group: critic + actions: + joint_pos: + _target_: unilab.tasks.locomotion.t800.manager_terms.T800JointPositionActionCfg + entity_name: robot + actuator_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + held_actuator_names: [J12_TORSO_YAW, J23_HEAD_PITCH, J24_HEAD_YAW] + scale: + J00_HIP_PITCH_L: 0.5 + J01_HIP_ROLL_L: 0.2 + J02_HIP_YAW_L: 0.2 + J03_KNEE_PITCH_L: 0.5 + J04_ANKLE_PITCH_L: 0.5 + J05_ANKLE_ROLL_L: 0.2 + J06_HIP_PITCH_R: 0.5 + J07_HIP_ROLL_R: 0.2 + J08_HIP_YAW_R: 0.2 + J09_KNEE_PITCH_R: 0.5 + J10_ANKLE_PITCH_R: 0.5 + J11_ANKLE_ROLL_R: 0.2 + J13_SHOULDER_PITCH_L: 0.2 + J14_SHOULDER_ROLL_L: 0.2 + J15_SHOULDER_YAW_L: 0.05 + J16_ELBOW_PITCH_L: 0.2 + J17_ELBOW_YAW_L: 0.05 + J18_SHOULDER_PITCH_R: 0.2 + J19_SHOULDER_ROLL_R: 0.2 + J20_SHOULDER_YAW_R: 0.05 + J21_ELBOW_PITCH_R: 0.2 + J22_ELBOW_YAW_R: 0.05 + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + tilt: + func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded + params: {max_tilt_deg: 25.0} + base_height: + func: unilab.envs.mdp.root_height_below_minimum + params: {minimum_height: 0.7165} + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel + weight: 2.0 + params: {tracking_sigma: 0.25, command_name: twist} + tracking_ang_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel + weight: 0.2 + params: {tracking_sigma: 0.25, command_name: twist} + feet_phase: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase + weight: 1.5 + params: + frequency: 1.5 + swing_height: 0.13 + tracking_sigma: 0.014 + min_forward_speed: 0.0 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.g1.manager_terms.lin_vel_z + weight: -1.0 + ang_vel_xy: + func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy + weight: -0.25 + base_height: + func: unilab.tasks.locomotion.g1.manager_terms.base_height + weight: -500.0 + params: {target_height: 1.0165} + orientation: + func: unilab.tasks.locomotion.g1.manager_terms.orientation + weight: -5.0 + penalty_feet_ori: + func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori + weight: -10.0 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.01 + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.1 + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/ppo/task/t800_walk_flat/mujoco.yaml b/conf/ppo/task/t800_walk_flat/mujoco.yaml new file mode 100644 index 000000000..b58966d5b --- /dev/null +++ b/conf/ppo/task/t800_walk_flat/mujoco.yaml @@ -0,0 +1,28 @@ +# @package _global_ +# MuJoCo owner for the standalone T800 Manager-Based PPO task. +defaults: + - /task/t800_walk_flat/base + - _self_ + +training: + task_name: T800WalkFlat + sim_backend: mujoco + play_steps: 2000 + +algo: + num_envs: 2048 + max_iterations: 5000 + empirical_normalization: false + obs_groups: + actor: + - policy + critic: + - critic + policy: + actor_hidden_dims: [512, 256, 128] + critic_hidden_dims: [512, 256, 128] + +play_profile: + enabled: true + env: + render_spacing: 2.0 diff --git a/conf/ppo/task/x2_wall_flip_tracking/motrix.yaml b/conf/ppo/task/x2_wall_flip_tracking/motrix.yaml index d2a45c89e..9b0aecd0a 100644 --- a/conf/ppo/task/x2_wall_flip_tracking/motrix.yaml +++ b/conf/ppo/task/x2_wall_flip_tracking/motrix.yaml @@ -1,70 +1,15 @@ # @package _global_ +defaults: + - /task/x2_wall_flip_tracking/mujoco + - _self_ + training: task_name: X2WallFlipTracking sim_backend: motrix - play_steps: 300 # 6s play video @ ctrl_dt=0.02 (fps 50) - play_env_num: 16 - render_spacing: 3.0 - cam_distance: 14.0 - cam_azimuth: 225.0 - cam_elevation: -18.0 - cam_lookat: [4.5, 4.5, 1.0] -interactive: - action_mode: policy -algo: - num_envs: 1024 - max_iterations: 9500 - save_interval: 500 - empirical_normalization: true - obs_groups: - actor: - - actor - critic: - - critic - algorithm: - entropy_coef: 0.005 - desired_kl: 0.01 + env: motrix_max_iterations: 3 - sampling_mode: start - truncate_on_clip_end: false - sim_dt: 0.005 - control_config: - action_scale: - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + play_profile: enabled: true env: @@ -76,25 +21,3 @@ play_profile: skybox_rgb1: [0.90, 0.90, 0.91] skybox_rgb2: [0.68, 0.68, 0.70] ground_texrepeat: [0.25, 0.25] -reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 diff --git a/conf/ppo/task/x2_wall_flip_tracking/mujoco.yaml b/conf/ppo/task/x2_wall_flip_tracking/mujoco.yaml index 83511b770..50922321b 100644 --- a/conf/ppo/task/x2_wall_flip_tracking/mujoco.yaml +++ b/conf/ppo/task/x2_wall_flip_tracking/mujoco.yaml @@ -1,94 +1,208 @@ # @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + training: task_name: X2WallFlipTracking sim_backend: mujoco - play_steps: 300 # 6s play video @ ctrl_dt=0.02 (fps 50) - # Offline-render (play) only — does not affect the training loop. 16 envs are - # laid out on a 4x4 grid; render_spacing must exceed the per-env wall reach - # (~2.14m toward -Y) so neighbouring cells don't overlap. The oblique - # azimuth (225) views the grid corner-on with each robot in front of its - # wall (azimuth 90 would put the walls between camera and robots); lookat is - # the grid centre (offsets span 0..9m in X/Y at spacing 3.0). + play_steps: 300 play_env_num: 16 render_spacing: 3.0 cam_distance: 14.0 cam_azimuth: 225.0 cam_elevation: -18.0 cam_lookat: [4.5, 4.5, 1.0] + interactive: action_mode: policy + algo: num_envs: 1024 max_iterations: 9500 save_interval: 500 empirical_normalization: true obs_groups: - actor: - - actor - critic: - - critic + actor: [actor] + critic: [critic] algorithm: entropy_coef: 0.005 desired_kl: 0.01 + +play_profile: + enabled: false + env: null + env: - sampling_mode: start - truncate_on_clip_end: false + scene: + model_file: src/unilab/assets/robots/x2/scene_flat_with_wall.xml + visual_model_file: src/unilab/assets/robots/x2/scene_flat_with_wall_visual.xml + default_keyframe_name: home + entities: + robot: + root_body_name: pelvis + joint_names: &x2_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_pitch_joint + - waist_roll_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_yaw_joint + - left_wrist_pitch_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_yaw_joint + - right_wrist_pitch_joint + - right_wrist_roll_joint + actuator_names: *x2_joints + geom_names: null + body_names: &tracked_bodies + - pelvis + - left_hip_pitch_link + - left_hip_roll_link + - left_hip_yaw_link + - left_knee_link + - left_ankle_pitch_link + - left_ankle_roll_link + - right_hip_pitch_link + - right_hip_roll_link + - right_hip_yaw_link + - right_knee_link + - right_ankle_pitch_link + - right_ankle_roll_link + - waist_yaw_link + - waist_pitch_link + - torso_link + - left_shoulder_pitch_link + - left_shoulder_roll_link + - left_shoulder_yaw_link + - left_elbow_link + - left_wrist_yaw_link + - left_wrist_pitch_link + - left_wrist_roll_link + - right_shoulder_pitch_link + - right_shoulder_roll_link + - right_shoulder_yaw_link + - right_elbow_link + - right_wrist_yaw_link + - right_wrist_pitch_link + - right_wrist_roll_link sim_dt: 0.005 - control_config: - action_scale: - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - - 0.25 - anchor_pos_z_threshold: 0.5 - ee_body_pos_z_threshold: 0.5 - terminate_on_undesired_contacts: true - noise_config: - level: 0.0 + observations: + actor: + terms: + motion_anchor_pos_b: null + base_lin_vel: null + base_ang_vel: + params: {sensor_name: body-angular-velocity} + critic: + terms: + base_lin_vel: + params: {sensor_name: body-linear-vel} + base_ang_vel: + params: {sensor_name: body-angular-velocity} + commands: + motion: + params: + motion_file: motions/x2/tictacflip_6-3_g1format.npz + anchor_body_name: torso_link + body_names: *tracked_bodies + sampling_mode: start + truncate_on_clip_end: false + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_pitch_link + - left_hip_roll_link + - left_hip_yaw_link + - left_knee_link + - left_ankle_pitch_link + - right_hip_pitch_link + - right_hip_roll_link + - right_hip_yaw_link + - right_knee_link + - right_ankle_pitch_link + - waist_yaw_link + - waist_pitch_link + - torso_link + - left_shoulder_pitch_link + - left_shoulder_roll_link + - left_shoulder_yaw_link + - left_elbow_link + - left_wrist_pitch_link + - left_wrist_roll_link + - right_shoulder_pitch_link + - right_shoulder_roll_link + - right_shoulder_yaw_link + - right_elbow_link + - right_wrist_pitch_link + - right_wrist_roll_link + reward: - scales: - motion_global_root_pos: 0.5 - motion_global_root_ori: 0.5 - motion_body_pos: 2.0 - motion_body_ori: 1.5 - motion_body_lin_vel: 1.0 - motion_body_ang_vel: 1.0 - motion_ee_body_pos_z: 2.0 - motion_joint_pos: 0.5 - motion_joint_vel: 0.25 - action_rate_l2: -0.005 - joint_limit: -10.0 - undesired_contacts: -0.1 - std_root_pos: 0.3 - std_root_ori: 0.4 - std_body_pos: 0.3 - std_body_ori: 0.4 - std_body_lin_vel: 1.0 - std_body_ang_vel: 3.14 - std_joint_pos: 0.2 - std_joint_vel: 1.0 + motion_body_pos: + weight: 2.0 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 + action_rate_l2: + weight: -0.005 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/ppo_cse/config.yaml b/conf/ppo_cse/config.yaml new file mode 100644 index 000000000..0c680449d --- /dev/null +++ b/conf/ppo_cse/config.yaml @@ -0,0 +1,89 @@ +defaults: + - _self_ + - task: a2arm_pos_force/mujoco + +algo: + algo_log_name: ppo_cse + seed: 1 + num_envs: 1024 + num_steps_per_env: 24 + max_iterations: 20000 + save_interval: 500 + empirical_normalization: false + load_run: "-1" + checkpoint: -1 + num_one_step_obs: 73 + num_actor_history: 32 + num_critic_history: 3 + policy: + init_noise_std: 1.0 + actor_hidden_dims: [512, 256, 128] + critic_hidden_dims: [512, 256, 128] + activation: elu + estimator: + num_pred: 12 + enc_hidden_dims: [512, 256, 128] + latent_dim: 64 + dec_hidden_dims: [128, 64] + learning_rate: 1.0e-5 + target_start: 0 + target_group_sizes: [3, 3, 3, 3] + target_weights: [0.2, 0.2, 1.0, 1.0] + algorithm: + value_loss_coef: 1.0 + use_clipped_value_loss: true + clip_param: 0.2 + entropy_coef: 1.0e-2 + num_learning_epochs: 5 + num_mini_batches: 4 + learning_rate: 5.0e-4 + schedule: adaptive + desired_kl: 0.01 + min_learning_rate: 1.0e-5 + max_learning_rate: 1.0e-2 + min_policy_std: 1.0e-2 + max_policy_std: 0.7 + gamma: 0.99 + lam: 0.95 + max_grad_norm: 1.0 + use_amp: false + +training: + task_name: A2ArmPosForce + device: null + logger: tensorboard + wandb_project: unilab + sim_backend: mujoco + play_only: false + no_play: false + play_env_num: 4 + play_steps: 800 + render_spacing: 1.0 + cam_distance: 6.0 + cam_elevation: -20.0 + cam_azimuth: 90.0 + cam_lookat: null + cam_tracking: false + cam_tracking_env_idx: 0 + cam_tracking_extra_envs: 2 + log_root: null + num_timesteps: null + log_dir: null + nan_guard: + enabled: true + buffer_size: 100 + max_envs_to_dump: 5 + output_dir: null + +env: + post_step_forward_sensor: false + +hydra: + run: + dir: . + output_subdir: null + job: + chdir: false + job_logging: + root: + handlers: [console] diff --git a/conf/ppo_cse/task/a2arm_pos_force/base.yaml b/conf/ppo_cse/task/a2arm_pos_force/base.yaml new file mode 100644 index 000000000..620cd1d44 --- /dev/null +++ b/conf/ppo_cse/task/a2arm_pos_force/base.yaml @@ -0,0 +1,121 @@ +# @package _global_ +env: + scene: + model_file: src/unilab/assets/robots/a2arm/scene_pos_force.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base_link + joint_names: [FL_hip_joint, FL_thigh_joint, FL_calf_joint, FR_hip_joint, FR_thigh_joint, FR_calf_joint, RL_hip_joint, RL_thigh_joint, RL_calf_joint, RR_hip_joint, RR_thigh_joint, RR_calf_joint, joint1, joint2, joint4, joint6, joint7] + body_names: [base_link, FL_hip, FL_thigh, FL_calf, FR_hip, FR_thigh, FR_calf, RL_hip, RL_thigh, RL_calf, RR_hip, RR_thigh, RR_calf] + geom_names: [FL, FR, RL, RR, floor] + actuator_names: [FL_hip, FL_thigh, FL_calf, FR_hip, FR_thigh, FR_calf, RL_hip, RL_thigh, RL_calf, RR_hip, RR_thigh, RR_calf, joint1, joint2, joint4, joint6, joint7] + end_effector: + body_names: [end_link] + sim_dt: 0.005 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + history: + _target_: unilab.tasks.locomotion.a2arm.observations.A2ArmActorHistoryCfg + func: unilab.tasks.locomotion.a2arm.observations.A2ArmActorHistory + params: + clip: 100.0 + actor_noise: true + history_length: "${oc.select:algo.num_actor_history,32}" + critic: + terms: + history: + _target_: unilab.tasks.locomotion.a2arm.observations.A2ArmCriticHistoryCfg + func: unilab.tasks.locomotion.a2arm.observations.A2ArmCriticHistory + params: + clip: 100.0 + actor_noise: false + history_length: "${oc.select:algo.num_critic_history,3}" + actions: + joint_pd: + _target_: unilab.tasks.locomotion.a2arm.actions.A2ArmPdActionCfg + entity_name: robot + action_delay_steps: 0 + randomize_motor_strength: true + leg_motor_strength_range: [0.85, 1.15] + arm_motor_strength_range: [0.85, 1.15] + commands: + task_state: + _target_: unilab.tasks.locomotion.a2arm.state.A2ArmPosForceCommandCfg + entity_name: robot + ctrl_dt: 0.02 + command_resample_steps: 250 + force_start_step: 192000 + force_curriculum_scales: [] + force_curriculum_stage_steps: 72000 + max_push_force_gripper_cmd: [-18.0, 15.0] + max_push_force_gripper_ext: [-30.0, 30.0] + max_push_force_base_cmd: [-25.0, 25.0] + max_push_force_base_ext: [-20.0, 20.0] + base_probability_cmd: 0.8 + base_probability_ext: 0.8 + gripper_settling: 25 + base_settling: 50 + force_z_gripper_cmd_scale: 0.33 + force_z_gripper_ext_scale: 0.33 + root_yaw_range: 1.5707963267948966 + randomize_base_mass: true + added_mass_range: [0.0, 4.0] + random_com: true + com_offset_x: [-0.08, 0.08] + com_offset_y: [-0.08, 0.08] + com_offset_z: [-0.08, 0.08] + randomize_foot_friction: true + foot_friction_range: [0.5, 1.8] + randomize_gripper_mass: true + gripper_added_mass_range: [0.0, 0.10] + velocity_push: true + push_interval: 400 + max_push_vel_xy: 0.3 + velocity_push_standing_scale: 1.0 + soft_dof_pos_limit: 0.9 + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + fallen: + func: unilab.tasks.locomotion.a2arm.rewards.a2arm_termination + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel_force_world: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 2.0, params: {name: tracking_lin_vel_force_world, sigma: 0.25}} + tracking_ee_force_world: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 2.0, params: {name: tracking_ee_force_world, sigma: 1.0}} + tracking_ang_vel: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 1.0, params: {name: tracking_ang_vel, sigma: 0.25}} + orientation: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -1.5, params: {name: orientation}} + ref_dof_leg: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 3.0, params: {name: ref_dof_leg, scale: 0.1}} + alive: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 1.5, params: {name: alive}} + lin_vel_z: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -1.5, params: {name: lin_vel_z}} + ang_vel_xy: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -0.1, params: {name: ang_vel_xy}} + action_rate: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -0.02, params: {name: action_rate}} + action_rate_arm: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -0.09, params: {name: action_rate_arm}} + torques: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -5.0e-6, params: {name: torques}} + dof_vel: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -8.0e-4, params: {name: dof_vel}} + dof_vel_arm: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -2.0e-4, params: {name: dof_vel_arm}} + dof_acc: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -2.5e-7, params: {name: dof_acc}} + dof_acc_arm: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -4.5e-7, params: {name: dof_acc_arm}} + base_height: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -2.0, params: {name: base_height, target: 0.435}} + hip_pos: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -0.5, params: {name: hip_pos}} + torque_limits: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -0.005, params: {name: torque_limits, soft_limit: 0.9}} + dof_pos_limits: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -10.0, params: {name: dof_pos_limits}} + stand_still: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 0.5, params: {name: stand_still, scale: 0.05}} + collision: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -5.0, params: {name: collision}} + feet_contact_number: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 2.0, params: {name: feet_contact_number}} + feet_air_time: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 1.0, params: {name: feet_air_time, threshold: 0.5}} + feet_height: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: 1.0, params: {name: feet_height, target: 0.12}} + feet_height_high: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -15.0, params: {name: feet_height_high, target: 0.24}} + feet_pos_xy: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -3.0, params: {name: feet_pos_xy}} + feet_drag: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -8.0e-4, params: {name: feet_drag}} + feet_contact_forces: {func: unilab.tasks.locomotion.a2arm.rewards.a2arm_reward, weight: -1.0e-3, params: {name: feet_contact_forces, threshold: 200.0}} diff --git a/conf/ppo_cse/task/a2arm_pos_force/mujoco.yaml b/conf/ppo_cse/task/a2arm_pos_force/mujoco.yaml new file mode 100644 index 000000000..c02bbbace --- /dev/null +++ b/conf/ppo_cse/task/a2arm_pos_force/mujoco.yaml @@ -0,0 +1,32 @@ +# @package _global_ +defaults: + - /task/a2arm_pos_force/base + - _self_ +training: + task_name: A2ArmPosForce + sim_backend: mujoco +interactive_play_profile: + enabled: true + env: + terminations: + time_out: null + fallen: null +algo: + num_envs: 1024 + max_iterations: 20000 + num_one_step_obs: 73 + num_actor_history: 32 + num_critic_history: 3 + empirical_normalization: false + policy: + init_noise_std: 1.0 + algorithm: + learning_rate: 5.0e-4 + entropy_coef: 1.0e-2 + num_learning_epochs: 5 + desired_kl: 0.01 + min_learning_rate: 1.0e-5 + estimator: + target_start: 0 + target_group_sizes: [3, 3, 3, 3] + target_weights: [0.2, 0.2, 1.0, 1.0] diff --git a/conf/sac/config.yaml b/conf/sac/config.yaml new file mode 100644 index 000000000..59d03e9ff --- /dev/null +++ b/conf/sac/config.yaml @@ -0,0 +1,142 @@ +defaults: + - _self_ + - task: g1_walk_flat/mujoco + +algo: + algo: sac + algo_log_name: fast_sac + runtime_impl: null + runtime_resolver: null + load_run: "-1" + seed: 1 + num_envs: 4096 + # Learner batch size for one SAC update. + batch_size: 8192 + replay_buffer_n: 512 + updates_per_step: 4 + learning_starts: 1 + policy_frequency: 4 + max_iterations: 500 + save_interval: 500 + gamma: 0.97 + tau: 0.125 + actor_lr: 3.0e-4 + critic_lr: 3.0e-4 + actor_hidden_dim: 512 + critic_hidden_dim: 768 + num_atoms: 101 + obs_normalization: false + use_layer_norm: true + actor: {} + algo_params: + alpha_lr: 3.0e-4 + alpha_init: 0.01 + target_entropy_ratio: 0.0 + max_grad_norm: 0.0 + amp_dtype: auto + use_compile: true + use_cuda_graph_critic: false + use_cuda_graph_actor: false + use_cuda_graph_critic_packed_staging: false + use_cuda_graph_actor_packed_staging: false + +training: + task_name: G1WalkFlat + # list[int] | null; null/[] = auto-selected single-device behavior; + # [d0] = explicit single CUDA device; [d0..dN-1] = N-way data parallel, + # rank i trains on cuda:devices[i]. + devices: null + # list[list[int]] | null; one CPU-id segment per rank for the collector's + # MuJoCo pool; null = auto partition of cpu_count // world_size per rank. + dp_collector_cpu_ids: null + logger: tensorboard + wandb_project: unilab + wandb_entity: null + wandb_group: null + wandb_job_type: null + wandb_name: null + wandb_tags: [] + wandb_notes: null + wandb_mode: null + sim_backend: mujoco + nan_guard: + enabled: true + buffer_size: 100 + max_envs_to_dump: 5 + output_dir: null + use_amp: true + play_only: false + no_play: false + sim2sim_strict: true + play_render_mode: auto + export_onnx: true + play_env_num: 16 + play_steps: 800 + cam_distance: 6.0 + cam_elevation: -20.0 + cam_azimuth: 90.0 + log_root: null + log_dir: null + env_steps_per_sync: 1 + trace_enabled: false + trace_output_dir: null + trace_thread_time: false + trace_cuda_events: true + nvtx_profile_ranges: false + replay_prefetch_mode: one_tick + torch_threads: + enabled: true + # "auto" resolves per process role from host CPU count with conservative caps. + # Override these from Hydra when benchmarking a specific machine. + learner_num_threads: auto + collector_num_threads: auto + learner_num_interop_threads: 1 + collector_num_interop_threads: 1 + compile_threads: auto + set_env_vars: true + +interactive: + action_mode: zero + policy_obs_mode: auto + show_target_bodies: false + show_reward_debug: false + target_show_axes: false + target_body_names: "" + target_max_bodies: 0 + target_marker_radius: 0.02 + target_axis_length: 0.08 + target_marker_alpha: 0.75 + reward_debug_show_velocity: false + reward_debug_lin_vel_scale: 0.08 + reward_debug_ang_vel_scale: 0.05 + reward_debug_show_connectors: false + reward_debug_show_global_anchor: false + camera_follow_body: true + camera_focus_body_name: "" + camera_height_offset: 0.15 + camera_distance: null + camera_elevation: null + camera_azimuth: null + use_env_visual_model: true + speed: 1.0 + start_paused: false + keyboard: false + keyboard_step_lin: 0.1 + keyboard_step_ang: 0.2 + +env: + post_step_forward_sensor: false + # adaptive_chunk_size: auto-tune the MuJoCo BatchEnvPool chunk_size at materialize + # (cache-backed). chunk_size (int) manually overrides and wins; null => use default. + adaptive_chunk_size: true + chunk_size: null + +hydra: + run: + dir: . + output_subdir: null + job: + chdir: false + job_logging: + root: + handlers: [console] diff --git a/conf/sac/task/g1_23dof_flip_tracking/mujoco.yaml b/conf/sac/task/g1_23dof_flip_tracking/mujoco.yaml new file mode 100644 index 000000000..70ce6e8e7 --- /dev/null +++ b/conf/sac/task/g1_23dof_flip_tracking/mujoco.yaml @@ -0,0 +1,105 @@ +# @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + +training: + task_name: G1FlipTrackingSAC23Dof + sim_backend: mujoco + play_steps: 1000 + +algo: + num_envs: 4096 + max_iterations: 25000 + save_interval: 1000 + gamma: 0.99 + tau: 0.05 + num_atoms: 501 + updates_per_step: 4 + policy_frequency: 2 + algo_params: + alpha_init: 0.005 + target_entropy_ratio: 0.05 + max_grad_norm: 10.0 + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + sim_dt: 0.005 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_wrist_roll_joint": 0.43857731392336724 + commands: + motion: + params: + motion_file: motions/g1/flip_360_001__A304_23dof.npz + sampling_mode: mixed + sampling_start_ratio: 0.1 + truncate_on_clip_end: true + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + +reward: + motion_global_root_pos: + weight: 0.5 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + action_rate_l2: + weight: -0.005 + joint_limit: + weight: -10.0 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/sac/task/g1_23dof_motion_tracking/motrix.yaml b/conf/sac/task/g1_23dof_motion_tracking/motrix.yaml new file mode 100644 index 000000000..735e89be0 --- /dev/null +++ b/conf/sac/task/g1_23dof_motion_tracking/motrix.yaml @@ -0,0 +1,9 @@ +# @package _global_ +# Motrix is the sim2sim eval owner for MuJoCo-trained WBT checkpoints. +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + +training: + task_name: G1MotionTrackingSAC23Dof + sim_backend: motrix diff --git a/conf/sac/task/g1_23dof_motion_tracking/mujoco.yaml b/conf/sac/task/g1_23dof_motion_tracking/mujoco.yaml new file mode 100644 index 000000000..da1338c9e --- /dev/null +++ b/conf/sac/task/g1_23dof_motion_tracking/mujoco.yaml @@ -0,0 +1,76 @@ +# @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + +training: + task_name: G1MotionTrackingSAC23Dof + sim_backend: mujoco + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: &g1_23dof_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: *g1_23dof_joints + body_names: &tracked_bodies_23dof + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_roll_rubber_hand + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_roll_rubber_hand + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + ".*_(shoulder_(pitch|roll|yaw)|elbow)_joint": 0.43857731392336724 + ".*_wrist_roll_joint": 0.07450087032950714 + commands: + motion: + params: + motion_file: motions/g1/dance1_subject2_part_23dof.npz + body_names: *tracked_bodies_23dof + terminations: + ee_body_pos: + params: + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand diff --git a/conf/sac/task/g1_23dof_walk_flat/base.yaml b/conf/sac/task/g1_23dof_walk_flat/base.yaml new file mode 100644 index 000000000..c7571348f --- /dev/null +++ b/conf/sac/task/g1_23dof_walk_flat/base.yaml @@ -0,0 +1,66 @@ +# @package _global_ +# Canonical G1 23-DoF walk Manager-Based task declaration (off-policy owners). +# Inherits the 29-DoF off-policy contract and swaps the scene to the 23-DoF +# model (no waist roll/pitch, no wrist pitch/yaw) with 23-entry pose weights. +defaults: + - /task/g1_walk_flat/base + - _self_ + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + +reward: + pose: + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/sac/task/g1_23dof_walk_flat/motrix.yaml b/conf/sac/task/g1_23dof_walk_flat/motrix.yaml new file mode 100644 index 000000000..99d75a942 --- /dev/null +++ b/conf/sac/task/g1_23dof_walk_flat/motrix.yaml @@ -0,0 +1,44 @@ +# @package _global_ +# SAC Motrix 23-DoF owner: keeps DENYLIST parity with the MuJoCo owner and +# retunes reward shaping / disables kp/kd randomization for Motrix. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofFlat + sim_backend: motrix +algo: + num_envs: 2048 + learning_starts: 1 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null +reward: + tracking_lin_vel: + weight: 2.2 + tracking_ang_vel: + weight: 1.8 + penalty_ang_vel_xy: + weight: -1.2 + penalty_orientation: + weight: -12.0 + penalty_action_rate: + weight: -2.5 + pose: + weight: -0.6 + penalty_feet_ori: + weight: -5.0 + feet_phase: + weight: 6.0 + params: + tracking_sigma: 0.008 + alive: + weight: 12.0 diff --git a/conf/sac/task/g1_23dof_walk_flat/mujoco.yaml b/conf/sac/task/g1_23dof_walk_flat/mujoco.yaml new file mode 100644 index 000000000..d06148b02 --- /dev/null +++ b/conf/sac/task/g1_23dof_walk_flat/mujoco.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# SAC MuJoCo 23-DoF owner: inherits the 23-DoF off-policy Manager-Based +# contract and only carries backend/algo identity. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofFlat + sim_backend: mujoco +algo: + num_envs: 2048 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 diff --git a/conf/sac/task/g1_23dof_walk_rough/motrix.yaml b/conf/sac/task/g1_23dof_walk_rough/motrix.yaml new file mode 100644 index 000000000..10899e153 --- /dev/null +++ b/conf/sac/task/g1_23dof_walk_rough/motrix.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# SAC Motrix 23-DoF rough owner: static-hfield rough scene, Motrix +# sim_dt=0.01, Motrix-direction reward retuning; kp/kd randomization disabled. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofRough + sim_backend: motrix +algo: + num_envs: 2048 + learning_starts: 1 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + sim_dt: 0.01 + scene: + model_file: src/unilab/assets/robots/g1/scene_rough_23dof.xml + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null +reward: + tracking_lin_vel: + weight: 2.2 + tracking_ang_vel: + weight: 1.8 + penalty_ang_vel_xy: + weight: -1.2 + penalty_orientation: + weight: -12.0 + penalty_action_rate: + weight: -2.5 + pose: + weight: -0.6 + penalty_feet_ori: + weight: -5.0 + feet_phase: + weight: 6.0 + params: + tracking_sigma: 0.008 + alive: + weight: 12.0 diff --git a/conf/sac/task/g1_23dof_walk_rough/mujoco.yaml b/conf/sac/task/g1_23dof_walk_rough/mujoco.yaml new file mode 100644 index 000000000..b6b0fb319 --- /dev/null +++ b/conf/sac/task/g1_23dof_walk_rough/mujoco.yaml @@ -0,0 +1,22 @@ +# @package _global_ +# SAC MuJoCo 23-DoF rough owner: 23-DoF off-policy contract plus the +# static-hfield rough scene. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofRough + sim_backend: mujoco +algo: + num_envs: 2048 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_rough_23dof.xml diff --git a/conf/sac/task/g1_23dof_wall_flip_tracking/mujoco.yaml b/conf/sac/task/g1_23dof_wall_flip_tracking/mujoco.yaml new file mode 100644 index 000000000..0a111f66a --- /dev/null +++ b/conf/sac/task/g1_23dof_wall_flip_tracking/mujoco.yaml @@ -0,0 +1,41 @@ +# @package _global_ +defaults: + - /task/g1_23dof_flip_tracking/mujoco + - _self_ + +training: + task_name: G1WallFlipTrackingSAC23Dof + sim_backend: mujoco + +algo: + algo_params: + target_entropy_ratio: 0.0 + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof_with_wall.xml + commands: + motion: + params: + motion_file: motions/g1/flip_from_wall_104__A304_23dof.npz + sampling_mode: uniform + sampling_start_ratio: 0.0 + terminations: + anchor_pos: + params: {command_name: motion, threshold: 1.0e9} + ee_body_pos: + params: + command_name: motion + threshold: 1.0e9 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + undesired_contacts: null + +reward: + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 diff --git a/conf/sac/task/g1_23dof_wbt_obs/mujoco.yaml b/conf/sac/task/g1_23dof_wbt_obs/mujoco.yaml new file mode 100644 index 000000000..bc9e7edfb --- /dev/null +++ b/conf/sac/task/g1_23dof_wbt_obs/mujoco.yaml @@ -0,0 +1,186 @@ +# @package _global_ +defaults: + - /task/g1_23dof_motion_tracking/mujoco + - _self_ + +training: + task_name: G1WBTObs23Dof + sim_backend: mujoco + +algo: + num_envs: 4096 + max_iterations: 140000 + save_interval: 1000 + gamma: 0.99 + tau: 0.05 + num_atoms: 501 + updates_per_step: 4 + policy_frequency: 2 + algo_params: + alpha_init: 0.1 + target_entropy_ratio: 0.5 + max_grad_norm: 10.0 + +env: + sim_dt: 0.005 + scene: + entities: + robot: + geom_names: + - left_foot1_collision + - left_foot2_collision + - left_foot3_collision + - left_foot4_collision + - left_foot5_collision + - left_foot6_collision + - left_foot7_collision + - right_foot1_collision + - right_foot2_collision + - right_foot3_collision + - right_foot4_collision + - right_foot5_collision + - right_foot6_collision + - right_foot7_collision + observations: + actor: + terms: + motion_anchor_pos_b: null + motion_anchor_ori_b: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_ori_b + params: {command_name: motion} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.05 + n_max: 0.05 + base_lin_vel: null + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_gyro} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.2 + n_max: 0.2 + history_length: 5 + joint_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_pos_rel_biased + params: {command_name: motion} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + history_length: 5 + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.5 + n_max: 0.5 + history_length: 5 + actions: + func: unilab.envs.mdp.last_action + history_length: 5 + critic: + terms: + base_ang_vel: + params: {sensor_name: pelvis_gyro} + actions: + joint_pos: + scale: 2.0 + simulate_action_latency: true + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.4} + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_roll_rubber_hand + - right_wrist_roll_rubber_hand + events: + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + body_names: pelvis + mass_distribution_params: [-1.0, 1.0] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + body_names: pelvis + com_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.05, 0.05] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.85, 1.15] + operation: scale + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + actuator_names: ".*" + foot_friction: + func: unilab.envs.mdp.geom_friction + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + geom_names: "^(left|right)_foot[1-7]_collision$" + ranges: [0.3, 1.2] + operation: abs + shared_random: true + encoder_bias: + func: unilab.tasks.motion_tracking.g1.manager_terms.randomize_encoder_bias + mode: reset + params: + bias_range: [-0.01, 0.01] + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: ".*" + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [4.0, 4.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + +reward: + motion_global_root_ori: + weight: 1.0 + motion_body_pos: + weight: 1.0 + action_rate_l2: + weight: -0.1 + joint_limit: + weight: -5.0 + joint_acc_l2: + func: unilab.tasks.motion_tracking.g1.manager_terms.joint_acc_l2 + weight: -2.5e-7 + joint_torque_l2: + func: unilab.tasks.motion_tracking.g1.manager_terms.joint_torque_l2 + weight: -1.0e-5 + params: {action_name: joint_pos} diff --git a/conf/sac/task/g1_flip_tracking/mujoco.yaml b/conf/sac/task/g1_flip_tracking/mujoco.yaml new file mode 100644 index 000000000..5e89be9b3 --- /dev/null +++ b/conf/sac/task/g1_flip_tracking/mujoco.yaml @@ -0,0 +1,107 @@ +# @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + +training: + task_name: G1FlipTrackingSAC + sim_backend: mujoco + play_steps: 1000 + +algo: + num_envs: 4096 + max_iterations: 25000 + save_interval: 1000 + gamma: 0.99 + tau: 0.05 + num_atoms: 501 + updates_per_step: 4 + policy_frequency: 2 + algo_params: + alpha_init: 0.005 + target_entropy_ratio: 0.05 + max_grad_norm: 10.0 + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + sim_dt: 0.005 + actions: + joint_pos: + scale: + ".*_(hip_pitch|hip_yaw)_joint": 0.5475464629911068 + ".*_(hip_roll|knee)_joint": 0.35066146637882434 + ".*_ankle_(pitch|roll)_joint": 0.43857731392336724 + "waist_yaw_joint": 0.5475464629911068 + "waist_(roll|pitch)_joint": 0.43857731392336724 + ".*_(shoulder_(pitch|roll|yaw)|elbow|wrist_roll)_joint": 0.43857731392336724 + ".*_wrist_(pitch|yaw)_joint": 0.07450087032950714 + commands: + motion: + params: + motion_file: motions/g1/flip_360_001__A304.npz + sampling_mode: mixed + sampling_start_ratio: 0.1 + truncate_on_clip_end: true + pose_range: &zero_pose + x: [0.0, 0.0] + y: [0.0, 0.0] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + velocity_range: *zero_pose + joint_position_range: [0.0, 0.0] + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.5} + anchor_ori: + params: + command_name: motion + threshold: 1.0e9 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: &ee_bodies + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_undesired_body_contacts + params: + command_name: motion + threshold: 0.05 + body_names: &undesired_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link + +reward: + motion_global_root_pos: + weight: 0.5 + motion_body_ori: + weight: 1.5 + motion_ee_body_pos_z: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_z_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3, body_names: *ee_bodies} + action_rate_l2: + weight: -0.005 + joint_limit: + weight: -10.0 + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: {command_name: motion, threshold: 0.05, body_names: *undesired_bodies} diff --git a/conf/sac/task/g1_motion_tracking/mjwarp.yaml b/conf/sac/task/g1_motion_tracking/mjwarp.yaml new file mode 100644 index 000000000..3f5ca8dc6 --- /dev/null +++ b/conf/sac/task/g1_motion_tracking/mjwarp.yaml @@ -0,0 +1,14 @@ +# @package _global_ +# mjwarp (GPU) owner for G1 WBT. Benchmark/support scope: collector active-window +# throughput measurement for issue #1292; mujoco-warp + warp-lang stay optional +# deps. Inherits the MuJoCo owner verbatim so sim2sim DENYLIST fields keep parity; +# mjwarp capacity knobs stay at backend defaults (nconmax/njmax=512). +# NOTE: mjwarp rejects reset/interval domain randomization and push terms; this +# owner inherits the MuJoCo DR-free config, so no extra disables are needed. +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + +training: + task_name: G1MotionTrackingSAC + sim_backend: mjwarp diff --git a/conf/sac/task/g1_motion_tracking/motrix.yaml b/conf/sac/task/g1_motion_tracking/motrix.yaml new file mode 100644 index 000000000..85904de4b --- /dev/null +++ b/conf/sac/task/g1_motion_tracking/motrix.yaml @@ -0,0 +1,9 @@ +# @package _global_ +# Motrix is the sim2sim eval owner for MuJoCo-trained WBT checkpoints. +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + +training: + task_name: G1MotionTrackingSAC + sim_backend: motrix diff --git a/conf/sac/task/g1_motion_tracking/mujoco.yaml b/conf/sac/task/g1_motion_tracking/mujoco.yaml new file mode 100644 index 000000000..f0d7555a7 --- /dev/null +++ b/conf/sac/task/g1_motion_tracking/mujoco.yaml @@ -0,0 +1,268 @@ +# @package _global_ +# G1 Whole-Body Tracking (WBT) with FastSAC on MuJoCo. +# Hyperparameters aligned with holosoma g1-29dof-wbt-fast-sac. +training: + task_name: G1MotionTrackingSAC + sim_backend: mujoco + +algo: + num_envs: 2048 + max_iterations: 25000 + save_interval: 1000 + gamma: 0.99 + tau: 0.05 + num_atoms: 501 + updates_per_step: 4 + policy_frequency: 2 + algo_params: + alpha_init: 0.1 + target_entropy_ratio: 0.5 + max_grad_norm: 10.0 + +env: + seed: null + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: &g1_joints + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: *g1_joints + body_names: &tracked_bodies + - pelvis + - left_hip_roll_link + - left_knee_link + - left_ankle_roll_link + - right_hip_roll_link + - right_knee_link + - right_ankle_roll_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - left_wrist_yaw_link + - right_shoulder_roll_link + - right_elbow_link + - right_wrist_yaw_link + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 10.0 + observations: + actor: + enable_corruption: true + terms: + command: &command_obs + func: unilab.envs.mdp.generated_commands + params: {command_name: motion} + motion_anchor_pos_b: &anchor_pos_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_pos_b + params: {command_name: motion} + motion_anchor_ori_b: &anchor_ori_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_ori_b + params: {command_name: motion} + base_lin_vel: &base_lin_vel_obs + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.1 + n_max: 0.1 + base_ang_vel: &base_ang_vel_obs + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.2 + n_max: 0.2 + joint_pos: &joint_pos_obs + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_pos_rel + params: {command_name: motion} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + joint_vel: &joint_vel_obs + func: unilab.envs.mdp.joint_vel_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -1.5 + n_max: 1.5 + actions: &actions_obs + func: unilab.envs.mdp.last_action + critic: + terms: + command: *command_obs + motion_anchor_pos_b: *anchor_pos_obs + motion_anchor_ori_b: *anchor_ori_obs + base_lin_vel: *base_lin_vel_obs + base_ang_vel: *base_ang_vel_obs + joint_pos: *joint_pos_obs + joint_vel: *joint_vel_obs + actions: *actions_obs + body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.robot_body_pos_b + params: {command_name: motion} + body_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.robot_body_ori_b + params: {command_name: motion} + sac_base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + actions: + joint_pos: + _target_: unilab.tasks.motion_tracking.common.manager_terms.MotionJointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 2.0 + use_default_offset: true + command_name: motion + commands: + motion: + _target_: unilab.tasks.motion_tracking.common.manager_terms.MotionCommandCfg + entity_name: robot + resampling_time_range: [1.0e9, 1.0e9] + params: + motion_file: motions/g1/dance1_subject2_part.npz + anchor_body_name: torso_link + body_names: *tracked_bodies + sampling_mode: adaptive + sampling_start_ratio: 0.0 + truncate_on_clip_end: true + pose_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.01, 0.01] + roll: [-0.1, 0.1] + pitch: [-0.1, 0.1] + yaw: [-0.2, 0.2] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.2, 0.2] + roll: [-0.52, 0.52] + pitch: [-0.52, 0.52] + yaw: [-0.78, 0.78] + joint_position_range: [-0.1, 0.1] + joint_default_position_range: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + motion_clip_end: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_clip_end + time_out: true + params: {command_name: motion} + anchor_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_anchor_pos_z_only + params: {command_name: motion, threshold: 0.5} + anchor_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_anchor_ori + params: + command_name: motion + threshold: 0.8 + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + ee_body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.bad_motion_body_pos_z_only + params: + command_name: motion + threshold: 0.5 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + policy_observation_group: actor + critic_observation_group: critic + +reward: + motion_global_root_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_anchor_position_error_exp + weight: 1.0 + params: {command_name: motion, std: 0.3} + motion_global_root_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_anchor_orientation_error_exp + weight: 0.5 + params: {command_name: motion, std: 0.4} + motion_body_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_position_error_exp + weight: 2.0 + params: {command_name: motion, std: 0.3} + motion_body_ori: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_relative_body_orientation_error_exp + weight: 1.0 + params: {command_name: motion, std: 0.4} + motion_body_lin_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_body_linear_velocity_error_exp + weight: 1.0 + params: {command_name: motion, std: 1.0} + motion_body_ang_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_global_body_angular_velocity_error_exp + weight: 1.0 + params: {command_name: motion, std: 3.14} + motion_joint_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_position_error_exp + weight: 0.0 + params: {command_name: motion, std: 0.2} + motion_joint_vel: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_velocity_error_exp + weight: 0.0 + params: {command_name: motion, std: 1.0} + action_rate_l2: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.1 + joint_limit: + func: unilab.tasks.motion_tracking.common.manager_terms.joint_pos_limits + weight: -2.0 + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: ".*" + undesired_contacts: + func: unilab.tasks.motion_tracking.common.manager_terms.undesired_body_contacts + weight: -0.1 + params: + command_name: motion + threshold: 0.05 + body_names: + - pelvis + - left_hip_roll_link + - left_knee_link + - right_hip_roll_link + - right_knee_link + - torso_link + - left_shoulder_roll_link + - left_elbow_link + - right_shoulder_roll_link + - right_elbow_link diff --git a/conf/sac/task/g1_walk_flat/base.yaml b/conf/sac/task/g1_walk_flat/base.yaml new file mode 100644 index 000000000..a058d94f5 --- /dev/null +++ b/conf/sac/task/g1_walk_flat/base.yaml @@ -0,0 +1,272 @@ +# @package _global_ +# Canonical G1 29-DoF walk Manager-Based task declaration (off-policy owners). +# Backend owner leaves inherit this file and only override backend/algo tuning +# or explicitly disabled terms. Observation scaling follows the walk profile +# (gyro x0.25, joint velocity x0.05, critic linear velocity x2.0); every +# off-policy owner carries the penalty curriculum. +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + body_names: [pelvis] + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + # Observation noise matches the legacy off-policy noise_config (level=1.0, + # actor-only; gyro/gravity/linvel scales were 0.0 there): joint pos + # +/-0.01 and joint vel +/-0.1, applied before term scaling. + enable_corruption: true + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + operation: add + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.1 + n_max: 0.1 + operation: add + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + scale: 2.0 + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 1.0 + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + tilt: + func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded + params: + max_tilt_deg: 65.0 + base_height: + func: unilab.envs.mdp.root_height_below_minimum + params: + minimum_height: 0.3 + curriculum: + penalty_scaling: + func: unilab.tasks.locomotion.g1.manager_terms.G1PenaltyCurriculum + # Effective schedule matches the tuned legacy baseline: the legacy env + # halved the shared override dict once per env construction (two probe + # envs + the collector in every off-policy runner), so collectors actually + # trained at 1/8 initial / 1/4 cap of these YAML weights. The manager + # runtime isolates each env, so the tuned effective range is declared + # explicitly here. + params: + initial_scale: 0.125 + min_scale: 0.125 + max_scale: 0.25 + level_down_threshold: 150.0 + level_up_threshold: 750.0 + degree: 0.001 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel + weight: 2.0 + params: + tracking_sigma: 0.25 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel + weight: 1.5 + params: + tracking_sigma: 0.25 + command_name: twist + penalty_ang_vel_xy: + func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy + weight: -1.0 + penalty_orientation: + func: unilab.tasks.locomotion.g1.manager_terms.orientation + weight: -10.0 + penalty_action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -4.0 + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.5 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori + weight: -20.0 + feet_phase: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase + weight: 5.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.04 + min_forward_speed: 0.0 + command_name: twist + alive: + func: unilab.tasks.locomotion.g1.manager_terms.alive + weight: 10.0 diff --git a/conf/sac/task/g1_walk_flat/mjwarp.yaml b/conf/sac/task/g1_walk_flat/mjwarp.yaml new file mode 100644 index 000000000..80702f2c1 --- /dev/null +++ b/conf/sac/task/g1_walk_flat/mjwarp.yaml @@ -0,0 +1,29 @@ +# @package _global_ +# Configured-only SAC mjwarp owner for the unified host contract adapter. Keeps +# DENYLIST parity with the MuJoCo owner plus the mjwarp capacity knobs; legacy +# kp/kd randomization is disabled. Offline record reuses MuJoCo rendering; +# native playback and device-resident runtime are intentionally absent. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: mjwarp + play_render_mode: record +algo: + num_envs: 2048 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + mjwarp_nconmax: 128 + mjwarp_njmax: 256 + render_spacing: 2.0 + events: + # Legacy mjwarp owners disable kp/kd and armature randomization. + pd_gains: null diff --git a/conf/sac/task/g1_walk_flat/motrix.yaml b/conf/sac/task/g1_walk_flat/motrix.yaml new file mode 100644 index 000000000..bc58ce374 --- /dev/null +++ b/conf/sac/task/g1_walk_flat/motrix.yaml @@ -0,0 +1,44 @@ +# @package _global_ +# SAC Motrix owner: keeps DENYLIST parity with the MuJoCo owner and retunes +# reward shaping / disables kp/kd randomization for the Motrix direction. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: motrix +algo: + num_envs: 2048 + learning_starts: 1 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null +reward: + tracking_lin_vel: + weight: 2.2 + tracking_ang_vel: + weight: 1.8 + penalty_ang_vel_xy: + weight: -1.2 + penalty_orientation: + weight: -12.0 + penalty_action_rate: + weight: -2.5 + pose: + weight: -0.6 + penalty_feet_ori: + weight: -5.0 + feet_phase: + weight: 6.0 + params: + tracking_sigma: 0.008 + alive: + weight: 12.0 diff --git a/conf/sac/task/g1_walk_flat/mujoco.yaml b/conf/sac/task/g1_walk_flat/mujoco.yaml new file mode 100644 index 000000000..7c98fc381 --- /dev/null +++ b/conf/sac/task/g1_walk_flat/mujoco.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# SAC MuJoCo owner: inherits the 29-DoF off-policy Manager-Based contract from +# the shared base and only carries backend/algo identity. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: mujoco +algo: + num_envs: 2048 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 diff --git a/conf/sac/task/g1_walk_rough/motrix.yaml b/conf/sac/task/g1_walk_rough/motrix.yaml new file mode 100644 index 000000000..e73c5b962 --- /dev/null +++ b/conf/sac/task/g1_walk_rough/motrix.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# SAC Motrix rough owner: static-hfield rough scene, Motrix sim_dt=0.01, and +# the Motrix-direction reward retuning; kp/kd randomization stays disabled. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkRough + sim_backend: motrix +algo: + num_envs: 2048 + learning_starts: 1 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + sim_dt: 0.01 + scene: + model_file: src/unilab/assets/robots/g1/scene_rough.xml + events: + # Legacy Motrix owners disable kp/kd randomization. + pd_gains: null +reward: + tracking_lin_vel: + weight: 2.2 + tracking_ang_vel: + weight: 1.8 + penalty_ang_vel_xy: + weight: -1.2 + penalty_orientation: + weight: -12.0 + penalty_action_rate: + weight: -2.5 + pose: + weight: -0.6 + penalty_feet_ori: + weight: -5.0 + feet_phase: + weight: 6.0 + params: + tracking_sigma: 0.008 + alive: + weight: 12.0 diff --git a/conf/sac/task/g1_walk_rough/mujoco.yaml b/conf/sac/task/g1_walk_rough/mujoco.yaml new file mode 100644 index 000000000..cce086903 --- /dev/null +++ b/conf/sac/task/g1_walk_rough/mujoco.yaml @@ -0,0 +1,23 @@ +# @package _global_ +# SAC MuJoCo rough owner: inherits the 29-DoF off-policy Manager-Based contract +# and swaps the scene to the static-hfield rough XML (no height-scan +# observation and no terrain curriculum, matching the legacy rough task). +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkRough + sim_backend: mujoco +algo: + num_envs: 2048 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_rough.xml diff --git a/conf/sac/task/g1_wall_flip_tracking/mujoco.yaml b/conf/sac/task/g1_wall_flip_tracking/mujoco.yaml new file mode 100644 index 000000000..68ddd8efd --- /dev/null +++ b/conf/sac/task/g1_wall_flip_tracking/mujoco.yaml @@ -0,0 +1,41 @@ +# @package _global_ +defaults: + - /task/g1_flip_tracking/mujoco + - _self_ + +training: + task_name: G1WallFlipTrackingSAC + sim_backend: mujoco + +algo: + algo_params: + target_entropy_ratio: 0.0 + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_with_wall.xml + commands: + motion: + params: + motion_file: motions/g1/flip_from_wall_104__A304.npz + sampling_mode: uniform + sampling_start_ratio: 0.0 + terminations: + anchor_pos: + params: {command_name: motion, threshold: 1.0e9} + ee_body_pos: + params: + command_name: motion + threshold: 1.0e9 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + undesired_contacts: null + +reward: + motion_joint_pos: + weight: 0.5 + motion_joint_vel: + weight: 0.25 diff --git a/conf/sac/task/g1_wbt_obs/mujoco.yaml b/conf/sac/task/g1_wbt_obs/mujoco.yaml new file mode 100644 index 000000000..4c16924bd --- /dev/null +++ b/conf/sac/task/g1_wbt_obs/mujoco.yaml @@ -0,0 +1,185 @@ +# @package _global_ +defaults: + - /task/g1_motion_tracking/mujoco + - _self_ + +training: + task_name: G1WBTObs + sim_backend: mujoco + +algo: + num_envs: 4096 + max_iterations: 140000 + save_interval: 1000 + gamma: 0.99 + tau: 0.05 + num_atoms: 501 + updates_per_step: 4 + policy_frequency: 2 + algo_params: + alpha_init: 0.1 + target_entropy_ratio: 0.5 + max_grad_norm: 10.0 + +env: + sim_dt: 0.005 + scene: + entities: + robot: + geom_names: + - left_foot1_collision + - left_foot2_collision + - left_foot3_collision + - left_foot4_collision + - left_foot5_collision + - left_foot6_collision + - left_foot7_collision + - right_foot1_collision + - right_foot2_collision + - right_foot3_collision + - right_foot4_collision + - right_foot5_collision + - right_foot6_collision + - right_foot7_collision + observations: + actor: + terms: + motion_anchor_pos_b: null + motion_anchor_ori_b: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_anchor_ori_b + params: {command_name: motion} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.05 + n_max: 0.05 + base_lin_vel: null + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_gyro} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.2 + n_max: 0.2 + history_length: 5 + joint_pos: + func: unilab.tasks.motion_tracking.common.manager_terms.motion_joint_pos_rel_biased + params: {command_name: motion} + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + history_length: 5 + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.5 + n_max: 0.5 + history_length: 5 + actions: + func: unilab.envs.mdp.last_action + history_length: 5 + critic: + terms: + base_ang_vel: + params: {sensor_name: pelvis_gyro} + actions: + joint_pos: + simulate_action_latency: true + terminations: + anchor_pos: + params: {command_name: motion, threshold: 0.4} + ee_body_pos: + params: + command_name: motion + threshold: 0.5 + body_names: + - left_ankle_roll_link + - right_ankle_roll_link + - left_wrist_yaw_link + - right_wrist_yaw_link + events: + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + body_names: pelvis + mass_distribution_params: [-1.0, 1.0] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + body_names: pelvis + com_range: + x: [-0.05, 0.05] + y: [-0.05, 0.05] + z: [-0.05, 0.05] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.85, 1.15] + operation: scale + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + actuator_names: ".*" + foot_friction: + func: unilab.envs.mdp.geom_friction + mode: reset + params: + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + geom_names: "^(left|right)_foot[1-7]_collision$" + ranges: [0.3, 1.2] + operation: abs + shared_random: true + encoder_bias: + func: unilab.tasks.motion_tracking.g1.manager_terms.randomize_encoder_bias + mode: reset + params: + bias_range: [-0.01, 0.01] + asset_cfg: + _target_: unilab.managers.SceneEntityCfg + name: robot + joint_names: ".*" + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [4.0, 4.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + +reward: + motion_global_root_ori: + weight: 1.0 + motion_body_pos: + weight: 1.0 + action_rate_l2: + weight: -0.1 + joint_limit: + weight: -5.0 + joint_acc_l2: + func: unilab.tasks.motion_tracking.g1.manager_terms.joint_acc_l2 + weight: -2.5e-7 + joint_torque_l2: + func: unilab.tasks.motion_tracking.g1.manager_terms.joint_torque_l2 + weight: -1.0e-5 + params: {action_name: joint_pos} diff --git a/conf/sac/task/go2_footstand/base.yaml b/conf/sac/task/go2_footstand/base.yaml new file mode 100644 index 000000000..08095758c --- /dev/null +++ b/conf/sac/task/go2_footstand/base.yaml @@ -0,0 +1,217 @@ +# @package _global_ +# Canonical Go2 footstand Manager-Based declaration for off-policy training. +# Keep this task surface aligned with the PPO owner; Hydra remains the sole entry. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + body_names: + - base + - FL_hip + - FL_thigh + - FL_calf + - FR_hip + - FR_thigh + - FR_calf + - RL_hip + - RL_thigh + - RL_calf + - RR_hip + - RR_thigh + - RR_calf + geom_names: [floor] + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.004 + ctrl_dt: 0.02 + max_episode_seconds: 10.0 + adaptive_chunk_size: false + observations: + policy: + enable_corruption: true + terms: + frame: + func: unilab.tasks.locomotion.go2.footstand.frame_observation + params: + action_name: joint_pos + noise: + _target_: UniformNoiseCfg + n_min: [-0.1, -0.1, -0.1, -0.2, -0.2, -0.2, -0.05, -0.05, -0.05, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -0.01, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, -1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + n_max: [0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + history_length: 15 + critic: + terms: + frame: + func: unilab.tasks.locomotion.go2.footstand.frame_observation + params: + action_name: joint_pos + history_length: 15 + privileged: + func: unilab.tasks.locomotion.go2.footstand.privileged_observation + params: + action_name: joint_pos + actions: + joint_pos: + _target_: unilab.tasks.locomotion.go2.footstand.FootstandIncrementalActionCfg + entity_name: robot + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + joint_position_limits: + - [-1.0472, 1.0472] + - [-1.5708, 3.4907] + - [-2.7227, -0.83776] + - [-1.0472, 1.0472] + - [-1.5708, 3.4907] + - [-2.7227, -0.83776] + - [-1.0472, 1.0472] + - [-0.5236, 4.5379] + - [-2.7227, -0.83776] + - [-1.0472, 1.0472] + - [-0.5236, 4.5379] + - [-2.7227, -0.83776] + action_scale: 0.3 + clip_actions: 1.0 + kp: 35.0 + kd: 0.5 + simulate_action_latency: false + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + reset_joints: + func: unilab.tasks.locomotion.go2.footstand.FootstandJointReset + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*" + position_offset_range: [-0.05, 0.05] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [35.0, 35.0] + kd_range: [0.5, 0.5] + operation: abs + floor_friction: null + link_mass: null + torso_com: null + joint_armature: null + terminations: + footstand: + func: unilab.tasks.locomotion.go2.footstand.FootstandTermination + params: + action_name: joint_pos + grace_steps: 100 + height_fraction: 0.8 + orientation_threshold: 0.2 + energy_threshold: 200.0 + time_out: + func: unilab.envs.mdp.time_out + time_out: true + policy_observation_group: policy + critic_observation_group: critic + scale_rewards_by_dt: true + +reward: + footstand: + func: unilab.tasks.locomotion.go2.footstand.FootstandReward + weight: 1.0 + params: + state_term_name: footstand + scales: + height: 2.0 + orientation: 2.0 + contact: -1.0 + action_rate: -0.01 + termination: -2.0 + dof_pos_limits: -0.5 + torques: 0.0 + pose: -0.1 + penalty_contact: -0.2 + tar: 0.8 + rear_feet_contact: 0.5 + rear_leg_symmetry: -0.2 + front_leg_motion: -0.05 + upright_stability: -0.2 + knee_clearance: -0.5 + stay_still: -0.1 + energy: -0.003 + dof_acc: -2.5e-7 + soft_joint_pos_limit_factor: 0.9 + knee_height_target: 0.08 + front_feet_min_separation: 0.16 + front_feet_side_margin: 0.04 + rear_hip_abduction_margin: 0.25 + rear_foot_slip_deadband: 0.02 + rear_foot_anchor_radius: 0.03 diff --git a/conf/sac/task/go2_footstand/drake.yaml b/conf/sac/task/go2_footstand/drake.yaml new file mode 100644 index 000000000..3c9fd8b7c --- /dev/null +++ b/conf/sac/task/go2_footstand/drake.yaml @@ -0,0 +1,41 @@ +# @package _global_ +defaults: + - /task/go2_footstand/base + - _self_ + +training: + task_name: Go2FootStand + sim_backend: drake + no_play: true + play_steps: 400 + play_env_num: 1 + play_render_mode: record + +algo: + algo_log_name: fast_sac_drake + num_envs: 4096 + batch_size: 1024 + replay_buffer_n: 512 + updates_per_step: 2 + learning_starts: 2 + max_iterations: 300 + save_interval: 100 + actor_hidden_dim: 256 + critic_hidden_dim: 512 + obs_normalization: true + use_layer_norm: true + algo_params: + alpha_init: 0.005 + target_entropy_ratio: 0.0 + use_compile: false + +env: + drake_backend_mode: batch + drake_nthread: 20 + events: + # Drake keeps the task joint reset and rejects unsupported payload fields. + pd_gains: null + floor_friction: null + link_mass: null + torso_com: null + joint_armature: null diff --git a/conf/sac/task/go2_joystick_flat/base.yaml b/conf/sac/task/go2_joystick_flat/base.yaml new file mode 100644 index 000000000..409129c16 --- /dev/null +++ b/conf/sac/task/go2_joystick_flat/base.yaml @@ -0,0 +1,206 @@ +# @package _global_ +# Canonical Go2 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [31.5, 38.5] + kd_range: [0.45, 0.55] + operation: abs + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + weight: 0.24 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/offpolicy/task/sac/go2_joystick_flat/drake.yaml b/conf/sac/task/go2_joystick_flat/drake.yaml similarity index 57% rename from conf/offpolicy/task/sac/go2_joystick_flat/drake.yaml rename to conf/sac/task/go2_joystick_flat/drake.yaml index 6829cfe93..b709df1b7 100644 --- a/conf/offpolicy/task/sac/go2_joystick_flat/drake.yaml +++ b/conf/sac/task/go2_joystick_flat/drake.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + training: task_name: Go2JoystickFlat sim_backend: drake @@ -28,23 +32,5 @@ algo: env: drake_backend_mode: batch drake_nthread: 20 - scene: - model_file: src/unilab/assets/robots/go2/scene_flat.xml - domain_rand: - randomize_kp: false - randomize_kd: false - push_robots: false - -reward: - scales: - tracking_lin_vel: 1.0 - tracking_ang_vel: 0.2 - lin_vel_z: -5.0 - ang_vel_xy: -0.1 - base_height: -100.0 - action_rate: -0.005 - similar_to_default: -0.1 - contact: 0.24 - swing_feet_z: 4.0 - tracking_sigma: 0.25 - base_height_target: 0.3 + events: + pd_gains: null diff --git a/conf/sac/task/go2w_joystick_flat/base.yaml b/conf/sac/task/go2w_joystick_flat/base.yaml new file mode 100644 index 000000000..3d057e1ee --- /dev/null +++ b/conf/sac/task/go2w_joystick_flat/base.yaml @@ -0,0 +1,263 @@ +# @package _global_ +# Canonical Go2W flat Manager-Based task declaration. Backend leaves only own +# backend identity and backend-specific rendering/runtime settings. +env: + scene: + model_file: src/unilab/assets/robots/go2w/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base_link + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - FR_wheel_joint + - FL_wheel_joint + - RR_wheel_joint + - RL_wheel_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + - FR_wheel + - FL_wheel + - RR_wheel + - RL_wheel + body_names: [base_link] + sim_dt: 0.005 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + leg_joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + leg_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + wheel_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_wheel_joint" + actions: + func: unilab.envs.mdp.last_action + params: + action_name: motor + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + leg_joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + leg_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + wheel_joint_vel: + func: unilab.envs.mdp.joint_vel_rel + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_wheel_joint" + actions: + func: unilab.envs.mdp.last_action + params: + action_name: motor + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + motor_torque: + func: unilab.tasks.locomotion.go2w.manager_terms.motor_torque + params: + action_name: motor + actions: + motor: + _target_: unilab.tasks.locomotion.go2w.manager_terms.Go2WMixedActionCfg + entity_name: robot + actuator_names: [".*"] + leg_action_scale: 0.5 + wheel_action_scale: 10.0 + leg_kp: 50.0 + leg_kd: 1.5 + wheel_kd: 0.5 + clip_actions: 1.0 + simulate_action_latency: false + commands: + twist: + _target_: unilab.tasks.locomotion.go2w.manager_terms.Go2WVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [0.0, 1.0] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [-1.0, 1.0] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + motor_gains: + func: unilab.tasks.locomotion.go2w.manager_terms.randomize_motor_gains + mode: reset + params: + action_name: motor + kp_multiplier_range: [1.0, 1.0] + kd_multiplier_range: [1.0, 1.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.75 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.4 + orientation: + func: unilab.envs.mdp.flat_orientation_l2 + weight: -2.0 + action_rate: + func: unilab.tasks.locomotion.go2w.manager_terms.clipped_action_rate_l2 + weight: -0.005 + params: + action_name: motor + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.5 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_(hip|thigh|calf)_joint" + torques: + func: unilab.tasks.locomotion.go2w.manager_terms.motor_torque_l2 + weight: -0.0002 + params: + action_name: motor + wheel_vel: + func: unilab.envs.mdp.joint_vel_l2 + weight: 0.0 + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + joint_names: ".*_wheel_joint" + alive: + func: unilab.tasks.locomotion.go2w.manager_terms.constant_alive + weight: 0.5 + upward: + func: unilab.tasks.locomotion.go2w.manager_terms.upward_l2 + weight: 1.0 diff --git a/conf/sac/task/go2w_joystick_flat/drake.yaml b/conf/sac/task/go2w_joystick_flat/drake.yaml new file mode 100644 index 000000000..644465051 --- /dev/null +++ b/conf/sac/task/go2w_joystick_flat/drake.yaml @@ -0,0 +1,34 @@ +# @package _global_ +defaults: + - /task/go2w_joystick_flat/base + - _self_ + +training: + task_name: Go2WJoystickFlat + sim_backend: drake + no_play: true + play_steps: 400 + play_env_num: 1 + play_render_mode: record + +algo: + algo_log_name: fast_sac_drake + num_envs: 512 + batch_size: 1024 + replay_buffer_n: 512 + updates_per_step: 2 + learning_starts: 2 + max_iterations: 300 + save_interval: 100 + actor_hidden_dim: 256 + critic_hidden_dim: 512 + obs_normalization: true + use_layer_norm: true + algo_params: + alpha_init: 0.005 + target_entropy_ratio: 0.0 + use_compile: false + +env: + drake_backend_mode: batch + drake_nthread: 20 diff --git a/conf/offpolicy/task/sac/sharpa_inhand/mujoco_hora.yaml b/conf/sac/task/sharpa_inhand/mujoco_hora.yaml similarity index 96% rename from conf/offpolicy/task/sac/sharpa_inhand/mujoco_hora.yaml rename to conf/sac/task/sharpa_inhand/mujoco_hora.yaml index aa69445f8..7035d538b 100644 --- a/conf/offpolicy/task/sac/sharpa_inhand/mujoco_hora.yaml +++ b/conf/sac/task/sharpa_inhand/mujoco_hora.yaml @@ -24,7 +24,7 @@ interactive: algo: algo_log_name: hora_sac runtime_impl: hora_sac - runtime_resolver: unilab.algos.torch.hora.sac:resolve_hora_sac_runtime + runtime_resolver: unilab.algos.hora.sac:resolve_hora_sac_runtime num_envs: 1024 batch_size: 2048 replay_buffer_n: 1280 @@ -35,7 +35,6 @@ algo: save_interval: 896 actor_lr: 4.5e-4 critic_lr: 4.5e-4 - use_symmetry: false actor: priv_info_embed_dim: 9 priv_mlp_hidden_dims: [256, 128, 9] diff --git a/conf/sac/task/stewart_balance/base.yaml b/conf/sac/task/stewart_balance/base.yaml new file mode 100644 index 000000000..e38b87656 --- /dev/null +++ b/conf/sac/task/stewart_balance/base.yaml @@ -0,0 +1,116 @@ +# @package _global_ +# Off-policy copy of the canonical Stewart Manager-Based declaration. Hydra +# config groups have separate search roots, so this intentionally mirrors PPO. +env: + scene: + model_file: src/unilab/assets/robots/stewart/scene.xml + entities: + stewart: + root_body_name: ball + actuator_names: [a0, a1, a2, a3, a4, a5] + body_names: + - ball + - top + - leg00 + - leg10 + - leg01 + - leg11 + - leg02 + - leg12 + - top_connect00 + - top_connect10 + - top_connect01 + - top_connect11 + - top_connect02 + - top_connect12 + sim_dt: 0.004 + ctrl_dt: 0.02 + max_episode_seconds: 24.0 + render_spacing: 4.5 + observations: + policy: + terms: + balance: + func: unilab.tasks.manipulation.stewart.balance.StewartObservation + params: + entity_name: stewart + action_name: tilt + ball_body_name: ball + top_body_name: top + target_rotation_limit_deg: 6.0 + vel_smooth: 0.25 + actions: + tilt: + _target_: unilab.tasks.manipulation.stewart.balance.StewartTiltActionCfg + entity_name: stewart + actuator_names: [a0, a1, a2, a3, a4, a5] + top_body_name: top + ball_body_name: ball + leg_body_names: [leg00, leg10, leg01, leg11, leg02, leg12] + top_connect_body_names: + - top_connect00 + - top_connect10 + - top_connect01 + - top_connect11 + - top_connect02 + - top_connect12 + raw_action_clip: [-1.0, 1.0] + target_rotation_limit_deg: 6.0 + action_smooth: 0.60 + center_control_radius: 0.25 + center_control_min_gain: 0.15 + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_ball: + func: unilab.tasks.manipulation.stewart.balance.StewartBallReset + mode: reset + params: + entity_name: stewart + platform_radius: 0.8 + init_ball_radius_ratio: 0.18 + ball_home_z: 1.2 + terminations: + balance_state: + func: unilab.tasks.manipulation.stewart.balance.StewartBalanceState + params: + observation_group: policy + observation_term: balance + platform_radius: 0.8 + fall_radius: 0.5 + top_center_z: 1.0 + still_xy: 0.12 + still_vel: 0.07 + still_xy_hysteresis: 1.15 + still_vel_hysteresis: 1.20 + zero_vel_thresh: 0.07 + still_steps_needed: 5 + time_out: + func: unilab.envs.mdp.time_out + time_out: true + scale_rewards_by_dt: false + policy_observation_group: policy + critic_observation_group: null + +reward: + center: + func: unilab.tasks.manipulation.stewart.balance.center_reward + weight: 0.7 + params: + state_term_name: balance_state + progress: + func: unilab.tasks.manipulation.stewart.balance.progress_reward + weight: 0.6 + params: + state_term_name: balance_state + still: + func: unilab.tasks.manipulation.stewart.balance.still_reward + weight: 3.0 + params: + state_term_name: balance_state + fall: + func: unilab.tasks.manipulation.stewart.balance.fall_reward + weight: -6.0 + params: + state_term_name: balance_state diff --git a/conf/offpolicy/task/sac/stewart_balance/drake.yaml b/conf/sac/task/stewart_balance/drake.yaml similarity index 87% rename from conf/offpolicy/task/sac/stewart_balance/drake.yaml rename to conf/sac/task/stewart_balance/drake.yaml index 5a672e516..4992f2146 100644 --- a/conf/offpolicy/task/sac/stewart_balance/drake.yaml +++ b/conf/sac/task/stewart_balance/drake.yaml @@ -1,4 +1,8 @@ # @package _global_ +defaults: + - /task/stewart_balance/base + - _self_ + training: task_name: StewartBalance sim_backend: drake @@ -29,10 +33,3 @@ algo: env: drake_backend_mode: batch drake_nthread: 20 - -reward: - scales: - center: 0.7 - progress: 0.6 - still: 3.0 - fall_penalty: -6.0 diff --git a/conf/sac/task/t800_walk_flat/base.yaml b/conf/sac/task/t800_walk_flat/base.yaml new file mode 100644 index 000000000..a2afb9829 --- /dev/null +++ b/conf/sac/task/t800_walk_flat/base.yaml @@ -0,0 +1,111 @@ +# @package _global_ +# Standalone EngineAI T800 25-DoF Manager-Based SAC walk-flat owner. +env: + scene: + model_file: src/unilab/assets/robots/t800/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: LINK_BASE + body_names: [LINK_BASE] + joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J12_TORSO_YAW, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R, J23_HEAD_PITCH, J24_HEAD_YAW] + actuator_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J12_TORSO_YAW, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R, J23_HEAD_PITCH, J24_HEAD_YAW] + sim_dt: 0.002 + ctrl_dt: 0.01 + max_episode_seconds: 20.0 + observations: + policy: + enable_corruption: true + terms: + base_ang_vel: {func: unilab.envs.mdp.builtin_sensor, params: {sensor_name: torso_gyro}, scale: 0.25} + projected_gravity: {func: unilab.envs.mdp.projected_gravity_from_sensor, params: {sensor_name: torso_upvector}} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: {_target_: unilab.managers.SceneEntityCfg, name: robot, joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R]} + noise: {_target_: unilab.managers._noise.UniformNoiseCfg, n_min: -0.01, n_max: 0.01, operation: add} + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + params: + asset_cfg: {_target_: unilab.managers.SceneEntityCfg, name: robot, joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R]} + noise: {_target_: unilab.managers._noise.UniformNoiseCfg, n_min: -0.1, n_max: 0.1, operation: add} + actions: {func: unilab.envs.mdp.last_action, params: {action_name: joint_pos}} + command: {func: unilab.envs.mdp.generated_commands, params: {command_name: twist}} + gait_phase: {func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase, params: {frequency: 1.5, init_mode: offset_phase}} + critic: + enable_corruption: false + terms: + base_ang_vel: {func: unilab.envs.mdp.builtin_sensor, params: {sensor_name: torso_gyro}, scale: 0.25} + projected_gravity: {func: unilab.envs.mdp.projected_gravity_from_sensor, params: {sensor_name: torso_upvector}} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + params: + asset_cfg: {_target_: unilab.managers.SceneEntityCfg, name: robot, joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R]} + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + params: + asset_cfg: {_target_: unilab.managers.SceneEntityCfg, name: robot, joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R]} + actions: {func: unilab.envs.mdp.last_action, params: {action_name: joint_pos}} + command: {func: unilab.envs.mdp.generated_commands, params: {command_name: twist}} + gait_phase: {func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase, params: {frequency: 1.5, init_mode: offset_phase}} + base_lin_vel: {func: unilab.envs.mdp.builtin_sensor, params: {sensor_name: pelvis_local_linvel}, scale: 2.0} + policy_observation_group: policy + critic_observation_group: critic + actions: + joint_pos: + _target_: unilab.tasks.locomotion.t800.manager_terms.T800JointPositionActionCfg + entity_name: robot + actuator_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R] + held_actuator_names: [J12_TORSO_YAW, J23_HEAD_PITCH, J24_HEAD_YAW] + scale: {J00_HIP_PITCH_L: 1.0, J01_HIP_ROLL_L: 1.0, J02_HIP_YAW_L: 1.0, J03_KNEE_PITCH_L: 1.0, J04_ANKLE_PITCH_L: 1.0, J05_ANKLE_ROLL_L: 1.0, J06_HIP_PITCH_R: 1.0, J07_HIP_ROLL_R: 1.0, J08_HIP_YAW_R: 1.0, J09_KNEE_PITCH_R: 1.0, J10_ANKLE_PITCH_R: 1.0, J11_ANKLE_ROLL_R: 1.0, J13_SHOULDER_PITCH_L: 0.2, J14_SHOULDER_ROLL_L: 0.2, J15_SHOULDER_YAW_L: 0.05, J16_ELBOW_PITCH_L: 0.2, J17_ELBOW_YAW_L: 0.05, J18_SHOULDER_PITCH_R: 0.2, J19_SHOULDER_ROLL_R: 0.2, J20_SHOULDER_YAW_R: 0.05, J21_ELBOW_PITCH_R: 0.2, J22_ELBOW_YAW_R: 0.05} + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: {lin_vel_x: [-0.6, 1.0], lin_vel_y: [-0.4, 0.4], ang_vel_z: [-0.8, 0.8]} + events: + reset_scene_to_default: {func: unilab.envs.mdp.reset_scene_to_default, mode: reset} + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: {x: [-0.5, 0.5], y: [-0.5, 0.5], z: [0.0, 0.0], roll: [0.0, 0.0], pitch: [0.0, 0.0], yaw: [-3.141592653589793, 3.141592653589793]} + velocity_range: {x: [-0.5, 0.5], y: [-0.5, 0.5], z: [-0.5, 0.5], roll: [-0.5, 0.5], pitch: [-0.5, 0.5], yaw: [-0.5, 0.5]} + pd_gains: {func: unilab.envs.mdp.pd_gains, mode: reset, params: {kp_range: [0.9, 1.1], kd_range: [0.9, 1.1], operation: scale}} + terminations: + time_out: {func: unilab.envs.mdp.time_out, time_out: true} + tilt: {func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded, params: {max_tilt_deg: 65.0}} + base_height: {func: unilab.envs.mdp.root_height_below_minimum, params: {minimum_height: 0.3}} + curriculum: + penalty_scaling: + func: unilab.tasks.locomotion.g1.manager_terms.G1PenaltyCurriculum + params: {initial_scale: 0.125, min_scale: 0.125, max_scale: 0.25, level_down_threshold: 150.0, level_up_threshold: 750.0, degree: 0.001} + +reward: + tracking_lin_vel: {func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel, weight: 2.0, params: {tracking_sigma: 0.25, command_name: twist}} + tracking_ang_vel: {func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel, weight: 1.5, params: {tracking_sigma: 0.25, command_name: twist}} + penalty_ang_vel_xy: {func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy, weight: -1.0} + penalty_orientation: {func: unilab.tasks.locomotion.g1.manager_terms.orientation, weight: -10.0} + penalty_action_rate: {func: unilab.envs.mdp.action_rate_l2, weight: -4.0} + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.5 + params: + asset_cfg: {_target_: unilab.managers.SceneEntityCfg, name: robot, joint_names: [J00_HIP_PITCH_L, J01_HIP_ROLL_L, J02_HIP_YAW_L, J03_KNEE_PITCH_L, J04_ANKLE_PITCH_L, J05_ANKLE_ROLL_L, J06_HIP_PITCH_R, J07_HIP_ROLL_R, J08_HIP_YAW_R, J09_KNEE_PITCH_R, J10_ANKLE_PITCH_R, J11_ANKLE_ROLL_R, J13_SHOULDER_PITCH_L, J14_SHOULDER_ROLL_L, J15_SHOULDER_YAW_L, J16_ELBOW_PITCH_L, J17_ELBOW_YAW_L, J18_SHOULDER_PITCH_R, J19_SHOULDER_ROLL_R, J20_SHOULDER_YAW_R, J21_ELBOW_PITCH_R, J22_ELBOW_YAW_R]} + pose_weights: [0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 0.01, 2.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: {func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori, weight: -20.0} + penalty_close_feet_lateral: {func: unilab.tasks.locomotion.t800.manager_terms.penalty_close_feet_lateral, weight: -5.0, params: {min_width: 0.18, sigma: 0.04}} + feet_phase: {func: unilab.tasks.locomotion.g1.manager_terms.feet_phase, weight: 5.0, params: {frequency: 1.5, swing_height: 0.11, tracking_sigma: 0.014, min_forward_speed: 0.0, command_name: twist}} + alive: {func: unilab.tasks.locomotion.g1.manager_terms.alive, weight: 10.0} diff --git a/conf/sac/task/t800_walk_flat/mjwarp.yaml b/conf/sac/task/t800_walk_flat/mjwarp.yaml new file mode 100644 index 000000000..333317edb --- /dev/null +++ b/conf/sac/task/t800_walk_flat/mjwarp.yaml @@ -0,0 +1,31 @@ +# @package _global_ +# Configured-only SAC mjwarp owner for the T800 Manager-Based task. Keeps +# DENYLIST parity with the MuJoCo owner plus the mjwarp capacity knobs; kp/kd +# randomization is disabled because the mjwarp backend does not advertise gain +# DR support. Offline record reuses MuJoCo rendering; native playback and +# device-resident runtime are intentionally absent. +defaults: + - /task/t800_walk_flat/base + - _self_ + +training: + task_name: T800WalkFlat + sim_backend: mjwarp + play_render_mode: record +algo: + num_envs: 2048 + max_iterations: 10000 + save_interval: 1000 + learning_starts: 10 + updates_per_step: 8 + gamma: 0.98488578 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 +env: + mjwarp_nconmax: 128 + mjwarp_njmax: 256 + render_spacing: 2.0 + events: + # mjwarp does not advertise kp/kd gain DR support. + pd_gains: null diff --git a/conf/sac/task/t800_walk_flat/mujoco.yaml b/conf/sac/task/t800_walk_flat/mujoco.yaml new file mode 100644 index 000000000..84a26f846 --- /dev/null +++ b/conf/sac/task/t800_walk_flat/mujoco.yaml @@ -0,0 +1,20 @@ +# @package _global_ +# MuJoCo owner for the standalone T800 Manager-Based SAC task. +defaults: + - /task/t800_walk_flat/base + - _self_ + +training: + task_name: T800WalkFlat + sim_backend: mujoco + +algo: + num_envs: 2048 + max_iterations: 100000 + save_interval: 1000 + learning_starts: 10 + updates_per_step: 8 + gamma: 0.98488578 + algo_params: + alpha_init: 0.001 + target_entropy_ratio: 0.0 diff --git a/conf/offpolicy/config.yaml b/conf/td3/config.yaml similarity index 80% rename from conf/offpolicy/config.yaml rename to conf/td3/config.yaml index 908b3cb53..909355b08 100644 --- a/conf/offpolicy/config.yaml +++ b/conf/td3/config.yaml @@ -1,7 +1,39 @@ defaults: - _self_ - - algo: sac - - task: ${algo}/g1_walk_flat/mujoco + - task: g1_walk_flat/mujoco + +algo: + algo: td3 + algo_log_name: fast_td3 + load_run: "-1" + seed: 1 + num_envs: 4096 + batch_size: 8192 + replay_buffer_n: 1000 + updates_per_step: 4 + learning_starts: 1 + policy_frequency: 2 + max_iterations: 5000 + save_interval: 500 + gamma: 0.97 + tau: 0.1 + actor_lr: 3.0e-4 + critic_lr: 3.0e-4 + actor_hidden_dim: 512 + critic_hidden_dim: 1024 + num_atoms: 101 + obs_normalization: true + use_layer_norm: false + algo_params: + weight_decay: 0.1 + v_min: -10.0 + v_max: 10.0 + init_scale: 0.01 + log_std_min: -1.6 + log_std_max: -0.22 + policy_noise: 0.2 + noise_clip: 0.5 + use_cdq: true training: task_name: G1WalkFlat diff --git a/conf/td3/task/g1_23dof_walk_flat/base.yaml b/conf/td3/task/g1_23dof_walk_flat/base.yaml new file mode 100644 index 000000000..c7571348f --- /dev/null +++ b/conf/td3/task/g1_23dof_walk_flat/base.yaml @@ -0,0 +1,66 @@ +# @package _global_ +# Canonical G1 23-DoF walk Manager-Based task declaration (off-policy owners). +# Inherits the 29-DoF off-policy contract and swaps the scene to the 23-DoF +# model (no waist roll/pitch, no wrist pitch/yaw) with 23-entry pose weights. +defaults: + - /task/g1_walk_flat/base + - _self_ + +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat_23dof.xml + entities: + robot: + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + +reward: + pose: + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] diff --git a/conf/td3/task/g1_23dof_walk_flat/mujoco.yaml b/conf/td3/task/g1_23dof_walk_flat/mujoco.yaml new file mode 100644 index 000000000..52c728a45 --- /dev/null +++ b/conf/td3/task/g1_23dof_walk_flat/mujoco.yaml @@ -0,0 +1,12 @@ +# @package _global_ +# TD3 MuJoCo 23-DoF owner: inherits the 23-DoF off-policy Manager-Based +# contract and only carries backend/algo identity. +defaults: + - /task/g1_23dof_walk_flat/base + - _self_ + +training: + task_name: G1Walk23DofFlat + sim_backend: mujoco +algo: + max_iterations: 100000 diff --git a/conf/td3/task/g1_walk_flat/base.yaml b/conf/td3/task/g1_walk_flat/base.yaml new file mode 100644 index 000000000..a058d94f5 --- /dev/null +++ b/conf/td3/task/g1_walk_flat/base.yaml @@ -0,0 +1,272 @@ +# @package _global_ +# Canonical G1 29-DoF walk Manager-Based task declaration (off-policy owners). +# Backend owner leaves inherit this file and only override backend/algo tuning +# or explicitly disabled terms. Observation scaling follows the walk profile +# (gyro x0.25, joint velocity x0.05, critic linear velocity x2.0); every +# off-policy owner carries the penalty curriculum. +env: + scene: + model_file: src/unilab/assets/robots/g1/scene_flat.xml + default_keyframe_name: stand + entities: + robot: + root_body_name: pelvis + joint_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + actuator_names: + - left_hip_pitch_joint + - left_hip_roll_joint + - left_hip_yaw_joint + - left_knee_joint + - left_ankle_pitch_joint + - left_ankle_roll_joint + - right_hip_pitch_joint + - right_hip_roll_joint + - right_hip_yaw_joint + - right_knee_joint + - right_ankle_pitch_joint + - right_ankle_roll_joint + - waist_yaw_joint + - waist_roll_joint + - waist_pitch_joint + - left_shoulder_pitch_joint + - left_shoulder_roll_joint + - left_shoulder_yaw_joint + - left_elbow_joint + - left_wrist_roll_joint + - left_wrist_pitch_joint + - left_wrist_yaw_joint + - right_shoulder_pitch_joint + - right_shoulder_roll_joint + - right_shoulder_yaw_joint + - right_elbow_joint + - right_wrist_roll_joint + - right_wrist_pitch_joint + - right_wrist_yaw_joint + body_names: [pelvis] + sim_dt: 0.006666666666666667 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + # Observation noise matches the legacy off-policy noise_config (level=1.0, + # actor-only; gyro/gravity/linvel scales were 0.0 there): joint pos + # +/-0.01 and joint vel +/-0.1, applied before term scaling. + enable_corruption: true + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.01 + n_max: 0.01 + operation: add + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + noise: + _target_: unilab.managers._noise.UniformNoiseCfg + n_min: -0.1 + n_max: 0.1 + operation: add + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: torso_gyro} + scale: 0.25 + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: {sensor_name: torso_upvector} + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + scale: 0.05 + actions: + func: unilab.envs.mdp.last_action + params: {action_name: joint_pos} + command: + func: unilab.envs.mdp.generated_commands + params: {command_name: twist} + gait_phase: + func: unilab.tasks.locomotion.g1.manager_terms.G1GaitPhase + params: + frequency: 1.5 + init_mode: offset_phase + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: {sensor_name: pelvis_local_linvel} + scale: 2.0 + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 1.0 + use_default_offset: true + commands: + twist: + _target_: unilab.tasks.locomotion.g1.manager_terms.G1VelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + planar_dead_zone: 0.2 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [0.9, 1.1] + kd_range: [0.9, 1.1] + operation: scale + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + tilt: + func: unilab.tasks.locomotion.g1.manager_terms.g1_tilt_exceeded + params: + max_tilt_deg: 65.0 + base_height: + func: unilab.envs.mdp.root_height_below_minimum + params: + minimum_height: 0.3 + curriculum: + penalty_scaling: + func: unilab.tasks.locomotion.g1.manager_terms.G1PenaltyCurriculum + # Effective schedule matches the tuned legacy baseline: the legacy env + # halved the shared override dict once per env construction (two probe + # envs + the collector in every off-policy runner), so collectors actually + # trained at 1/8 initial / 1/4 cap of these YAML weights. The manager + # runtime isolates each env, so the tuned effective range is declared + # explicitly here. + params: + initial_scale: 0.125 + min_scale: 0.125 + max_scale: 0.25 + level_down_threshold: 150.0 + level_up_threshold: 750.0 + degree: 0.001 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_lin_vel + weight: 2.0 + params: + tracking_sigma: 0.25 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.g1.manager_terms.track_ang_vel + weight: 1.5 + params: + tracking_sigma: 0.25 + command_name: twist + penalty_ang_vel_xy: + func: unilab.tasks.locomotion.g1.manager_terms.ang_vel_xy + weight: -1.0 + penalty_orientation: + func: unilab.tasks.locomotion.g1.manager_terms.orientation + weight: -10.0 + penalty_action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -4.0 + pose: + func: unilab.tasks.locomotion.g1.manager_terms.weighted_pose + weight: -0.5 + params: + pose_weights: [0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 0.01, 1.0, 5.0, 0.01, 5.0, 5.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + penalty_feet_ori: + func: unilab.tasks.locomotion.g1.manager_terms.penalty_feet_ori + weight: -20.0 + feet_phase: + func: unilab.tasks.locomotion.g1.manager_terms.feet_phase + weight: 5.0 + params: + frequency: 1.5 + swing_height: 0.09 + tracking_sigma: 0.04 + min_forward_speed: 0.0 + command_name: twist + alive: + func: unilab.tasks.locomotion.g1.manager_terms.alive + weight: 10.0 diff --git a/conf/td3/task/g1_walk_flat/mujoco.yaml b/conf/td3/task/g1_walk_flat/mujoco.yaml new file mode 100644 index 000000000..b4f669538 --- /dev/null +++ b/conf/td3/task/g1_walk_flat/mujoco.yaml @@ -0,0 +1,12 @@ +# @package _global_ +# TD3 MuJoCo owner: inherits the 29-DoF off-policy Manager-Based contract and +# only carries backend/algo identity. +defaults: + - /task/g1_walk_flat/base + - _self_ + +training: + task_name: G1WalkFlat + sim_backend: mujoco +algo: + max_iterations: 100000 diff --git a/conf/td3/task/go1_joystick_flat/base.yaml b/conf/td3/task/go1_joystick_flat/base.yaml new file mode 100644 index 000000000..8a2f3bf57 --- /dev/null +++ b/conf/td3/task/go1_joystick_flat/base.yaml @@ -0,0 +1,244 @@ +# @package _global_ +# Canonical Go1 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go1/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: trunk + joint_names: + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + body_names: [trunk] + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + base_mass: + func: unilab.envs.mdp.randomize_rigid_body_mass + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: trunk + mass_distribution_params: [-1.5, 1.5] + operation: add + recompute_inertia: false + base_com: + func: unilab.envs.mdp.randomize_rigid_body_com + mode: reset + params: + asset_cfg: + _target_: SceneEntityCfg + name: robot + body_names: trunk + com_range: + x: [-0.05, 0.05] + y: [0.0, 0.0] + z: [0.0, 0.0] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [35.0, 35.0] + kd_range: [0.5, 0.5] + operation: abs + push_robot: + func: unilab.envs.mdp.push_by_setting_velocity + mode: interval + interval_range_s: [15.0, 15.0] + is_global_time: true + params: + velocity_range: + x: [-1.0, 1.0] + y: [-1.0, 1.0] + z: [-0.5, 0.5] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [0.0, 0.0] + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + # Legacy Go1 sums four matching feet while this community term returns their mean. + weight: 0.96 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/td3/task/go1_joystick_flat/motrix.yaml b/conf/td3/task/go1_joystick_flat/motrix.yaml new file mode 100644 index 000000000..fb408b50b --- /dev/null +++ b/conf/td3/task/go1_joystick_flat/motrix.yaml @@ -0,0 +1,27 @@ +# @package _global_ +defaults: + - /task/go1_joystick_flat/base + - _self_ + +training: + task_name: Go1JoystickFlat + sim_backend: motrix +algo: + num_envs: 1024 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + batch_size: 8192 + replay_buffer_n: 1024 +env: + commands: + twist: + ranges: + lin_vel_x: [0.5, 0.5] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + push_robot: null +reward: + contact: null diff --git a/conf/td3/task/go2_joystick_flat/base.yaml b/conf/td3/task/go2_joystick_flat/base.yaml new file mode 100644 index 000000000..409129c16 --- /dev/null +++ b/conf/td3/task/go2_joystick_flat/base.yaml @@ -0,0 +1,206 @@ +# @package _global_ +# Canonical Go2 flat Manager-Based task declaration. Backend owner leaves inherit +# this file and only override backend/algo tuning or explicitly disabled terms. +env: + scene: + model_file: src/unilab/assets/robots/go2/scene_flat.xml + default_keyframe_name: home + entities: + robot: + root_body_name: base + joint_names: + - FL_hip_joint + - FL_thigh_joint + - FL_calf_joint + - FR_hip_joint + - FR_thigh_joint + - FR_calf_joint + - RL_hip_joint + - RL_thigh_joint + - RL_calf_joint + - RR_hip_joint + - RR_thigh_joint + - RR_calf_joint + actuator_names: + - FR_hip + - FR_thigh + - FR_calf + - FL_hip + - FL_thigh + - FL_calf + - RR_hip + - RR_thigh + - RR_calf + - RL_hip + - RL_thigh + - RL_calf + sim_dt: 0.01 + ctrl_dt: 0.02 + max_episode_seconds: 20.0 + observations: + policy: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + critic: + terms: + base_ang_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: gyro + projected_gravity: + func: unilab.envs.mdp.projected_gravity_from_sensor + params: + sensor_name: upvector + joint_pos: + func: unilab.envs.mdp.joint_pos_rel + joint_vel: + func: unilab.envs.mdp.joint_vel_rel + actions: + func: unilab.envs.mdp.last_action + command: + func: unilab.envs.mdp.generated_commands + params: + command_name: twist + gait_phase: + func: unilab.tasks.locomotion.common.manager_terms.quadruped_gait_phase + params: + frequency: 2.0 + base_lin_vel: + func: unilab.envs.mdp.builtin_sensor + params: + sensor_name: local_linvel + actions: + joint_pos: + _target_: unilab.envs.mdp.JointPositionActionCfg + entity_name: robot + actuator_names: [".*"] + scale: 0.25 + use_default_offset: true + commands: + twist: + _target_: unilab.envs.mdp.UniformVelocityCommandCfg + entity_name: robot + resampling_time_range: [20.0, 20.0] + heading_command: false + heading_control_stiffness: 0.5 + rel_standing_envs: 0.0 + rel_heading_envs: 0.0 + rel_world_envs: 0.0 + rel_forward_envs: 0.0 + init_velocity_prob: 0.0 + ranges: + lin_vel_x: [-0.6, 1.0] + lin_vel_y: [-0.4, 0.4] + ang_vel_z: [-0.8, 0.8] + events: + reset_scene_to_default: + func: unilab.envs.mdp.reset_scene_to_default + mode: reset + reset_root_state_uniform: + func: unilab.envs.mdp.reset_root_state_uniform + mode: reset + params: + pose_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [0.0, 0.0] + roll: [0.0, 0.0] + pitch: [0.0, 0.0] + yaw: [-3.141592653589793, 3.141592653589793] + velocity_range: + x: [-0.5, 0.5] + y: [-0.5, 0.5] + z: [-0.5, 0.5] + roll: [-0.5, 0.5] + pitch: [-0.5, 0.5] + yaw: [-0.5, 0.5] + pd_gains: + func: unilab.envs.mdp.pd_gains + mode: reset + params: + kp_range: [31.5, 38.5] + kd_range: [0.45, 0.55] + operation: abs + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + bad_orientation: + func: unilab.envs.mdp.bad_orientation + params: + limit_angle: 1.0471975511965976 + policy_observation_group: policy + critic_observation_group: critic + +reward: + tracking_lin_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_lin_vel_xy_exp + weight: 1.0 + params: + std: 0.5 + command_name: twist + tracking_ang_vel: + func: unilab.tasks.locomotion.common.manager_terms.track_ang_vel_z_exp + weight: 0.2 + params: + std: 0.5 + command_name: twist + lin_vel_z: + func: unilab.tasks.locomotion.common.manager_terms.lin_vel_z_l2 + weight: -5.0 + ang_vel_xy: + func: unilab.tasks.locomotion.common.manager_terms.ang_vel_xy_l2 + weight: -0.1 + base_height: + func: unilab.tasks.locomotion.common.manager_terms.base_height_l2 + weight: -100.0 + params: + target_height: 0.3 + action_rate: + func: unilab.envs.mdp.action_rate_l2 + weight: -0.005 + similar_to_default: + func: unilab.tasks.locomotion.common.manager_terms.joint_deviation_l1 + weight: -0.1 + contact: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_contact + weight: 0.24 + params: + frequency: 2.0 + sensor_names: + - FL_foot_contact + - FR_foot_contact + - RL_foot_contact + - RR_foot_contact + contact_threshold: 0.1 + stance_threshold: 0.6 + swing_feet_z: + func: unilab.tasks.locomotion.common.manager_terms.feet_phase_swing_height + weight: 4.0 + params: + frequency: 2.0 + sensor_names: [FL_pos, FR_pos, RL_pos, RR_pos] + target_height: 0.1 + kernel: 0.01 + swing_start: 0.6 diff --git a/conf/td3/task/go2_joystick_flat/motrix.yaml b/conf/td3/task/go2_joystick_flat/motrix.yaml new file mode 100644 index 000000000..a12a474be --- /dev/null +++ b/conf/td3/task/go2_joystick_flat/motrix.yaml @@ -0,0 +1,25 @@ +# @package _global_ +defaults: + - /task/go2_joystick_flat/base + - _self_ + +training: + task_name: Go2JoystickFlat + sim_backend: motrix +algo: + num_envs: 1024 + learning_starts: 10 + max_iterations: 5000 + save_interval: 1000 + updates_per_step: 8 + batch_size: 8192 + replay_buffer_n: 1024 +env: + commands: + twist: + ranges: + lin_vel_x: [0.5, 0.5] + lin_vel_y: [0.0, 0.0] + ang_vel_z: [0.0, 0.0] + events: + pd_gains: null diff --git a/docs/sphinx/AGENTS.md b/docs/sphinx/AGENTS.md index cf2413609..cf519a82e 100644 --- a/docs/sphinx/AGENTS.md +++ b/docs/sphinx/AGENTS.md @@ -117,7 +117,7 @@ language-independent absolute path. creating a duplicate. 4. Gather evidence near the claim: - algorithms and tasks: `conf/`, `scripts/train_*.py`, `src/unilab/algos/` - - env contract: `src/unilab/base/np_env.py`, `src/unilab/training/rsl_rl.py` + - env contract: `src/unilab/base/np_env.py`, `src/unilab/algos/rsl_rl.py` - backend contract: `src/unilab/base/backend/base.py` - registry: `src/unilab/base/registry.py` - runner/IPC: `src/unilab/ipc/`, `src/unilab/training/run.py` diff --git a/docs/sphinx/source/adr/ADR-0000-index.md b/docs/sphinx/source/adr/ADR-0000-index.md index 007ef110a..772be3814 100644 --- a/docs/sphinx/source/adr/ADR-0000-index.md +++ b/docs/sphinx/source/adr/ADR-0000-index.md @@ -19,6 +19,7 @@ orphan: true | [ADR-0003 Task Owner And Config Compose Contract](ADR-0003-task-owner-and-config-compose-contract.md) | Config owner | Accepted | | [ADR-0004 Registry Bootstrap Contract](ADR-0004-registry-bootstrap-contract.md) | Registry bootstrap | Accepted | | [ADR-0005 Unified Obs Critic Env And IPC Contract](ADR-0005-unified-obs-critic-env-and-ipc-contract.md) | Observation / IPC | Accepted | +| [ADR-0006 Community Manager API On NumPy Runtime](ADR-0006-community-manager-api-on-numpy-runtime.md) | Manager API / NumPy runtime | Accepted | ## ADR Governance diff --git a/docs/sphinx/source/adr/ADR-0002-backend-capability-boundary-for-play-and-snapshot.md b/docs/sphinx/source/adr/ADR-0002-backend-capability-boundary-for-play-and-snapshot.md index 323f55982..a6dbce37c 100644 --- a/docs/sphinx/source/adr/ADR-0002-backend-capability-boundary-for-play-and-snapshot.md +++ b/docs/sphinx/source/adr/ADR-0002-backend-capability-boundary-for-play-and-snapshot.md @@ -53,7 +53,7 @@ MuJoCo 与 Motrix 的渲染路径和输出形式不同。仓库当前存在两 - 后端文档与矩阵: `docs/sphinx/source/zh_CN/2-user_guide/3-backends/0-index.md` - Backend 抽象: `src/unilab/base/backend/base.py` -- 训练入口与 play 边界: `scripts/train_rsl_rl.py`, `scripts/train_appo.py`, `scripts/train_offpolicy.py` +- 训练入口与 play 边界: `scripts/train_rsl_rl.py`, `scripts/train_appo.py`, `scripts/train_sac.py`, `scripts/train_td3.py`, `scripts/train_flashsac.py` ## Related Documents diff --git a/docs/sphinx/source/adr/ADR-0003-task-owner-and-config-compose-contract.md b/docs/sphinx/source/adr/ADR-0003-task-owner-and-config-compose-contract.md index 99d5525f8..b2811370d 100644 --- a/docs/sphinx/source/adr/ADR-0003-task-owner-and-config-compose-contract.md +++ b/docs/sphinx/source/adr/ADR-0003-task-owner-and-config-compose-contract.md @@ -25,12 +25,17 @@ orphan: true 2. owner YAML 直接持有 `training.task_name`、`training.sim_backend`、`reward`、`env` 及 task-specific `algo`。 3. `training.sim_backend` 是 owner 身份字段，不是独立 backend switch。 4. CLI override 允许参数覆盖，但不能破坏 task owner 的 backend identity。 +5. Manager-Based production task 也不例外：owner YAML 完整持有 manager/term/callable 与 + observation mapping，compose 后在 Registry 冷路径物化为 typed cfg；Python 不保存 + task-specific config mirror。 ## Stable Contracts - PPO/APPO owner 路径: `conf/{ppo,appo}/task//.yaml` -- Offpolicy owner 路径: `conf/offpolicy/task///.yaml` +- Offpolicy owner 路径: `conf/{sac,td3,flashsac}/task//.yaml`（每个 off-policy 算法一棵独立配置树） - reward 注入与 backend 差异表达必须在 owner YAML 层显式存在。 +- Manager-Based cfg 使用 Hydra `_target_` 与 dotted callable reference；解析失败或类型错误 + 必须在 env/backend 构造前报错。 ## Consequences diff --git a/docs/sphinx/source/adr/ADR-0004-registry-bootstrap-contract.md b/docs/sphinx/source/adr/ADR-0004-registry-bootstrap-contract.md index 7c96f790c..5e0691f82 100644 --- a/docs/sphinx/source/adr/ADR-0004-registry-bootstrap-contract.md +++ b/docs/sphinx/source/adr/ADR-0004-registry-bootstrap-contract.md @@ -35,7 +35,7 @@ UniLab 的 env 注册依赖 `@registry.envcfg(...)` 与 `@registry.env(...)` dec ## Consequences -- 新增 env package 时，需要同步声明 bootstrap modules。 +- 新增 task leaf 时，需要在 `unilab.tasks` 中同步声明 bootstrap module。 - registry 相关回归可以在 `ensure_registries()` 边界直接测试，不必依赖顶层训练脚本间接发现。 - 文档可以把 registry bootstrap 作为正式架构引用，而不是“当前实现细节”。 @@ -48,7 +48,7 @@ UniLab 的 env 注册依赖 `@registry.envcfg(...)` 与 `@registry.env(...)` dec - Registry 入口: `src/unilab/base/registry.py` - Bootstrap helper: `src/unilab/base/registry.py` -- Env package 入口: `src/unilab/envs/locomotion/__init__.py`, `src/unilab/envs/motion_tracking/__init__.py`, `src/unilab/envs/manipulation/__init__.py` +- Task package 入口: `src/unilab/tasks/__init__.py` - Bootstrap tests: `tests/utils/test_algo_utils.py`, `tests/base/test_registry.py` ## Related Documents diff --git a/docs/sphinx/source/adr/ADR-0006-community-manager-api-on-numpy-runtime.md b/docs/sphinx/source/adr/ADR-0006-community-manager-api-on-numpy-runtime.md new file mode 100644 index 000000000..c7eb30d17 --- /dev/null +++ b/docs/sphinx/source/adr/ADR-0006-community-manager-api-on-numpy-runtime.md @@ -0,0 +1,272 @@ +--- +orphan: true +--- + +# ADR-0006 Community Manager API On NumPy Runtime + +语言: 简体中文 + +- Status: Accepted +- Date: 2026-08-17 +- Owners: Env / Config / Backend maintainers +- Supersedes: None +- Superseded by: None + +## Context + +UniLab 的 `NpEnv`、Hydra owner YAML、registry、`SimBackend` 与 heterogeneous +training runtime 已形成稳定 contract，但 task 的 observation、action、reward、 +termination、event、command 与 curriculum 仍主要由各 env 的私有方法组装。用户迁移 +Isaac Lab 或 mjlab task 时，需要重写 manager term、配置和 lifecycle。 + +本决策采用 mjlab v1.6.0 的 manager package 作为可逐文件审查的迁移基线： + +- repository: `mujocolab/mjlab` +- commit: `0fb8a681136be94ffc636a3dd423cabb97d91f10` +- source: `src/mjlab/managers/` 的 12 个 Python 文件 +- license: Apache-2.0；上游 `LICENSE` 声明 + `Copyright 2025, The mjlab Developers` + +该基线只定义 manager-facing API 与语义。它不把 mjlab 的 Torch、Warp、scene +composer、viewer、simulation 或 training runtime 带入 UniLab，也不恢复或参考 UniLab +历史上的 Manager-Based API 实现。 + +## Decision + +### 1. Source-aligned public surface + +`src/unilab/managers/` 按 pinned mjlab package 的模块职责和 exports 直接迁移。以下名称 +是 canonical public surface；Torch 类型替换为 NumPy 类型不构成改名： + +| Module | Canonical exports | +| --- | --- | +| `manager_base` | `ManagerBase`, `ManagerTermBase`, `ManagerTermBaseCfg` | +| `action_manager` | `ActionManager`, `ActionTerm`, `ActionTermCfg` | +| `observation_manager` | `ObservationManager`, `ObservationGroupCfg`, `ObservationTermCfg` | +| `reward_manager` | `RewardManager`, `RewardTermCfg` | +| `termination_manager` | `TerminationManager`, `TerminationTermCfg` | +| `event_manager` | `EventManager`, `EventMode`, `EventTermCfg` | +| `command_manager` | `CommandManager`, `CommandTerm`, `CommandTermCfg`, `NullCommandManager` | +| `curriculum_manager` | `CurriculumManager`, `CurriculumTermCfg`, `NullCurriculumManager` | +| `metrics_manager` | `MetricsManager`, `MetricsTermCfg`, `NullMetricsManager` | +| `recorder_manager` | `RecorderManager`, `RecorderTerm`, `RecorderTermCfg`, `NullRecorderManager` | +| `scene_entity_config` | `SceneEntityCfg` | + +Manager cfg 使用 plain dataclass instance；term 集合使用保持插入顺序的 typed `dict`。 +`func + params`、function/class term、class term 的 `(cfg, env)` 构造和局部 +`reset(env_ids)` 语义保持不变。显式空配置或 term 值为 `None` 表示用户选择禁用，允许 +使用 upstream Null manager/no-op 语义。 + +未来 env lifecycle 的 canonical 名称沿用迁移源的 `ManagerBasedRlEnv` 与 +`ManagerBasedRlEnvCfg`。如果为 Isaac Lab 拼写提供 `ManagerBasedRLEnv` / +`ManagerBasedRLEnvCfg`，它们必须是同一对象的无分支 alias，不能形成第二套实现。 +其他别名必须由实际 migration fixture 证明有价值，不能预先扩张 API。 + +### 2. NumPy runtime boundary + +Manager-facing tensor、buffer、term return、env IDs 和 entity view 使用 +`np.ndarray` 或 `slice`。Torch 的 `device`、`.to()`、`.cpu()` 与 Tensor-only API 不属于 +UniLab manager contract；manager package 不能 import Torch、runner、learner 或 IPC。 + +数值转换保持下列语义： + +- shape、dtype 和更新时序与上游一致；action history 与 observation history 不改变顺序； +- buffer 在 manager 构造或 reset owner 边界分配，step 热路径复用； +- 随机采样使用由 env 拥有并可复现的 NumPy generator，不依赖进程全局 RNG； +- shape 不匹配以及非有限 term 输出在最近 manager/term 边界直接报错；reward 不使用 + `nan_to_num` 把非法值静默变成零； +- observation 明确配置的 noise/delay/history/NaN policy 可以保留，但默认不能掩盖非法 + 输出。 + +### 3. UniLab env、config 与 IPC boundary + +Managers 只依赖一个 typed env context。P0 context 包含 `num_envs`、physics/control dt、 +episode counters、NumPy RNG、各 manager 属性，以及正式 scene/entity facade；不能要求 +`device` 或 backend 私有对象。 + +Manager 内可以使用社区常见的 `policy` / `actor` / `critic` observation group。env owner +必须显式把 actor-facing group 映射为 `NpEnvState.obs["obs"]`，并把可选 critic group +映射为 `NpEnvState.obs["critic"]`。runner、learner 与 IPC 不推断、不拼接 group。 +`reset() -> (obs_dict, info_dict)`、final observation 与 `obs_groups_spec` 保持现有 contract。 + +Production task 的唯一配置 source of truth 是 Hydra owner YAML。它完整声明 scene/backend +tuning、manager/group/term 的顺序与启停、具体 cfg 类型、callable、params、weight 和 +observation group mapping；compose 后按以下冷路径进入现有 registry： + +`owner YAML -> DictConfig -> typed config materialization -> Registry factory -> ManagerBasedRlEnv` + +具体 cfg 类型使用 Hydra `_target_`，term callable 使用完整 dotted reference。通用 +materializer 将其解析为 plain dataclass instance，并在未知字段、target/callable 解析失败、 +抽象或错误 term cfg 类型及缺少必填字段时 fail-closed。Python 只拥有 term 实现、公共 cfg +类型和通用 factory，不保存第二份 task-specific term 清单或默认值；直接构造 typed cfg +只用于底层单测。DictConfig 与解析逻辑不能进入 reset/step 热路径，scripts 不解释 term +业务规则。 + +### 4. Scene/entity owner boundary + +`SceneEntityCfg` 和 term 所需的最小 NumPy entity facade 属于 `src/unilab/base/` 公共 +contract；backend 负责通过 `SimBackend` materialize 名称、ID 和 state/control view，env +负责把 facade 组合进 manager context。该决策解决 #586 的 owner 问题，但不引入完整 +scene composer 或通用 asset hierarchy。 + +- entity name 以及 joint/body/geom/site/actuator selector 在 init/materialization/cache + 冷路径解析一次；热路径只持有已解析 `list[int]`、`np.ndarray` 或 `slice`； +- selector 保留上游 name/regex、`preserve_order`、names/IDs consistency check 和全选压缩为 + `slice(None)` 的语义； +- root/joint/body/site/geom/control 能力只能来自 `SimBackend` 已声明方法；不暴露 backend + model/data 私有对象； +- tendon/camera/light/material/texture/pair 等迁移表面可以存在，但 backend 未声明能力时在 + resolve/materialization 直接 `NotImplementedError`，不能返回空 ID 或跳过； +- 新 backend 能力必须作为独立 child 扩展 `SimBackend` 并补 conformance tests，不能在 + manager 或 env 中用 `getattr` / `hasattr` 探测私有实现。 + +Named sensor 使用 `SimBackend.bind_sensor_data(names)` 在 materialization 冷路径校验名称、 +每个 sensor 的展平宽度、batch shape 与 finite 值，并返回 immutable +`BackendSensorView`。term 热路径只调用 `view.read()`；MuJoCo、Drake 和 MJWarp adapter +分别保留已解析的 host-cache slice 或数值 slot。MotrixSim 当前公开接口只提供 named +sensor accessor、没有数值 sensor ID，因此 Motrix adapter 在 scene materialization 时缓存 +可用名称，并把原生批量 accessor 与 immutable 名称 tuple 封装为 backend-owned opaque +reader；term 不接触名称解析、XML 或 model metadata，未知名称在进入原生调用前 fail-closed。 + +这是 pinned mjlab sensor-facing 语义的 intentional NumPy/backend adaptation：社区侧的 +tensor/device view 在 UniLab 表达为按请求名称顺序拼接的二维 NumPy batch +`(num_envs, sum(sensor_widths))`。名称顺序、单 sensor 宽度和当前值可见，Torch device、 +backend model/data 与原生 handle 不属于 manager contract。 + +### 5. Fail-closed capability rule + +用户显式禁用与实现缺失是两种不同状态。前者允许 Null manager；后者必须失败： + +| Failure | Required behavior | +| --- | --- | +| cfg/term 类型错误、签名或 shape 不匹配 | `TypeError` / `ValueError`，包含 manager 与 term | +| term 输出 NaN/Inf | `ValueError`，包含 manager、group/term 与非法值类别 | +| backend/entity capability 未实现 | `NotImplementedError`，包含 manager、term、capability 与 backend | +| selector name/ID 不存在或不一致 | `KeyError` / `ValueError`，包含 entity 与 selector | + +不得 warning 后 skip、返回零/旧值、自动换 backend、禁用 feature 或回退到旧 env。当前 +不新增公共 exception hierarchy；只有 consumer 证明需要 machine-readable 分类时再单独 +决策。 + +### 6. Performance and deletion policy + +优先级固定为：社区 Manager-Based API 语义与结构一致性，优先于改变公共设计的局部性能 +优化。在此约束下，生产级 NumPy 热路径不能引入明显可避免的重复解析、逐环境 Python +循环、数组复制或临时分配。优化必须由同配置、同硬件 benchmark 证明有足够收益，并优先 +保持在内部预解析、预分配和批量 NumPy 实现；低收益但增加专用 fast path、缓存协议或长期 +复杂度的方案不采用。 + +Production task 迁移后必须在同一 task-family child 删除被替代的旧 dispatch、重复 +reward/config helper 和 bridge。Umbrella 完成时只保留一套 manager lifecycle，不保留 +fallback 到旧单体 env 的永久兼容路径。 + +## Stable Contracts + +### Compatibility matrix + +状态只表示本 ADR 固定的迁移目标；实际 support claim 仍需要代码、注册、配置和测试证据。 + +| Surface | Target | Notes | +| --- | --- | --- | +| manager modules、class/config names、dict order | Compatible | 直接保留 pinned mjlab 1.6.0 表面 | +| function/class term、`params`、local reset | Compatible | class term 在冷路径实例化 | +| action split/apply/history、reward dt scaling、termination timeout split | Compatible | NumPy 实现保持时序 | +| observation groups、clip/scale/noise/delay/history | Adapted | 数值为 NumPy；group 在 env boundary 显式映射 | +| manager buffers、env IDs、RNG | Adapted | Torch→NumPy；无 device API | +| `ManagerBasedRlEnv` return | Adapted | 保留 `NpEnvState` 与 UniLab reset/final-observation contract | +| config container | Adapted | Hydra owner YAML 唯一持有 task 配置，冷路径物化为 plain typed instances | +| `SceneEntityCfg` selectors | Adapted | 语义保留；只解析 `SimBackend` 已声明能力 | +| named sensor view | Adapted | 冷路径 bind；有序展平 NumPy batch；reader 由 backend 拥有 | +| event/domain randomization | Adapted | 调度语义保留；mutation 走 backend DR/capability contract | +| Metrics/Recorder | Adapted | lifecycle hook 存在时启用；缺失时显式失败或显式空配置 | +| Torch device、Warp mutation、viewer glue | Unsupported | 不进入 manager core，不提供静默替代 | +| Omniverse/USD/mjlab Scene/Simulation | Unsupported | 不属于 UniLab runtime | + +### Mechanical migration example + +迁移前的 mjlab term： + +```python +import torch +from mjlab.managers import RewardTermCfg + +def joint_error(env) -> torch.Tensor: + return torch.square(env.joint_pos - env.target_joint_pos).sum(dim=1) + +term = RewardTermCfg(func=joint_error, weight=-1.0) +``` + +迁移后的 UniLab term 只改 import、数值类型和对应 NumPy 运算： + +```python +import numpy as np +from unilab.managers import RewardTermCfg + +def joint_error(env) -> np.ndarray: + return np.square(env.joint_pos - env.target_joint_pos).sum(axis=1) + +term = RewardTermCfg(func=joint_error, weight=-1.0) +``` + +如果迁移还要求重写 term 结构、增加 backend 分支或改 runner/IPC，说明 adapter boundary +不够薄，必须停止并拆出 owner child。 + +### Provenance and change accounting + +每个 source-derived Python 文件必须注明上游 repository、tag/commit、原始路径、 +Apache-2.0 和 UniLab 的修改类别。实现 PR 分别报告： + +1. source-derived：保留的上游结构/语义； +2. mechanical：import、typing、Torch→NumPy 和格式转换； +3. UniLab-specific glue：新 facade、contract adapter 或行为； +4. deleted：删除的上游不适用代码和 UniLab 旧实现。 + +不得通过重新分类隐藏 glue 超预算；不建立长期 upstream mirror 或自动 sync tooling。 + +## Alternatives Considered + +- 重新设计一套更适合 UniLab 的 managers，再提供兼容 facade。拒绝：会形成 + UniLab-only 方言和两套行为，增加用户迁移与长期维护成本。 +- 在 manager 热路径保留 Torch。拒绝：破坏 NumPy runtime、backend isolation 与 + heterogeneous CPU physics → accelerator learner 数据面。 +- 一次迁移完整 mjlab scene/simulation/entity runtime。拒绝：复制第二套 backend/scene + abstraction，并引入 Warp/MuJoCo/Viewer 假设。 +- 先设计 compiler、fused term protocol 或专用 fast path。拒绝：在 benchmark 证明瓶颈前 + 增加结构复杂度，并可能牺牲社区 term 语义。 +- 缺失能力 warning + skip 或回退旧 env。拒绝：配置表面与真实执行不一致，不能用于生产。 +- task-owned Python factory 声明 callable/term，Hydra 只做字段 overlay。拒绝：会让同一 task + 在 Python 和 YAML 中拥有两份配置，增加迁移 friction 和语义漂移。 + +## Consequences + +- Manager port 的审查基线是 pinned upstream diff，而不是重新解释每个 manager 的职责。 +- NumPy、UniLab env/config contract 和显式 unsupported 是允许的偏离；其他偏离必须在 + compatibility matrix 中先记录。 +- Scene/entity 采用最小 base facade，#586 不再阻塞 manager port；真实 backend 能力仍按 + 独立 child 和 conformance evidence 接入。 +- Config/Registry 永久维护一个通用 Hydra `_target_` / dotted callable 到 typed manager cfg + 的冷路径 materializer；production task 不维护 Python config mirror。 +- 迁移初期允许 production 旧 task 与未接入的 manager package 同时存在，但 task 一旦迁移 + 就必须删除对应旧实现；umbrella 结束时不能保留双 lifecycle。 +- 性能 gate 关注明显低效与实测瓶颈，不以复杂度换取未经证明的小收益。 + +## Evidence In Repo + +- Env contract: `src/unilab/base/np_env.py` +- Backend contract: `src/unilab/base/backend/base.py` +- Scene config owner: `src/unilab/base/scene.py` +- Config schema and registry: `src/unilab/structured_configs.py`, + `src/unilab/base/config_materialization.py`, `src/unilab/base/registry.py`, `conf/` +- Observation/IPC contract: `docs/sphinx/source/adr/ADR-0005-unified-obs-critic-env-and-ipc-contract.md` +- Layer boundary: `docs/sphinx/source/adr/ADR-0001-runtime-model-and-layer-boundaries.md` +- Upstream checkout used for the decision: + `/home/user/ws/simulator/mjlab/src/mjlab/managers/` at `0fb8a681` + +## Related Documents + +- {doc}`ADR Index ` +- {doc}`Manager-Based API contract ` +- {doc}`RL Infrastructure 开发标准 ` +- [Roadmap #1042](https://github.com/unilabsim/UniLab/issues/1042) +- [Implementation issue #1043](https://github.com/unilabsim/UniLab/issues/1043) +- [Entity abstraction decision #586](https://github.com/unilabsim/UniLab/issues/586) diff --git a/docs/sphinx/source/adr/README.md b/docs/sphinx/source/adr/README.md index 007ef110a..772be3814 100644 --- a/docs/sphinx/source/adr/README.md +++ b/docs/sphinx/source/adr/README.md @@ -19,6 +19,7 @@ orphan: true | [ADR-0003 Task Owner And Config Compose Contract](ADR-0003-task-owner-and-config-compose-contract.md) | Config owner | Accepted | | [ADR-0004 Registry Bootstrap Contract](ADR-0004-registry-bootstrap-contract.md) | Registry bootstrap | Accepted | | [ADR-0005 Unified Obs Critic Env And IPC Contract](ADR-0005-unified-obs-critic-env-and-ipc-contract.md) | Observation / IPC | Accepted | +| [ADR-0006 Community Manager API On NumPy Runtime](ADR-0006-community-manager-api-on-numpy-runtime.md) | Manager API / NumPy runtime | Accepted | ## ADR Governance diff --git a/docs/sphinx/source/api_reference/algos/index.md b/docs/sphinx/source/api_reference/algos/index.md index 41c42cb8e..94a4ff6ab 100644 --- a/docs/sphinx/source/api_reference/algos/index.md +++ b/docs/sphinx/source/api_reference/algos/index.md @@ -1,22 +1,35 @@ # `unilab.algos` — Learning Algorithms -- **`unilab.algos.torch`** — PPO (RSL-RL), APPO, FastSAC, FastTD3, FlashSAC, +- **`unilab.algos`** — PPO (RSL-RL), APPO, FastSAC, FastTD3, FlashSAC, HIM-PPO, HORA + distillation, generic off-policy runner. All trainers conform to a single runner contract — see {doc}`../../en/4-developer_guide/2-contracts/5-runner_lifecycle`. -```{toctree} -:maxdepth: 2 - -torch -``` - ```{eval-rst} .. autosummary:: :toctree: _autosummary :template: autosummary/module.rst :recursive: - unilab.algos + unilab.algos.common + unilab.algos.appo + unilab.algos.fast_sac + unilab.algos.fast_td3 + unilab.algos.flash_sac + unilab.algos.him_ppo + unilab.algos.hora + unilab.algos.offpolicy +``` + +## Standalone PPO entrypoints + +```{eval-rst} +.. automodule:: unilab.algos.rsl_rl_ppo + :members: +``` + +```{eval-rst} +.. automodule:: unilab.algos.rsl_rl_runtime + :members: ``` diff --git a/docs/sphinx/source/api_reference/algos/torch.md b/docs/sphinx/source/api_reference/algos/torch.md deleted file mode 100644 index 347557974..000000000 --- a/docs/sphinx/source/api_reference/algos/torch.md +++ /dev/null @@ -1,29 +0,0 @@ -# `unilab.algos.torch` - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :template: autosummary/module.rst - :recursive: - - unilab.algos.torch.common - unilab.algos.torch.appo - unilab.algos.torch.fast_sac - unilab.algos.torch.fast_td3 - unilab.algos.torch.flash_sac - unilab.algos.torch.him_ppo - unilab.algos.torch.hora - unilab.algos.torch.offpolicy -``` - -## Standalone PPO entrypoints - -```{eval-rst} -.. automodule:: unilab.algos.torch.rsl_rl_ppo - :members: -``` - -```{eval-rst} -.. automodule:: unilab.algos.torch.rsl_rl_runtime - :members: -``` diff --git a/docs/sphinx/source/api_reference/base/index.md b/docs/sphinx/source/api_reference/base/index.md index 96ec2b764..94553193d 100644 --- a/docs/sphinx/source/api_reference/base/index.md +++ b/docs/sphinx/source/api_reference/base/index.md @@ -10,7 +10,6 @@ in this reference, read this one. | `Registry` | Task / backend / algorithm registration and lookup | | `Scene` | Cold-path scene materialization | | `observations`, `final_observation` | Observation builders & terminal handling | -| `augmentation` | Symmetry / mirror augmentation utilities | | `curriculum` | Curriculum schedule primitives | ```{eval-rst} diff --git a/docs/sphinx/source/api_reference/envs/index.md b/docs/sphinx/source/api_reference/envs/index.md index 6665cd124..f7eb11999 100644 --- a/docs/sphinx/source/api_reference/envs/index.md +++ b/docs/sphinx/source/api_reference/envs/index.md @@ -1,21 +1,11 @@ -# `unilab.envs` — Tasks +# `unilab.envs` — Environment runtime -Concrete RL tasks split by family: +Task-agnostic Manager-Based environment runtime and reusable MDP terms. +Concrete task implementations are owned by {doc}`../tasks/index`. -- **locomotion** — Go1, Go2, Go2w, Go2 + Airbot, Unitree G1 -- **manipulation** — Allegro / Sharpa in-hand cube -- **motion_tracking** — G1 whole-body motion tracking + flips - -Every env inherits `NpEnv` and is registered into the task `Registry` so it -can be selected via `uv run train --algo --task --sim `. - -```{toctree} -:maxdepth: 2 - -locomotion -manipulation -motion_tracking -``` +`ManagerBasedRLEnv` preserves UniLab's NumPy `NpEnv` contract while executing +community-style action, observation, reward, termination, event, command, and +curriculum managers. ```{eval-rst} .. autosummary:: diff --git a/docs/sphinx/source/api_reference/envs/locomotion.md b/docs/sphinx/source/api_reference/envs/locomotion.md deleted file mode 100644 index 1a9d5107a..000000000 --- a/docs/sphinx/source/api_reference/envs/locomotion.md +++ /dev/null @@ -1,15 +0,0 @@ -# `unilab.envs.locomotion` - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :template: autosummary/module.rst - :recursive: - - unilab.envs.locomotion.common - unilab.envs.locomotion.g1 - unilab.envs.locomotion.go1 - unilab.envs.locomotion.go2 - unilab.envs.locomotion.go2_arm - unilab.envs.locomotion.go2w -``` diff --git a/docs/sphinx/source/api_reference/envs/manipulation.md b/docs/sphinx/source/api_reference/envs/manipulation.md deleted file mode 100644 index 5e2b2b73b..000000000 --- a/docs/sphinx/source/api_reference/envs/manipulation.md +++ /dev/null @@ -1,12 +0,0 @@ -# `unilab.envs.manipulation` - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :template: autosummary/module.rst - :recursive: - - unilab.envs.manipulation.allegro_inhand - unilab.envs.manipulation.sharpa_inhand - unilab.envs.manipulation.stewart -``` diff --git a/docs/sphinx/source/api_reference/envs/motion_tracking.md b/docs/sphinx/source/api_reference/envs/motion_tracking.md deleted file mode 100644 index c4371bd6b..000000000 --- a/docs/sphinx/source/api_reference/envs/motion_tracking.md +++ /dev/null @@ -1,13 +0,0 @@ -# `unilab.envs.motion_tracking` - -Whole-body motion tracking tasks. G1 humanoid currently ships flip tracking -plus general motion tracking (PPO + SAC variants). - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :template: autosummary/module.rst - :recursive: - - unilab.envs.motion_tracking.g1 -``` diff --git a/docs/sphinx/source/api_reference/index.md b/docs/sphinx/source/api_reference/index.md index df6920ca1..194fa57e8 100644 --- a/docs/sphinx/source/api_reference/index.md +++ b/docs/sphinx/source/api_reference/index.md @@ -30,10 +30,16 @@ The contracts everything else depends on: `NpEnv`, `SimBackend`, `Registry`, :::{grid-item-card} 🧪 `unilab.envs` :link: envs/index :link-type: doc -Concrete tasks — locomotion, manipulation, motion tracking — layered on +Manager-Based environment runtime and task-agnostic MDP terms layered on top of `base`. ::: +:::{grid-item-card} 🤖 `unilab.tasks` +:link: tasks/index +:link-type: doc +Concrete locomotion, manipulation, and motion-tracking task packages. +::: + :::: ## Learning stack @@ -91,12 +97,6 @@ Procedural and heightfield terrain generators. Scene rendering and viser bridges. ::: -:::{grid-item-card} 🔧 `unilab.tools` -:link: tools/index -:link-type: doc -Scene export, NaN visualizer, ONNX export. -::: - :::{grid-item-card} 🧰 `unilab.utils` :link: utils/index :link-type: doc @@ -125,6 +125,7 @@ top_level base/index envs/index +tasks/index ``` ```{toctree} @@ -144,7 +145,6 @@ backend/index dr/index terrains/index visualization/index -tools/index utils/index logging/index ``` diff --git a/docs/sphinx/source/api_reference/tasks/index.md b/docs/sphinx/source/api_reference/tasks/index.md new file mode 100644 index 000000000..eb4ed03d1 --- /dev/null +++ b/docs/sphinx/source/api_reference/tasks/index.md @@ -0,0 +1,27 @@ +# `unilab.tasks` — Concrete tasks + +Concrete RL tasks split by family: + +- **locomotion** — A2, Go1, Go2, Go2w, Go2 + Airbot, and Unitree G1 +- **manipulation** — Allegro / Sharpa in-hand cube and Stewart balance +- **motion_tracking** — G1 and X2 whole-body motion tracking + +Every task is registered into the task `Registry` so it can be selected via +`uv run train --algo --task --sim `. + +```{toctree} +:maxdepth: 2 + +locomotion +manipulation +motion_tracking +``` + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :template: autosummary/module.rst + :recursive: + + unilab.tasks +``` diff --git a/docs/sphinx/source/api_reference/tasks/locomotion.md b/docs/sphinx/source/api_reference/tasks/locomotion.md new file mode 100644 index 000000000..c4e504491 --- /dev/null +++ b/docs/sphinx/source/api_reference/tasks/locomotion.md @@ -0,0 +1,16 @@ +# `unilab.tasks.locomotion` + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :template: autosummary/module.rst + :recursive: + + unilab.tasks.locomotion.a2 + unilab.tasks.locomotion.common + unilab.tasks.locomotion.g1 + unilab.tasks.locomotion.go1 + unilab.tasks.locomotion.go2 + unilab.tasks.locomotion.go2_arm + unilab.tasks.locomotion.go2w +``` diff --git a/docs/sphinx/source/api_reference/tasks/manipulation.md b/docs/sphinx/source/api_reference/tasks/manipulation.md new file mode 100644 index 000000000..328e820de --- /dev/null +++ b/docs/sphinx/source/api_reference/tasks/manipulation.md @@ -0,0 +1,12 @@ +# `unilab.tasks.manipulation` + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :template: autosummary/module.rst + :recursive: + + unilab.tasks.manipulation.allegro_inhand + unilab.tasks.manipulation.sharpa_inhand + unilab.tasks.manipulation.stewart +``` diff --git a/docs/sphinx/source/api_reference/tasks/motion_tracking.md b/docs/sphinx/source/api_reference/tasks/motion_tracking.md new file mode 100644 index 000000000..8ca1f557d --- /dev/null +++ b/docs/sphinx/source/api_reference/tasks/motion_tracking.md @@ -0,0 +1,14 @@ +# `unilab.tasks.motion_tracking` + +Whole-body motion-tracking tasks for G1 and X2 robots. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :template: autosummary/module.rst + :recursive: + + unilab.tasks.motion_tracking.common + unilab.tasks.motion_tracking.g1 + unilab.tasks.motion_tracking.x2 +``` diff --git a/docs/sphinx/source/api_reference/tools/index.md b/docs/sphinx/source/api_reference/tools/index.md deleted file mode 100644 index a8e1918a5..000000000 --- a/docs/sphinx/source/api_reference/tools/index.md +++ /dev/null @@ -1,15 +0,0 @@ -# `unilab.tools` — CLI Tools - -Console-script entrypoints registered in `pyproject.toml`: - -- `unilab-viz-nan` — interactive NaN trace viewer for failed runs. -- `unilab-export-scene` — dump the resolved scene of a task to disk. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :template: autosummary/module.rst - :recursive: - - unilab.tools -``` diff --git a/docs/sphinx/source/api_reference/training/index.md b/docs/sphinx/source/api_reference/training/index.md index 5e497f661..80d7238f5 100644 --- a/docs/sphinx/source/api_reference/training/index.md +++ b/docs/sphinx/source/api_reference/training/index.md @@ -1,8 +1,10 @@ # `unilab.training` — Training Runtime -Glue between `algos`, `envs` and `ipc`: experiment lifecycle, metric -monitoring, reward bookkeeping, seeding, and the top-level `run` helpers -invoked by the `train` / `eval` / `demo` CLI entrypoints. +Glue between `algos`, `envs` and `ipc`: experiment lifecycle and the +top-level `run` helpers invoked by the `train` / `eval` / `demo` CLI +entrypoints. Layer-0 helpers (seeding, monitoring, reward bookkeeping, +checkpoint resolution, sim2sim contracts) live in `unilab.utils`; resolved +env config adaptation lives in `unilab.base.config_adapter`. ```{eval-rst} .. autosummary:: diff --git a/docs/sphinx/source/api_reference/utils/index.md b/docs/sphinx/source/api_reference/utils/index.md index 99e6e4270..30a8b0133 100644 --- a/docs/sphinx/source/api_reference/utils/index.md +++ b/docs/sphinx/source/api_reference/utils/index.md @@ -1,7 +1,9 @@ # `unilab.utils` — Utilities -Device probing, tensor helpers, support-matrix bookkeeping, NaN guards, -and pure-numpy geometry/rotation helpers shared across envs and scripts. +Device probing, tensor helpers, training seeding, hardware monitoring, +reward bookkeeping, checkpoint resolution, sim2sim contract checks, NaN +guards, and pure-numpy geometry/rotation helpers shared across envs and +scripts. ```{eval-rst} .. autosummary:: diff --git a/docs/sphinx/source/conf.py b/docs/sphinx/source/conf.py index b818430fa..65effaca0 100644 --- a/docs/sphinx/source/conf.py +++ b/docs/sphinx/source/conf.py @@ -276,6 +276,9 @@ # map is computed below. _LANGUAGE_PATH_FORWARD: dict[str, str] = { "en/1-getting_started/5-faq": "zh_CN/1-getting_started/5-faq", + "en/4-developer_guide/1-architecture/6-manager_based_api": ( + "zh_CN/4-developer_guide/1-architecture/6-manager_based_api" + ), } # Keyed by (current_pagename, target_language) → target_pagename. _LANGUAGE_PATH_MAP: dict[tuple[str, str], str] = {} diff --git a/docs/sphinx/source/en/1-getting_started/3-evaluation_and_playback.md b/docs/sphinx/source/en/1-getting_started/3-evaluation_and_playback.md index 42f6f70bd..574026df1 100644 --- a/docs/sphinx/source/en/1-getting_started/3-evaluation_and_playback.md +++ b/docs/sphinx/source/en/1-getting_started/3-evaluation_and_playback.md @@ -23,7 +23,8 @@ Render modes: - `none` — skip rendering, just compute metrics. `training.export_onnx=false` currently applies only to the off-policy playback path -(`scripts/train_offpolicy.py` and CLI runs with `--algo sac|td3|flashsac`). It skips +(`scripts/train_sac.py` / `scripts/train_td3.py` / `scripts/train_flashsac.py` +and CLI runs with `--algo sac|td3|flashsac`). It skips `policy.onnx` export and verification but still runs playback and video recording. ## MuJoCo Viewer Scripts diff --git a/docs/sphinx/source/en/1-getting_started/4-project_structure.md b/docs/sphinx/source/en/1-getting_started/4-project_structure.md index 40ab3d3d2..364244a49 100644 --- a/docs/sphinx/source/en/1-getting_started/4-project_structure.md +++ b/docs/sphinx/source/en/1-getting_started/4-project_structure.md @@ -24,8 +24,9 @@ The main config roots are: - `conf/ppo/config.yaml` for torch PPO. - `conf/appo/config.yaml` for APPO. -- `conf/offpolicy/config.yaml` plus `conf/offpolicy/algo/*.yaml` for SAC, - TD3, and FlashSAC. +- `conf/sac/config.yaml`, `conf/td3/config.yaml`, and + `conf/flashsac/config.yaml` for SAC, TD3, and FlashSAC, each with its + algorithm hyperparameters inlined. - `conf/ppo_him/config.yaml` and `conf/hora_distill/config.yaml` for the specialized HIM-PPO and HORA paths. diff --git a/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md b/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md index 5809775a8..4d728c632 100644 --- a/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md +++ b/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md @@ -9,9 +9,9 @@ keeps the lower-level scripts available for debugging Hydra composition. | --- | --- | --- | | PPO | `uv run train --algo ppo --task --sim ` | `scripts/train_rsl_rl.py` | | APPO | `uv run train --algo appo --task --sim ` | `scripts/train_appo.py` | -| SAC | `uv run train --algo sac --task --sim ` | `scripts/train_offpolicy.py` | -| TD3 | `uv run train --algo td3 --task --sim ` | `scripts/train_offpolicy.py` | -| FlashSAC | `uv run train --algo flashsac --task --sim ` | `scripts/train_offpolicy.py` | +| SAC | `uv run train --algo sac --task --sim ` | `scripts/train_sac.py` | +| TD3 | `uv run train --algo td3 --task --sim ` | `scripts/train_td3.py` | +| FlashSAC | `uv run train --algo flashsac --task --sim ` | `scripts/train_flashsac.py` | Examples: @@ -123,8 +123,8 @@ The lower-level scripts remain available when you need to inspect Hydra config groups or reproduce a script-level issue. For normal usage, keep route-defining values in the unified CLI flags above. -For off-policy routes, keep `--algo` aligned with the owner tree under -`conf/offpolicy/task//`; do not include the algorithm name in `--task`. +For off-policy routes, `--algo` selects the per-algorithm owner tree +`conf//`; do not include the algorithm name in `--task`. ## Common Overrides diff --git a/docs/sphinx/source/en/2-user_guide/1-training/2-hydra_config.md b/docs/sphinx/source/en/2-user_guide/1-training/2-hydra_config.md index 9fb08cfed..447e870ad 100644 --- a/docs/sphinx/source/en/2-user_guide/1-training/2-hydra_config.md +++ b/docs/sphinx/source/en/2-user_guide/1-training/2-hydra_config.md @@ -9,7 +9,7 @@ identity of the task, backend, reward, scene, and task-specific runtime fields. | --- | --- | | PPO | `conf/ppo/task//.yaml` | | APPO | `conf/appo/task//.yaml` | -| SAC / TD3 / FlashSAC | `conf/offpolicy/task///.yaml` | +| SAC / TD3 / FlashSAC | `conf//task//.yaml` | | HIM-PPO | `conf/ppo_him/task//.yaml` | | HORA distillation | `conf/hora_distill/task//.yaml` | @@ -21,8 +21,8 @@ uv run train --algo ppo --task go2_joystick_flat --sim motrix uv run train --algo sac --task g1_walk_flat --sim mujoco ``` -For off-policy, `--algo` selects the first owner-path segment under -`conf/offpolicy/task//`; do not include the algorithm name in `--task`. +For off-policy, `--algo` selects the per-algorithm config tree `conf//`; +do not include the algorithm name in `--task`. ## Safe Overrides diff --git a/docs/sphinx/source/en/2-user_guide/1-training/3-logging.md b/docs/sphinx/source/en/2-user_guide/1-training/3-logging.md index a766838ba..8dcd29d68 100644 --- a/docs/sphinx/source/en/2-user_guide/1-training/3-logging.md +++ b/docs/sphinx/source/en/2-user_guide/1-training/3-logging.md @@ -25,9 +25,9 @@ stack overrides `training.log_root` or `training.log_dir`: | --- | --- | --- | | PPO | `logs/rsl_rl_ppo//` | `conf/ppo/config.yaml` | | APPO | `logs/appo//` | `conf/appo/config.yaml` | -| SAC | `logs/fast_sac//` | `conf/offpolicy/algo/sac.yaml` | -| FlashSAC | `logs/flash_sac//` | `conf/offpolicy/algo/flashsac.yaml` | -| TD3 | `logs/fast_td3//` | `conf/offpolicy/algo/td3.yaml` | +| SAC | `logs/fast_sac//` | `conf/sac/config.yaml` | +| FlashSAC | `logs/flash_sac//` | `conf/flashsac/config.yaml` | +| TD3 | `logs/fast_td3//` | `conf/td3/config.yaml` | A run directory is named `YYYY-MM-DD_HH-MM-SS_`, for example `2026-03-09_18-30-00_mujoco`. Common artifacts include `run_config.json`, diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/0-index.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/0-index.md index d9ff6c63b..73adb82d2 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/0-index.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/0-index.md @@ -8,9 +8,9 @@ lives, and which command shape selects it. For general flags, see | --- | --- | --- | --- | | PPO | synchronous on-policy | `scripts/train_rsl_rl.py` | `conf/ppo/config.yaml` | | APPO | async on-policy | `scripts/train_appo.py` | `conf/appo/config.yaml` | -| SAC | off-policy | `scripts/train_offpolicy.py` | `conf/offpolicy/algo/sac.yaml` | -| TD3 | off-policy | `scripts/train_offpolicy.py` | `conf/offpolicy/algo/td3.yaml` | -| FlashSAC | off-policy | `scripts/train_offpolicy.py` | `conf/offpolicy/algo/flashsac.yaml` | +| SAC | off-policy | `scripts/train_sac.py` | `conf/sac/config.yaml` | +| TD3 | off-policy | `scripts/train_td3.py` | `conf/td3/config.yaml` | +| FlashSAC | off-policy | `scripts/train_flashsac.py` | `conf/flashsac/config.yaml` | | HIM-PPO | height-estimator PPO path | `scripts/train_him_ppo.py` | `conf/ppo_him/config.yaml` | | HORA | teacher/student distillation path | `scripts/train_hora_distill.py` | `conf/hora_distill/config.yaml` | diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/1-ppo.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/1-ppo.md index e52e20183..bbbca179e 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/1-ppo.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/1-ppo.md @@ -2,7 +2,7 @@ PPO is the default synchronous on-policy training path. It uses `scripts/train_rsl_rl.py`, composes from `conf/ppo/config.yaml`, and runs the -RSL-RL adapter code in `src/unilab/algos/torch/rsl_rl_ppo.py` and +RSL-RL adapter code in `src/unilab/algos/rsl_rl_ppo.py` and `src/unilab/training/rsl_rl.py`. ## Quick Start diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/2-appo.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/2-appo.md index aea0cc9c9..f095dab65 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/2-appo.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/2-appo.md @@ -1,7 +1,7 @@ # APPO APPO is UniLab's asynchronous PPO path. It uses `scripts/train_appo.py`, -`conf/appo/config.yaml`, and the runtime under `src/unilab/algos/torch/appo/`. +`conf/appo/config.yaml`, and the runtime under `src/unilab/algos/appo/`. The config exposes `algo.steps_per_env`, `training.collector_device`, and `training.replay_queue_size`; the algorithm config includes V-trace clipping fields. diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/3-sac.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/3-sac.md index 4c46cce36..cdd80dd00 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/3-sac.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/3-sac.md @@ -1,13 +1,13 @@ # SAC -SAC is selected through the shared off-policy entrypoint -`scripts/train_offpolicy.py`, which TD3 and FlashSAC share as well. The main -config is `conf/offpolicy/config.yaml`, and the SAC algorithm defaults live in -`conf/offpolicy/algo/sac.yaml`. The current log name is `fast_sac`. +SAC runs through `scripts/train_sac.py`; TD3 and FlashSAC have their own +entrypoints and per-algorithm config trees. The main config is +`conf/sac/config.yaml`, with the SAC algorithm defaults inlined there. The +current log name is `fast_sac`. ## Runtime Model -The off-policy runner decouples CPU simulation from accelerator learning through +The off-policy runner decouples simulation collection from accelerator learning through bounded shared memory. A collector subprocess publishes packed transitions through two ingress slots, while the complete replay ring is authoritative on one CUDA or Apple MPS learner device. Host replay allocation therefore does not @@ -25,7 +25,7 @@ uv run train --algo sac --task g1_walk_rough --sim motrix training.no_play=true ## Key Fields -For the off-policy playback path (`scripts/train_offpolicy.py` / CLI `--algo sac`), +For the off-policy playback path (`scripts/train_sac.py` / CLI `--algo sac`), set `training.export_onnx=false` to skip `policy.onnx` export while still recording playback video. See {doc}`/en/1-getting_started/3-evaluation_and_playback`. @@ -33,7 +33,7 @@ playback video. See {doc}`/en/1-getting_started/3-evaluation_and_playback`. - `algo.num_envs=4096` - `algo.batch_size=8192` is the learner batch per update. - `algo.max_iterations=500` -- `training.use_amp=true` in the shared off-policy config +- `training.use_amp=true` in `conf/sac/config.yaml` The off-policy device replay path uses synchronized, learner-owned inference: collectors exchange observations and actions through shared memory and do not own an actor. @@ -44,3 +44,17 @@ uv run train --algo sac --task g1_walk_flat --sim mujoco \ algo.max_iterations=1000 \ training.no_play=true ``` + +## Single-node multi-GPU device placement + +`training.devices` assigns rank i's learner to `cuda:devices[i]`; each rank owns one +collector. For mjwarp, the rank process and its collector process explicitly bind Warp's +default/current device to that same learner device before probe or production environment +materialization. The collector therefore does not fall back to Warp's fresh-process default +of `cuda:0`. The local binding is recorded as `collector_backend_device` in the runtime +manifest. + +MuJoCo has a committed multi-GPU scaling benchmark. The mjwarp per-rank placement contract is +covered by `tests/base/backend/test_process_device.py` and the off-policy runner/worker unit +tests; the repository does not currently contain an mjwarp multi-GPU throughput or convergence +benchmark. diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/4-td3.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/4-td3.md index 5b75c187e..d219a5bf4 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/4-td3.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/4-td3.md @@ -1,7 +1,7 @@ # TD3 -TD3 shares the off-policy training script with SAC and FlashSAC. Select it -with `--algo td3`; owner YAML evidence lives under `conf/offpolicy/task/td3/`. +TD3 runs through `scripts/train_td3.py` in its own config tree. Select it +with `--algo td3`; owner YAML evidence lives under `conf/td3/task/`. ## Quick Start @@ -11,11 +11,11 @@ uv run train --algo td3 --task g1_walk_flat --sim mujoco ## Key Fields -For the off-policy playback path (`scripts/train_offpolicy.py` / CLI `--algo td3`), +For the off-policy playback path (`scripts/train_td3.py` / CLI `--algo td3`), set `training.export_onnx=false` to skip `policy.onnx` export while still recording playback video. See {doc}`/en/1-getting_started/3-evaluation_and_playback`. -- Defaults live in `conf/offpolicy/algo/td3.yaml`. +- Defaults are inlined in `conf/td3/config.yaml`. - `algo.algo_log_name=fast_td3`. - `algo.max_iterations=5000`. - `algo.policy_frequency=2`. diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/5-flash_sac.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/5-flash_sac.md index f1ce54b40..fdf221b5b 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/5-flash_sac.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/5-flash_sac.md @@ -1,11 +1,11 @@ # FlashSAC -FlashSAC is the third algorithm on the shared off-policy entrypoint. Select it -with `--algo flashsac`; defaults live in -`conf/offpolicy/algo/flashsac.yaml`, and the implementation lives under -`src/unilab/algos/torch/flash_sac/`. +FlashSAC runs through `scripts/train_flashsac.py` in its own config tree. +Select it with `--algo flashsac`; defaults are inlined in +`conf/flashsac/config.yaml`, and the implementation lives under +`src/unilab/algos/flash_sac/`. -It shares the off-policy training script with SAC and TD3, but does not use the +It shares the off-policy runner design with SAC and TD3, but does not use the same default networks: the actor uses a block-based structure and the critic uses a distributional (categorical) Q variant. @@ -18,7 +18,7 @@ uv run train --algo flashsac --task go2_joystick_flat --sim mujoco training.no_p ## Key Fields -For the off-policy playback path (`scripts/train_offpolicy.py` / CLI `--algo flashsac`), +For the off-policy playback path (`scripts/train_flashsac.py` / CLI `--algo flashsac`), set `training.export_onnx=false` to skip `policy.onnx` export while still recording playback video. See {doc}`/en/1-getting_started/3-evaluation_and_playback`. diff --git a/docs/sphinx/source/en/2-user_guide/2-algorithms/7-hora.md b/docs/sphinx/source/en/2-user_guide/2-algorithms/7-hora.md index 808ccf28b..d320f633e 100644 --- a/docs/sphinx/source/en/2-user_guide/2-algorithms/7-hora.md +++ b/docs/sphinx/source/en/2-user_guide/2-algorithms/7-hora.md @@ -13,7 +13,7 @@ uv run train --algo appo --task sharpa_inhand --sim mujoco --profile hora traini ``` The HORA PPO owner sets `algo.algo_log_name=hora_ppo` and resolves the runtime -through `unilab.algos.torch.hora.rsl_rl:resolve_hora_ppo_runtime`. The APPO +through `unilab.algos.hora.rsl_rl:resolve_hora_ppo_runtime`. The APPO variant sets `algo.algo_log_name=hora_appo`. ## Student Distillation @@ -24,5 +24,5 @@ CLI does not currently declare a separate HORA distillation `--algo` route, so the public CLI examples on this page stay on the teacher path above. Teacher checkpoint resolution is implemented in -`src/unilab/algos/torch/hora/distill_config.py`. The student log family is +`src/unilab/algos/hora/distill_config.py`. The student log family is `hora_distill`. diff --git a/docs/sphinx/source/en/2-user_guide/3-backends/0-index.md b/docs/sphinx/source/en/2-user_guide/3-backends/0-index.md index 6f6f9a122..a775e3b0c 100644 --- a/docs/sphinx/source/en/2-user_guide/3-backends/0-index.md +++ b/docs/sphinx/source/en/2-user_guide/3-backends/0-index.md @@ -23,7 +23,7 @@ uv run train --algo ppo --task go1_joystick_flat --sim motrix Owner YAML locations: - PPO / APPO: `conf/{ppo,appo}/task//.yaml` -- Off-policy: `conf/offpolicy/task///.yaml` +- Off-policy (SAC / TD3 / FlashSAC): `conf//task//.yaml` The selected owner YAML sets `training.sim_backend` as an identity field. diff --git a/docs/sphinx/source/en/2-user_guide/4-tasks/1-locomotion.md b/docs/sphinx/source/en/2-user_guide/4-tasks/1-locomotion.md index 94818f8e7..ed04ce19e 100644 --- a/docs/sphinx/source/en/2-user_guide/4-tasks/1-locomotion.md +++ b/docs/sphinx/source/en/2-user_guide/4-tasks/1-locomotion.md @@ -1,7 +1,7 @@ # Locomotion -Locomotion tasks are registered in `src/unilab/envs/locomotion/` and -`src/unilab/envs/motion_tracking/`. The available owner YAMLs under `conf/` +Locomotion tasks are registered in `src/unilab/tasks/locomotion/` and +`src/unilab/tasks/motion_tracking/`. The available owner YAMLs under `conf/` define which algorithm and backend combinations are runnable. ## Families @@ -29,12 +29,14 @@ backend: {doc}`../../5-reference/5-support_matrix`. ## Go2 FootStand -`go2_footstand` is the Go2 front-feet-stand task. It is **MuJoCo-only**. +`go2_footstand` is the Go2 front-feet-stand task. Its PPO owner YAMLs +register MuJoCo, Motrix, and Drake; the SAC owner currently targets Drake. -- PPO config: `conf/ppo/task/go2_footstand/mujoco.yaml` -- Registered env: `Go2FootStand` (registered for `sim_backend="mujoco"`) -- Implementation: `src/unilab/envs/locomotion/go2/footstand.py` - (extends the Go2 base task) +- Canonical PPO task config: `conf/ppo/task/go2_footstand/base.yaml` +- Backend owners: `conf/ppo/task/go2_footstand/{mujoco,motrix,drake}.yaml` +- Registered env: `Go2FootStand` (MuJoCo, Motrix, and Drake) +- Implementation: `src/unilab/tasks/locomotion/go2/footstand.py` + (task-owned NumPy manager terms on the generic Manager-Based runtime) - Go2 model XML: `src/unilab/assets/robots/go2/go2.xml` ```bash @@ -66,14 +68,14 @@ The full FootStand recipe is a three-stage teacher-student pipeline; the shipped ### Observation Layout The `Go2FootStand` policy (actor) observation uses 15 history frames of 45 dims -each (`_FOOTSTAND_FRAME_OBS_DIM = 45`): +each (`FRAME_OBS_DIM = 45`): ```text linvel(3) + gyro(3) + gravity(3) + joint_position_delta(12) + joint_velocity(12) + last_action(12) ``` So the policy observation is `45 * 15 = 675`. The value (critic) observation -appends the current-step privileged tail (`_FOOTSTAND_PRIVILEGED_TAIL_DIM = 49`) +appends the current-step privileged tail (`PRIVILEGED_OBS_DIM = 49`) after that history: ```text @@ -84,23 +86,24 @@ The value observation is therefore `675 + 49 = 724`. ### Rewards And Terminations -Defaults come from `conf/ppo/task/go2_footstand/mujoco.yaml`. The reward scales +Defaults come from `conf/ppo/task/go2_footstand/base.yaml`; backend leaves only +override backend-specific terms and tuning. The reward scales include stand `height`, `orientation`, `rear_feet_contact`, target front-leg angle (`tar`), `action_rate`, `dof_pos_limits`, `front_leg_motion`, `rear_leg_symmetry`, `knee_clearance`, `upright_stability`, `stay_still`, `pose`, plus `energy` and `dof_acc` penalties; `termination` and `penalty_contact` drive the termination / penalty paths (front-leg / front-body contact, low height, bad orientation, and a -high-energy cutoff via `energy_termination_threshold`). +high-energy cutoff in the `footstand` termination term). ### Tuning Keys -- `env.obs_history_len`: policy observation history length; config default is `15`. -- `env.energy_termination_threshold`: high-energy termination cutoff; config - default is `200.0`. -- `env.domain_rand`: floor friction, link mass, torso CoM, dof armature, and reset - joint qpos randomization. -- `reward.scales.height` / `orientation` / `rear_feet_contact`: stand pose and - rear-foot contact weights. +- `env.observations.policy.terms.frame.history_length`: policy history length + (default `15`). +- `env.terminations.footstand.params.energy_threshold`: high-energy cutoff + (default `200.0`). +- `env.events`: reset and domain-randomization terms. Backend owners explicitly + set unsupported model-field terms to `null`. +- `reward.footstand.params.scales`: stand, contact, motion, and energy weights. ### Near-Risk Validation diff --git a/docs/sphinx/source/en/2-user_guide/4-tasks/2-motion_tracking.md b/docs/sphinx/source/en/2-user_guide/4-tasks/2-motion_tracking.md index 7b78c58a5..4dc8e15e0 100644 --- a/docs/sphinx/source/en/2-user_guide/4-tasks/2-motion_tracking.md +++ b/docs/sphinx/source/en/2-user_guide/4-tasks/2-motion_tracking.md @@ -1,40 +1,43 @@ # Motion Tracking -G1 motion tracking tasks live under `src/unilab/envs/motion_tracking/` and are +G1 motion tracking tasks live under `src/unilab/tasks/motion_tracking/` and are selected through task owner YAMLs in `conf/ppo/`, `conf/appo/`, and selected off-policy paths. > **Motion assets moved to Hugging Face.** The `.npz` clips are no longer shipped > in the repository. On first use `MotionLoader` -> (`src/unilab/envs/motion_tracking/g1/motion_loader.py`) downloads them on demand +> (`src/unilab/tasks/motion_tracking/common/motion_loader.py`) downloads them on demand > from [unilabsim/unilab-motions](https://huggingface.co/datasets/unilabsim/unilab-motions) > via `src/unilab/assets/hub.py` (`_HF_MOTIONS_REPO_ID`). `uv sync` already installs > the required `huggingface_hub` dependency. ## Task Owners -Each task ships a default motion clip defined in the env config dataclass: +Each task ships a default motion clip in its Hydra task-owner YAML. Hydra is the +configuration entry point; the selected owner is materialized into the shared +`ManagerBasedRlEnvCfg` and then consumed by the NumPy Manager-Based runtime. | CLI Task | Registered Env | Default Motion | Owner Evidence | | --- | --- | --- | --- | | `g1_motion_tracking` | `G1MotionTracking` | `dance1_subject2_part.npz` | `conf/ppo/task/g1_motion_tracking/`, `conf/appo/task/g1_motion_tracking/` | | `g1_flip_tracking` | `G1FlipTracking` | `flip_360_001__A304.npz` | `conf/ppo/task/g1_flip_tracking/`, `conf/appo/task/g1_flip_tracking/` | | `g1_wall_flip_tracking` | `G1WallFlipTracking` | `flip_from_wall_104__A304.npz` | `conf/ppo/task/g1_wall_flip_tracking/`, `conf/appo/task/g1_wall_flip_tracking/` | -| `x2_wall_flip_tracking` | `X2WallFlipTracking` | `tictacflip_6-3_g1format.npz` | `conf/ppo/task/x2_wall_flip_tracking/` (MuJoCo only) | -| `g1_climb_tracking` | G1 climb tracking env | clip from env config | `conf/ppo/task/g1_climb_tracking/`, `conf/appo/task/g1_climb_tracking/` | -| `g1_box_tracking` | G1 box tracking env | clip from env config | `conf/ppo/task/g1_box_tracking/` | -| `g1_wbt_obs` | `G1MotionTrackingSAC` | shared with `g1_motion_tracking` | `conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml` | +| `x2_wall_flip_tracking` | `X2WallFlipTracking` | `tictacflip_6-3_g1format.npz` | `conf/ppo/task/x2_wall_flip_tracking/` | +| `g1_climb_tracking` | `G1ClimbTracking` | `climb_20_z_scale_1.0.npz` | `conf/ppo/task/g1_climb_tracking/`, `conf/appo/task/g1_climb_tracking/` | +| `g1_box_tracking` | `G1BoxTracking` | `sub3_largebox_003_boxconverted.npz` | `conf/ppo/task/g1_box_tracking/` | +| `g1_wbt_obs` | `G1WBTObs` | `dance1_subject2_part.npz` | `conf/sac/task/g1_wbt_obs/mujoco.yaml` | -The defaults are set in code: `dance1_subject2_part.npz` -(`g1/tracking.py`), `flip_360_001__A304.npz` and `flip_from_wall_104__A304.npz` -(`g1/flip_tracking.py`), and `tictacflip_6-3_g1format.npz` (`x2/flip_tracking.py`). +The 23-DoF task-owner directories select their matching 23-DoF scene, motion, +entity, and action declarations. Profile differences remain in Hydra. The G1 +identities use the shared manager factory; X2 adds only a cold-path mesh resolver +before delegating to that factory. ## PPO And APPO PPO owner iteration budgets (the `--sim mujoco` owner YAMLs): `g1_motion_tracking` runs `algo.max_iterations=15000`; `g1_flip_tracking` and `g1_wall_flip_tracking` -run `20000`; the MuJoCo-only `x2_wall_flip_tracking` runs `9500`. (The Motrix -owner YAML for `g1_flip_tracking` raises this to `30000`.) +run `20000`; `x2_wall_flip_tracking` runs `9500`. (The Motrix owner YAML for +`g1_flip_tracking` raises this to `30000`.) ```bash uv run train --algo ppo --task g1_motion_tracking --sim mujoco @@ -57,11 +60,13 @@ uv run train --algo sac --task g1_motion_tracking --sim mujoco training.use_amp= uv run train --algo sac --task g1_wbt_obs --sim mujoco training.use_amp=true ``` -The `g1_wbt_obs` owner is the deploy-aligned off-policy observation profile: a -pelvis IMU state (`pelvis_local_linvel` / `pelvis_gyro` / `pelvis_upvector`) plus -per-term observation history (`noise_config.obs_history_length: 5`), byte-aligned -with the deploy-time `ObservationManager`. Deploy tooling lives under -`scripts/deploy/`, and the observation alignment is cross-checked by +The `g1_wbt_obs` owner is the deploy-aligned off-policy observation profile. Its +actor keeps the command and anchor-orientation terms at one step while the +`base_ang_vel`, `joint_pos`, `joint_vel`, and `actions` terms declare +`history_length: 5`. `ObservationManager` owns and flattens those per-term +histories; the actor uses the configured encoder-biased joint-position term while +the critic keeps the clean term. Deploy tooling lives under `scripts/deploy/`, and +the observation alignment is cross-checked by `tests/scripts/test_obs_alignment_g1_wbt.py`. When a Motrix sim2sim replay needs a checkpoint from another log root, pass the absolute path through `uv run eval`: @@ -72,16 +77,20 @@ uv run eval --algo sac --task g1_motion_tracking --sim motrix \ ## Motion Files -Motion NPZ files are read through `env.motion_file`, which also accepts a list of +Motion NPZ files are selected through +`env.commands.motion.params.motion_file`, which accepts one path or a list of paths. A standard clip must contain the seven keys `fps`, `joint_pos`, `joint_vel`, `body_pos_w`, `body_quat_w`, `body_lin_vel_w`, and `body_ang_vel_w` -(validated in `g1/motion_loader.py`): +(validated in `common/motion_loader.py`): ```yaml env: - motion_file: - - src/unilab/assets/motions/g1/dance1_subject2_part.npz - - src/unilab/assets/motions/g1/walk1_subject5_from_csv.npz + commands: + motion: + params: + motion_file: + - motions/g1/dance1_subject2_part.npz + - motions/g1/walk1_subject5_from_csv.npz ``` Conversion and inspection helpers are in `scripts/motion/`: @@ -115,21 +124,22 @@ randomization so the precise clip start state is reused: ```bash CUDA_VISIBLE_DEVICES=1 uv run train --algo sac --task g1_motion_tracking --sim mujoco \ training.use_amp=true algo.seed=1 \ - +env.motion_file=src/unilab/assets/motions/g1/motion_crawl_slope_uni.npz \ - +env.scene.model_file=src/unilab/assets/robots/g1/scene_crawl_slope.xml \ - +env.sampling_mode=start \ - env.truncate_on_clip_end=true \ - +env.max_episode_seconds=20.0 \ - '+env.pose_randomization={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ - '+env.velocity_randomization={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ - '+env.joint_position_range=[0,0]' + env.commands.motion.params.motion_file=motions/g1/motion_crawl_slope_uni.npz \ + env.scene.model_file=src/unilab/assets/robots/g1/scene_crawl_slope.xml \ + env.commands.motion.params.sampling_mode=start \ + env.commands.motion.params.truncate_on_clip_end=true \ + env.max_episode_seconds=20.0 \ + 'env.commands.motion.params.pose_range={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ + 'env.commands.motion.params.velocity_range={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ + 'env.commands.motion.params.joint_position_range=[0,0]' ``` -Key overrides: `env.motion_file` selects the crawl-slope clip; -`env.scene.model_file` switches to the slope scene (`scene_crawl_slope.xml` exists -under `src/unilab/assets/robots/g1/`); `sampling_mode=start` plus -`truncate_on_clip_end=true` starts from the clip beginning and truncates there; and -zeroing the randomization ranges reuses the exact clip initial state. +Key overrides: `env.commands.motion.params.motion_file` selects the crawl-slope +clip; `env.scene.model_file` switches to the slope scene +(`scene_crawl_slope.xml` exists under `src/unilab/assets/robots/g1/`); +`sampling_mode=start` plus `truncate_on_clip_end=true` starts from the clip +beginning and truncates there; and zeroing the command reset ranges reuses the +exact clip initial state. ## Interactive Debugging diff --git a/docs/sphinx/source/en/2-user_guide/4-tasks/3-manipulation.md b/docs/sphinx/source/en/2-user_guide/4-tasks/3-manipulation.md index 33d577191..c1a60a289 100644 --- a/docs/sphinx/source/en/2-user_guide/4-tasks/3-manipulation.md +++ b/docs/sphinx/source/en/2-user_guide/4-tasks/3-manipulation.md @@ -1,7 +1,7 @@ # Manipulation -Manipulation tasks live in `src/unilab/envs/manipulation/` and the Go2 arm -manip-loco env lives in `src/unilab/envs/locomotion/go2_arm/`. +Manipulation tasks live in `src/unilab/tasks/manipulation/` and the Go2 arm +manip-loco env lives in `src/unilab/tasks/locomotion/go2_arm/`. ## In-Hand diff --git a/docs/sphinx/source/en/2-user_guide/5-domain_randomization/0-index.md b/docs/sphinx/source/en/2-user_guide/5-domain_randomization/0-index.md index 3ee8a2c20..5caf9e859 100644 --- a/docs/sphinx/source/en/2-user_guide/5-domain_randomization/0-index.md +++ b/docs/sphinx/source/en/2-user_guide/5-domain_randomization/0-index.md @@ -1,9 +1,14 @@ # Domain Randomization -This page only describes the current status of tasks in the repo that are already registered and already wired to a DR provider. All conclusions come from the code; nothing is inferred from design intent. +This page only describes the current domain randomization status of registered tasks in the repo. All conclusions come from the code; nothing is inferred from design intent. -The current unified entry point lives in `NpEnv._init_domain_randomization()` and `DomainRandomizationManager`: +Two DR declaration paths exist today: + +- **Manager-Based (Compatible) tasks**: reset / interval randomization is declared through Hydra `events:` manager terms in the owner YAML; reset-lifecycle events sample at reset, interval-lifecycle events perturb between steps. See the `events:` block of `conf/ppo/task/go1_joystick_flat/base.yaml` for an example. +- **Legacy provider path**: only the 3 Adapted families (`sharpa_inhand` / `sharpa_inhand_grasp` / `go2_arm_manip_loco`, including their appo / hora / ppo_him owners) still declare `env.domain_rand.*` configuration through a `DomainRandomizationProvider` + `DomainRandomizationManager`. + +The unified entry point of the legacy provider path lives in `NpEnv._init_domain_randomization()` and `DomainRandomizationManager`: - init path: the task provider produces an `InitRandomizationPlan`; the manager calls the backend's `apply_init_randomization(...)` during env initialization - reset path: the task provider produces a `ResetPlan`; the manager validates capability and then calls the backend's `set_state(..., randomization=...)` @@ -17,62 +22,71 @@ These three paths correspond to three lifecycle classes: ## Status Conclusions -1. All tasks currently wired to a DR provider use the unified DR entry point; no task bypasses `DomainRandomizationManager` to run a separate DR flow inside `reset()`. -2. They are all roughly structured: task files define a `domain_rand` config dataclass, a `DomainRandomizationProvider`, and a `ResetPlan`; `G1WalkFlat` reuses `G1Walk`'s provider. -3. What is "unified" today is mainly the entry point and execution flow, not every randomization item itself. The shared helper `build_common_reset_randomization()` currently generates `base_mass_delta`, `base_com_offset`, `gravity`, `kp`, `kd`; the shared interval helper currently only generates push. +1. Manager-Based tasks do not register a DR provider; their reset/interval randomization consists of `events:` manager terms in the owner YAML, executed uniformly by the manager lifecycle. Only the frozen compatibility factories of the Adapted families still go through the `DomainRandomizationManager` unified entry point. +2. Adapted-family owners define a `domain_rand` config dataclass, a `DomainRandomizationProvider`, and a `ResetPlan`; Manager-Based owners declare reset behavior through Hydra command/event terms. G1 motion reset perturbations belong to `MotionCommandCfg`, while WBT adds `EventTermCfg` reset and interval terms. +3. What is "unified" today is mainly the entry point and execution flow, not every randomization item itself. The legacy path's shared helper `build_common_reset_randomization()` currently generates `base_mass_delta`, `base_com_offset`, `gravity`, `kp`, `kd`; the shared interval helper currently only generates push. 4. `ResetRandomizationPayload` can already express `gravity`, `body_iquat`, `body_inertia`, `kp`, `kd`, and `MuJoCoBackend` has declared support. Whether these are actually used still depends on whether the task provider samples and dispatches them. 5. `MotrixBackend` currently supports `base_mass_delta`, `base_com_offset`, `kp`, `kd`, and interval push; and it requires all model actuators to be position actuators during initialization. 6. `geom_size` is not a reset-lifecycle field; Sharpa-hand object geom scale is handled by init-lifecycle model materialization. ## Uniformity Assessment Table -| Task | Uses unified DR entry? | Structured form? | reset form | interval form | Code | +| Task | Declaration path | Structured form? | reset form | interval form | Code | | --- | --- | --- | --- | --- | --- | -| `Go1JoystickFlat` | Yes | Yes: `Domain_Rand + Provider + ResetPlan` | task state sampling + common payload | push | `go1/joystick.py` | -| `Go2JoystickFlat` | Yes | Yes: `Domain_Rand + Provider + ResetPlan` | task state sampling + common payload | push | `go2/joystick.py` | -| `G1WalkFlat` | Yes | Yes: `Domain_Rand + Provider + ResetPlan` | task state sampling + common payload | push | `g1/joystick.py` | -| `G1WalkRough` | Yes | Yes: reuses `G1WalkDomainRandomizationProvider` | task state sampling + common payload | push | `g1/joystick.py` | -| `G1MotionTracking` | Yes | Yes: `Domain_Rand + Provider + ResetPlan` | extensive task-specific reset sampling + common payload | push | `motion_tracking/g1/tracking.py` | -| `AllegroInhandRotation` | Yes | Yes: `DomainRandConfig + Provider + ResetPlan` | task-specific reset sampling + common payload | none | `allegro_inhand/rotation.py` | -| `SharpaInhandRotation` | Yes | Yes: `InitRandomizationPlan + ResetPlan + IntervalRandomizationPlan` | grasp cache sampling + common payload | object `body_force` | `sharpa_inhand/rotation.py` | -| `SharpaInhandRotationGrasp` | Yes | Yes: reuses the Sharpa rotation provider and overrides reset sampling | grasp collection reset + common payload | none | `sharpa_inhand/grasp_gen.py` | +| `Go1JoystickFlat` | Hydra `events:` terms | Yes: owner YAML declares reset/interval events | root-state reset + base mass/COM + `pd_gains` | `push_by_setting_velocity` event | `conf/ppo/task/go1_joystick_flat/base.yaml` | +| `Go2JoystickFlat` | Hydra `events:` terms | Yes: owner YAML declares reset events | root-state reset + `pd_gains` kp/kd | none | `conf/ppo/task/go2_joystick_flat/base.yaml` | +| `G1WalkFlat` | Hydra `events:` terms | Yes: Hydra `EventTermCfg` + Manager-Based reset terms | root-state reset + kp/kd via `pd_gains` | none | `g1/manager_terms.py` | +| `G1WalkRough` | Hydra `events:` terms | Yes: same Manager-Based event terms as `G1WalkFlat` | root-state reset + kp/kd via `pd_gains` | none | `g1/manager_terms.py` | +| `G1MotionTracking` | Hydra command term | Yes: Hydra `MotionCommandCfg` + Manager-Based command reset | motion frame, root pose/velocity, and joint-position sampling | none | `motion_tracking/common/manager_terms.py` | +| `G1WBTObs` | Hydra `events:` terms | Yes: same motion command + Hydra `EventTermCfg` | motion reset plus mass/COM/PD/friction/encoder-bias events | interval velocity kick | `motion_tracking/g1/manager_terms.py` | +| `AllegroInhandRotation` | Hydra `events:` terms | Yes: Hydra `EventTermCfg` + Manager-Based reset term | entity-scoped hand/ball reset | none | `allegro_inhand/manager_terms.py` | +| `AllegroInhandRotationGrasp` | Hydra `events:` terms | Yes: reuses the rotation reset event + `RecorderTermCfg` | noisy hand reset + grasp collection | none | `allegro_inhand/grasp_gen.py` | +| `SharpaInhandRotation` | legacy provider | Yes: `InitRandomizationPlan + ResetPlan + IntervalRandomizationPlan` | grasp cache sampling + common payload | object `body_force` | `sharpa_inhand/rotation.py` | +| `SharpaInhandRotationGrasp` | legacy provider | Yes: reuses the Sharpa rotation provider and overrides reset sampling | grasp collection reset + common payload | none | `sharpa_inhand/grasp_gen.py` | +| `Go2ArmManipLoco` | legacy provider | Yes: `DomainRandConfig + LocomotionDRProvider subclass + ResetPlan` | task state sampling + common payload | push | `go2_arm/manip_loco.py` | ## Per-task Domain Randomization List | Task | Currently implemented reset domain randomization | Currently implemented interval domain randomization | Default state | | --- | --- | --- | --- | -| `Go1JoystickFlat` | base xy; base yaw; base qvel; command sampling; `current_actions/last_actions` zeroed; optional `base_mass_delta`; optional `base_com_offset`; optional `gravity` | `push_robots` | `base_mass_delta`, `base_com_offset`, and push enabled by default; `gravity` disabled by default | -| `Go2JoystickFlat` | base xy; base yaw; base qvel; command sampling; `current_actions/last_actions` zeroed; kp/kd randomization (enabled by default); optional `base_mass_delta`; optional `base_com_offset`; optional `gravity` | `push_robots` | kp/kd enabled by default; common payload and push disabled by default | -| `G1WalkFlat` | base xy; base yaw; base qvel sampled by `reset_base_qvel_limit`; command sampling; `gait_phase` sampling; `current_actions/last_actions` zeroed; kp/kd randomization (enabled by default); optional `base_mass_delta`; optional `base_com_offset`; optional `gravity` | `push_robots` | kp/kd enabled by default; common payload and push disabled by default | -| `G1WalkRough` | Same as `G1WalkFlat`, directly reuses the same provider | `push_robots` | kp/kd enabled by default; common payload and push disabled by default | -| `G1MotionTracking` | motion frame sampling; root pose perturbation `x/y/z/roll/pitch/yaw`; root velocity perturbation `x/y/z/roll/pitch/yaw`; joint position noise; under MuJoCo clipped by joint range; `current_actions/last_actions` zeroed; optional `base_mass_delta`; optional `base_com_offset`; optional `gravity` | `push_robots` | `pose_randomization`, `velocity_randomization`, `joint_position_range` have non-zero perturbations by default; common payload and push disabled by default | -| `AllegroInhandRotation` | If a grasp cache exists, sample a grasp randomly; otherwise apply `joint_noise` to hand joints and `ball_z_offset` to the ball; always apply `ball_vel_noise` to ball linear velocity; optional common reset randomization payload (incl. `gravity`) | none | If the grasp cache path is available it is sampled by default; `joint_noise`, `ball_vel_noise`, `ball_z_offset` default to 0; common payload disabled by default | +| `Go1JoystickFlat` | base xy/yaw and base qvel via `reset_root_state_uniform`; command sampling (`UniformVelocityCommandCfg`); base mass via `randomize_rigid_body_mass`; base COM via `randomize_rigid_body_com`; kp/kd via `pd_gains` | `push_by_setting_velocity` interval event | all listed event terms are declared and enabled by default in `conf/ppo/task/go1_joystick_flat/base.yaml` | +| `Go2JoystickFlat` | base xy/yaw and base qvel via `reset_root_state_uniform`; command sampling; kp/kd via `pd_gains` | none | event terms declared and enabled by default in `conf/ppo/task/go2_joystick_flat/base.yaml` | +| `G1WalkFlat` | base xy/yaw and base qvel via `reset_root_state_uniform`; command sampling with a planar dead zone; `gait_phase` sampling; kp/kd randomization via `pd_gains` | none | kp/kd enabled on mujoco owners by default; disabled on motrix/mjwarp owners | +| `G1WalkRough` | Same as `G1WalkFlat` (shared owner bases, rough scene) | none | Same defaults as `G1WalkFlat` | +| `G1MotionTracking` | Motion-command frame sampling; root pose perturbation `x/y/z/roll/pitch/yaw`; root velocity perturbation `x/y/z/roll/pitch/yaw`; joint-position noise clipped through the public entity soft limits; action-manager state reset | none | `pose_range`, `velocity_range`, and `joint_position_range` have non-zero perturbations in the base owner | +| `G1WBTObs` | Same motion reset plus base mass, base COM, PD gain, foot friction, and encoder-bias event terms | `push_by_setting_velocity` | The WBT owner explicitly enables all listed event terms; unsupported capabilities raise rather than fall back | +| `AllegroInhandRotation` | Entity-scoped hand/ball reset; an explicitly configured grasp cache is sampled, otherwise `null` explicitly selects the model home pose; optional `joint_noise`, `ball_velocity_noise`, and `ball_z_offset` | none | owner YAML explicitly selects the home pose and zero reset noise; a configured missing or malformed cache fails closed | +| `AllegroInhandRotationGrasp` | Reuses the rotation reset with `joint_noise=0.25`; Manager-Based termination checks fingertip distance, contact count, and ball height; recorder stores successful timeout rows | none | generates the 50k-row Allegro grasp cache and raises `RunComplete` after a successful save | | `SharpaInhandRotation` | grasp cache bucketed sampling by `scale_ids`; object pose / quat reset; optional common reset randomization payload (incl. `gravity`) | object `body_force` direct force disturbance | `domain_rand.scale_list` defaults come from the owner YAML; under MuJoCo, object geom scale is materialized during init; common payload disabled by default; object force enabled by default via the Sharpa owner YAML | | `SharpaInhandRotationGrasp` | hand pose reset; object pose / quat reset; collects successful grasps and stores them bucketed by `scale_ids`; optional `base_mass_delta`; optional `base_com_offset`; optional `gravity` | none | Used by default to generate the Sharpa grasp cache; cache filename includes the single scale value; common payload disabled by default | ## Current Unified DR Capabilities and Boundaries -### 1. Unified Entry Point Is Complete +### 1. The Legacy Provider Entry Point Is Unified -The unified entry point is guaranteed by `NpEnv` and `DomainRandomizationManager`: +The unified entry point of the legacy provider path is guaranteed by `NpEnv` +and `DomainRandomizationManager`: - Tasks only need to register a provider - The manager uniformly performs capability validation - The backend is uniformly responsible for actually applying the randomization payload -So from an execution-path perspective, the tasks are already unified. +So from an execution-path perspective, the Adapted families still on this path +are unified; Manager-Based tasks instead execute the `events:` terms declared +in the owner YAML through the manager lifecycle. ### 2. The Shared Helpers Are Still Narrow -`dr_utils.py` currently has only two classes of shared helpers: +The legacy path's `dr_utils.py` currently has only two classes of shared helpers: - reset common payload: `base_mass_delta`, `base_com_offset`, `gravity`, `kp`, `kd` - interval common payload: push This means: -- Although locomotion tasks all go through the unified entry point, their base xy, yaw, qvel, command, and gait phase are still sampled directly inside each provider -- `G1MotionTracking`'s pose / velocity / joint noise is also task-specific logic +- The go2_arm / sharpa families still on the legacy provider path sample their + task-specific state directly inside each provider +- `G1MotionTracking`'s pose / velocity / joint noise is owned by its manager command - Allegro's grasp / object initial state sampling is entirely task-specific logic - Sharpa's `geom_size` scale is init-lifecycle model materialization and is not part of the reset common payload @@ -112,7 +126,10 @@ But on the task side, the current reality is: not every provider constructs thes - Lifecycle: only sampled and written at reset; the env retains that gravity until the next reset re-samples it. - Backend: currently in UniLab, only the MuJoCo backend declares support for this reset term; the Motrix backend does not. Some tasks filter it by capability and skip it; others raise an error in the validate stage. -The config entry is under each task's `env.domain_rand`: +The config entry exists only under `env.domain_rand` of the Adapted-family +owners still on the legacy provider path (`sharpa_inhand_grasp`, +`go2_arm_manip_loco`, and their hora / appo / ppo_him variants); Manager-Based +tasks have no `env.domain_rand`: ```yaml env: @@ -132,7 +149,7 @@ Field semantics: If you only want to randomize the magnitude while keeping the vertical-down direction, only open up the `z` component: ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[0.0,0.0,-10.5],[0.0,0.0,-8.5]]' ``` @@ -140,7 +157,7 @@ uv run train --algo ppo --task g1_walk_flat --sim mujoco \ If you want to randomize both direction and magnitude, open up `x/y/z`: ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[-0.3,-0.3,-10.5],[0.3,0.3,-8.5]]' ``` @@ -155,7 +172,13 @@ Notes: ## Interval push Usage -Tasks supporting interval push configure it under `env.domain_rand`: +The `env.domain_rand.push_robots` family of fields exists only in the go2_arm +Adapted-family owners (`conf/ppo/task/go2_arm_manip_loco/mujoco.yaml` etc.); +Manager-Based tasks declare push through a `push_by_setting_velocity` interval +event term instead (for example `conf/ppo/task/go1_joystick_flat/base.yaml` and +`conf/ppo/task/quadruped_joystick_rough/base.yaml`). + +The go2_arm owners configure push under `env.domain_rand`: ```yaml env: @@ -172,11 +195,11 @@ env: - `push_body_name`: the target body / link to apply the force to. Defaults to `null`, meaning the backend's `base_name` is used. ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task go2_arm_manip_loco --sim mujoco \ env.domain_rand.push_robots=true \ env.domain_rand.push_interval=500 \ 'env.domain_rand.max_force=[20.0,20.0,5.0]' \ - env.domain_rand.push_body_name=torso_link + env.domain_rand.push_body_name=base ``` Notes: diff --git a/docs/sphinx/source/en/2-user_guide/5-domain_randomization/1-configuration.md b/docs/sphinx/source/en/2-user_guide/5-domain_randomization/1-configuration.md index bc1ee6ae0..ca555110b 100644 --- a/docs/sphinx/source/en/2-user_guide/5-domain_randomization/1-configuration.md +++ b/docs/sphinx/source/en/2-user_guide/5-domain_randomization/1-configuration.md @@ -1,11 +1,19 @@ # Configuration -Domain randomization is configured inside the selected task owner YAML, usually -under `env.domain_rand`. Use `--task` and `--sim` to select backend-specific -behavior first, then override fields inside that selected owner. +Domain randomization is configured inside the selected task owner YAML. Use +`--task` and `--sim` to select backend-specific behavior first, then override +fields inside that selected owner. + +Two declaration paths exist today: + +- Manager-Based (Compatible) tasks declare reset / interval randomization + through Hydra `events:` manager terms in the owner YAML, for example + `conf/ppo/task/go1_joystick_flat/base.yaml`. +- Only the Adapted families (sharpa / go2_arm and their hora / appo / ppo_him + owners) still configure legacy provider fields under `env.domain_rand`. ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[0.0,0.0,-10.5],[0.0,0.0,-8.5]]' ``` @@ -21,25 +29,30 @@ Common lifecycle boundaries: The detailed task status and field semantics are in {doc}`0-index`. Domain randomization is split by lifecycle: init, reset, and interval. The -manager path is `src/unilab/dr/manager.py`; task providers live near the env -owners, and backend capabilities are declared through +legacy path's manager is `src/unilab/dr/manager.py`; task providers live near +the env owners, and backend capabilities are declared through `src/unilab/base/backend/base.py`. ## Reset Gravity Use `--sim mujoco` when enabling gravity reset randomization; Motrix does not -advertise the same gravity capability in the current backend. +advertise the same gravity capability in the current backend. This item is only +available on the legacy provider path (Adapted-family owners). ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[0.0,0.0,-10.5],[0.0,0.0,-8.5]]' ``` ## Interval Push +Manager-Based tasks declare push through a `push_by_setting_velocity` interval +event term; `env.domain_rand.push_robots` is only available on the go2_arm +Adapted-family owners. + ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task go2_arm_manip_loco --sim mujoco \ env.domain_rand.push_robots=true \ env.domain_rand.push_interval=500 \ 'env.domain_rand.max_force=[20.0,20.0,5.0]' @@ -48,8 +61,10 @@ uv run train --algo ppo --task g1_walk_flat --sim mujoco \ ## Owner-Local Defaults Keep ranges in the task owner YAML when they are part of the task contract. For -example, `conf/ppo/task/go2_joystick_rough/mujoco.yaml` enables base mass, -center-of-mass, kp/kd, and push randomization, while +example, the rough quadruped family's base mass, center-of-mass, kp/kd, and +push randomization are declared as event terms in the shared base +`conf/ppo/task/quadruped_joystick_rough/base.yaml` (the `go2_joystick_rough` +backend owners compose it through Hydra defaults), while `conf/ppo/task/sharpa_inhand/mujoco.yaml` configures object scale, friction, and force disturbance for Sharpa. diff --git a/docs/sphinx/source/en/2-user_guide/5-domain_randomization/2-writing_providers.md b/docs/sphinx/source/en/2-user_guide/5-domain_randomization/2-writing_providers.md index 36f74c627..061ab04e6 100644 --- a/docs/sphinx/source/en/2-user_guide/5-domain_randomization/2-writing_providers.md +++ b/docs/sphinx/source/en/2-user_guide/5-domain_randomization/2-writing_providers.md @@ -1,5 +1,12 @@ # Writing Providers +This page describes the legacy provider path: only the 3 Adapted families +(`sharpa_inhand` / `sharpa_inhand_grasp` / `go2_arm_manip_loco`) still declare +domain randomization through a task-level `DomainRandomizationProvider`. +Migrated Manager-Based tasks do not write providers; they declare randomization +through Hydra `events:` manager terms in the owner YAML (see {doc}`0-index` +and {doc}`1-configuration`). + Task-level domain randomization providers live with the task env owner. They sample task-specific state and return plans consumed by `DomainRandomizationManager`. @@ -25,13 +32,13 @@ The shared types live in `src/unilab/dr/types.py`, and the manager lives in ## Evidence -Representative provider implementations are in: +Representative provider implementations are in (all on the Adapted-family +compatibility path): -- `src/unilab/envs/locomotion/go1/joystick.py` -- `src/unilab/envs/locomotion/g1/joystick.py` -- `src/unilab/envs/motion_tracking/g1/tracking.py` -- `src/unilab/envs/manipulation/allegro_inhand/rotation.py` -- `src/unilab/envs/manipulation/sharpa_inhand/rotation.py` +- `src/unilab/tasks/locomotion/common/dr_provider.py` (`LocomotionDRProvider`, + used by the go2_arm family) +- `src/unilab/tasks/locomotion/go2_arm/manip_loco.py` +- `src/unilab/tasks/manipulation/sharpa_inhand/rotation.py` Developer contract details are in {doc}`../../4-developer_guide/2-contracts/4-dr_contract`. diff --git a/docs/sphinx/source/en/2-user_guide/6-terrain/2-heightfield_import.md b/docs/sphinx/source/en/2-user_guide/6-terrain/2-heightfield_import.md index ef5d80fa6..32a17a7d0 100644 --- a/docs/sphinx/source/en/2-user_guide/6-terrain/2-heightfield_import.md +++ b/docs/sphinx/source/en/2-user_guide/6-terrain/2-heightfield_import.md @@ -10,7 +10,7 @@ example is `Go2JoystickRough`, with owners in - `src/unilab/terrains/heightfield_terrains.py` - `src/unilab/terrains/terrain_generator.py` -- `src/unilab/envs/locomotion/go2/rough.py` +- `src/unilab/tasks/locomotion/go2/rough.py` - `src/unilab/base/backend/mujoco/xml.py` - `src/unilab/base/backend/motrix/scene.py` diff --git a/docs/sphinx/source/en/2-user_guide/7-tooling/3-nan_visualizer.md b/docs/sphinx/source/en/2-user_guide/7-tooling/3-nan_visualizer.md index 53495316c..bab4bef06 100644 --- a/docs/sphinx/source/en/2-user_guide/7-tooling/3-nan_visualizer.md +++ b/docs/sphinx/source/en/2-user_guide/7-tooling/3-nan_visualizer.md @@ -12,7 +12,7 @@ uv run train --algo ppo --task go2_joystick_flat --sim mujoco \ training.nan_guard.output_dir=/tmp/unilab/nan_dumps ``` -The viewer implementation is `src/unilab/tools/viz_nan.py`, registered as the +The viewer implementation is `src/unilab/utils/nan_viz.py`, registered as the `unilab-viz-nan` console entry. It replays a dump path and lets you select the environment index. Dump format and round-trip loading are covered by `tests/test_nan_guard.py`. diff --git a/docs/sphinx/source/en/2-user_guide/7-tooling/4-scene_export.md b/docs/sphinx/source/en/2-user_guide/7-tooling/4-scene_export.md index f92fa64b7..dfeea30da 100644 --- a/docs/sphinx/source/en/2-user_guide/7-tooling/4-scene_export.md +++ b/docs/sphinx/source/en/2-user_guide/7-tooling/4-scene_export.md @@ -1,6 +1,6 @@ # Scene Export -Scene export is implemented by `src/unilab/tools/export_scene.py` and registered +Scene export is implemented by `src/unilab/base/backend/mujoco/export_scene.py` and registered as the `unilab-export-scene` console entry in `pyproject.toml`. It accepts a MuJoCo XML or MJB model path, writes `scene.xml`, copies mesh assets when they are discoverable, and can create a zip archive. diff --git a/docs/sphinx/source/en/2-user_guide/7-tooling/5-robot_import.md b/docs/sphinx/source/en/2-user_guide/7-tooling/5-robot_import.md index 5c657bb03..888be3ff6 100644 --- a/docs/sphinx/source/en/2-user_guide/7-tooling/5-robot_import.md +++ b/docs/sphinx/source/en/2-user_guide/7-tooling/5-robot_import.md @@ -30,7 +30,7 @@ Prefer MuJoCo/MJCF `.xml`, copied according to the contract above. If the source is URDF-only, convert it with the repository script: ```bash -uv run unilab-import-robot [robot_name] +uv run scripts/tools/import_robot.py [robot_name] ``` ```{important} @@ -44,7 +44,7 @@ where possible. only for position-control owners. - If the robot must preserve torque/motor actuator semantics, later task extension should follow the control pattern in - `src/unilab/envs/locomotion/go2w/`: keep action interpretation, PD/torque + `src/unilab/tasks/locomotion/go2w/`: keep action interpretation, PD/torque control, and the actuator contract inside the robot owner boundary. - After conversion, `mujoco.viewer` opens automatically to show the converted result and proceed to keyframe adjustment. @@ -72,7 +72,7 @@ When checking `home`, confirm at least: ## Output Artifacts -After running `uv run unilab-import-robot [robot_name]`, the script +After running `uv run scripts/tools/import_robot.py [robot_name]`, the script generates: - `src/unilab/assets/robots//assets/`: converted and organized mesh diff --git a/docs/sphinx/source/en/3-deployment/1-sim_to_real/1-overview.md b/docs/sphinx/source/en/3-deployment/1-sim_to_real/1-overview.md index 38a1bc8b9..b1d3d14d4 100644 --- a/docs/sphinx/source/en/3-deployment/1-sim_to_real/1-overview.md +++ b/docs/sphinx/source/en/3-deployment/1-sim_to_real/1-overview.md @@ -70,10 +70,10 @@ flowchart LR runtime (units, frame, filter cutoffs). Log the first deploy-side observation window and compare it with a sim rollout built from the same owner YAML. -- **Action latency.** Some task configs expose one-step delayed action - execution through `control_config.simulate_action_latency`. Measure the - deploy loop and make the training owner match that contract before a - hardware run. See {doc}`8-latency_budget`. +- **Action latency.** Some task owners expose one-step delayed action execution + through a control config or Manager-Based action term. Measure the deploy loop + and make the training owner match that contract before a hardware run. See + {doc}`8-latency_budget`. - **Friction / damping mismatch.** Especially for in-hand manipulation. Sweep friction in DR; cross-check via {doc}`../2-sim_to_sim/3-contact_and_friction_alignment`. - **Reset transients.** Sim resets to a stable pose; deployment starts from a diff --git a/docs/sphinx/source/en/3-deployment/1-sim_to_real/2-g1_whole_body.md b/docs/sphinx/source/en/3-deployment/1-sim_to_real/2-g1_whole_body.md index e4ba8b088..905ac9736 100644 --- a/docs/sphinx/source/en/3-deployment/1-sim_to_real/2-g1_whole_body.md +++ b/docs/sphinx/source/en/3-deployment/1-sim_to_real/2-g1_whole_body.md @@ -115,7 +115,7 @@ clip. On hardware you need a wall-clock → phase mapping that is: - **Bounded rate** — clip dφ/dt to the value the policy was trained with (the motion loader records this; load `reference_motion.npz`). -See `unilab.envs.motion_tracking.g1.motion_loader` for the sim-side +See `unilab.tasks.motion_tracking.common.motion_loader` for the sim-side loader you should mirror on hardware. ## 5. Safety layer diff --git a/docs/sphinx/source/en/3-deployment/1-sim_to_real/4-allegro_inhand.md b/docs/sphinx/source/en/3-deployment/1-sim_to_real/4-allegro_inhand.md index 604b3b72e..e6351dde7 100644 --- a/docs/sphinx/source/en/3-deployment/1-sim_to_real/4-allegro_inhand.md +++ b/docs/sphinx/source/en/3-deployment/1-sim_to_real/4-allegro_inhand.md @@ -60,17 +60,19 @@ that samples plausible initial hand configurations. The hardware-side equivalent is the operator placing the cube in the hand — verify your distribution of starting configurations matches the trained env's grasp generator output (see -`unilab.envs.manipulation.allegro_inhand.grasp_gen`). +`unilab.tasks.manipulation.allegro_inhand.grasp_gen`). If your real-world starting grip differs systematically, **add those poses to the grasp generator**, retrain, and try again. ## Action interface -The manipulation envs map policy actions to joint position targets through the -task control config (`src/unilab/envs/manipulation/allegro_inhand/base.py` and -`src/unilab/envs/manipulation/sharpa_inhand/base.py`). The deploy controller -must use the same joint order, action scale, and limit policy. +The manipulation envs map policy actions to joint position targets through their +task control config. Allegro owns this declaration in +`conf/ppo/task/allegro_inhand/base.yaml` and its Manager-Based action term; +Sharpa currently owns it in `src/unilab/tasks/manipulation/sharpa_inhand/base.py`. +The deploy controller must use the same joint order, action scale, and limit +policy. ## Failure recovery diff --git a/docs/sphinx/source/en/3-deployment/1-sim_to_real/5-onnx_runtime.md b/docs/sphinx/source/en/3-deployment/1-sim_to_real/5-onnx_runtime.md index 92c101742..719af2dc8 100644 --- a/docs/sphinx/source/en/3-deployment/1-sim_to_real/5-onnx_runtime.md +++ b/docs/sphinx/source/en/3-deployment/1-sim_to_real/5-onnx_runtime.md @@ -12,7 +12,7 @@ graph when that path implements ONNX Runtime checking. | PPO (torch) | `scripts/train_rsl_rl.py` | `EXPORT_POLICY=True` in the script entrypoint; playback calls `runner.export_policy_to_onnx(...)` and `runner.export_policy_to_jit(...)`. | | HIM-PPO | `scripts/train_him_ppo.py` | Same script-level export pattern as PPO. | | APPO | `scripts/train_appo.py` | Playback writes `policy.onnx` and verifies ONNX Runtime output against PyTorch. | -| SAC / TD3 / FlashSAC | `scripts/train_offpolicy.py` | Playback writes `policy.onnx`; SAC and FlashSAC use `actor.as_export_module()` before export. | +| SAC / TD3 / FlashSAC | `scripts/train_sac.py` / `scripts/train_td3.py` / `scripts/train_flashsac.py` | Playback writes `policy.onnx`; SAC and FlashSAC use `actor.as_export_module()` before export. | ## Commands diff --git a/docs/sphinx/source/en/3-deployment/1-sim_to_real/6-domain_randomization.md b/docs/sphinx/source/en/3-deployment/1-sim_to_real/6-domain_randomization.md index 1a4512a13..64de13a12 100644 --- a/docs/sphinx/source/en/3-deployment/1-sim_to_real/6-domain_randomization.md +++ b/docs/sphinx/source/en/3-deployment/1-sim_to_real/6-domain_randomization.md @@ -45,7 +45,7 @@ range in the task owner only after recording why that range is plausible. Tasks that use DR attach a provider through the env initialization path: ```python -from unilab.envs.locomotion.common.dr_provider import LocomotionDRProvider +from unilab.tasks.locomotion.common.dr_provider import LocomotionDRProvider class MyTaskEnv(NpEnv): def __init__(self, cfg): diff --git a/docs/sphinx/source/en/3-deployment/1-sim_to_real/8-latency_budget.md b/docs/sphinx/source/en/3-deployment/1-sim_to_real/8-latency_budget.md index 7e3177b7f..16a04eff0 100644 --- a/docs/sphinx/source/en/3-deployment/1-sim_to_real/8-latency_budget.md +++ b/docs/sphinx/source/en/3-deployment/1-sim_to_real/8-latency_budget.md @@ -8,25 +8,26 @@ budgets as robot-specific measurements, not UniLab defaults. | Surface | Repo evidence | What it covers | | --- | --- | --- | -| One-step action delay | `control_config.simulate_action_latency` in locomotion and G1 motion-tracking envs | Executes the previous action instead of the current action. | -| G1 WBT observation history | `noise_config.obs_history_length` and `scripts/deploy/export_deploy_config.py` | Exports per-term `obs_layout` history for `gyro`, `joint_pos_rel`, `dof_vel`, and `last_actions`. | +| One-step action delay | Manager action term `simulate_action_latency` declarations in task owners | Executes the previous action instead of the current action. | +| G1 WBT observation history | Per-term `history_length` in `conf/sac/task/g1_wbt_obs/mujoco.yaml` and `scripts/deploy/export_deploy_config.py` | Exports per-term `obs_layout` history for `gyro`, `joint_pos_rel`, `dof_vel`, and `last_actions`. | | Sharpa tactile contact latency | `domain_rand.contact_latency` in Sharpa in-hand configs | Keeps previous tactile contact values for sampled contact channels. | | Deploy-side ONNX contract check | `scripts/deploy/sim_prototype.py` | Validates `obs_layout`, `obs_dim`, ONNX input width, clipping, and EMA action smoothing for the G1 WBT path. | ## Action Latency -For tasks that expose `control_config.simulate_action_latency`, the env applies -`last_actions` when the flag is enabled. Keep this in the selected task owner -YAML instead of adding deploy-only behavior later. +For Manager-Based tasks that enable action latency, the action manager applies +the previous action when the flag is enabled. Keep this in the selected task +owner YAML instead of adding deploy-only behavior later. ```yaml env: - control_config: - simulate_action_latency: true + actions: + joint_pos: + simulate_action_latency: true ``` The checked-in G1 WBT owner enables this flag in -`conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml`. +`conf/sac/task/g1_wbt_obs/mujoco.yaml`. ## Observation Lag And History diff --git a/docs/sphinx/source/en/3-deployment/2-sim_to_sim/7-config_guard.md b/docs/sphinx/source/en/3-deployment/2-sim_to_sim/7-config_guard.md index ae58f3f13..975a8ad99 100644 --- a/docs/sphinx/source/en/3-deployment/2-sim_to_sim/7-config_guard.md +++ b/docs/sphinx/source/en/3-deployment/2-sim_to_sim/7-config_guard.md @@ -18,7 +18,7 @@ uv run eval --algo ppo --task go2_joystick_flat --sim motrix --load-run -1 1. **At train time**: `ExperimentTracker` snapshots the contract fields that define policy I/O into `contract_snapshot` in `run_config.json` (the checkpoint format is untouched, so historical checkpoints stay compatible). 2. **At replay time**: `eval` loads the **target backend** owner config selected by `--sim` (e.g. `conf/ppo/task/go2_joystick_flat/motrix.yaml`) and injects `training.play_only=true`. -3. **Before env creation**: the four play entrypoints (rsl_rl / appo / offpolicy / him_ppo) call `resolve_sim2sim_config`, comparing the target config against the source run's contract snapshot field by field. +3. **Before env creation**: the play entrypoints (rsl_rl / appo / sac / td3 / flashsac / him_ppo) call `resolve_sim2sim_config`, comparing the target config against the source run's contract snapshot field by field. 4. **At weight load**: `policy_load_dim_guard` wraps checkpoint loading, re-raising cryptic tensor shape-mismatch errors as a clear sim2sim diagnostic. ## What the guard covers @@ -27,7 +27,7 @@ Fields are classified by dotted path into three tiers (see `src/unilab/training/ | Tier | Behavior | Fields | |---|---|---| -| **DENYLIST** | Mismatch → `CrossBackendIncompatibleError`, aborts | `algo.obs_groups`, `env.control_config.action_scale`, `algo.policy.actor_hidden_dims` / `critic_hidden_dims`, `algo.empirical_normalization` / `algo.obs_normalization`, `env.sampling_mode` | +| **DENYLIST** | Mismatch → `CrossBackendIncompatibleError`, aborts | `algo.obs_groups`, legacy `env.control_config.action_scale`, Manager-Based `env.observations` / `env.actions` / policy and critic group mapping, `algo.policy.actor_hidden_dims` / `critic_hidden_dims`, `algo.empirical_normalization` / `algo.obs_normalization`, `env.sampling_mode` | | **WARNING_LIST** | Prints a warning, continues | `reward.*`, `env.control_config.simulate_action_latency`, `env.ctrl_dt` | | **ALLOWLIST** | Free to override, not checked | `training.sim_backend`, `env.scene`, `training.play_steps`, `env.domain_rand`, `env.noise_config`, `env.commands.vel_limit` | @@ -40,8 +40,14 @@ If the target backend's DENYLIST fields differ from training (e.g. a task whose > Legacy runs: if `run_config.json` has no `contract_snapshot` (older training), the guard skips with a warning instead of breaking your workflow. +Manager-Based snapshots store the complete typed observation and action declarations from +Hydra. A snapshot from before those fields existed cannot prove that its policy I/O is +equivalent to a Manager-Based target, so asymmetric presence fails closed. Set +`training.sim2sim_strict=false` only as an explicit user override; the load-time dimension +guard still remains active. + ## See also - {doc}`1-backend_swap` - {doc}`4-reward_parity` -- {doc}`../../4-developer_guide/9-sim2sim_contract_status` +- {doc}`/zh_CN/4-developer_guide/9-sim2sim_contract_status` diff --git a/docs/sphinx/source/en/3-deployment/3-framework_migration/0-index.md b/docs/sphinx/source/en/3-deployment/3-framework_migration/0-index.md index 85047eeb2..6d62a0de5 100644 --- a/docs/sphinx/source/en/3-deployment/3-framework_migration/0-index.md +++ b/docs/sphinx/source/en/3-deployment/3-framework_migration/0-index.md @@ -9,7 +9,7 @@ contract-driven layout. :::{grid-item-card} From Isaac Lab :link: 1-from_isaac_lab :link-type: doc -Map GPU-resident task structure to UniLab's CPU sim and learner split. +Keep Manager-Based terms while adapting Hydra config, NumPy execution, and scene access. ::: :::{grid-item-card} From Legged Gym diff --git a/docs/sphinx/source/en/3-deployment/3-framework_migration/1-from_isaac_lab.md b/docs/sphinx/source/en/3-deployment/3-framework_migration/1-from_isaac_lab.md index f2a4c6fbb..39a78b4a4 100644 --- a/docs/sphinx/source/en/3-deployment/3-framework_migration/1-from_isaac_lab.md +++ b/docs/sphinx/source/en/3-deployment/3-framework_migration/1-from_isaac_lab.md @@ -1,93 +1,198 @@ # Migrating from Isaac Lab -If you have an Isaac Lab task you want to run in UniLab, this page tells -you what stays the same, what changes, and where the sharp edges are. - -## What stays the same - -- Gymnasium-style env interface (`reset`, `step`, `obs/reward/info`). -- Hydra-based configuration. Most of your existing YAML can be ported with - field-name remapping. -- The general idea of a "task" that composes scene + reward + DR + obs. -- PPO as the default algo — UniLab ships RSL-RL's PPO out of the box. +Port an Isaac Lab Manager-Based task to UniLab by keeping its manager and term +structure, then adapting configuration, numeric execution, and scene access at +their owner boundaries. Do not rewrite it as a monolithic `NpEnv` subclass. + +This is source-compatible migration, not a promise that an arbitrary Isaac Lab +task runs unchanged. The target path is: + +```text +Hydra owner YAML + -> plain ManagerBasedRlEnvCfg + -> Registry + make_manager_based_rl_env + -> ManagerBasedRlEnv on the NumPy/SimBackend runtime + -> NpEnvState for the existing training and IPC path +``` -## What changes +## Compatibility boundary ```{list-table} :header-rows: 1 -:widths: 30 35 35 - -* - Isaac Lab concept - - UniLab equivalent - - Notes -* - `DirectRLEnv` - - `unilab.base.np_env.NpEnv` - - UniLab obs is always a **dict**, not a tensor. -* - `RigidBody.cfg` - - Task-side asset import + scene composition - - See {doc}`../../4-developer_guide/1-architecture/4-scene_composition`. -* - GPU PhysX backend - - CPU MuJoCo / Motrix + GPU learner - - Architectural inversion — see below. -* - `RandomizationCfg` - - {doc}`../../4-developer_guide/2-contracts/4-dr_contract` - - UniLab DR runs in cold-path resampling only. -* - `RewardManager` chains - - Reward composition in env, plus - `unilab.training.reward` bookkeeping - - Reward terms still keyed for component-wise logging. -* - `EventCfg` event-driven hooks - - Phase + curriculum + DR providers - - Hooks are explicit, not implicit. +:widths: 28 24 48 + +* - Isaac Lab surface + - UniLab status + - Migration rule +* - Manager categories, term names and dictionary order + - Compatible + - Keep observation, action, event, reward, termination, command, and + curriculum terms in the same order. +* - Function/class terms and `func + params` + - Compatible + - Change imports to `unilab.managers`; keep term boundaries and partial + `reset(env_ids)` semantics. +* - `ManagerBasedRLEnv` / `ManagerBasedRLEnvCfg` + - Compatible spelling aliases + - The canonical UniLab names are `ManagerBasedRlEnv` and + `ManagerBasedRlEnvCfg`; the aliases point to the same implementation. +* - Tensor values and operations + - Adapted + - Replace `torch.Tensor` with `np.ndarray` and use vectorized NumPy. There is + no manager-facing device API. +* - Nested `@configclass` task configuration + - Adapted + - Move the complete task declaration to one Hydra owner YAML. `_target_` + selects concrete config dataclasses and dotted `func` values select terms. +* - `InteractiveSceneCfg`, USD, and PhysX views + - Adapted or unsupported + - Declare a task-owned `SceneCfg` and `EntityCfg`; access state and control + only through `SceneEntityCfg` and the public entity facade. Unsupported + capabilities raise during cold-path binding. +* - Omniverse, Isaac renderer, and Torch/PhysX mutation + - Unsupported + - UniLab does not install or silently emulate these runtimes. +``` + +The normative boundary is +{doc}`ADR-0006 `. Only +surfaces backed by registration, configuration, and tests should be described +as compatible. + +## Migration procedure + +### 1. Inventory the source task + +Pin the Isaac Lab revision and list the source manager groups, term names, term +order, parameters, observation dimensions, action dimensions, reset behavior, +and episode timing. Classify each dependency before writing code: + +- reuse an existing `unilab.managers` config or `unilab.envs.mdp` term; +- adapt a task-specific term from Torch to NumPy; +- stop if the term requires a capability absent from the public entity or + `SimBackend` contract. + +Do not probe backend objects with `getattr`/`hasattr`, return zeros, or route the +task back to a legacy environment. + +### 2. Port scene and assets on the cold path + +Replace Isaac Lab's USD/`InteractiveSceneCfg` declaration with a task-owned +`SceneCfg`. Declare every entity and selector needed by terms. The +`SceneEntityCfg` selector resolves names and regular expressions once during +materialization; reset and step reuse cached IDs and NumPy views. + +The Cartpole fixture uses a minimal task-owned MJCF asset. More complex assets +must follow +{doc}`scene composition <../../4-developer_guide/1-architecture/4-scene_composition>` +and the selected backend's formal capabilities. + +### 3. Port term code, not the manager structure + +Keep each function/class term and its parameters. Replace Torch types and +operators mechanically with NumPy, preserve batch shapes, and return one value +per environment where the source term does. Stateful terms resolve selectors +and allocate buffers in their constructor, then update only NumPy buffers on +the hot path. + +Python owns term implementations and reusable config dataclasses. It must not +hold a second task-specific list of enabled terms or default weights. + +### 4. Make Hydra the only task configuration owner + +Declare scene, timing, groups, terms, concrete config types, callables, +parameters, weights, and observation mapping in the owner YAML. For example: + +```yaml +env: + observations: + policy: + terms: + joint_pos_rel: + func: unilab.envs.mdp.joint_pos_rel + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + policy_observation_group: policy + critic_observation_group: null + +reward: + alive: + func: unilab.envs.mdp.is_alive + weight: 1.0 +``` + +Manager mappings whose value type is a single concrete config dataclass +(observations / events / rewards / terminations / curriculum / metrics / +recorders) may omit `_target_`; materialization infers it from the field type +annotation. `actions` / `commands` have abstract base configs, so they must +still declare a concrete `_target_` (for example +`unilab.envs.mdp.JointPositionActionCfg`). Config classes under +`unilab.managers.` (such as `SceneEntityCfg`) may be referenced by their bare +class name. + +Hydra composition materializes this declaration into plain typed config on the +cold path. Unknown fields, unresolved `_target_`/`func` references, and wrong +config types fail before reset or step. Direct Python config construction is +reserved for focused lower-level tests. + +### 5. Register one generic runtime path + +The task module registers `ManagerBasedRlEnvCfg` and +`make_manager_based_rl_env` for each backend that the repository actually +supports. Backend owner YAMLs carry backend identity and tuning. Users select +the composed owner through the normal CLI, for example: + +```bash +uv run train --algo ppo --task --sim mujoco ``` -## The architectural inversion - -Isaac Lab places the simulator on GPU and lets you batch thousands of -envs in PhysX. UniLab places the simulator on CPU (often multithread) and -batches across worker **processes**, sharing memory with a single GPU -learner. - -Implications: - -- **Per-env step time** in UniLab is comparable or worse than Isaac on a - single env. **Throughput** comes from process parallelism + asynchrony - (see `unilab.ipc.async_runner`). -- You can run on **MPS, ROCm, XPU** as the learner device — Isaac is - CUDA-only. -- **No GPU contention** between simulator and learner — your trainer's - memory usage is predictable. - -## Step-by-step migration - -1. **Audit observations.** Make sure every observation key is a vector - you can express without GPU PhysX queries. If not, add a state - estimator or move the query to cold path. -2. **Port the asset.** UniLab consumes MJCF as its source of truth. If - you have USD, convert to MJCF first. -3. **Port the env.** Subclass `unilab.base.np_env.NpEnv`. Move - reward computation into the env's `compute_reward()`. -4. **Port the YAML.** Map Isaac Lab's `EnvCfg` fields to UniLab task owner - YAML following the table in - {doc}`5-task_config_translation`. -5. **Port the reward.** Use the cookbook at - {doc}`6-reward_porting`. -6. **Validate.** Train a small run, compare reward curves against your - Isaac baseline. - -## What you'll miss (and how to compensate) - -- **Isaac Sim renderer.** Use Motrix's headless video export or build a - viser scene (`unilab.visualization.viser_scene`). -- **Per-env tensor obs.** UniLab gives you dict-of-arrays; wrap with your - own `obs_to_tensor` if you need a tensor. -- **Built-in GPU-side DR.** UniLab DR is CPU-side per process. For most - tasks this is plenty; for extreme parallelism use more worker - processes. +Do not add a task-specific training-script branch, environment factory, runner, +or IPC path. + +Two maintainer-approved factory wrappers are the only registered exceptions to +the generic-factory rule: `make_g1_walk_env` +(`src/unilab/tasks/locomotion/g1/manager_terms.py`) constructs the +`G1WalkManagerBasedEnv` subclass that owns the G1 walk manager-based runtime, +and `make_x2_wall_flip_env` +(`src/unilab/tasks/motion_tracking/x2/__init__.py`) resolves untracked X2 +meshes on the cold path before delegating to `make_manager_based_rl_env`. +Every other Compatible task registers `make_manager_based_rl_env` directly. + +### 6. Validate near each adaptation + +Test Hydra composition and typed materialization, term order and math, selector +failure, observation/action shapes, partial reset, and at least one real +registered backend transition. Compare behavior with the pinned source task; +benchmark only after semantic migration is complete. + +## Final task status + +The #1042 migration closeout covers 39 production tasks and 86 task/backend +registrations. The fail-closed source of truth is +`src/unilab/tasks/migration_matrix.py`: `migration_record()` raises `KeyError` +for a production task name with no entry, so adding a production registration +requires an explicit migration decision. + +- 36 tasks are **Compatible** (`target=complete`): the Hydra owner YAML + materializes the canonical NumPy Manager-Based runtime. +- 3 tasks are **Adapted** (`target=compatibility`): `Go2ArmManipLoco`, + `SharpaInhandRotation`, and `SharpaInhandRotationGrasp` keep custom + IK/history or tactile/contact/cache behavior behind one frozen compatibility + factory each; they migrate only when the formal capability exists. + +## Repository evidence + +`tests/fixtures/isaac_lab_cartpole/` ports the Manager-Based Cartpole task from +Isaac Lab commit `b0542fe2d45bf91c4e1d9ef6952b9c709c80b4e8`. It preserves all +12 source term names and their order while adapting Torch to NumPy, nested +config objects to Hydra YAML, and the scene/action/reset boundaries to a +fixture-local MJCF implementation. It is test-only evidence, not a production +task or a blanket Isaac Lab support claim. ## See also -- {doc}`2-from_legged_gym` -- {doc}`3-from_rsl_rl` -- {doc}`5-task_config_translation` -- {doc}`6-reward_porting` +- {doc}`Manager-Based API <../../4-developer_guide/1-architecture/6-manager_based_api>` +- {doc}`Environment contract <../../4-developer_guide/2-contracts/1-env_contract>` +- {doc}`ADR-0006 ` diff --git a/docs/sphinx/source/en/3-deployment/3-framework_migration/2-from_legged_gym.md b/docs/sphinx/source/en/3-deployment/3-framework_migration/2-from_legged_gym.md index 650c908ba..16d82862b 100644 --- a/docs/sphinx/source/en/3-deployment/3-framework_migration/2-from_legged_gym.md +++ b/docs/sphinx/source/en/3-deployment/3-framework_migration/2-from_legged_gym.md @@ -9,12 +9,12 @@ mostly mechanical. | Legged Gym | UniLab | |---|---| -| `LeggedRobot` env class | `unilab.envs.locomotion.common.base` | +| `LeggedRobot` env class | `unilab.tasks.locomotion.common.base` | | `compute_observations()` | env-side obs builder + `unilab.base.observations` | | `_reward_*` methods | env's `compute_reward()` + reward term registry | | `command_ranges` | task owner YAML's `commands` block | | Terrain curriculum | {doc}`../../2-user_guide/6-terrain/1-procedural` | -| RSL-RL PPO | `unilab.algos.torch.rsl_rl_ppo` | +| RSL-RL PPO | `unilab.algos.rsl_rl_ppo` | ## What's new @@ -22,7 +22,7 @@ mostly mechanical. + Motrix. Pick one (or both) before porting; see {doc}`../2-sim_to_sim/1-backend_swap`. - **Async collection.** Legged Gym collects on-GPU synchronously; UniLab's - APPO (`unilab.algos.torch.appo`) decouples collectors from + APPO (`unilab.algos.appo`) decouples collectors from learner. If wall-clock matters, port to APPO once your reward parity is established. - **Hardware deployment.** Legged Gym → real-world deployment is a @@ -32,7 +32,7 @@ mostly mechanical. ## Migration checklist 1. Copy your URDF / MJCF assets under `src/unilab/assets/robots//`. -2. Create a task module under `src/unilab/envs/locomotion//`. +2. Create a task module under `src/unilab/tasks/locomotion//`. 3. Mirror your reward terms; keep the same names so reward parity is diff-able. 4. Translate command sampling — Legged Gym's `_resample_commands` becomes diff --git a/docs/sphinx/source/en/3-deployment/3-framework_migration/3-from_rsl_rl.md b/docs/sphinx/source/en/3-deployment/3-framework_migration/3-from_rsl_rl.md index c8d2c5f9f..a104ceb59 100644 --- a/docs/sphinx/source/en/3-deployment/3-framework_migration/3-from_rsl_rl.md +++ b/docs/sphinx/source/en/3-deployment/3-framework_migration/3-from_rsl_rl.md @@ -1,7 +1,7 @@ # Migrating from RSL-RL You're already using RSL-RL standalone? Good news: UniLab ships RSL-RL PPO -as one of its supported algorithms (`unilab.algos.torch.rsl_rl_ppo`) +as one of its supported algorithms (`unilab.algos.rsl_rl_ppo`) and it's nearly drop-in. ## What you gain by moving inside UniLab @@ -13,7 +13,7 @@ and it's nearly drop-in. backend / task / algo selection. No more bespoke train scripts per robot. 3. **Async runner.** Wrap RSL-RL PPO inside - `unilab.algos.torch.appo` for higher throughput on machines + `unilab.algos.appo` for higher throughput on machines with many CPU cores. 4. **Deployment story.** ONNX export with the right wrapper, safety layer documentation, and the diff --git a/docs/sphinx/source/en/3-deployment/3-framework_migration/4-from_skrl.md b/docs/sphinx/source/en/3-deployment/3-framework_migration/4-from_skrl.md index 55d748e1e..c4d7616e6 100644 --- a/docs/sphinx/source/en/3-deployment/3-framework_migration/4-from_skrl.md +++ b/docs/sphinx/source/en/3-deployment/3-framework_migration/4-from_skrl.md @@ -8,7 +8,7 @@ deployment path. | skrl | UniLab | |---|---| -| `Agent` (PPO, SAC, …) | `unilab.algos.torch.*` | +| `Agent` (PPO, SAC, …) | `unilab.algos.*` | | `RolloutMemory` | `unilab.ipc.rollout_ring_buffer` | | `ReplayMemory` | `unilab.ipc.replay_buffer` | | `Trainer` | `unilab.training.run` | diff --git a/docs/sphinx/source/en/3-deployment/3-framework_migration/6-reward_porting.md b/docs/sphinx/source/en/3-deployment/3-framework_migration/6-reward_porting.md index 72e11d3a3..267e1efb9 100644 --- a/docs/sphinx/source/en/3-deployment/3-framework_migration/6-reward_porting.md +++ b/docs/sphinx/source/en/3-deployment/3-framework_migration/6-reward_porting.md @@ -38,7 +38,7 @@ Notes: - UniLab's `state` carries `prev_contact` so you don't need to manage edge detection yourself. See - `unilab.envs.locomotion.common.rewards`. + `unilab.tasks.locomotion.common.rewards`. ## Pattern: action smoothness penalty @@ -47,7 +47,7 @@ def reward_action_rate(self, state): return -np.sum((state.action - state.prev_action) ** 2, axis=1) ``` -Already a stock helper in `unilab.envs.locomotion.common.rewards`. +Already a stock helper in `unilab.tasks.locomotion.common.rewards`. ## Pattern: posture penalty @@ -76,5 +76,5 @@ def reward_termination(self, state): ## See also - {doc}`5-task_config_translation` -- `unilab.training.reward` -- `unilab.envs.locomotion.common.rewards` +- `unilab.utils.reward` +- `unilab.tasks.locomotion.common.rewards` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/0-index.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/0-index.md index f31456d43..9e48c9faf 100644 --- a/docs/sphinx/source/en/4-developer_guide/1-architecture/0-index.md +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/0-index.md @@ -36,6 +36,12 @@ Scene fragments, assets, and cold-path materialization. Bootstrap imports and env/backend registration. ::: +:::{grid-item-card} Manager-Based API +:link: 6-manager_based_api +:link-type: doc +Community manager semantics, NumPy runtime, and fail-closed boundaries. +::: + :::: ```{toctree} @@ -46,4 +52,5 @@ Bootstrap imports and env/backend registration. 3-layer_boundaries 4-scene_composition 5-registry +6-manager_based_api ``` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/1-overview.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/1-overview.md index 987f67999..da683c0d1 100644 --- a/docs/sphinx/source/en/4-developer_guide/1-architecture/1-overview.md +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/1-overview.md @@ -79,7 +79,9 @@ Use `make test` for the fast path and `make test-all` (`make check`, - `scripts/train_rsl_rl.py` - `scripts/train_appo.py` -- `scripts/train_offpolicy.py` +- `scripts/train_sac.py` +- `scripts/train_td3.py` +- `scripts/train_flashsac.py` - `src/unilab/base/np_env.py` - `src/unilab/base/backend/base.py` - `src/unilab/base/registry.py` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/2-runtime_model.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/2-runtime_model.md index b2989ba31..423775d28 100644 --- a/docs/sphinx/source/en/4-developer_guide/1-architecture/2-runtime_model.md +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/2-runtime_model.md @@ -41,8 +41,8 @@ CPU physics env loop -> shared IPC buffer -> learner ## Evidence In Repo - PPO entrypoint: `scripts/train_rsl_rl.py` -- APPO runner: `src/unilab/algos/torch/appo/runner.py` -- Off-policy runner: `src/unilab/algos/torch/offpolicy/double_buffer_runner.py` +- APPO runner: `src/unilab/algos/appo/runner.py` +- Off-policy runner: `src/unilab/algos/offpolicy/double_buffer_runner.py` - IPC primitives: `src/unilab/ipc/async_runner.py`, `src/unilab/ipc/rollout_ring_buffer.py`, `src/unilab/ipc/replay_buffer.py`, `src/unilab/ipc/weight_sync.py` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/3-layer_boundaries.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/3-layer_boundaries.md index 007682817..164b2b8a7 100644 --- a/docs/sphinx/source/en/4-developer_guide/1-architecture/3-layer_boundaries.md +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/3-layer_boundaries.md @@ -34,4 +34,5 @@ standard is {doc}`/zh_CN/4-developer_guide/0-index`. - Env state contract: `src/unilab/base/np_env.py` - Registry construction path: `src/unilab/base/registry.py` - Training entrypoints: `scripts/train_rsl_rl.py`, - `scripts/train_appo.py`, `scripts/train_offpolicy.py` + `scripts/train_appo.py`, `scripts/train_sac.py`, + `scripts/train_td3.py`, `scripts/train_flashsac.py` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/4-scene_composition.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/4-scene_composition.md index 872ac1a37..c921febca 100644 --- a/docs/sphinx/source/en/4-developer_guide/1-architecture/4-scene_composition.md +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/4-scene_composition.md @@ -105,7 +105,7 @@ Disallowed on hot paths: The current procedural terrain user-facing path is Go2 rough terrain: -- Env owner: `src/unilab/envs/locomotion/go2/rough.py` +- Task owner: `src/unilab/tasks/locomotion/go2/rough.py` - Terrain generator: `src/unilab/terrains/terrain_generator.py` - MuJoCo materializer: `src/unilab/base/backend/mujoco/xml.py` - Motrix materializer: `src/unilab/base/backend/motrix/scene.py` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/5-registry.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/5-registry.md index 4b6538cd1..3bf01b4eb 100644 --- a/docs/sphinx/source/en/4-developer_guide/1-architecture/5-registry.md +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/5-registry.md @@ -8,11 +8,9 @@ defined by {doc}`/adr/ADR-0004-registry-bootstrap-contract` and implemented in 1. Training entrypoints call `unilab.training.common.ensure_registries()`. 2. That helper delegates to `unilab.base.registry.ensure_registries()`. -3. The registry imports declared bootstrap packages: - `unilab.envs.locomotion`, `unilab.envs.manipulation`, and - `unilab.envs.motion_tracking`. -4. Each package exposes `__unilab_registry_modules__`, a tuple of modules that - contain registration side effects. +3. The registry imports its sole declared bootstrap package, `unilab.tasks`. +4. `unilab.tasks` exposes `__unilab_registry_modules__`, an explicit tuple of + task leaf modules that contain registration side effects. 5. Imported modules register configs with `@registry.envcfg(...)` and env implementations with `@registry.env(..., sim_backend=...)` or `registry.register_env(...)`. @@ -22,9 +20,8 @@ defined by {doc}`/adr/ADR-0004-registry-bootstrap-contract` and implemented in ## Extension Rules -- Add new env modules to the package-level `__unilab_registry_modules__` tuple - if they live in a new module that is not imported by an existing bootstrap - entry. +- Add new task leaves to `unilab.tasks.__unilab_registry_modules__` when they + are not imported by an existing bootstrap entry. - Keep registration cheap. Scene materialization, XML processing, asset access, and backend construction belong after `registry.make(...)`, not in decorator registration. @@ -35,7 +32,5 @@ defined by {doc}`/adr/ADR-0004-registry-bootstrap-contract` and implemented in - Bootstrap helper: `src/unilab/base/registry.py` - Training helper: `src/unilab/training/common.py` -- Package declarations: `src/unilab/envs/locomotion/__init__.py`, - `src/unilab/envs/manipulation/__init__.py`, - `src/unilab/envs/motion_tracking/__init__.py` +- Task bootstrap declaration: `src/unilab/tasks/__init__.py` - Tests: `tests/base/test_registry.py`, `tests/utils/test_algo_utils.py` diff --git a/docs/sphinx/source/en/4-developer_guide/1-architecture/6-manager_based_api.md b/docs/sphinx/source/en/4-developer_guide/1-architecture/6-manager_based_api.md new file mode 100644 index 000000000..d22db32e1 --- /dev/null +++ b/docs/sphinx/source/en/4-developer_guide/1-architecture/6-manager_based_api.md @@ -0,0 +1,35 @@ +# Manager-Based API + +UniLab uses a community-compatible manager API on its NumPy runtime. Manager modules, +term configs, function/class terms, lifecycle ordering, and reset semantics follow the +pinned mjlab 1.6.0 source. Numeric execution uses NumPy while preserving UniLab's +`NpEnvState`, Hydra owner YAML, `SimBackend`, registry, and IPC contracts. + +The normative compatibility matrix and mechanical migration example are in +{doc}`ADR-0006 `. + +## Invariants + +- Community manager semantics and a general structure take priority over local + optimizations that would create a UniLab-only term API. +- Manager buffers, term returns, environment IDs, and entity views use `np.ndarray` + or `slice`; manager core does not depend on Torch, Warp, runners, learners, or IPC. +- `SceneEntityCfg` resolves through a base-owned scene/entity facade on the cold path. + The facade uses only the public `SimBackend` contract, and hot paths reuse cached IDs + and views. +- Named-sensor observation terms bind a backend-owned view through + `EntityScene.bind_sensor_data(...)` during construction; their hot path only reads + that view and never re-resolves sensor names or XML/model metadata. +- `ManagerBasedRlEnv` owns backend materialization exactly once: manager construction + and startup events run first, then `SimBackend.materialize()` completes before any + reset or step can execute. +- Explicitly empty configuration may use a Null manager. A requested capability that + is unavailable fails at the nearest boundary; it is never skipped, zero-filled, or + routed back to a legacy environment. +- Hot paths avoid obvious repeated parsing, per-environment Python loops, copies, and + temporary allocations. Further optimization requires benchmark evidence and must + not add disproportionate structural complexity. + +Only surfaces backed by registration, configuration, and tests may be called +Compatible. NumPy/env/config adapters are Adapted; capabilities without a formal +backend contract are Unsupported and fail closed. diff --git a/docs/sphinx/source/en/4-developer_guide/2-contracts/2-backend_contract.md b/docs/sphinx/source/en/4-developer_guide/2-contracts/2-backend_contract.md index f7c021958..2012bae3b 100644 --- a/docs/sphinx/source/en/4-developer_guide/2-contracts/2-backend_contract.md +++ b/docs/sphinx/source/en/4-developer_guide/2-contracts/2-backend_contract.md @@ -17,6 +17,9 @@ Optional capabilities are explicit: physics-state playback, and native video capture support. - `BackendHeightScanner` and `create_hfield_scanner(...)` expose terrain scan support through a reusable backend-owned object. +- `BackendSensorView` and `bind_sensor_data(...)` validate ordered named sensors + on the cold path and retain a backend-owned reader for finite, shape-stable + NumPy batches. Manager hot paths do not inspect XML or model metadata. - Domain randomization support is surfaced through `get_dr_capabilities()` and the init, reset, and interval randomization methods. - Unsupported optional methods raise `NotImplementedError` from the base class. @@ -37,5 +40,6 @@ Optional capabilities are explicit: - Backend factory: `src/unilab/base/backend/__init__.py` - MuJoCo backend: `src/unilab/base/backend/mujoco/backend.py` - Motrix backend: `src/unilab/base/backend/motrix/backend.py` -- Backend contract tests: `tests/base/test_sim_backend.py`, +- Backend contract tests: `tests/base/test_backend_sensor_view.py`, + `tests/base/test_backend_conformance.py`, `tests/base/test_sim_backend.py`, `tests/base/test_backend_imports.py`, `tests/base/test_motrix_backend_options.py` diff --git a/docs/sphinx/source/en/4-developer_guide/2-contracts/3-task_owner.md b/docs/sphinx/source/en/4-developer_guide/2-contracts/3-task_owner.md index 8c9ca61c9..2b71bff19 100644 --- a/docs/sphinx/source/en/4-developer_guide/2-contracts/3-task_owner.md +++ b/docs/sphinx/source/en/4-developer_guide/2-contracts/3-task_owner.md @@ -8,8 +8,8 @@ contract is recorded in - PPO and APPO owner YAMLs use `conf/{ppo,appo}/task//.yaml`. -- Off-policy owner YAMLs include the algorithm dimension: - `conf/offpolicy/task///.yaml`. +- Off-policy algorithms (SAC / TD3 / FlashSAC) each have their own config + tree: `conf//task//.yaml`. - Other existing config roots, such as `conf/ppo_him/` and `conf/hora_distill/`, follow the same owner-YAML identity rule for their supported tasks. @@ -19,8 +19,8 @@ contract is recorded in - Use public CLI flags to switch backend, for example `uv run train --algo ppo --task go2_joystick_flat --sim mujoco` or `uv run train --algo ppo --task go2_joystick_flat --sim motrix`. -- For off-policy entrypoints, keep `--algo ` aligned with the internal - owner YAML path `conf/offpolicy/task///.yaml`. +- For off-policy entrypoints, `--algo ` selects the per-algorithm config + tree; the owner YAML path is `conf//task//.yaml`. - `training.sim_backend` is an identity field inside the selected owner YAML. It is not an independent backend switch. - Backend-specific reward, env, scene, and algorithm differences belong in the @@ -32,8 +32,7 @@ contract is recorded in - PPO owner example: `conf/ppo/task/go2_joystick_flat/mujoco.yaml` - APPO config root: `conf/appo/config.yaml` -- Off-policy config root: `conf/offpolicy/config.yaml` -- Off-policy task/algo guard: `src/unilab/training/common.py` +- Off-policy config roots: `conf/{sac,td3,flashsac}/config.yaml` - Config tests: `tests/config/test_config_system.py`, `tests/scripts/test_train_script_configs.py`, `tests/envs/locomotion/g1/test_g1_owner_contract.py` diff --git a/docs/sphinx/source/en/4-developer_guide/2-contracts/4-dr_contract.md b/docs/sphinx/source/en/4-developer_guide/2-contracts/4-dr_contract.md index 39fd08277..644ecd7ae 100644 --- a/docs/sphinx/source/en/4-developer_guide/2-contracts/4-dr_contract.md +++ b/docs/sphinx/source/en/4-developer_guide/2-contracts/4-dr_contract.md @@ -100,6 +100,6 @@ payloads. - DR types: `src/unilab/dr/types.py` - DR manager: `src/unilab/dr/manager.py` - Backend interface: `src/unilab/base/backend/base.py` -- Example providers: `src/unilab/envs/locomotion/g1/joystick.py`, - `src/unilab/envs/motion_tracking/g1/tracking.py`, - `src/unilab/envs/manipulation/sharpa_inhand/rotation.py` +- Example providers: `src/unilab/tasks/locomotion/common/dr_provider.py`, + `src/unilab/tasks/locomotion/go2_arm/manip_loco.py`, + `src/unilab/tasks/manipulation/sharpa_inhand/rotation.py` diff --git a/docs/sphinx/source/en/4-developer_guide/2-contracts/5-runner_lifecycle.md b/docs/sphinx/source/en/4-developer_guide/2-contracts/5-runner_lifecycle.md index f1875f5cc..29b422ee0 100644 --- a/docs/sphinx/source/en/4-developer_guide/2-contracts/5-runner_lifecycle.md +++ b/docs/sphinx/source/en/4-developer_guide/2-contracts/5-runner_lifecycle.md @@ -21,7 +21,8 @@ The training scripts follow the same high-level sequence: `OnPolicyRunner`. - `scripts/train_appo.py` uses `APPORunner`, `RolloutRingBuffer`, and `SharedWeightSync`. -- `scripts/train_offpolicy.py` uses off-policy runners with `ReplayBuffer` and +- `scripts/train_sac.py`, `scripts/train_td3.py`, and + `scripts/train_flashsac.py` use off-policy runners with `ReplayBuffer` and `SharedWeightSync`. - `AsyncRunner` owns collector process lifecycle and shared-resource cleanup for async runners. diff --git a/docs/sphinx/source/en/4-developer_guide/3-extending/1-new_task.md b/docs/sphinx/source/en/4-developer_guide/3-extending/1-new_task.md index 37924785a..e73f74009 100644 --- a/docs/sphinx/source/en/4-developer_guide/3-extending/1-new_task.md +++ b/docs/sphinx/source/en/4-developer_guide/3-extending/1-new_task.md @@ -20,8 +20,8 @@ Start from the contracts: {doc}`../2-contracts/1-env_contract`, `reset(env_indices)` returns `(obs_dict, info_dict)`, and `step(actions)` returns `NpEnvState`. 7. Add owner YAMLs under the relevant config root, such as - `conf/ppo/task//.yaml` or - `conf/offpolicy/task///.yaml`. + `conf/ppo/task//.yaml` or, for an off-policy algorithm, + `conf//task//.yaml`. 8. Put task or scene keyframes in task/scene XML fragments referenced through `SceneCfg.fragment_files`; do not put task-level keyframes in `robot.xml`. @@ -38,5 +38,5 @@ Start from the contracts: {doc}`../2-contracts/1-env_contract`, - Registry API: `src/unilab/base/registry.py` - Env state contract: `src/unilab/base/np_env.py` - Scene config: `src/unilab/base/scene.py` -- Existing task examples: `src/unilab/envs/locomotion/go2/joystick.py`, - `src/unilab/envs/manipulation/allegro_inhand/rotation.py` +- Existing task examples: `src/unilab/tasks/locomotion/go2/joystick.py`, + `src/unilab/tasks/manipulation/allegro_inhand/rotation.py` diff --git a/docs/sphinx/source/en/4-developer_guide/3-extending/3-new_algorithm.md b/docs/sphinx/source/en/4-developer_guide/3-extending/3-new_algorithm.md index 8d15edeec..fec9181d9 100644 --- a/docs/sphinx/source/en/4-developer_guide/3-extending/3-new_algorithm.md +++ b/docs/sphinx/source/en/4-developer_guide/3-extending/3-new_algorithm.md @@ -8,15 +8,17 @@ Algorithm work must preserve the env, config, and runner contracts. Start with - Synchronous on-policy example: `scripts/train_rsl_rl.py`. - Async on-policy example: `scripts/train_appo.py` with `APPORunner`. -- Off-policy examples: `scripts/train_offpolicy.py` with SAC, TD3, and - FlashSAC configs under `conf/offpolicy/`. +- Off-policy examples: `scripts/train_sac.py`, `scripts/train_td3.py`, and + `scripts/train_flashsac.py`, each with its own config tree under + `conf//`. ## Implementation Checklist 1. Put reusable learner or runner code under `src/unilab/algos/`. 2. Add Hydra config under the owning config root. A new off-policy variant should - add `conf/offpolicy/algo/.yaml` and matching - `conf/offpolicy/task///.yaml` owner YAMLs. + add its own config tree: `conf//config.yaml` with the algorithm + hyperparameters inlined, plus matching + `conf//task//.yaml` owner YAMLs. 3. If a new top-level training script is required, keep it as assembly: compose Hydra, call `ensure_registries()`, construct the env through the registry path, then hand control to the runner or trainer. @@ -25,9 +27,9 @@ Algorithm work must preserve the env, config, and runner contracts. Start with 5. For async algorithms, reuse `AsyncRunner`, `ReplayBuffer` or `RolloutRingBuffer`, and `SharedWeightSync` instead of creating a new IPC lifecycle. -6. For off-policy algorithms, keep the CLI `--algo ` selection aligned - with the owner YAML path `conf/offpolicy/task///.yaml`; - `assert_offpolicy_task_choice_matches_algo` enforces this guard. +6. For off-policy algorithms, the CLI `--algo ` selection maps to the + per-algorithm config tree; owner YAMLs live at + `conf//task//.yaml`. ## Validation Near Risk @@ -41,4 +43,4 @@ Algorithm work must preserve the env, config, and runner contracts. Start with - Structured config dataclasses: `src/unilab/structured_configs.py` - Training helpers: `src/unilab/training/common.py`, `src/unilab/training/run.py` -- Existing algorithm packages: `src/unilab/algos/torch/` +- Existing algorithm packages: `src/unilab/algos/` diff --git a/docs/sphinx/source/en/4-developer_guide/3-extending/4-new_terrain.md b/docs/sphinx/source/en/4-developer_guide/3-extending/4-new_terrain.md index 23ae12c30..de2dd4cc4 100644 --- a/docs/sphinx/source/en/4-developer_guide/3-extending/4-new_terrain.md +++ b/docs/sphinx/source/en/4-developer_guide/3-extending/4-new_terrain.md @@ -35,4 +35,4 @@ materialization out of `step()`, `reset()`, and hot domain-randomization loops. - Terrain configs and presets: `src/unilab/terrains/config.py` - Terrain generator: `src/unilab/terrains/terrain_generator.py` - Heightfield terrain types: `src/unilab/terrains/heightfield_terrains.py` -- Height-scan helper: `src/unilab/envs/locomotion/common/height_scan.py` +- Height-scan helper: `src/unilab/tasks/locomotion/common/height_scan.py` diff --git a/docs/sphinx/source/en/4-developer_guide/6-agent_quick_reference.md b/docs/sphinx/source/en/4-developer_guide/6-agent_quick_reference.md index c27f11854..9512aac70 100644 --- a/docs/sphinx/source/en/4-developer_guide/6-agent_quick_reference.md +++ b/docs/sphinx/source/en/4-developer_guide/6-agent_quick_reference.md @@ -11,7 +11,8 @@ repo facts. - Algorithms index: {doc}`../2-user_guide/2-algorithms/0-index` - PPO entrypoint: `scripts/train_rsl_rl.py` - APPO entrypoint: `scripts/train_appo.py` -- SAC / TD3 / FlashSAC entrypoint: `scripts/train_offpolicy.py` +- SAC / TD3 / FlashSAC entrypoints: `scripts/train_sac.py` / + `scripts/train_td3.py` / `scripts/train_flashsac.py` - HIM-PPO entrypoint: `scripts/train_him_ppo.py` - HORA distillation entrypoint: `scripts/train_hora_distill.py` diff --git a/docs/sphinx/source/en/4-developer_guide/7-motion_assets.md b/docs/sphinx/source/en/4-developer_guide/7-motion_assets.md index ded0b552f..cca872197 100644 --- a/docs/sphinx/source/en/4-developer_guide/7-motion_assets.md +++ b/docs/sphinx/source/en/4-developer_guide/7-motion_assets.md @@ -82,45 +82,59 @@ Alternatively, pre-download into the in-repo directory with `--local-dir` 3. Reference the new file path in the env config. -## Robot Mesh Assets +## Robot Binary Assets -Robot binary meshes (`.STL`) are externalized the same way, on the Hugging Face -dataset repo +Robot binary meshes and textures (for example `.STL`, `.obj`, and `.png`) are +externalized the same way, on the Hugging Face dataset repo [unilabsim/unilab-robots](https://huggingface.co/datasets/unilabsim/unilab-robots). X2 meshes download lazily on first use and land under their original path -`src/unilab/assets/robots/x2/meshes/`, so the XML `meshdir` references resolve -unchanged. Pre-fetch them without running a task: +`src/unilab/assets/robots/x2/meshes/`. T800 OBJ files and textures land under +`robots/t800/assets/` and `robots/t800/textures/`, respectively, so the original +relative XML paths remain valid. Pre-fetch them without running a task: ```bash uv run unilab-pull-assets --robot x2 +uv run unilab-pull-assets --robot t800 ``` -To add a new robot's meshes: +To add a new robot's binary assets: -1. Upload to the HF repo, keeping the directory layout identical: +1. Upload each directory to the HF repo while keeping the directory layout + identical. A robot with multiple asset directories requires one upload per + directory. For example, T800 uses: ```bash - huggingface-cli upload unilabsim/unilab-robots \ - src/unilab/assets/robots//meshes robots//meshes \ + uv run hf upload unilabsim/unilab-robots \ + src/unilab/assets/robots/t800/assets robots/t800/assets \ + --repo-type dataset + uv run hf upload unilabsim/unilab-robots \ + src/unilab/assets/robots/t800/textures robots/t800/textures \ --repo-type dataset ``` -2. Ignore the local `*.STL` in `.gitignore` (keep a `.gitkeep` so the directory - persists). -3. Resolve the directory once on a cold path from the env, e.g. - `resolve_robot_asset_dir("robots//meshes", marker=".STL")`. +2. Ignore the downloaded directory contents in `.gitignore` and keep a + `.gitkeep` so each directory persists. +3. Resolve every referenced directory on a cold path before the backend parses + the XML. The current API resolves one directory per call, so a T800 task uses: + + ```python + resolve_robot_asset_dir("robots/t800/assets", marker="LINK_BASE.obj") + resolve_robot_asset_dir("robots/t800/textures", marker="LINK_BASE.png") + ``` ## Architecture Notes - Asset resolver module: `src/unilab/assets/hub.py` (`resolve_motion_files`). - Single integration point: `MotionLoader.__init__` in - `src/unilab/envs/motion_tracking/g1/motion_loader.py`, which calls the + `src/unilab/tasks/motion_tracking/common/motion_loader.py`, which calls the resolver once on a cold path. - Hot paths (`step` / `reset`) never trigger any file download or parsing. - `ASSETS_ROOT_PATH` is unchanged, so the download target matches the original local path exactly. -- Robot meshes use the same directory resolver (`resolve_robot_asset_dir`), - integrated at `X2WallFlipTrackingEnv.__init__` in - `src/unilab/envs/motion_tracking/x2/flip_tracking.py`, and exposed as the - `unilab-pull-assets` CLI. +- Robot binary assets use the same directory resolver + (`resolve_robot_asset_dir`). The + thin `make_x2_wall_flip_env` factory in + `src/unilab/tasks/motion_tracking/x2/__init__.py` resolves them once before + delegating to the shared manager environment factory. The resolver is also + exposed through the `unilab-pull-assets` CLI. diff --git a/docs/sphinx/source/en/4-developer_guide/8-motrix_contact_sensor.md b/docs/sphinx/source/en/4-developer_guide/8-motrix_contact_sensor.md index 2dd6d323c..805493dce 100644 --- a/docs/sphinx/source/en/4-developer_guide/8-motrix_contact_sensor.md +++ b/docs/sphinx/source/en/4-developer_guide/8-motrix_contact_sensor.md @@ -91,7 +91,7 @@ contact-frame data (one normal scalar plus two tangent scalars). The env reads tactile force through `_read_tactile_force()` → `_extract_sensor_scalar()` in -`src/unilab/envs/manipulation/sharpa_inhand/base.py`. That helper currently +`src/unilab/tasks/manipulation/sharpa_inhand/base.py`. That helper currently collapses any `(N, >=3)` array with `np.linalg.norm(data[:, :3], axis=1)`. If the env still routes both backend shapes through that one branch, the @@ -131,8 +131,8 @@ backend subclass. | File | Role | | --- | --- | -| `src/unilab/envs/manipulation/sharpa_inhand/base.py` | `_extract_sensor_scalar()`, `_read_tactile_force()` | -| `src/unilab/envs/manipulation/sharpa_inhand/rotation.py` | reward computation, virtual torque | +| `src/unilab/tasks/manipulation/sharpa_inhand/base.py` | `_extract_sensor_scalar()`, `_read_tactile_force()` | +| `src/unilab/tasks/manipulation/sharpa_inhand/rotation.py` | reward computation, virtual torque | | `src/unilab/assets/robots/sharpa_wave/right_sharpa_wave.xml` | contact-sensor XML definitions | | `src/unilab/base/backend/motrix/backend.py` | Motrix `get_sensor_data()` | | `src/unilab/base/backend/mujoco/backend.py` | MuJoCo `get_sensor_data()` | diff --git a/docs/sphinx/source/en/5-reference/5-support_matrix.md b/docs/sphinx/source/en/5-reference/5-support_matrix.md index 5c9c3119a..cd37cb7af 100644 --- a/docs/sphinx/source/en/5-reference/5-support_matrix.md +++ b/docs/sphinx/source/en/5-reference/5-support_matrix.md @@ -3,12 +3,17 @@ This matrix is generated conceptually from registry entries, owner YAMLs, and tests. The generator implementation is `src/unilab/utils/support_matrix.py`; the write target for the generated block is currently the Chinese reference page -`docs/sphinx/source/zh_CN/5-reference/5-support_matrix.md`. +`docs/sphinx/source/zh_CN/5-reference/5-support_matrix.md`. This English page +mirrors that generated content. ## Backend Selection Rules - The default backend is `mujoco`. - Switch to Motrix with `--sim motrix` on the unified CLI. +- `--sim mjwarp` has completed training validation only on the `g1_walk_flat` + host adapter, where PPO (torch) and SAC (torch) are Tested; the SAC + `t800_walk_flat` mjwarp owner is Configured only, other entrypoints follow + the matrix below, and using it requires installing the `mjwarp` extra. - `--algo`, `--task`, and `--sim` jointly select the owner YAML. - Do not treat `training.sim_backend` as a standalone backend switch. @@ -17,70 +22,125 @@ write target for the generated block is currently the Chinese reference page - `mujoco`: `--render-mode auto` exports `play_video.mp4`. - `motrix`: `--render-mode auto` opens an interactive renderer window; it does not record a video and is not bound by `play_steps`. -- `--render-mode record`: both backends record a video only. +- `mjwarp`: only supports explicit, finite-step `record`, rendered offline + through the task owner's MuJoCo visual model; `auto`, interactive, and + native renderers are not supported. +- `--render-mode record`: MuJoCo, mjwarp, and Motrix all record a video only. - `--render-mode none`: no playback. ## Evidence Grades | Grade | Repository Evidence | | --- | --- | -| `Registered` | The env/backend pair appears after `registry.ensure_registries()`. | -| `Configured` | A matching owner YAML exists under `conf/ppo/task`, `conf/appo/task`, or `conf/offpolicy/task`. | -| `Tested` | Automated tests cover the entrypoint/task-owner/backend combination through config compose or runtime smoke. | +| `Registered` | The env/backend pair exists in `registry.list_registered_envs()` after `ensure_registries()`. | +| `Configured` | A matching owner YAML exists under `conf/{ppo,appo,sac,td3,flashsac}/task/...`. | +| `Tested` | Automated tests under `tests/` cover the entrypoint/task-owner/backend combination, or an explicit maintainer full-training validation with near-risk automated tests exists. `Tested` here does not mean the default recommended path. | | `Benchmarked` | A checked-in benchmark manifest exists for the combination. | | `Recommended` | Explicit recommendation metadata exists in the repo. | -The current generator reports no checked-in benchmark manifest and no separate -recommendation metadata, so rows do not auto-promote to `Benchmarked` or +`Tested` only describes existing automated coverage or explicit maintainer +training validation; it does not imply the combination has all the backend +capabilities of the same-named MuJoCo owner. For example, a phase-1 Motrix +owner may only cover training smoke and an explicitly enabled DR subset. + +`mjwarp` has completed training validation only on the `g1_walk_flat` host +adapter: the PPO (torch) and SAC (torch) owners have completed training +validation and have backend, contract, and playback automated coverage, so +they are marked `Tested`. The SAC `t800_walk_flat` mjwarp owner only has an +owner YAML and compose coverage, so it is marked `Configured`, which does not +imply training validation. mjwarp playback +only supports explicit, finite-step `record` and reuses the MuJoCo offline +renderer; it does not support `auto`, interactive, or native playback. A +`Registered` mark on other entrypoints only denotes env/backend registry +identity, not support for the corresponding algorithm, terrain, full DR, or +production training. + +No checked-in benchmark manifest bound to these combinations has been detected, +so rows do not auto-promote to `Benchmarked`. There is also no separate +recommendation metadata in the repo, so rows do not auto-promote to `Recommended`. ## Entrypoint x Task Owner -| Entrypoint | Task owner | MuJoCo | Motrix | -| --- | --- | --- | --- | -| PPO (torch) | `go1_joystick_flat` | Tested | Tested | -| PPO (torch) | `go2_joystick_flat` | Tested | Tested | -| PPO (torch) | `go2_joystick_rough` | Tested | Tested | -| PPO (torch) | `g1_walk_flat` | Tested | Tested | -| PPO (torch) | `g1_motion_tracking` | Tested | Tested | -| PPO (torch) | `g1_flip_tracking` | Tested | Tested | -| PPO (torch) | `g1_wall_flip_tracking` | Tested | Tested | -| PPO (torch) | `allegro_inhand` | Tested | Tested | -| PPO (torch) | `sharpa_inhand` | Tested | Tested | -| PPO (torch) | `sharpa_inhand_grasp` | Tested | Tested | -| PPO (torch) | `allegro_inhand_grasp` | Tested | Tested | -| PPO (torch) | `g1_box_tracking` | Tested | Tested | -| PPO (torch) | `g1_climb_tracking` | Tested | Tested | -| PPO (torch) | `g1_motion_tracking_deploy` | Tested | Registered | -| PPO (torch) | `go1_joystick_rough` | Tested | Tested | -| PPO (torch) | `go2_arm_manip_loco` | Tested | - | -| PPO (torch) | `go2_footstand` | Tested | - | -| PPO (torch) | `go2w_joystick_flat` | Tested | Tested | -| PPO (torch) | `go2w_joystick_rough` | Tested | Tested | -| APPO (torch) | `go1_joystick_flat` | Tested | Registered | -| APPO (torch) | `go2_joystick_flat` | Tested | Registered | -| APPO (torch) | `g1_walk_flat` | Tested | Registered | -| APPO (torch) | `g1_motion_tracking` | Tested | Tested | -| APPO (torch) | `g1_flip_tracking` | Tested | Tested | -| APPO (torch) | `g1_wall_flip_tracking` | Tested | Tested | -| APPO (torch) | `allegro_inhand` | Tested | Tested | -| APPO (torch) | `sharpa_inhand` | Tested | Registered | -| APPO (torch) | `g1_climb_tracking` | Tested | Tested | -| SAC (torch) | `g1_walk_flat` | Tested | Tested | -| SAC (torch) | `g1_walk_rough` | Tested | Tested | -| SAC (torch) | `g1_motion_tracking` | Tested | Tested | -| SAC (torch) | `g1_wbt_obs` | Tested | Registered | -| TD3 (torch) | `go1_joystick_flat` | Registered | Tested | -| TD3 (torch) | `go2_joystick_flat` | Registered | Tested | -| TD3 (torch) | `g1_walk_flat` | Tested | Registered | -| FlashSAC (torch) | `go2_joystick_flat` | Tested | Registered | -| FlashSAC (torch) | `g1_walk_flat` | Tested | Registered | +| Entrypoint | Task owner | MuJoCo | mjwarp | Motrix | +| --- | --- | --- | --- | --- | +| PPO (torch) | `go1_joystick_flat` (Go1 joystick) | Tested | - | Tested | +| PPO (torch) | `go2_joystick_flat` (Go2 joystick) | Tested | - | Tested | +| PPO (torch) | `go2_joystick_rough` (Go2 joystick rough) | Tested | - | Tested | +| PPO (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Tested | Tested | +| PPO (torch) | `g1_motion_tracking` (G1 motion tracking) | Tested | - | Tested | +| PPO (torch) | `g1_flip_tracking` (G1 flip tracking) | Tested | - | Tested | +| PPO (torch) | `g1_wall_flip_tracking` (G1 wall flip tracking) | Tested | - | Tested | +| PPO (torch) | `x2_wall_flip_tracking` (X2 wall flip tracking) | Tested | - | Tested | +| PPO (torch) | `allegro_inhand` (Allegro in-hand) | Tested | - | Tested | +| PPO (torch) | `sharpa_inhand` (Sharpa in-hand) | Tested | - | Tested | +| PPO (torch) | `sharpa_inhand_grasp` (Sharpa in-hand grasp) | Tested | - | Tested | +| PPO (torch) | `a2_joystick_flat` (a2 joystick flat) | Tested | - | - | +| PPO (torch) | `allegro_inhand_grasp` (allegro inhand grasp) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_box_tracking` (g1 23dof box tracking) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_climb_tracking` (g1 23dof climb tracking) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_flip_tracking` (g1 23dof flip tracking) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_motion_tracking` (g1 23dof motion tracking) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_motion_tracking_deploy` (g1 23dof motion tracking deploy) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_walk_flat` (g1 23dof walk flat) | Tested | - | Tested | +| PPO (torch) | `g1_23dof_walk_rough` (g1 23dof walk rough) | Tested | - | Registered | +| PPO (torch) | `g1_23dof_wall_flip_tracking` (g1 23dof wall flip tracking) | Tested | - | Tested | +| PPO (torch) | `g1_box_tracking` (g1 box tracking) | Tested | - | Tested | +| PPO (torch) | `g1_climb_tracking` (g1 climb tracking) | Tested | - | Tested | +| PPO (torch) | `g1_motion_tracking_deploy` (g1 motion tracking deploy) | Tested | - | Tested | +| PPO (torch) | `go1_joystick_rough` (go1 joystick rough) | Tested | - | Tested | +| PPO (torch) | `go2_arm_manip_loco` (go2 arm manip loco) | Tested | - | Tested | +| PPO (torch) | `go2_footstand` (go2 footstand) | Tested | - | Tested | +| PPO (torch) | `go2w_joystick_flat` (go2w joystick flat) | Tested | - | Tested | +| PPO (torch) | `go2w_joystick_rough` (go2w joystick rough) | Tested | - | Tested | +| PPO (torch) | `stewart_balance` (stewart balance) | Tested | - | Tested | +| PPO (torch) | `t800_walk_flat` (t800 walk flat) | Tested | Registered | - | +| APPO (torch) | `go1_joystick_flat` (Go1 joystick) | Tested | - | Tested | +| APPO (torch) | `go2_joystick_flat` (Go2 joystick) | Tested | - | Tested | +| APPO (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Registered | Registered | +| APPO (torch) | `g1_motion_tracking` (G1 motion tracking) | Tested | - | Tested | +| APPO (torch) | `g1_flip_tracking` (G1 flip tracking) | Tested | - | Tested | +| APPO (torch) | `g1_wall_flip_tracking` (G1 wall flip tracking) | Tested | - | Tested | +| APPO (torch) | `allegro_inhand` (Allegro in-hand) | Tested | - | Tested | +| APPO (torch) | `sharpa_inhand` (Sharpa in-hand) | Tested | - | Tested | +| APPO (torch) | `g1_23dof_climb_tracking` (g1 23dof climb tracking) | Tested | - | Tested | +| APPO (torch) | `g1_23dof_flip_tracking` (g1 23dof flip tracking) | Tested | - | Tested | +| APPO (torch) | `g1_23dof_motion_tracking` (g1 23dof motion tracking) | Tested | - | Tested | +| APPO (torch) | `g1_23dof_walk_flat` (g1 23dof walk flat) | Tested | - | Registered | +| APPO (torch) | `g1_23dof_wall_flip_tracking` (g1 23dof wall flip tracking) | Tested | - | Tested | +| APPO (torch) | `g1_climb_tracking` (g1 climb tracking) | Tested | - | Tested | +| SAC (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Tested | Tested | +| SAC (torch) | `g1_walk_rough` (G1 walk rough) | Tested | - | Tested | +| SAC (torch) | `g1_motion_tracking` (G1 motion tracking) | Tested | - | Tested | +| SAC (torch) | `g1_flip_tracking` (G1 flip tracking) | Tested | - | Registered | +| SAC (torch) | `g1_wall_flip_tracking` (G1 wall flip tracking) | Tested | - | Registered | +| SAC (torch) | `g1_23dof_flip_tracking` (g1 23dof flip tracking) | Tested | - | Registered | +| SAC (torch) | `g1_23dof_motion_tracking` (g1 23dof motion tracking) | Tested | - | Tested | +| SAC (torch) | `g1_23dof_walk_flat` (g1 23dof walk flat) | Tested | - | Tested | +| SAC (torch) | `g1_23dof_walk_rough` (g1 23dof walk rough) | Tested | - | Tested | +| SAC (torch) | `g1_23dof_wall_flip_tracking` (g1 23dof wall flip tracking) | Tested | - | Registered | +| SAC (torch) | `g1_23dof_wbt_obs` (g1 23dof wbt obs) | Tested | - | Registered | +| SAC (torch) | `g1_wbt_obs` (g1 wbt obs) | Tested | - | Registered | +| SAC (torch) | `t800_walk_flat` (t800 walk flat) | Tested | Configured | - | +| TD3 (torch) | `go1_joystick_flat` (Go1 joystick) | Registered | - | Tested | +| TD3 (torch) | `go2_joystick_flat` (Go2 joystick) | Registered | - | Tested | +| TD3 (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Registered | Registered | +| TD3 (torch) | `g1_23dof_walk_flat` (g1 23dof walk flat) | Tested | - | Registered | +| FlashSAC (torch) | `go2_joystick_flat` (Go2 joystick) | Tested | - | Registered | +| FlashSAC (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Configured | Tested | +| FlashSAC (torch) | `g1_23dof_walk_flat` (g1 23dof walk flat) | Tested | - | Tested | ## Source Index -- Registry bootstrap: `src/unilab/envs/**` registrations via +- Registry bootstrap: `src/unilab/envs/**` decorators via `unilab.base.registry.ensure_registries()`. - Owner YAML scan: `conf/ppo/task/**`, `conf/appo/task/**`, - `conf/offpolicy/task/**`. + `conf/sac/task/**`, `conf/td3/task/**`, `conf/flashsac/task/**`. - Generic compose coverage: `tests/config/test_config_system.py::test_supported_task_composes`. +- Validated mjwarp entrypoints are explicitly recorded in + `_MAINTAINER_VALIDATED_MJWARP_ENTRYPOINT_TASKS`; near-risk coverage lives in + `tests/base/test_mjwarp_backend.py`, + `tests/base/test_backend_conformance.py`, + `tests/base/test_mjwarp_differential.py`, and + `tests/base/test_mjwarp_playback.py`. diff --git a/docs/sphinx/source/glossary.md b/docs/sphinx/source/glossary.md index fdea68676..ae90f0a8e 100644 --- a/docs/sphinx/source/glossary.md +++ b/docs/sphinx/source/glossary.md @@ -46,8 +46,8 @@ Cold path 指 init、materialization、cache build 等低频路径。Hot path 顶层 CLI 通过 `--algo --task --sim ` 选择最终 task owner 配置。PPO / APPO 路径位于 -`conf/{ppo,appo}/task//.yaml`，off-policy 路径位于 -`conf/offpolicy/task///.yaml`。 +`conf/{ppo,appo}/task//.yaml`，off-policy 算法（SAC / TD3 / +FlashSAC）各自有独立的配置树，路径位于 `conf//task//.yaml`。 owner YAML 直接持有 `training.task_name`、`training.sim_backend`、`reward`、`env` 以及 task-specific `algo`。`training.sim_backend` 是 owner YAML 的身份字段，不是独立 backend switch。 diff --git a/docs/sphinx/source/zh_CN/1-getting_started/3-evaluation_and_playback.md b/docs/sphinx/source/zh_CN/1-getting_started/3-evaluation_and_playback.md index 40af2bd2b..379d667ca 100644 --- a/docs/sphinx/source/zh_CN/1-getting_started/3-evaluation_and_playback.md +++ b/docs/sphinx/source/zh_CN/1-getting_started/3-evaluation_and_playback.md @@ -23,7 +23,8 @@ uv run demo dance - `none` — 跳过渲染，仅计算指标。 `training.export_onnx=false` 目前仅适用于 off-policy 回放路径 -（`scripts/train_offpolicy.py` 以及使用 `--algo sac|td3|flashsac` 的 CLI 运行）。它会跳过 +（`scripts/train_sac.py` / `scripts/train_td3.py` / `scripts/train_flashsac.py` +以及使用 `--algo sac|td3|flashsac` 的 CLI 运行）。它会跳过 `policy.onnx` 的导出与校验，但仍会执行回放和视频录制。 ## MuJoCo viewer 可视化脚本 diff --git a/docs/sphinx/source/zh_CN/1-getting_started/4-project_structure.md b/docs/sphinx/source/zh_CN/1-getting_started/4-project_structure.md index 799d1cef5..f093387f8 100644 --- a/docs/sphinx/source/zh_CN/1-getting_started/4-project_structure.md +++ b/docs/sphinx/source/zh_CN/1-getting_started/4-project_structure.md @@ -22,8 +22,8 @@ UniLab 将运行时 contract、配置、训练脚本和文档分置于不同的 - `conf/ppo/config.yaml`，用于 torch PPO。 - `conf/appo/config.yaml`，用于 APPO。 -- `conf/offpolicy/config.yaml` 加上 `conf/offpolicy/algo/*.yaml`，用于 SAC、 - TD3 和 FlashSAC。 +- `conf/sac/config.yaml`、`conf/td3/config.yaml` 和 `conf/flashsac/config.yaml`， + 分别用于 SAC、TD3 和 FlashSAC，算法超参数内联在各自的 config.yaml 中。 - `conf/ppo_him/config.yaml` 和 `conf/hora_distill/config.yaml`，用于 专门的 HIM-PPO 和 HORA 路径。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md b/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md index 71252e4db..ee0b96385 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md @@ -9,9 +9,9 @@ Hydra 组合。 | --- | --- | --- | | PPO | `uv run train --algo ppo --task --sim ` | `scripts/train_rsl_rl.py` | | APPO | `uv run train --algo appo --task --sim ` | `scripts/train_appo.py` | -| SAC | `uv run train --algo sac --task --sim ` | `scripts/train_offpolicy.py` | -| TD3 | `uv run train --algo td3 --task --sim ` | `scripts/train_offpolicy.py` | -| FlashSAC | `uv run train --algo flashsac --task --sim ` | `scripts/train_offpolicy.py` | +| SAC | `uv run train --algo sac --task --sim ` | `scripts/train_sac.py` | +| TD3 | `uv run train --algo td3 --task --sim ` | `scripts/train_td3.py` | +| FlashSAC | `uv run train --algo flashsac --task --sim ` | `scripts/train_flashsac.py` | 示例： @@ -116,8 +116,8 @@ demo 入口由 `src/unilab/demo.py` 实现，并从 `src/unilab/cli.py` 路由 当你需要检查 Hydra 配置组或复现脚本层面的问题时，底层脚本仍然可用。在正常使用 中，请将定义路由的取值保留在上面的统一 CLI flag 中。 -对于 off-policy 路由，请保持 `--algo` 与 `conf/offpolicy/task//` 下的 -owner 树对齐；不要在 `--task` 中包含算法名称。 +对于 off-policy 路由，`--algo` 选择按算法划分的 owner 树 `conf//`； +不要在 `--task` 中包含算法名称。 ## 常用 Override diff --git a/docs/sphinx/source/zh_CN/2-user_guide/1-training/2-hydra_config.md b/docs/sphinx/source/zh_CN/2-user_guide/1-training/2-hydra_config.md index 19070967a..e1c26a1ff 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/1-training/2-hydra_config.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/1-training/2-hydra_config.md @@ -9,7 +9,7 @@ reward、scene 以及 task 专属运行时字段的身份标识。 | --- | --- | | PPO | `conf/ppo/task//.yaml` | | APPO | `conf/appo/task//.yaml` | -| SAC / TD3 / FlashSAC | `conf/offpolicy/task///.yaml` | +| SAC / TD3 / FlashSAC | `conf//task//.yaml` | | HIM-PPO | `conf/ppo_him/task//.yaml` | | HORA 蒸馏 | `conf/hora_distill/task//.yaml` | @@ -21,8 +21,8 @@ uv run train --algo ppo --task go2_joystick_flat --sim motrix uv run train --algo sac --task g1_walk_flat --sim mujoco ``` -对于 off-policy，`--algo` 选择 `conf/offpolicy/task//` 下 owner 路径的第一 -个分段；不要在 `--task` 中包含算法名称。 +对于 off-policy，`--algo` 选择按算法划分的配置树 `conf//`；不要在 +`--task` 中包含算法名称。 ## 安全的 Override diff --git a/docs/sphinx/source/zh_CN/2-user_guide/1-training/3-logging.md b/docs/sphinx/source/zh_CN/2-user_guide/1-training/3-logging.md index 81c178336..bc967e272 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/1-training/3-logging.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/1-training/3-logging.md @@ -23,9 +23,9 @@ uv run train --algo ppo --task go2_joystick_flat --sim mujoco | --- | --- | --- | | PPO | `logs/rsl_rl_ppo//` | `conf/ppo/config.yaml` | | APPO | `logs/appo//` | `conf/appo/config.yaml` | -| SAC | `logs/fast_sac//` | `conf/offpolicy/algo/sac.yaml` | -| FlashSAC | `logs/flash_sac//` | `conf/offpolicy/algo/flashsac.yaml` | -| TD3 | `logs/fast_td3//` | `conf/offpolicy/algo/td3.yaml` | +| SAC | `logs/fast_sac//` | `conf/sac/config.yaml` | +| FlashSAC | `logs/flash_sac//` | `conf/flashsac/config.yaml` | +| TD3 | `logs/fast_td3//` | `conf/td3/config.yaml` | 单个 run 目录名为 `YYYY-MM-DD_HH-MM-SS_`，例如 `2026-03-09_18-30-00_mujoco`。常见产物包括 `run_config.json`、 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/0-index.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/0-index.md index c5c9c62d3..08d4c6c5a 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/0-index.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/0-index.md @@ -7,9 +7,9 @@ | --- | --- | --- | --- | | PPO | 同步 on-policy | `scripts/train_rsl_rl.py` | `conf/ppo/config.yaml` | | APPO | 异步 on-policy | `scripts/train_appo.py` | `conf/appo/config.yaml` | -| SAC | off-policy | `scripts/train_offpolicy.py` | `conf/offpolicy/algo/sac.yaml` | -| TD3 | off-policy | `scripts/train_offpolicy.py` | `conf/offpolicy/algo/td3.yaml` | -| FlashSAC | off-policy | `scripts/train_offpolicy.py` | `conf/offpolicy/algo/flashsac.yaml` | +| SAC | off-policy | `scripts/train_sac.py` | `conf/sac/config.yaml` | +| TD3 | off-policy | `scripts/train_td3.py` | `conf/td3/config.yaml` | +| FlashSAC | off-policy | `scripts/train_flashsac.py` | `conf/flashsac/config.yaml` | | HIM-PPO | 高度估计器 PPO 路径 | `scripts/train_him_ppo.py` | `conf/ppo_him/config.yaml` | | HORA | teacher/student 蒸馏路径 | `scripts/train_hora_distill.py` | `conf/hora_distill/config.yaml` | diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/1-ppo.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/1-ppo.md index 5a900bfc1..61afe0bc7 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/1-ppo.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/1-ppo.md @@ -1,7 +1,7 @@ # PPO PPO 是默认的同步 on-policy 训练路径。它使用 `scripts/train_rsl_rl.py`，从 -`conf/ppo/config.yaml` 组合配置，并运行 `src/unilab/algos/torch/rsl_rl_ppo.py` +`conf/ppo/config.yaml` 组合配置，并运行 `src/unilab/algos/rsl_rl_ppo.py` 和 `src/unilab/training/rsl_rl.py` 中的 RSL-RL 适配代码。 ## 快速开始 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/2-appo.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/2-appo.md index d15b568b5..251e352c1 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/2-appo.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/2-appo.md @@ -1,7 +1,7 @@ # APPO APPO 是 UniLab 的异步 PPO 路径。它使用 `scripts/train_appo.py`、 -`conf/appo/config.yaml` 以及 `src/unilab/algos/torch/appo/` 下的运行时。该配置暴露 +`conf/appo/config.yaml` 以及 `src/unilab/algos/appo/` 下的运行时。该配置暴露 了 `algo.steps_per_env`、`training.collector_device` 和 `training.replay_queue_size`；算法配置中包含 V-trace 裁剪字段。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/3-sac.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/3-sac.md index 897fa9601..374f98a4b 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/3-sac.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/3-sac.md @@ -1,12 +1,12 @@ # SAC -SAC 通过共享的 off-policy 入口 `scripts/train_offpolicy.py` 选择，TD3 与 FlashSAC -也共用该脚本。主配置为 `conf/offpolicy/config.yaml`，SAC 算法的默认值位于 -`conf/offpolicy/algo/sac.yaml`。当前的日志名称为 `fast_sac`。 +SAC 通过 `scripts/train_sac.py` 运行；TD3 与 FlashSAC 各有独立的入口与按算法 +划分的配置树。主配置为 `conf/sac/config.yaml`，SAC 算法的默认值内联在其中。 +当前的日志名称为 `fast_sac`。 ## 运行模型 -off-policy runner 通过有界 shared memory 把 CPU 仿真与 accelerator 学习解耦。 +off-policy runner 通过有界 shared memory 把仿真采集与 accelerator 学习解耦。 collector 子进程通过两个 packed ingress slot 发布 transition，完整 replay ring 只由 一个 CUDA 或 Apple MPS learner device 持有。因此 host replay 分配不随 replay capacity 增长；slot 的 device copy 完成后才推进 `ptr` 与 `size`。CUDA 通过 side @@ -22,7 +22,7 @@ uv run train --algo sac --task g1_walk_rough --sim motrix training.no_play=true ## 关键字段 -对于 off-policy 回放路径（`scripts/train_offpolicy.py` / CLI `--algo sac`），设置 +对于 off-policy 回放路径（`scripts/train_sac.py` / CLI `--algo sac`），设置 `training.export_onnx=false` 可在仍然录制回放视频的同时跳过 `policy.onnx` 导出。参 见 {doc}`/zh_CN/1-getting_started/3-evaluation_and_playback`。 @@ -51,6 +51,12 @@ rank 子进程。启动时 rank 0 一次性广播 actor、critic、target critic 执行阻塞式 flat-gradient `all_reduce(SUM) / world_size`。各 rank 不交换 replay 数据， 在相同初值、平均梯度和更新顺序下各自维护一致的 optimizer 状态。 +每个 rank 当前只创建一个 collector。使用 mjwarp 时，rank i 会在 probe env 和 collector +正式 env materialization 之前，把 Warp 的进程默认/当前 device 显式绑定到该 rank 的 +learner device `cuda:devices[i]`；因此 collector 不依赖 Warp 新进程默认的 `cuda:0`，也不 +会跨 rank 集中到同一张卡。runtime manifest 的 `collector_backend_device` 记录本 rank 的 +实际绑定。 + off-policy 只公开 `training.devices` 这一个设备字段：`null` 或 `[]` 自动选择单个 learner device，`[0]` 显式选择 `cuda:0`，两个以上索引才启动多卡拓扑。 @@ -70,7 +76,9 @@ rank 子目录或任何日志文件。 `training.log_dir` 保持原样。 collector 的 CPU 亲和按 rank 自动均分（`cpu_count // world_size` 一段），可用 -`training.dp_collector_cpu_ids` 显式指定。 +`training.dp_collector_cpu_ids` 显式指定。该核区经 `EnvCfg.cpu_ids` 生效：除 +MuJoCo worker 线程逐核绑定外，collector 进程本身（含 Numba 并行 kernel 线程池，池 +大小取核区长度）也被限制在同一核区内，避免跨 rank 抢占。 当前限制： @@ -86,7 +94,9 @@ collector 的 CPU 亲和按 rank 自动均分（`cpu_count // world_size` 一段 默认 stream 与 side stream 均通过；跳过 warmup 时首个 all-reduce 会在 capture 中报 `operation not permitted when stream is capturing`。有限超时的最小复现见 `scripts/benchmark/rl/reproduce_nccl_cuda_graph_capture.py`。 -- 仅验证过 `mujoco` backend。 +- MuJoCo 有已提交的多卡 scaling benchmark；mjwarp 的 per-rank device placement 有 + `tests/base/backend/test_process_device.py` 与 off-policy runner/worker 单测覆盖，但仓库中 + 尚无 mjwarp 多卡吞吐或收敛 benchmark。 - 仅单节点：rank 之间通过 run 目录里的 FileStore rendezvous，NCCL 走 TCP loopback（默认 `NCCL_P2P_DISABLE=1` / `NCCL_SHM_DISABLE=1`，环境变量显式设置 时优先）——部分机型（如 RTX 6000D）的 NCCL P2P/SHM peer transport 不可靠， diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/4-td3.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/4-td3.md index 236d694af..08350383b 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/4-td3.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/4-td3.md @@ -1,7 +1,7 @@ # TD3 -TD3 与 SAC、FlashSAC 共用 off-policy 训练脚本。使用 `--algo td3` 选择它；owner -YAML 证据位于 `conf/offpolicy/task/td3/` 下。 +TD3 通过 `scripts/train_td3.py` 运行，拥有独立的配置树。使用 `--algo td3` +选择它；owner YAML 证据位于 `conf/td3/task/` 下。 ## 快速开始 @@ -11,11 +11,11 @@ uv run train --algo td3 --task g1_walk_flat --sim mujoco ## 关键字段 -对于 off-policy 回放路径（`scripts/train_offpolicy.py` / CLI `--algo td3`），设置 +对于 off-policy 回放路径（`scripts/train_td3.py` / CLI `--algo td3`），设置 `training.export_onnx=false` 可在仍然录制回放视频的同时跳过 `policy.onnx` 导出。参 见 {doc}`/zh_CN/1-getting_started/3-evaluation_and_playback`。 -- 默认值位于 `conf/offpolicy/algo/td3.yaml`。 +- 默认值内联在 `conf/td3/config.yaml` 中。 - `algo.algo_log_name=fast_td3`。 - `algo.max_iterations=5000`。 - `algo.policy_frequency=2`。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/5-flash_sac.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/5-flash_sac.md index e1f8c5911..b93e6f9e7 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/5-flash_sac.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/5-flash_sac.md @@ -1,11 +1,11 @@ # FlashSAC -FlashSAC 是共享 off-policy 入口上的第三个算法。使用 `--algo flashsac` 选择它；默认 -值位于 `conf/offpolicy/algo/flashsac.yaml`，实现位于 -`src/unilab/algos/torch/flash_sac/` 下。 +FlashSAC 通过 `scripts/train_flashsac.py` 运行，拥有独立的配置树。使用 +`--algo flashsac` 选择它；默认值内联在 `conf/flashsac/config.yaml` 中，实现位于 +`src/unilab/algos/flash_sac/` 下。 -它与 SAC、TD3 共用 off-policy 训练脚本，但默认网络并不相同：actor 使用 block-based -结构，critic 使用 distributional（categorical）Q 变体。 +它与 SAC、TD3 共用 off-policy runner 设计，但默认网络并不相同：actor 使用 +block-based 结构，critic 使用 distributional（categorical）Q 变体。 ## 快速开始 @@ -16,7 +16,7 @@ uv run train --algo flashsac --task go2_joystick_flat --sim mujoco training.no_p ## 关键字段 -对于 off-policy 回放路径（`scripts/train_offpolicy.py` / CLI `--algo flashsac`），设 +对于 off-policy 回放路径（`scripts/train_flashsac.py` / CLI `--algo flashsac`），设 置 `training.export_onnx=false` 可在仍然录制回放视频的同时跳过 `policy.onnx` 导出。 参见 {doc}`/zh_CN/1-getting_started/3-evaluation_and_playback`。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/7-hora.md b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/7-hora.md index bf623244b..1cddce6ed 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/7-hora.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/7-hora.md @@ -13,7 +13,7 @@ uv run train --algo appo --task sharpa_inhand --sim mujoco --profile hora traini ``` HORA PPO owner 设置 `algo.algo_log_name=hora_ppo`，并通过 -`unilab.algos.torch.hora.rsl_rl:resolve_hora_ppo_runtime` 解析运行时。APPO 变体设置 +`unilab.algos.hora.rsl_rl:resolve_hora_ppo_runtime` 解析运行时。APPO 变体设置 `algo.algo_log_name=hora_appo`。 ## Student 蒸馏 @@ -22,5 +22,5 @@ student 蒸馏由 `scripts/train_hora_distill.py` 实现，并由 `conf/hora_distill/task/sharpa_inhand/mujoco.yaml` 配置。顶层 CLI 目前没有声明独立的 HORA 蒸馏 `--algo` 路由，因此本页的公开 CLI 示例仍保持在上面的 teacher 路径上。 -teacher 检查点的解析在 `src/unilab/algos/torch/hora/distill_config.py` 中实现。 +teacher 检查点的解析在 `src/unilab/algos/hora/distill_config.py` 中实现。 student 日志族为 `hora_distill`。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/3-backends/0-index.md b/docs/sphinx/source/zh_CN/2-user_guide/3-backends/0-index.md index 111d7f24a..c319a6d2b 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/3-backends/0-index.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/3-backends/0-index.md @@ -22,7 +22,7 @@ uv run train --algo ppo --task go1_joystick_flat --sim motrix Owner YAML 位置： - PPO / APPO：`conf/{ppo,appo}/task//.yaml` -- Off-policy：`conf/offpolicy/task///.yaml` +- Off-policy（SAC / TD3 / FlashSAC）：`conf//task//.yaml` 被选中的 owner YAML 将 `training.sim_backend` 设为身份字段。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/1-locomotion.md b/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/1-locomotion.md index ba6885967..f8987b5da 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/1-locomotion.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/1-locomotion.md @@ -1,7 +1,7 @@ # 运动控制 -运动控制任务注册在 `src/unilab/envs/locomotion/` 和 -`src/unilab/envs/motion_tracking/` 中。`conf/` 下可用的 owner YAML +运动控制任务注册在 `src/unilab/tasks/locomotion/` 和 +`src/unilab/tasks/motion_tracking/` 中。`conf/` 下可用的 owner YAML 定义了哪些算法与后端组合是可运行的。 ## 系列 @@ -29,11 +29,14 @@ uv run train --algo sac --task g1_walk_flat --sim mujoco ## Go2 FootStand -`go2_footstand` 是 Go2 前足站立任务，**仅支持 MuJoCo**。 +`go2_footstand` 是 Go2 前足站立任务。PPO owner YAML 已注册 MuJoCo、 +Motrix 和 Drake；当前 SAC owner 使用 Drake。 -- PPO 配置：`conf/ppo/task/go2_footstand/mujoco.yaml` -- 环境注册名：`Go2FootStand`（注册于 `sim_backend="mujoco"`） -- 环境实现：`src/unilab/envs/locomotion/go2/footstand.py`（继承 Go2 基础任务） +- PPO canonical 配置：`conf/ppo/task/go2_footstand/base.yaml` +- 后端 owner：`conf/ppo/task/go2_footstand/{mujoco,motrix,drake}.yaml` +- 环境注册名：`Go2FootStand`（MuJoCo、Motrix、Drake） +- 环境实现：`src/unilab/tasks/locomotion/go2/footstand.py` + （通用 Manager-Based runtime 上的 task-owned NumPy manager terms） - Go2 模型 XML：`src/unilab/assets/robots/go2/go2.xml` ```bash @@ -59,14 +62,14 @@ FootStand 的完整流程是三阶段教师-学生 pipeline；当前仓库里的 ### 观测口径 `Go2FootStand` 的策略（actor）网络观测使用 15 帧历史，每帧 45 维 -（`_FOOTSTAND_FRAME_OBS_DIM = 45`）： +（`FRAME_OBS_DIM = 45`）： ```text linvel(3) + gyro(3) + gravity(3) + joint_position_delta(12) + joint_velocity(12) + last_action(12) ``` 因此策略网络观测维度是 `45 * 15 = 675`。价值（critic）网络在这段历史观测后追加当前时刻的 -特权观测尾部（`_FOOTSTAND_PRIVILEGED_TAIL_DIM = 49`）： +特权观测尾部（`PRIVILEGED_OBS_DIM = 49`）： ```text gyro(3) + accelerometer(3) + linvel(3) + global_angvel(3) + dof_pos(12) + dof_vel(12) + torques(12) + height(1) @@ -76,19 +79,23 @@ gyro(3) + accelerometer(3) + linvel(3) + global_angvel(3) + dof_pos(12) + dof_ve ### 奖励与终止项 -默认奖励来自 `conf/ppo/task/go2_footstand/mujoco.yaml`。奖励权重包括站立 `height`、 +默认奖励来自 `conf/ppo/task/go2_footstand/base.yaml`，后端 leaf 只覆盖 +后端专属 term 和调优项。奖励权重包括站立 `height`、 `orientation`、`rear_feet_contact`、前腿目标角度（`tar`）、`action_rate`、 `dof_pos_limits`、`front_leg_motion`、`rear_leg_symmetry`、`knee_clearance`、 `upright_stability`、`stay_still`、`pose`，以及 `energy` 和 `dof_acc` 惩罚； `termination` 与 `penalty_contact` 驱动终止/惩罚路径（前腿/前身体接触、低高度、坏朝向， -以及由 `energy_termination_threshold` 控制的高能耗截断）。 +以及 `footstand` termination term 中的高能耗截断）。 ### 调参提示 -- `env.obs_history_len`：策略观测历史长度，配置默认为 `15`。 -- `env.energy_termination_threshold`：高能耗终止阈值，配置默认为 `200.0`。 -- `env.domain_rand`：地面摩擦、连杆质量、机身质心、关节惯量和重置关节位置随机化。 -- `reward.scales.height` / `orientation` / `rear_feet_contact`：站立姿态和后脚接触权重。 +- `env.observations.policy.terms.frame.history_length`：策略观测历史长度， + 默认为 `15`。 +- `env.terminations.footstand.params.energy_threshold`：高能耗终止阈值， + 默认为 `200.0`。 +- `env.events`：重置和 domain randomization terms；后端 owner 对不支持的 + model-field term 显式设为 `null`。 +- `reward.footstand.params.scales`：站立、接触、运动和能耗权重。 ### 近风险检查 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/2-motion_tracking.md b/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/2-motion_tracking.md index cf7a16812..0b7c15bd2 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/2-motion_tracking.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/2-motion_tracking.md @@ -1,38 +1,40 @@ # 动作追踪 -G1 动作追踪任务位于 `src/unilab/envs/motion_tracking/` 下，并通过 +G1 动作追踪任务位于 `src/unilab/tasks/motion_tracking/` 下，并通过 `conf/ppo/`、`conf/appo/` 以及选定的 off-policy 路径中的 task owner YAML 选择。 > **Motion 资产已迁移到 Hugging Face。** `.npz` 片段不再随仓库分发，首次使用时由 -> `MotionLoader`（`src/unilab/envs/motion_tracking/g1/motion_loader.py`）按需从 +> `MotionLoader`（`src/unilab/tasks/motion_tracking/common/motion_loader.py`）按需从 > [unilabsim/unilab-motions](https://huggingface.co/datasets/unilabsim/unilab-motions) > 下载，下载逻辑在 `src/unilab/assets/hub.py`（`_HF_MOTIONS_REPO_ID`）。`uv sync` > 已自动安装所需的 `huggingface_hub` 依赖。 ## Task Owners -每个 task 在 env 配置 dataclass 中定义了默认 motion 片段： +每个 task 都在 Hydra task owner YAML 中定义默认 motion 片段。Hydra 是唯一配置入口； +选中的 owner 会被物化为共享的 `ManagerBasedRlEnvCfg`，再由 NumPy Manager-Based +runtime 执行。 | CLI Task | Registered Env | 默认 motion | Owner Evidence | | --- | --- | --- | --- | | `g1_motion_tracking` | `G1MotionTracking` | `dance1_subject2_part.npz` | `conf/ppo/task/g1_motion_tracking/`, `conf/appo/task/g1_motion_tracking/` | | `g1_flip_tracking` | `G1FlipTracking` | `flip_360_001__A304.npz` | `conf/ppo/task/g1_flip_tracking/`, `conf/appo/task/g1_flip_tracking/` | | `g1_wall_flip_tracking` | `G1WallFlipTracking` | `flip_from_wall_104__A304.npz` | `conf/ppo/task/g1_wall_flip_tracking/`, `conf/appo/task/g1_wall_flip_tracking/` | -| `x2_wall_flip_tracking` | `X2WallFlipTracking` | `tictacflip_6-3_g1format.npz` | `conf/ppo/task/x2_wall_flip_tracking/`（仅 MuJoCo） | -| `g1_climb_tracking` | G1 climb tracking env | 由 env 配置给出 | `conf/ppo/task/g1_climb_tracking/`, `conf/appo/task/g1_climb_tracking/` | -| `g1_box_tracking` | G1 box tracking env | 由 env 配置给出 | `conf/ppo/task/g1_box_tracking/` | -| `g1_wbt_obs` | `G1MotionTrackingSAC` | 与 `g1_motion_tracking` 共用 | `conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml` | +| `x2_wall_flip_tracking` | `X2WallFlipTracking` | `tictacflip_6-3_g1format.npz` | `conf/ppo/task/x2_wall_flip_tracking/` | +| `g1_climb_tracking` | `G1ClimbTracking` | `climb_20_z_scale_1.0.npz` | `conf/ppo/task/g1_climb_tracking/`, `conf/appo/task/g1_climb_tracking/` | +| `g1_box_tracking` | `G1BoxTracking` | `sub3_largebox_003_boxconverted.npz` | `conf/ppo/task/g1_box_tracking/` | +| `g1_wbt_obs` | `G1WBTObs` | `dance1_subject2_part.npz` | `conf/sac/task/g1_wbt_obs/mujoco.yaml` | -默认值在代码中设定：`dance1_subject2_part.npz`（`g1/tracking.py`）， -`flip_360_001__A304.npz` 与 `flip_from_wall_104__A304.npz`（`g1/flip_tracking.py`）， -以及 `tictacflip_6-3_g1format.npz`（`x2/flip_tracking.py`）。 +23-DoF task owner 目录选择对应的 23-DoF 场景、motion、entity 与 action 声明。 +profile 差异全部留在 Hydra 中。G1 identity 使用共享 manager factory；X2 只在委托给 +该 factory 前增加一层冷路径 mesh resolver。 ## PPO 与 APPO PPO owner 迭代预算（`--sim mujoco` owner YAML）：`g1_motion_tracking` 为 `algo.max_iterations=15000`；`g1_flip_tracking` 和 `g1_wall_flip_tracking` 为 -`20000`；仅 MuJoCo 的 `x2_wall_flip_tracking` 为 `9500`。（`g1_flip_tracking` 的 -Motrix owner YAML 将其提到 `30000`。） +`20000`；`x2_wall_flip_tracking` 为 `9500`。（`g1_flip_tracking` 的 Motrix owner +YAML 将其提到 `30000`。） ```bash uv run train --algo ppo --task g1_motion_tracking --sim mujoco @@ -55,12 +57,13 @@ uv run train --algo sac --task g1_motion_tracking --sim mujoco training.use_amp= uv run train --algo sac --task g1_wbt_obs --sim mujoco training.use_amp=true ``` -`g1_wbt_obs` owner 是与部署对齐的 off-policy 观测配置：pelvis IMU 状态 -（`pelvis_local_linvel` / `pelvis_gyro` / `pelvis_upvector`）加上 per-term 历史观测 -（`noise_config.obs_history_length: 5`），与部署侧的 `ObservationManager` 按字节对齐。 -部署工具在 `scripts/deploy/`，观测对齐由 `tests/scripts/test_obs_alignment_g1_wbt.py` -交叉校验。当 Motrix sim2sim 回放需要引用其他日志根目录下的 checkpoint 时，用 -`uv run eval` 透传绝对路径： +`g1_wbt_obs` owner 是与部署对齐的 off-policy 观测配置。actor 的 command 与 anchor +orientation term 保持单步，`base_ang_vel`、`joint_pos`、`joint_vel` 和 `actions` term +分别声明 `history_length: 5`。这些逐项历史由 `ObservationManager` 维护并展开；actor +使用配置中的 encoder-biased joint-position term，critic 则保留 clean term。部署工具在 +`scripts/deploy/`，观测对齐由 `tests/scripts/test_obs_alignment_g1_wbt.py` 交叉校验。 +当 Motrix sim2sim 回放需要引用其他日志根目录下的 checkpoint 时，用 `uv run eval` +透传绝对路径： ```bash uv run eval --algo sac --task g1_motion_tracking --sim motrix \ @@ -69,15 +72,19 @@ uv run eval --algo sac --task g1_motion_tracking --sim motrix \ ## 动作文件 -动作 NPZ 文件通过 `env.motion_file` 读取，也支持路径列表。标准片段必须包含七个 key： +动作 NPZ 文件通过 `env.commands.motion.params.motion_file` 选择，既可传单个路径， +也可传路径列表。标准片段必须包含七个 key： `fps`、`joint_pos`、`joint_vel`、`body_pos_w`、`body_quat_w`、`body_lin_vel_w`、 -`body_ang_vel_w`（在 `g1/motion_loader.py` 中校验）： +`body_ang_vel_w`（在 `common/motion_loader.py` 中校验）： ```yaml env: - motion_file: - - src/unilab/assets/motions/g1/dance1_subject2_part.npz - - src/unilab/assets/motions/g1/walk1_subject5_from_csv.npz + commands: + motion: + params: + motion_file: + - motions/g1/dance1_subject2_part.npz + - motions/g1/walk1_subject5_from_csv.npz ``` 转换与检查辅助工具在 `scripts/motion/` 中： @@ -109,20 +116,21 @@ uv run scripts/motion/replay_npz.py \ ```bash CUDA_VISIBLE_DEVICES=1 uv run train --algo sac --task g1_motion_tracking --sim mujoco \ training.use_amp=true algo.seed=1 \ - +env.motion_file=src/unilab/assets/motions/g1/motion_crawl_slope_uni.npz \ - +env.scene.model_file=src/unilab/assets/robots/g1/scene_crawl_slope.xml \ - +env.sampling_mode=start \ - env.truncate_on_clip_end=true \ - +env.max_episode_seconds=20.0 \ - '+env.pose_randomization={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ - '+env.velocity_randomization={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ - '+env.joint_position_range=[0,0]' + env.commands.motion.params.motion_file=motions/g1/motion_crawl_slope_uni.npz \ + env.scene.model_file=src/unilab/assets/robots/g1/scene_crawl_slope.xml \ + env.commands.motion.params.sampling_mode=start \ + env.commands.motion.params.truncate_on_clip_end=true \ + env.max_episode_seconds=20.0 \ + 'env.commands.motion.params.pose_range={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ + 'env.commands.motion.params.velocity_range={x:[0,0],y:[0,0],z:[0,0],roll:[0,0],pitch:[0,0],yaw:[0,0]}' \ + 'env.commands.motion.params.joint_position_range=[0,0]' ``` -关键覆写：`env.motion_file` 切爬坡动作；`env.scene.model_file` 切斜坡场景 -（`scene_crawl_slope.xml` 在 `src/unilab/assets/robots/g1/` 下）；`sampling_mode=start` -加 `truncate_on_clip_end=true` 从 clip 起点出发并在结尾截断；randomization 范围全置零 -复用 motion 精确初始状态。 +关键覆写：`env.commands.motion.params.motion_file` 切换爬坡动作； +`env.scene.model_file` 切换斜坡场景（`scene_crawl_slope.xml` 在 +`src/unilab/assets/robots/g1/` 下）；`sampling_mode=start` 加 +`truncate_on_clip_end=true` 从 clip 起点出发并在结尾截断；command reset 范围全置零 +即可复用 motion 的精确初始状态。 ## 交互式调试 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/3-manipulation.md b/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/3-manipulation.md index 6352b4f2a..c6d98ff36 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/3-manipulation.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/4-tasks/3-manipulation.md @@ -1,7 +1,7 @@ # 操作 -操作任务位于 `src/unilab/envs/manipulation/` 中，Go2 机械臂 manip-loco -env 位于 `src/unilab/envs/locomotion/go2_arm/` 中。 +操作任务位于 `src/unilab/tasks/manipulation/` 中，Go2 机械臂 manip-loco +env 位于 `src/unilab/tasks/locomotion/go2_arm/` 中。 ## 手内操作 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/0-index.md b/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/0-index.md index 2324ced4e..82bfcd39a 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/0-index.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/0-index.md @@ -1,9 +1,14 @@ # 域随机化 -本页仅描述仓库中那些已经注册、且已经接入 DR provider 的任务的当前状态。所有结论都来自代码；不从设计意图推断任何内容。 +本页仅描述仓库中已注册任务的域随机化现状。所有结论都来自代码；不从设计意图推断任何内容。 -当前统一的入口点位于 `NpEnv._init_domain_randomization()` 和 `DomainRandomizationManager`： +当前存在两条 DR 声明路径： + +- **Manager-Based（Compatible）任务**：reset / interval 随机化通过 owner YAML 中的 Hydra `events:` manager term 声明；reset 生命周期的 event 在 reset 时采样，interval 生命周期的 event 在 step 之间施加扰动。例如 `conf/ppo/task/go1_joystick_flat/base.yaml` 的 `events:` 段。 +- **legacy provider 路径**：只有 3 个 Adapted family（`sharpa_inhand` / `sharpa_inhand_grasp` / `go2_arm_manip_loco`，含 appo / hora / ppo_him owner）仍通过 `DomainRandomizationProvider` + `DomainRandomizationManager` 声明 `env.domain_rand.*` 配置。 + +legacy provider 路径的统一入口点位于 `NpEnv._init_domain_randomization()` 和 `DomainRandomizationManager`： - init 路径：task provider 产生一个 `InitRandomizationPlan`；manager 在 env 初始化期间调用后端的 `apply_init_randomization(...)` - reset 路径：task provider 产生一个 `ResetPlan`；manager 验证能力，然后调用后端的 `set_state(..., randomization=...)` @@ -17,62 +22,67 @@ ## 状态结论 -1. 当前所有接入 DR provider 的任务都使用统一的 DR 入口点；没有任何任务绕开 `DomainRandomizationManager` 在 `reset()` 内部运行单独的 DR 流程。 -2. 它们的结构都大致相同：task 文件定义一个 `domain_rand` 配置 dataclass、一个 `DomainRandomizationProvider` 和一个 `ResetPlan`；`G1WalkFlat` 复用 `G1Walk` 的 provider。 -3. 今天所"统一"的主要是入口点和执行流程，而不是每一个随机化项本身。共享辅助函数 `build_common_reset_randomization()` 目前生成 `base_mass_delta`、`base_com_offset`、`gravity`、`kp`、`kd`；共享的 interval 辅助函数目前只生成 push。 +1. Manager-Based 任务不注册 DR provider；它们的 reset/interval 随机化是 owner YAML 中的 `events:` manager term，由 manager 生命周期统一执行。只有 Adapted family 的冻结兼容工厂仍走 `DomainRandomizationManager` 统一入口。 +2. Adapted family owner 定义 `domain_rand` 配置 dataclass、`DomainRandomizationProvider` 和 `ResetPlan`；Manager-Based owner 则通过 Hydra command/event term 声明 reset 行为。G1 motion reset 扰动归 `MotionCommandCfg` 所有，WBT 另加 `EventTermCfg` reset 与 interval term。 +3. 今天所"统一"的主要是入口点和执行流程，而不是每一个随机化项本身。legacy 路径的共享辅助函数 `build_common_reset_randomization()` 目前生成 `base_mass_delta`、`base_com_offset`、`gravity`、`kp`、`kd`；共享的 interval 辅助函数目前只生成 push。 4. `ResetRandomizationPayload` 已经可以表达 `gravity`、`body_iquat`、`body_inertia`、`kp`、`kd`，并且 `MuJoCoBackend` 已声明支持。这些是否实际被使用，仍取决于 task provider 是否对它们进行采样和 dispatch。 5. `MotrixBackend` 目前支持 `base_mass_delta`、`base_com_offset`、`kp`、`kd` 和 interval push；并且它要求在初始化期间所有模型 actuator 都是 position actuator。 6. `geom_size` 不是 reset 生命周期字段；Sharpa 手物体的 geom 缩放由 init 生命周期的模型 materialization 处理。 ## 统一性评估表 -| Task | 使用统一 DR 入口？ | 结构化形式？ | reset 形式 | interval 形式 | Code | +| Task | 声明路径 | 结构化形式？ | reset 形式 | interval 形式 | Code | | --- | --- | --- | --- | --- | --- | -| `Go1JoystickFlat` | 是 | 是：`Domain_Rand + Provider + ResetPlan` | task 状态采样 + common payload | push | `go1/joystick.py` | -| `Go2JoystickFlat` | 是 | 是：`Domain_Rand + Provider + ResetPlan` | task 状态采样 + common payload | push | `go2/joystick.py` | -| `G1WalkFlat` | 是 | 是：`Domain_Rand + Provider + ResetPlan` | task 状态采样 + common payload | push | `g1/joystick.py` | -| `G1WalkRough` | 是 | 是：复用 `G1WalkDomainRandomizationProvider` | task 状态采样 + common payload | push | `g1/joystick.py` | -| `G1MotionTracking` | 是 | 是：`Domain_Rand + Provider + ResetPlan` | 大量 task 专属的 reset 采样 + common payload | push | `motion_tracking/g1/tracking.py` | -| `AllegroInhandRotation` | 是 | 是：`DomainRandConfig + Provider + ResetPlan` | task 专属的 reset 采样 + common payload | 无 | `allegro_inhand/rotation.py` | -| `SharpaInhandRotation` | 是 | 是：`InitRandomizationPlan + ResetPlan + IntervalRandomizationPlan` | grasp cache 采样 + common payload | 物体 `body_force` | `sharpa_inhand/rotation.py` | -| `SharpaInhandRotationGrasp` | 是 | 是：复用 Sharpa rotation provider 并 override reset 采样 | grasp 收集 reset + common payload | 无 | `sharpa_inhand/grasp_gen.py` | +| `Go1JoystickFlat` | Hydra `events:` term | 是：owner YAML 声明 reset/interval event | root-state reset + base mass/COM + `pd_gains` | `push_by_setting_velocity` event | `conf/ppo/task/go1_joystick_flat/base.yaml` | +| `Go2JoystickFlat` | Hydra `events:` term | 是：owner YAML 声明 reset event | root-state reset + `pd_gains` kp/kd | 无 | `conf/ppo/task/go2_joystick_flat/base.yaml` | +| `G1WalkFlat` | Hydra `events:` term | 是：Hydra `EventTermCfg` + Manager-Based reset term | root-state reset + 经 `pd_gains` 的 kp/kd | 无 | `g1/manager_terms.py` | +| `G1WalkRough` | Hydra `events:` term | 是：与 `G1WalkFlat` 相同的 Manager-Based event term | root-state reset + 经 `pd_gains` 的 kp/kd | 无 | `g1/manager_terms.py` | +| `G1MotionTracking` | Hydra command term | 是：Hydra `MotionCommandCfg` + Manager-Based command reset | motion frame、root pose/velocity 与 joint-position 采样 | 无 | `motion_tracking/common/manager_terms.py` | +| `G1WBTObs` | Hydra `events:` term | 是：同一 motion command + Hydra `EventTermCfg` | motion reset 加 mass/COM/PD/friction/encoder-bias event | interval velocity kick | `motion_tracking/g1/manager_terms.py` | +| `AllegroInhandRotation` | Hydra `events:` term | 是：Hydra `EventTermCfg` + Manager-Based reset term | entity 范围的手/球 reset | 无 | `allegro_inhand/manager_terms.py` | +| `AllegroInhandRotationGrasp` | Hydra `events:` term | 是：复用 rotation reset event + `RecorderTermCfg` | 带噪声的手部 reset + grasp 收集 | 无 | `allegro_inhand/grasp_gen.py` | +| `SharpaInhandRotation` | legacy provider | 是：`InitRandomizationPlan + ResetPlan + IntervalRandomizationPlan` | grasp cache 采样 + common payload | 物体 `body_force` | `sharpa_inhand/rotation.py` | +| `SharpaInhandRotationGrasp` | legacy provider | 是：复用 Sharpa rotation provider 并 override reset 采样 | grasp 收集 reset + common payload | 无 | `sharpa_inhand/grasp_gen.py` | +| `Go2ArmManipLoco` | legacy provider | 是：`DomainRandConfig + LocomotionDRProvider 子类 + ResetPlan` | task 状态采样 + common payload | push | `go2_arm/manip_loco.py` | ## 各任务域随机化清单 | Task | 当前已实现的 reset 域随机化 | 当前已实现的 interval 域随机化 | 默认状态 | | --- | --- | --- | --- | -| `Go1JoystickFlat` | base xy；base yaw；base qvel；command 采样；`current_actions/last_actions` 清零；可选 `base_mass_delta`；可选 `base_com_offset`；可选 `gravity` | `push_robots` | `base_mass_delta`、`base_com_offset` 和 push 默认启用；`gravity` 默认禁用 | -| `Go2JoystickFlat` | base xy；base yaw；base qvel；command 采样；`current_actions/last_actions` 清零；kp/kd 随机化（默认启用）；可选 `base_mass_delta`；可选 `base_com_offset`；可选 `gravity` | `push_robots` | kp/kd 默认启用；common payload 和 push 默认禁用 | -| `G1WalkFlat` | base xy；base yaw；由 `reset_base_qvel_limit` 采样的 base qvel；command 采样；`gait_phase` 采样；`current_actions/last_actions` 清零；kp/kd 随机化（默认启用）；可选 `base_mass_delta`；可选 `base_com_offset`；可选 `gravity` | `push_robots` | kp/kd 默认启用；common payload 和 push 默认禁用 | -| `G1WalkRough` | 与 `G1WalkFlat` 相同，直接复用同一 provider | `push_robots` | kp/kd 默认启用；common payload 和 push 默认禁用 | -| `G1MotionTracking` | 动作帧采样；root 位姿扰动 `x/y/z/roll/pitch/yaw`；root 速度扰动 `x/y/z/roll/pitch/yaw`；关节位置噪声；在 MuJoCo 下被关节范围 clip；`current_actions/last_actions` 清零；可选 `base_mass_delta`；可选 `base_com_offset`；可选 `gravity` | `push_robots` | `pose_randomization`、`velocity_randomization`、`joint_position_range` 默认有非零扰动；common payload 和 push 默认禁用 | -| `AllegroInhandRotation` | 若存在 grasp cache，则随机采样一个 grasp；否则对手部关节施加 `joint_noise` 并对球施加 `ball_z_offset`；始终对球的线速度施加 `ball_vel_noise`；可选 common reset 随机化 payload（含 `gravity`） | 无 | 若 grasp cache 路径可用则默认采样；`joint_noise`、`ball_vel_noise`、`ball_z_offset` 默认为 0；common payload 默认禁用 | +| `Go1JoystickFlat` | 经 `reset_root_state_uniform` 的 base xy/yaw 与 base qvel；command 采样（`UniformVelocityCommandCfg`）；经 `randomize_rigid_body_mass` 的 base mass；经 `randomize_rigid_body_com` 的 base COM；经 `pd_gains` 的 kp/kd | `push_by_setting_velocity` interval event | 上述 event term 全部在 `conf/ppo/task/go1_joystick_flat/base.yaml` 中默认声明并启用 | +| `Go2JoystickFlat` | 经 `reset_root_state_uniform` 的 base xy/yaw 与 base qvel；command 采样；经 `pd_gains` 的 kp/kd | 无 | event term 在 `conf/ppo/task/go2_joystick_flat/base.yaml` 中默认声明并启用 | +| `G1WalkFlat` | 经 `reset_root_state_uniform` 的 base xy/yaw 与 base qvel；带平面死区的 command 采样；`gait_phase` 采样；经 `pd_gains` 的 kp/kd 随机化 | 无 | mujoco owner 默认启用 kp/kd；motrix/mjwarp owner 默认禁用 | +| `G1WalkRough` | 与 `G1WalkFlat` 相同（共享 owner base，rough 场景） | 无 | 与 `G1WalkFlat` 相同的默认值 | +| `G1MotionTracking` | Motion-command frame 采样；root 位姿扰动 `x/y/z/roll/pitch/yaw`；root 速度扰动 `x/y/z/roll/pitch/yaw`；通过 public entity soft limit clip 的关节位置噪声；action-manager 状态 reset | 无 | base owner 中 `pose_range`、`velocity_range` 与 `joint_position_range` 默认有非零扰动 | +| `G1WBTObs` | 同一 motion reset 加 base mass、base COM、PD gain、足端摩擦和 encoder-bias event term | `push_by_setting_velocity` | WBT owner 显式启用上述全部 event term；能力不支持时直接报错，不回退 | +| `AllegroInhandRotation` | entity 范围的手/球 reset；显式配置 grasp cache 时进行采样，否则以 `null` 显式选择模型 home pose；可选 `joint_noise`、`ball_velocity_noise` 与 `ball_z_offset` | 无 | owner YAML 显式选择 home pose 与零 reset 噪声；配置的 cache 缺失或格式错误时 fail-closed | +| `AllegroInhandRotationGrasp` | 复用 rotation reset 并设置 `joint_noise=0.25`；Manager-Based termination 检查指尖距离、接触数和球高度；recorder 保存成功 timeout rows | 无 | 生成 5 万行 Allegro grasp cache，成功保存后抛出 `RunComplete` | | `SharpaInhandRotation` | grasp cache 按 `scale_ids` 分桶采样；物体位姿 / quat reset；可选 common reset 随机化 payload（含 `gravity`） | 物体 `body_force` 直接力扰动 | `domain_rand.scale_list` 默认值来自 owner YAML；在 MuJoCo 下，物体 geom 缩放在 init 期间 materialize；common payload 默认禁用；物体 force 通过 Sharpa owner YAML 默认启用 | | `SharpaInhandRotationGrasp` | 手部位姿 reset；物体位姿 / quat reset；收集成功的 grasp 并按 `scale_ids` 分桶存储；可选 `base_mass_delta`；可选 `base_com_offset`；可选 `gravity` | 无 | 默认用于生成 Sharpa grasp cache；cache 文件名包含单个 scale 值；common payload 默认禁用 | ## 当前统一 DR 的能力与边界 -### 1. 统一入口点是完整的 +### 1. legacy provider 入口是统一的 -统一入口点由 `NpEnv` 和 `DomainRandomizationManager` 保证： +legacy provider 路径的统一入口点由 `NpEnv` 和 `DomainRandomizationManager` 保证： - 任务只需注册一个 provider - manager 统一执行能力验证 - 后端统一负责实际施加随机化 payload -因此从执行路径的角度看，这些任务已经是统一的。 +因此从执行路径的角度看，仍走该路径的 Adapted family 是统一的；Manager-Based 任务则由 manager 生命周期统一执行 owner YAML 声明的 `events:` term。 ### 2. 共享辅助函数仍然较窄 -`dr_utils.py` 目前只有两类共享辅助函数： +legacy 路径的 `dr_utils.py` 目前只有两类共享辅助函数： - reset common payload：`base_mass_delta`、`base_com_offset`、`gravity`、`kp`、`kd` - interval common payload：push 这意味着： -- 尽管运动控制任务都走统一入口点，但它们的 base xy、yaw、qvel、command 和 gait phase 仍然直接在各自的 provider 内部采样 -- `G1MotionTracking` 的 pose / velocity / joint 噪声也是 task 专属逻辑 +- 仍走 legacy provider 的 go2_arm / sharpa family，其 task 专属状态仍直接在各自的 provider 内部采样 +- `G1MotionTracking` 的 pose / velocity / joint 噪声由其 manager command 所有 - Allegro 的 grasp / 物体初始状态采样完全是 task 专属逻辑 - Sharpa 的 `geom_size` 缩放是 init 生命周期的模型 materialization，不属于 reset common payload @@ -112,7 +122,7 @@ - 生命周期：仅在 reset 时采样和写入；env 会保留该重力，直到下一次 reset 重新采样。 - 后端：当前在 UniLab 中，只有 MuJoCo 后端声明支持该 reset 项；Motrix 后端不支持。一些任务按能力过滤并跳过它；另一些任务在 validate 阶段抛出错误。 -配置入口在每个任务的 `env.domain_rand` 下： +配置入口仅在仍走 legacy provider 路径的 Adapted family owner 的 `env.domain_rand` 下（`sharpa_inhand_grasp`、`go2_arm_manip_loco` 及对应 hora / appo / ppo_him 变体）；Manager-Based 任务没有 `env.domain_rand`： ```yaml env: @@ -132,7 +142,7 @@ env: 如果你只想随机化大小而保持竖直向下的方向，只开放 `z` 分量： ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[0.0,0.0,-10.5],[0.0,0.0,-8.5]]' ``` @@ -140,7 +150,7 @@ uv run train --algo ppo --task g1_walk_flat --sim mujoco \ 如果你想同时随机化方向和大小，开放 `x/y/z`： ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[-0.3,-0.3,-10.5],[0.3,0.3,-8.5]]' ``` @@ -155,7 +165,9 @@ uv run train --algo ppo --task g1_walk_flat --sim mujoco \ ## Interval push 用法 -支持 interval push 的任务在 `env.domain_rand` 下配置它： +`env.domain_rand.push_robots` 系列字段只存在于 go2_arm Adapted family 的 owner（`conf/ppo/task/go2_arm_manip_loco/mujoco.yaml` 等）；Manager-Based 任务改用 `push_by_setting_velocity` interval event term 声明 push（例如 `conf/ppo/task/go1_joystick_flat/base.yaml` 和 `conf/ppo/task/quadruped_joystick_rough/base.yaml`）。 + +go2_arm owner 在 `env.domain_rand` 下配置 push： ```yaml env: @@ -172,11 +184,11 @@ env: - `push_body_name`：施加力的目标 body / link。默认为 `null`，表示使用后端的 `base_name`。 ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task go2_arm_manip_loco --sim mujoco \ env.domain_rand.push_robots=true \ env.domain_rand.push_interval=500 \ 'env.domain_rand.max_force=[20.0,20.0,5.0]' \ - env.domain_rand.push_body_name=torso_link + env.domain_rand.push_body_name=base ``` 说明： diff --git a/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/1-configuration.md b/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/1-configuration.md index d83415575..2a3033b33 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/1-configuration.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/1-configuration.md @@ -1,11 +1,17 @@ # 配置 -域随机化在所选的 task owner YAML 内部配置，通常位于 -`env.domain_rand` 下。先使用 `--task` 和 `--sim` 选择后端专属行为， +域随机化在所选的 task owner YAML 内部配置。先使用 `--task` 和 `--sim` 选择后端专属行为， 然后在所选的 owner 内部 override 字段。 +当前有两条声明路径： + +- Manager-Based（Compatible）任务通过 owner YAML 的 `events:` manager term 声明 + reset / interval 随机化，例如 `conf/ppo/task/go1_joystick_flat/base.yaml`。 +- 只有 Adapted family（sharpa / go2_arm 及对应 hora / appo / ppo_him owner）仍在 + `env.domain_rand` 下配置 legacy provider 字段。 + ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[0.0,0.0,-10.5],[0.0,0.0,-8.5]]' ``` @@ -18,25 +24,29 @@ uv run train --algo ppo --task g1_walk_flat --sim mujoco \ 详细的任务状态和字段语义见 {doc}`0-index`。 -域随机化按生命周期划分：init、reset 和 interval。manager 路径是 +域随机化按生命周期划分：init、reset 和 interval。legacy 路径的 manager 位于 `src/unilab/dr/manager.py`；task provider 位于 env owner 附近， 后端能力通过 `src/unilab/base/backend/base.py` 声明。 ## Reset Gravity 在启用 gravity reset 随机化时使用 `--sim mujoco`；Motrix 在当前后端中 -未提供相同的 gravity 能力。 +未提供相同的 gravity 能力。该项只在 legacy provider 路径（Adapted family owner） +上可用。 ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task sharpa_inhand_grasp --sim mujoco \ env.domain_rand.randomize_gravity=true \ 'env.domain_rand.gravity_range=[[0.0,0.0,-10.5],[0.0,0.0,-8.5]]' ``` ## Interval Push +Manager-Based 任务通过 `push_by_setting_velocity` interval event term 声明 push； +`env.domain_rand.push_robots` 只在 go2_arm Adapted family owner 上可用。 + ```bash -uv run train --algo ppo --task g1_walk_flat --sim mujoco \ +uv run train --algo ppo --task go2_arm_manip_loco --sim mujoco \ env.domain_rand.push_robots=true \ env.domain_rand.push_interval=500 \ 'env.domain_rand.max_force=[20.0,20.0,5.0]' @@ -45,8 +55,9 @@ uv run train --algo ppo --task g1_walk_flat --sim mujoco \ ## Owner 本地默认值 当取值范围是任务 contract 的一部分时，将其保留在 task owner YAML 中。例如， -`conf/ppo/task/go2_joystick_rough/mujoco.yaml` 启用了 base mass、 -质心、kp/kd 和 push 随机化，而 +rough 四足家族的 base mass、质心、kp/kd 和 push 随机化作为 event term 声明在共享 base +`conf/ppo/task/quadruped_joystick_rough/base.yaml`（`go2_joystick_rough` 的 backend +owner 通过 Hydra defaults 组合它），而 `conf/ppo/task/sharpa_inhand/mujoco.yaml` 为 Sharpa 配置了物体缩放、摩擦和 力扰动。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/2-writing_providers.md b/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/2-writing_providers.md index 16c242981..901600ee3 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/2-writing_providers.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/5-domain_randomization/2-writing_providers.md @@ -1,5 +1,11 @@ # 编写 Provider +本页描述 legacy provider 路径：只有 3 个 Adapted family（`sharpa_inhand` / +`sharpa_inhand_grasp` / `go2_arm_manip_loco`）仍通过任务级 +`DomainRandomizationProvider` 声明域随机化。已迁移的 Manager-Based 任务不写 +provider；它们在 owner YAML 中通过 Hydra `events:` manager term 声明随机化（见 +{doc}`0-index` 与 {doc}`1-configuration`）。 + 任务级域随机化 provider 与 task env owner 放在一起。它们采样任务专属的 状态，并返回由 `DomainRandomizationManager` 消费的 plan。 @@ -24,13 +30,12 @@ ## 证据 -具有代表性的 provider 实现位于： +具有代表性的 provider 实现位于（全部属于 Adapted family 的兼容路径）： -- `src/unilab/envs/locomotion/go1/joystick.py` -- `src/unilab/envs/locomotion/g1/joystick.py` -- `src/unilab/envs/motion_tracking/g1/tracking.py` -- `src/unilab/envs/manipulation/allegro_inhand/rotation.py` -- `src/unilab/envs/manipulation/sharpa_inhand/rotation.py` +- `src/unilab/tasks/locomotion/common/dr_provider.py`（`LocomotionDRProvider`， + 由 go2_arm family 使用） +- `src/unilab/tasks/locomotion/go2_arm/manip_loco.py` +- `src/unilab/tasks/manipulation/sharpa_inhand/rotation.py` 开发者 contract 详情见 {doc}`../../4-developer_guide/2-contracts/4-dr_contract`。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/6-terrain/2-heightfield_import.md b/docs/sphinx/source/zh_CN/2-user_guide/6-terrain/2-heightfield_import.md index 4020a517d..64b439691 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/6-terrain/2-heightfield_import.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/6-terrain/2-heightfield_import.md @@ -6,7 +6,7 @@ - `src/unilab/terrains/heightfield_terrains.py` - `src/unilab/terrains/terrain_generator.py` -- `src/unilab/envs/locomotion/go2/rough.py` +- `src/unilab/tasks/locomotion/go2/rough.py` - `src/unilab/base/backend/mujoco/xml.py` - `src/unilab/base/backend/motrix/scene.py` diff --git a/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/3-nan_visualizer.md b/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/3-nan_visualizer.md index 7fe13f910..cd442ddd7 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/3-nan_visualizer.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/3-nan_visualizer.md @@ -8,4 +8,4 @@ uv run train --algo ppo --task go2_joystick_flat --sim mujoco \ training.nan_guard.output_dir=/tmp/unilab/nan_dumps ``` -viewer 的实现是 `src/unilab/tools/viz_nan.py`，注册为 `unilab-viz-nan` 控制台入口。它会回放一个 dump 路径，并让你选择环境索引。dump 格式和往返加载由 `tests/test_nan_guard.py` 覆盖。 +viewer 的实现是 `src/unilab/utils/nan_viz.py`，注册为 `unilab-viz-nan` 控制台入口。它会回放一个 dump 路径，并让你选择环境索引。dump 格式和往返加载由 `tests/test_nan_guard.py` 覆盖。 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/4-scene_export.md b/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/4-scene_export.md index 4d785c0c3..fff4b9de8 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/4-scene_export.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/4-scene_export.md @@ -1,6 +1,6 @@ # 场景导出 -场景导出由 `src/unilab/tools/export_scene.py` 实现，并在 `pyproject.toml` 中注册为 `unilab-export-scene` 控制台入口。它接受一个 MuJoCo XML 或 MJB 模型路径，写出 `scene.xml`，在能够发现 mesh asset 时复制它们，并且可以创建一个 zip 归档。 +场景导出由 `src/unilab/base/backend/mujoco/export_scene.py` 实现，并在 `pyproject.toml` 中注册为 `unilab-export-scene` 控制台入口。它接受一个 MuJoCo XML 或 MJB 模型路径，写出 `scene.xml`，在能够发现 mesh asset 时复制它们，并且可以创建一个 zip 归档。 对于 task 级别的实例化检查，请使用从 registry 和 owner config 构造 env 的脚本： diff --git a/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/5-robot_import.md b/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/5-robot_import.md index d74b5c620..cd3787a30 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/5-robot_import.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/7-tooling/5-robot_import.md @@ -25,7 +25,7 @@ task/reward/env 语义。 如果只有 URDF，使用仓库自带脚本进行转换： ```bash -uv run unilab-import-robot [robot_name] +uv run scripts/tools/import_robot.py [robot_name] ``` ```{important} @@ -36,7 +36,7 @@ visual mesh 作为 collision mesh，尽量把碰撞体简化为 box / capsule / - 默认自动导入会把 actuator 写成 `position`，这只适合位置控制 owner。 - 如果机器人必须保留 torque/motor actuator 语义，后续扩展任务时，需要参考 - `src/unilab/envs/locomotion/go2w/` 的控制方式，把 action 解释、PD/力矩控制和 + `src/unilab/tasks/locomotion/go2w/` 的控制方式，把 action 解释、PD/力矩控制和 actuator contract 放在机器人 owner 的控制边界内。 - 转换完成后，会自动弹出 `mujoco.viewer` 可视化界面展示转换结果，并进行下一步调整 Keyframe。 @@ -58,7 +58,7 @@ visual mesh 作为 collision mesh，尽量把碰撞体简化为 box / capsule / ## 输出产物 -使用 `uv run unilab-import-robot [robot_name]` 转换后，会在仓库内生成： +使用 `uv run scripts/tools/import_robot.py [robot_name]` 转换后，会在仓库内生成： - `src/unilab/assets/robots//assets/`：转换并整理后的 mesh 资产。 - `src/unilab/assets/robots//.xml`：机器人 MJCF 描述，只包含机器人 diff --git a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/1-overview.md b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/1-overview.md index 223c74a92..f385074e4 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/1-overview.md +++ b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/1-overview.md @@ -60,9 +60,9 @@ flowchart LR - **观测漂移。** 仿真与部署运行时之间的传感器预处理不同（单位、坐标系、滤波截止频率）。 记录第一段部署侧观测窗口，并与用同一份 owner YAML 构建的仿真回合作对比。 -- **动作延迟。** 一些任务配置通过 `control_config.simulate_action_latency` 暴露单步 - 延迟的动作执行。测量部署回路，并在硬件运行前让训练 owner 匹配该契约。见 - {doc}`8-latency_budget`。 +- **动作延迟。** 一些 task owner 通过 control config 或 Manager-Based action term + 暴露单步延迟的动作执行。测量部署回路，并在硬件运行前让训练 owner 匹配该契约。 + 见 {doc}`8-latency_budget`。 - **摩擦 / 阻尼不匹配。** 尤其对于手内操作。在 DR 中扫动摩擦；通过 {doc}`../2-sim_to_sim/3-contact_and_friction_alignment` 交叉核对。 - **复位瞬态。** 仿真复位到一个稳定姿态；部署则从一个控制器状态开始。安全层必须在 diff --git a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/2-g1_whole_body.md b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/2-g1_whole_body.md index df79c491e..0b7f9d12b 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/2-g1_whole_body.md +++ b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/2-g1_whole_body.md @@ -110,7 +110,7 @@ G1 部署原型将 actor 输出严格映射为： - **速率有界** —— 将 dφ/dt 钳制到策略训练时所用的值（运动加载器会记录这个值；加载 `reference_motion.npz`）。 -参见 `unilab.envs.motion_tracking.g1.motion_loader`，这是你应当在硬件上镜像的仿真侧 +参见 `unilab.tasks.motion_tracking.common.motion_loader`，这是你应当在硬件上镜像的仿真侧 加载器。 ## 5. 安全层 diff --git a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/4-allegro_inhand.md b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/4-allegro_inhand.md index d5df85edc..c325fdac2 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/4-allegro_inhand.md +++ b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/4-allegro_inhand.md @@ -55,17 +55,17 @@ owner 与部署运行时在观测时序上达成一致。见 `4-allegro_inhand` 与 `sharpa_inhand` 两个环境都自带一个**抓取生成器**，用于采样 合理的初始手部构型。硬件侧的等价物是操作员把方块放到手里 —— 请核实你的起始构型分布 与训练环境的抓取生成器输出相匹配（参见 -`unilab.envs.manipulation.allegro_inhand.grasp_gen`）。 +`unilab.tasks.manipulation.allegro_inhand.grasp_gen`）。 如果你真实世界的起始握持存在系统性差异，**把这些位姿加入抓取生成器**，重新训练， 然后再试。 ## 动作接口 -操作类环境通过任务控制配置把策略动作映射为关节位置目标 -（`src/unilab/envs/manipulation/allegro_inhand/base.py` 和 -`src/unilab/envs/manipulation/sharpa_inhand/base.py`）。部署控制器必须使用相同的 -关节顺序、动作缩放与限位策略。 +操作类环境通过任务控制配置把策略动作映射为关节位置目标。Allegro 的声明由 +`conf/ppo/task/allegro_inhand/base.yaml` 与其 Manager-Based action term 持有； +Sharpa 当前仍由 `src/unilab/tasks/manipulation/sharpa_inhand/base.py` 持有。 +部署控制器必须使用相同的关节顺序、动作缩放与限位策略。 ## 失败恢复 diff --git a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/5-onnx_runtime.md b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/5-onnx_runtime.md index 198f5e902..0b9eab473 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/5-onnx_runtime.md +++ b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/5-onnx_runtime.md @@ -11,7 +11,7 @@ owner；回放代码加载检查点、导出 `policy.onnx`，并在该路径实 | PPO（torch） | `scripts/train_rsl_rl.py` | 脚本入口处 `EXPORT_POLICY=True`；回放调用 `runner.export_policy_to_onnx(...)` 与 `runner.export_policy_to_jit(...)`。 | | HIM-PPO | `scripts/train_him_ppo.py` | 与 PPO 相同的脚本级导出模式。 | | APPO | `scripts/train_appo.py` | 回放写出 `policy.onnx` 并将 ONNX Runtime 输出与 PyTorch 比对校验。 | -| SAC / TD3 / FlashSAC | `scripts/train_offpolicy.py` | 回放写出 `policy.onnx`；SAC 与 FlashSAC 在导出前使用 `actor.as_export_module()`。 | +| SAC / TD3 / FlashSAC | `scripts/train_sac.py` / `scripts/train_td3.py` / `scripts/train_flashsac.py` | 回放写出 `policy.onnx`；SAC 与 FlashSAC 在导出前使用 `actor.as_export_module()`。 | ## 命令 diff --git a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/6-domain_randomization.md b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/6-domain_randomization.md index 53d864e55..6f642cdf3 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/6-domain_randomization.md +++ b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/6-domain_randomization.md @@ -43,7 +43,7 @@ 使用 DR 的任务通过环境初始化路径挂接一个 provider： ```python -from unilab.envs.locomotion.common.dr_provider import LocomotionDRProvider +from unilab.tasks.locomotion.common.dr_provider import LocomotionDRProvider class MyTaskEnv(NpEnv): def __init__(self, cfg): diff --git a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/8-latency_budget.md b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/8-latency_budget.md index 0d972f6f6..3be57cc32 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/8-latency_budget.md +++ b/docs/sphinx/source/zh_CN/3-deployment/1-sim_to_real/8-latency_budget.md @@ -7,23 +7,24 @@ | 面 | 仓库证据 | 它覆盖什么 | | --- | --- | --- | -| 单步动作延迟 | locomotion 与 G1 运动跟踪环境中的 `control_config.simulate_action_latency` | 执行上一步动作而非当前动作。 | -| G1 WBT 观测历史 | `noise_config.obs_history_length` 与 `scripts/deploy/export_deploy_config.py` | 为 `gyro`、`joint_pos_rel`、`dof_vel` 与 `last_actions` 导出逐项的 `obs_layout` 历史。 | +| 单步动作延迟 | task owner 中 Manager action term 的 `simulate_action_latency` 声明 | 执行上一步动作而非当前动作。 | +| G1 WBT 观测历史 | `conf/sac/task/g1_wbt_obs/mujoco.yaml` 中逐 term 的 `history_length` 与 `scripts/deploy/export_deploy_config.py` | 为 `gyro`、`joint_pos_rel`、`dof_vel` 与 `last_actions` 导出逐项的 `obs_layout` 历史。 | | Sharpa 触觉接触延迟 | Sharpa 手内配置中的 `domain_rand.contact_latency` | 为采样到的接触通道保留上一步的触觉接触值。 | | 部署侧 ONNX 契约检查 | `scripts/deploy/sim_prototype.py` | 为 G1 WBT 路径校验 `obs_layout`、`obs_dim`、ONNX 输入宽度、钳制以及 EMA 动作平滑。 | ## 动作延迟 -对于暴露 `control_config.simulate_action_latency` 的任务，当该开关启用时，环境会应用 -`last_actions`。把它保留在所选的任务 owner YAML 中，而不要事后添加仅部署的行为。 +对于启用 action latency 的 Manager-Based 任务，action manager 会在该开关开启时应用 +上一步 action。把它保留在所选的 task owner YAML 中，而不要事后添加仅部署的行为。 ```yaml env: - control_config: - simulate_action_latency: true + actions: + joint_pos: + simulate_action_latency: true ``` -已签入的 G1 WBT owner 在 `conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml` 中启用了 +已签入的 G1 WBT owner 在 `conf/sac/task/g1_wbt_obs/mujoco.yaml` 中启用了 该开关。 ## 观测滞后与历史 diff --git a/docs/sphinx/source/zh_CN/3-deployment/2-sim_to_sim/7-config_guard.md b/docs/sphinx/source/zh_CN/3-deployment/2-sim_to_sim/7-config_guard.md index 7c94210e1..f75f7721b 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/2-sim_to_sim/7-config_guard.md +++ b/docs/sphinx/source/zh_CN/3-deployment/2-sim_to_sim/7-config_guard.md @@ -18,7 +18,7 @@ uv run eval --algo ppo --task go2_joystick_flat --sim motrix --load-run -1 1. **训练时**：`ExperimentTracker` 把决定策略 I/O 的契约字段快照进 `run_config.json` 的 `contract_snapshot`（不改动 checkpoint 格式，历史 checkpoint 天然兼容）。 2. **回放时**：`eval` 读取 `--sim` 指定的**目标后端** owner 配置（如 `conf/ppo/task/go2_joystick_flat/motrix.yaml`），并注入 `training.play_only=true`。 -3. **建 env 前**：四个 play 入口（rsl_rl / appo / offpolicy / him_ppo）调用 `resolve_sim2sim_config`，把目标配置与源 run 的契约快照逐字段比对。 +3. **建 env 前**：各 play 入口（rsl_rl / appo / sac / td3 / flashsac / him_ppo）调用 `resolve_sim2sim_config`，把目标配置与源 run 的契约快照逐字段比对。 4. **加载权重时**：`policy_load_dim_guard` 包裹 checkpoint 加载，把底层 tensor 维度不匹配的晦涩报错重抛为清晰的 sim2sim 诊断。 ## 守卫的字段 @@ -27,7 +27,7 @@ uv run eval --algo ppo --task go2_joystick_flat --sim motrix --load-run -1 | 档位 | 行为 | 字段 | |---|---|---| -| **DENYLIST** | 差异即 `CrossBackendIncompatibleError`，中断 | `algo.obs_groups`、`env.control_config.action_scale`、`algo.policy.actor_hidden_dims` / `critic_hidden_dims`、`algo.empirical_normalization` / `algo.obs_normalization`、`env.sampling_mode` | +| **DENYLIST** | 差异即 `CrossBackendIncompatibleError`，中断 | `algo.obs_groups`、legacy `env.control_config.action_scale`、Manager-Based `env.observations` / `env.actions` / policy 与 critic group mapping、`algo.policy.actor_hidden_dims` / `critic_hidden_dims`、`algo.empirical_normalization` / `algo.obs_normalization`、`env.sampling_mode` | | **WARNING_LIST** | 仅打印 warning，继续 | `reward.*`、`env.control_config.simulate_action_latency`、`env.ctrl_dt` | | **ALLOWLIST** | 自由覆盖，不检查 | `training.sim_backend`、`env.scene`、`training.play_steps`、`env.domain_rand`、`env.noise_config`、`env.commands.vel_limit` | @@ -40,6 +40,10 @@ uv run eval --algo ppo --task go2_joystick_flat --sim motrix --load-run -1 > 兼容旧 run：若 `run_config.json` 没有 `contract_snapshot`（早期训练），守卫自动跳过并打印 warning，不会中断现有工作流。 +Manager-Based snapshot 会保存 Hydra 中完整的 typed observation/action 声明。缺少这些字段的旧 +snapshot 无法证明其 policy I/O 与 Manager-Based 目标等价，因此不对称出现时会 fail-closed。 +只有用户显式设置 `training.sim2sim_strict=false` 才会继续；加载权重时的维度守卫仍然生效。 + ## 另请参阅 - {doc}`1-backend_swap` diff --git a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/0-index.md b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/0-index.md index 3c60032d6..42d467c0c 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/0-index.md +++ b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/0-index.md @@ -8,7 +8,7 @@ :::{grid-item-card} 从 Isaac Lab 迁移 :link: 1-from_isaac_lab :link-type: doc -把 GPU 常驻的任务结构映射到 UniLab 的 CPU sim 与 learner 拆分。 +保留 Manager-Based term 结构，适配 Hydra 配置、NumPy 执行和场景访问。 ::: :::{grid-item-card} 从 Legged Gym 迁移 diff --git a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/1-from_isaac_lab.md b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/1-from_isaac_lab.md index 7eaaf7f6b..38ffe1aa1 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/1-from_isaac_lab.md +++ b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/1-from_isaac_lab.md @@ -1,83 +1,176 @@ # 从 Isaac Lab 迁移 -如果你有一个想在 UniLab 中运行的 Isaac Lab 任务，本页会告诉你哪些保持不变、 -哪些会改变，以及锋利的边角在哪里。 +把 Isaac Lab Manager-Based task 迁入 UniLab 时，应保留 manager 与 term 结构，只在各自 +owner 边界适配配置、数值执行和场景访问；不要把 task 重写成单体 `NpEnv` 子类。 -## 哪些保持不变 +这是基于源码的兼容迁移，不代表任意 Isaac Lab task 都能不修改直接运行。目标路径是： -- Gymnasium 风格的 env 接口（`reset`、`step`、`obs/reward/info`）。 -- 基于 Hydra 的配置。你现有的大部分 YAML 可以通过字段名重映射来移植。 -- "任务"由 scene + reward + DR + obs 组合而成这一总体思路。 -- PPO 作为默认算法 —— UniLab 开箱即带 RSL-RL 的 PPO。 +```text +Hydra owner YAML + -> plain ManagerBasedRlEnvCfg + -> Registry + make_manager_based_rl_env + -> NumPy/SimBackend runtime 上的 ManagerBasedRlEnv + -> 交给现有 training 和 IPC 路径的 NpEnvState +``` -## 哪些会改变 +## 兼容边界 ```{list-table} :header-rows: 1 -:widths: 30 35 35 - -* - Isaac Lab 概念 - - UniLab 对应物 - - 备注 -* - `DirectRLEnv` - - `unilab.base.np_env.NpEnv` - - UniLab 的 obs 始终是 **dict**，而不是 tensor。 -* - `RigidBody.cfg` - - 任务侧的 asset 导入 + 场景组合 - - 参见 {doc}`../../4-developer_guide/1-architecture/4-scene_composition`。 -* - GPU PhysX 后端 - - CPU MuJoCo / Motrix + GPU learner - - 架构倒置 —— 见下文。 -* - `RandomizationCfg` - - {doc}`../../4-developer_guide/2-contracts/4-dr_contract` - - UniLab 的 DR 只在冷路径重采样中运行。 -* - `RewardManager` 链 - - env 中的 reward 组合，外加 - `unilab.training.reward` 记账 - - reward 项仍然以 key 标识，以便分量级别的日志记录。 -* - `EventCfg` 事件驱动钩子 - - Phase + curriculum + DR provider - - 钩子是显式的，而非隐式的。 +:widths: 28 24 48 + +* - Isaac Lab 表面 + - UniLab 状态 + - 迁移规则 +* - Manager 类别、term 名称和字典顺序 + - Compatible + - 保持 observation、action、event、reward、termination、command 与 + curriculum term 的顺序。 +* - Function/class term 与 `func + params` + - Compatible + - 把 import 改为 `unilab.managers`；保留 term 边界和局部 + `reset(env_ids)` 语义。 +* - `ManagerBasedRLEnv` / `ManagerBasedRLEnvCfg` + - Compatible 拼写 alias + - UniLab canonical 名称是 `ManagerBasedRlEnv` 与 `ManagerBasedRlEnvCfg`；alias + 指向同一份实现。 +* - Tensor 数值与运算 + - Adapted + - 把 `torch.Tensor` 换成 `np.ndarray` 并使用向量化 NumPy；manager-facing + API 没有 device 接口。 +* - 嵌套 `@configclass` task 配置 + - Adapted + - 把完整 task 声明迁入唯一 Hydra owner YAML；用 `_target_` 选择具体 config + dataclass，用 dotted `func` 选择 term。 +* - `InteractiveSceneCfg`、USD 与 PhysX view + - Adapted 或 Unsupported + - 声明 task-owned `SceneCfg` 与 `EntityCfg`；状态和控制只通过 + `SceneEntityCfg` 与公共 entity facade 访问。不支持的能力在冷路径绑定时报错。 +* - Omniverse、Isaac renderer 与 Torch/PhysX mutation + - Unsupported + - UniLab 不安装这些 runtime，也不提供静默模拟或回退。 ``` -## 架构倒置 +规范边界见 {doc}`ADR-0006 `。 +只有已经被 registry、配置和测试覆盖的表面才能声明为 Compatible。 + +## 迁移步骤 + +### 1. 盘点来源 task + +固定 Isaac Lab revision，并列出来源 manager group、term 名称与顺序、参数、observation +维度、action 维度、reset 行为和 episode timing。写代码前逐项分类： + +- 复用已有 `unilab.managers` config 或 `unilab.envs.mdp` term； +- 把 task-specific term 从 Torch 适配为 NumPy； +- 如果 term 依赖公共 entity 或 `SimBackend` contract 尚未提供的能力，立即停止。 + +不能用 `getattr`/`hasattr` 探测 backend 对象、返回零，或把 task 路由回 legacy env。 + +### 2. 在冷路径迁移 scene 与 asset + +用 task-owned `SceneCfg` 代替 Isaac Lab 的 USD/`InteractiveSceneCfg` 声明，显式声明 term +需要的每个 entity 与 selector。`SceneEntityCfg` 在 materialization 时只解析一次名称和 +正则表达式；reset/step 复用缓存 ID 与 NumPy view。 + +Cartpole fixture 使用最小 task-owned MJCF。更复杂的 asset 必须遵守 +{doc}`场景组合 <../../4-developer_guide/1-architecture/4-scene_composition>`，并只使用所选 +backend 的正式能力。 + +### 3. 迁移 term 代码，不改 manager 结构 + +保留每个 function/class term 及其参数，机械地把 Torch 类型与运算改为 NumPy，保持 batch +shape，并在来源 term 返回每环境数值时继续返回每环境数值。Stateful term 在构造时解析 +selector、分配 buffer，热路径只更新 NumPy buffer。 + +Python 只拥有 term 实现和可复用 config dataclass，不能再保存一份 task-specific term +启停清单或默认 weight。 + +### 4. 让 Hydra 成为唯一 task 配置 owner + +在 owner YAML 中声明 scene、timing、group、term、具体 config 类型、callable、参数、 +weight 和 observation mapping。例如： + +```yaml +env: + observations: + policy: + terms: + joint_pos_rel: + func: unilab.envs.mdp.joint_pos_rel + terminations: + time_out: + func: unilab.envs.mdp.time_out + time_out: true + policy_observation_group: policy + critic_observation_group: null + +reward: + alive: + func: unilab.envs.mdp.is_alive + weight: 1.0 +``` + +值类型唯一且具体的 manager mapping（observations / events / rewards / +terminations / curriculum / metrics / recorders）可以省略 `_target_`，物化时按字段 +类型注解推断；`actions` / `commands` 的基类是抽象的，必须显式声明具体 `_target_` +（如 `unilab.envs.mdp.JointPositionActionCfg`）。`unilab.managers.` 下的 config 类 +（如 `SceneEntityCfg`）可以直接写裸类名。 + +Hydra compose 在冷路径把这份声明物化为 plain typed config。未知字段、无法解析的 +`_target_`/`func` 和错误 config 类型都会在 reset/step 之前报错。直接用 Python 构造 +config 只用于 focused 底层测试。 + +### 5. 只注册一条通用 runtime 路径 + +Task module 为仓库已经实际支持的每个 backend 注册 `ManagerBasedRlEnvCfg` 与 +`make_manager_based_rl_env`。Backend owner YAML 只承载 backend 身份与 tuning。用户通过 +标准 CLI 选择 compose owner，例如： + +```bash +uv run train --algo ppo --task --sim mujoco +``` + +不要增加 task-specific 训练脚本分支、env factory、runner 或 IPC 路径。 + +generic factory 规则只有两个 maintainer 批准的已注册例外： +`make_g1_walk_env`（`src/unilab/tasks/locomotion/g1/manager_terms.py`）构造 +`G1WalkManagerBasedEnv` 子类，承载 G1 walk 的 manager-based 生产 runtime； +`make_x2_wall_flip_env` +（`src/unilab/tasks/motion_tracking/x2/__init__.py`）在冷路径解析未跟踪的 X2 +mesh，然后委托给 `make_manager_based_rl_env`。其余所有 Compatible task 都直接注册 +`make_manager_based_rl_env`。 -Isaac Lab 把模拟器放在 GPU 上，让你在 PhysX 中批处理数千个 env。UniLab 把 -模拟器放在 CPU 上（通常是多线程），并跨 worker **进程**做批处理，与单个 GPU -learner 共享内存。 +### 6. 在适配风险附近验证 -由此带来的影响： +测试 Hydra compose 与 typed materialization、term 顺序与数学、selector 失败、 +observation/action shape、局部 reset，以及至少一个真实已注册 backend 的 transition。行为 +应与固定来源 task 对比；完成语义迁移后再做性能 benchmark。 -- 在单个 env 上，UniLab 的**每个 env 步进时间**与 Isaac 相当甚至更差。 - **吞吐量**来自进程并行 + 异步（参见 `unilab.ipc.async_runner`）。 -- 你可以用 **MPS、ROCm、XPU** 作为 learner 设备 —— Isaac 仅支持 CUDA。 -- 模拟器与 learner 之间**不存在 GPU 争用** —— 你的 trainer 内存占用是可预测的。 +## 任务迁移最终状态 -## 逐步迁移 +#1042 迁移收尾覆盖 39 个 production task、86 个 task/backend 注册。fail-closed 的 +source of truth 是 `src/unilab/tasks/migration_matrix.py`：`migration_record()` 对没有 +entry 的 production task 名称抛出 `KeyError`，因此新增 production 注册必须显式做出 +迁移决策。 -1. **审查观测。** 确保每个观测 key 都是一个无需 GPU PhysX 查询即可表达的向量。 - 如果不是，就添加一个状态估计器，或把该查询移到冷路径。 -2. **移植 asset。** UniLab 以 MJCF 作为唯一真实来源（source of truth）。如果你 - 有 USD，先转换为 MJCF。 -3. **移植 env。** 继承 `unilab.base.np_env.NpEnv`。把 reward 计算移进 env 的 - `compute_reward()`。 -4. **移植 YAML。** 按照 {doc}`5-task_config_translation` 中的表格，把 Isaac Lab - 的 `EnvCfg` 字段映射到 UniLab 任务 owner YAML。 -5. **移植 reward。** 使用 {doc}`6-reward_porting` 中的食谱。 -6. **验证。** 训练一个小规模运行，把 reward 曲线与你的 Isaac 基线对比。 +- 36 个 task 为 **Compatible**（`target=complete`）：Hydra owner YAML 物化 canonical + NumPy Manager-Based runtime。 +- 3 个 task 为 **Adapted**（`target=compatibility`）：`Go2ArmManipLoco`、 + `SharpaInhandRotation` 和 `SharpaInhandRotationGrasp` 各自把自定义 IK/history 或 + tactile/contact/cache 行为保留在一个冻结的兼容 factory 后面；只有当正式能力存在时 + 才迁移。 -## 你会失去什么（以及如何弥补） +## 仓库证据 -- **Isaac Sim 渲染器。** 使用 Motrix 的无头视频导出，或构建一个 viser 场景 - （`unilab.visualization.viser_scene`）。 -- **每个 env 的 tensor obs。** UniLab 给你的是 dict-of-arrays；如果你需要 tensor， - 用你自己的 `obs_to_tensor` 包一层。 -- **内置的 GPU 侧 DR。** UniLab 的 DR 是 CPU 侧、按进程进行的。对大多数任务来说 - 这已经足够；对于极端并行，使用更多的 worker 进程。 +`tests/fixtures/isaac_lab_cartpole/` 迁移了 Isaac Lab commit +`b0542fe2d45bf91c4e1d9ef6952b9c709c80b4e8` 的 Manager-Based Cartpole task。它保留 +全部 12 个来源 term 的名称和顺序，同时把 Torch 适配为 NumPy、嵌套 config object 适配 +为 Hydra YAML，并用 fixture-local MJCF 实现 scene/action/reset 边界。这只是 test-only +证据，不是 production task 或 Isaac Lab 全量支持声明。 ## 另请参阅 -- {doc}`2-from_legged_gym` -- {doc}`3-from_rsl_rl` -- {doc}`5-task_config_translation` -- {doc}`6-reward_porting` +- {doc}`Manager-Based API <../../4-developer_guide/1-architecture/6-manager_based_api>` +- {doc}`Env contract <../../4-developer_guide/2-contracts/1-env_contract>` +- {doc}`ADR-0006 ` diff --git a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/2-from_legged_gym.md b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/2-from_legged_gym.md index 93c01c49f..b4e4b775e 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/2-from_legged_gym.md +++ b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/2-from_legged_gym.md @@ -8,12 +8,12 @@ Legged Gym 曾是那套 GPU 常驻的 PPO 模板，教会了整个领域如何 | Legged Gym | UniLab | |---|---| -| `LeggedRobot` env 类 | `unilab.envs.locomotion.common.base` | +| `LeggedRobot` env 类 | `unilab.tasks.locomotion.common.base` | | `compute_observations()` | env 侧 obs 构建器 + `unilab.base.observations` | | `_reward_*` 方法 | env 的 `compute_reward()` + reward 项 registry | | `command_ranges` | 任务 owner YAML 的 `commands` 块 | | 地形课程 | {doc}`../../2-user_guide/6-terrain/1-procedural` | -| RSL-RL PPO | `unilab.algos.torch.rsl_rl_ppo` | +| RSL-RL PPO | `unilab.algos.rsl_rl_ppo` | ## 有哪些新东西 @@ -21,7 +21,7 @@ Legged Gym 曾是那套 GPU 常驻的 PPO 模板，教会了整个领域如何 在移植之前先选一个（或两个都选）；参见 {doc}`../2-sim_to_sim/1-backend_swap`。 - **异步采集。** Legged Gym 在 GPU 上同步采集；UniLab 的 - APPO（`unilab.algos.torch.appo`）把 collector 与 learner 解耦。如果你在意 + APPO（`unilab.algos.appo`）把 collector 与 learner 解耦。如果你在意 wall-clock 时间，在建立起 reward 一致性之后，就移植到 APPO。 - **硬件部署。** Legged Gym → 真实世界部署，是各实验室各自手工搭建的流程。UniLab 把 {doc}`../1-sim_to_real/1-overview` 流水线作为一等公民产物提供给你。 @@ -29,7 +29,7 @@ Legged Gym 曾是那套 GPU 常驻的 PPO 模板，教会了整个领域如何 ## 迁移清单 1. 把你的 URDF / MJCF asset 复制到 `src/unilab/assets/robots//` 下。 -2. 在 `src/unilab/envs/locomotion//` 下创建一个任务模块。 +2. 在 `src/unilab/tasks/locomotion//` 下创建一个任务模块。 3. 镜像你的 reward 项；保持名称相同，以便 reward 一致性可被 diff。 4. 翻译命令采样 —— Legged Gym 的 `_resample_commands` 在 UniLab 中变成一个 curriculum provider。 diff --git a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/3-from_rsl_rl.md b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/3-from_rsl_rl.md index a00068b26..a011bb933 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/3-from_rsl_rl.md +++ b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/3-from_rsl_rl.md @@ -1,7 +1,7 @@ # 从 RSL-RL 迁移 你已经在独立使用 RSL-RL 了？好消息：UniLab 把 RSL-RL PPO 作为其受支持算法之一 -（`unilab.algos.torch.rsl_rl_ppo`）提供，而且几乎是即插即用的。 +（`unilab.algos.rsl_rl_ppo`）提供，而且几乎是即插即用的。 ## 迁移进 UniLab 后你能获得什么 @@ -10,7 +10,7 @@ 更不容易出错。 2. **任务 owner。** 基于 Hydra 的配置组合，外加 registry 驱动的 backend / task / algo 选择。不再需要为每种机器人编写定制的训练脚本。 -3. **异步 runner。** 把 RSL-RL PPO 包进 `unilab.algos.torch.appo`，在拥有许多 +3. **异步 runner。** 把 RSL-RL PPO 包进 `unilab.algos.appo`，在拥有许多 CPU 核心的机器上获得更高吞吐量。 4. **部署流程。** 配合正确 wrapper 的 ONNX 导出、安全层文档，以及 {doc}`../1-sim_to_real/1-overview` 流水线。 diff --git a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/4-from_skrl.md b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/4-from_skrl.md index 01415f9ba..0eaf6de38 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/4-from_skrl.md +++ b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/4-from_skrl.md @@ -7,7 +7,7 @@ skrl 的强项在于算法广度。UniLab 专注于一组精选算法（PPO、SA | skrl | UniLab | |---|---| -| `Agent`（PPO、SAC……） | `unilab.algos.torch.*` | +| `Agent`（PPO、SAC……） | `unilab.algos.*` | | `RolloutMemory` | `unilab.ipc.rollout_ring_buffer` | | `ReplayMemory` | `unilab.ipc.replay_buffer` | | `Trainer` | `unilab.training.run` | diff --git a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/6-reward_porting.md b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/6-reward_porting.md index a86136ec9..cbc4fe732 100644 --- a/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/6-reward_porting.md +++ b/docs/sphinx/source/zh_CN/3-deployment/3-framework_migration/6-reward_porting.md @@ -37,7 +37,7 @@ def reward_feet_air_time(self, state): 注意： - UniLab 的 `state` 携带了 `prev_contact`，因此你无需自己管理边沿检测。参见 - `unilab.envs.locomotion.common.rewards`。 + `unilab.tasks.locomotion.common.rewards`。 ## 模式：动作平滑惩罚 @@ -46,7 +46,7 @@ def reward_action_rate(self, state): return -np.sum((state.action - state.prev_action) ** 2, axis=1) ``` -它已经是 `unilab.envs.locomotion.common.rewards` 中的现成辅助函数。 +它已经是 `unilab.tasks.locomotion.common.rewards` 中的现成辅助函数。 ## 模式：姿态惩罚 @@ -74,5 +74,5 @@ def reward_termination(self, state): ## 另请参阅 - {doc}`5-task_config_translation` -- `unilab.training.reward` -- `unilab.envs.locomotion.common.rewards` +- `unilab.utils.reward` +- `unilab.tasks.locomotion.common.rewards` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/0-index.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/0-index.md index 531791167..ee5e0fd29 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/0-index.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/0-index.md @@ -36,6 +36,12 @@ Runner 生命周期、worker/learner 拆分与数据流。 Bootstrap 导入与 env/backend 注册。 ::: +:::{grid-item-card} Manager-Based API +:link: 6-manager_based_api +:link-type: doc +社区 manager 语义、NumPy runtime 与 fail-closed 边界。 +::: + :::: ```{toctree} @@ -46,4 +52,5 @@ Bootstrap 导入与 env/backend 注册。 3-layer_boundaries 4-scene_composition 5-registry +6-manager_based_api ``` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/1-overview.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/1-overview.md index 9b1b65420..e9b0fb96a 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/1-overview.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/1-overview.md @@ -73,7 +73,9 @@ off-policy 算法则使用异步 runner、共享缓冲区，以及位于 `src/un - `scripts/train_rsl_rl.py` - `scripts/train_appo.py` -- `scripts/train_offpolicy.py` +- `scripts/train_sac.py` +- `scripts/train_td3.py` +- `scripts/train_flashsac.py` - `src/unilab/base/np_env.py` - `src/unilab/base/backend/base.py` - `src/unilab/base/registry.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/2-runtime_model.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/2-runtime_model.md index ca937096b..349b3431c 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/2-runtime_model.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/2-runtime_model.md @@ -50,8 +50,8 @@ CPU physics env loop -> shared IPC buffer -> learner ## 仓库中的证据 - PPO 入口：`scripts/train_rsl_rl.py` -- APPO runner：`src/unilab/algos/torch/appo/runner.py` -- Off-policy runner：`src/unilab/algos/torch/offpolicy/double_buffer_runner.py` +- APPO runner：`src/unilab/algos/appo/runner.py` +- Off-policy runner：`src/unilab/algos/offpolicy/double_buffer_runner.py` - IPC 原语：`src/unilab/ipc/async_runner.py`、 `src/unilab/ipc/rollout_ring_buffer.py`、`src/unilab/ipc/replay_buffer.py`、 `src/unilab/ipc/weight_sync.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/3-layer_boundaries.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/3-layer_boundaries.md index 826c9a988..e428e30cc 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/3-layer_boundaries.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/3-layer_boundaries.md @@ -34,4 +34,5 @@ - Env 状态契约：`src/unilab/base/np_env.py` - Registry 构造路径：`src/unilab/base/registry.py` - 训练入口：`scripts/train_rsl_rl.py`、 - `scripts/train_appo.py`、`scripts/train_offpolicy.py` + `scripts/train_appo.py`、`scripts/train_sac.py`、 + `scripts/train_td3.py`、`scripts/train_flashsac.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/4-scene_composition.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/4-scene_composition.md index 0b4abbb35..f3e0e662a 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/4-scene_composition.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/4-scene_composition.md @@ -98,7 +98,7 @@ materializer。 当前面向用户的程序化地形路径是 Go2 崎岖地形： -- Env owner：`src/unilab/envs/locomotion/go2/rough.py` +- Task owner：`src/unilab/tasks/locomotion/go2/rough.py` - 地形生成器：`src/unilab/terrains/terrain_generator.py` - MuJoCo materializer：`src/unilab/base/backend/mujoco/xml.py` - Motrix materializer：`src/unilab/base/backend/motrix/scene.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/5-registry.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/5-registry.md index 0df987d96..c96662e06 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/5-registry.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/5-registry.md @@ -8,10 +8,9 @@ Registry bootstrap 是一个针对环境的显式导入契约。它由 1. 训练入口调用 `unilab.training.common.ensure_registries()`。 2. 该 helper 委托给 `unilab.base.registry.ensure_registries()`。 -3. registry 导入已声明的 bootstrap 包： - `unilab.envs.locomotion`、`unilab.envs.manipulation` 与 - `unilab.envs.motion_tracking`。 -4. 每个包都暴露 `__unilab_registry_modules__`，即一个包含注册副作用的模块元组。 +3. registry 导入唯一声明的 bootstrap 包 `unilab.tasks`。 +4. `unilab.tasks` 暴露 `__unilab_registry_modules__`，即一个包含注册副作用的 + task leaf module 显式元组。 5. 被导入的模块通过 `@registry.envcfg(...)` 注册 config，并通过 `@registry.env(..., sim_backend=...)` 或 `registry.register_env(...)` 注册 env 实现。 @@ -20,8 +19,8 @@ Registry bootstrap 是一个针对环境的显式导入契约。它由 ## 扩展规则 -- 如果新的 env 模块位于某个尚未被现有 bootstrap 条目导入的新模块中，需将其加入 - 包级别的 `__unilab_registry_modules__` 元组。 +- 如果新的 task leaf 尚未被现有 bootstrap 条目导入，需将其加入 + `unilab.tasks.__unilab_registry_modules__`。 - 保持注册过程轻量。场景 materialization、XML 处理、资源访问以及 backend 构造 应放在 `registry.make(...)` 之后，而不是放在装饰器注册中。 - 重复的 env config 以及重复的 `(env, sim_backend)` 注册会在 @@ -31,7 +30,5 @@ Registry bootstrap 是一个针对环境的显式导入契约。它由 - Bootstrap helper：`src/unilab/base/registry.py` - 训练 helper：`src/unilab/training/common.py` -- 包声明：`src/unilab/envs/locomotion/__init__.py`、 - `src/unilab/envs/manipulation/__init__.py`、 - `src/unilab/envs/motion_tracking/__init__.py` +- Task bootstrap 声明：`src/unilab/tasks/__init__.py` - 测试：`tests/base/test_registry.py`、`tests/utils/test_algo_utils.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/6-manager_based_api.md b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/6-manager_based_api.md new file mode 100644 index 000000000..dcc9be804 --- /dev/null +++ b/docs/sphinx/source/zh_CN/4-developer_guide/1-architecture/6-manager_based_api.md @@ -0,0 +1,32 @@ +# Manager-Based API + +UniLab 采用“社区兼容 API + UniLab NumPy runtime”：manager-facing 模块、term cfg、 +function/class term、生命周期和顺序语义以固定的 mjlab 1.6.0 source 为基线；数值实现使用 +NumPy，并保留现有 `NpEnvState`、Hydra owner YAML、`SimBackend`、registry 与 IPC contract。 + +完整决策、兼容矩阵和机械迁移示例见 +{doc}`/adr/ADR-0006-community-manager-api-on-numpy-runtime`。 + +## 不变量 + +- 公共 manager 结构优先保持社区语义；不为局部性能制造 UniLab-only term API。 +- Production task 只从 Hydra owner YAML 配置：YAML 完整声明 manager/group/term、具体 cfg + `_target_`、dotted callable、params、weight 与 observation mapping；Registry 冷路径将其 + 物化为 plain typed cfg，Python 不保留 task config mirror。 +- 未知字段、无法解析的 target/callable、抽象或错误 cfg 类型直接报错；DictConfig 和解析 + 不进入 reset/step，scripts 不解释 task 业务规则。 +- manager buffer、term return、env ID 和 entity view 使用 `np.ndarray` / `slice`，core 不依赖 + Torch、Warp、runner、learner 或 IPC。 +- `SceneEntityCfg` 在冷路径通过 base scene/entity facade 解析；facade 只调用正式 + `SimBackend` contract，热路径复用缓存 ID/view。 +- named-sensor observation term 在构造时通过 `EntityScene.bind_sensor_data(...)` 绑定 + backend-owned view；热路径只读该 view，不重复解析 sensor 名称或 XML/model metadata。 +- `ManagerBasedRlEnv` 恰好拥有一次 backend 物化：先完成 manager 构造和 startup event， + 再调用 `SimBackend.materialize()`，任何 reset/step 都不能在物化前执行。 +- 用户显式空配置可以使用 Null manager；配置请求但 runtime/backend 不支持的能力必须在 + 最近边界报错，不能 warning、skip、返回零或回退旧 env。 +- 热路径避免明显的重复解析、逐环境 Python 循环、复制和临时分配；进一步优化需要 + benchmark 证明收益，且不能增加不成比例的结构复杂度。 + +只有被注册、配置和测试覆盖的表面才能声明 Compatible。NumPy/env/config adapter 标为 +Adapted；缺少正式 backend contract 的能力标为 Unsupported 并 fail-closed。 diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/3-task_owner.md b/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/3-task_owner.md index 6de0c852b..efecc177d 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/3-task_owner.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/3-task_owner.md @@ -7,8 +7,8 @@ - PPO 与 APPO 的 owner YAML 使用 `conf/{ppo,appo}/task//.yaml`。 -- Off-policy owner YAML 多出算法这一维度： - `conf/offpolicy/task///.yaml`。 +- Off-policy 算法（SAC / TD3 / FlashSAC）各自有独立的配置树： + `conf//task//.yaml`。 - 其他已有的 config 根目录，例如 `conf/ppo_him/` 与 `conf/hora_distill/`，对其 所支持的任务遵循相同的 owner YAML 身份规则。 @@ -17,8 +17,8 @@ - 使用对外的 CLI flag 切换 backend，例如 `uv run train --algo ppo --task go2_joystick_flat --sim mujoco` 或 `uv run train --algo ppo --task go2_joystick_flat --sim motrix`。 -- 对于 off-policy 入口，保持 `--algo ` 与内部 owner YAML 路径 - `conf/offpolicy/task///.yaml` 对齐。 +- 对于 off-policy 入口，`--algo ` 选择按算法划分的配置树；owner YAML + 路径为 `conf//task//.yaml`。 - `training.sim_backend` 是所选 owner YAML 内部的身份字段，而不是一个独立的 backend 切换开关。 - 与 backend 相关的 reward、env、scene 与算法差异属于 owner YAML，而不是训练 @@ -29,8 +29,7 @@ - PPO owner 示例：`conf/ppo/task/go2_joystick_flat/mujoco.yaml` - APPO config 根目录：`conf/appo/config.yaml` -- Off-policy config 根目录：`conf/offpolicy/config.yaml` -- Off-policy task/algo guard：`src/unilab/training/common.py` +- Off-policy config 根目录：`conf/{sac,td3,flashsac}/config.yaml` - Config 测试：`tests/config/test_config_system.py`、 `tests/scripts/test_train_script_configs.py`、 `tests/envs/locomotion/g1/test_g1_owner_contract.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/4-dr_contract.md b/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/4-dr_contract.md index cdb493364..1fc3f0f03 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/4-dr_contract.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/4-dr_contract.md @@ -90,6 +90,6 @@ actuator 的机制泄漏到共享 payload 里。 - DR 类型：`src/unilab/dr/types.py` - DR manager：`src/unilab/dr/manager.py` - Backend 接口：`src/unilab/base/backend/base.py` -- 示例 provider：`src/unilab/envs/locomotion/g1/joystick.py`、 - `src/unilab/envs/motion_tracking/g1/tracking.py`、 - `src/unilab/envs/manipulation/sharpa_inhand/rotation.py` +- 示例 provider：`src/unilab/tasks/locomotion/common/dr_provider.py`、 + `src/unilab/tasks/locomotion/go2_arm/manip_loco.py`、 + `src/unilab/tasks/manipulation/sharpa_inhand/rotation.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/5-runner_lifecycle.md b/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/5-runner_lifecycle.md index 24e1d94d7..06cfec2f8 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/5-runner_lifecycle.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/2-contracts/5-runner_lifecycle.md @@ -19,7 +19,8 @@ runner；它们不应另起第二套 collector/learner 协议。 `OnPolicyRunner`。 - `scripts/train_appo.py` 使用 `APPORunner`、`RolloutRingBuffer` 与 `SharedWeightSync`。 -- `scripts/train_offpolicy.py` 使用 off-policy runner，配合 `ReplayBuffer` 与 +- `scripts/train_sac.py`、`scripts/train_td3.py` 与 `scripts/train_flashsac.py` + 使用 off-policy runner，配合 `ReplayBuffer` 与 `SharedWeightSync`。 - `AsyncRunner` 为异步 runner 拥有 collector 进程生命周期与共享资源清理。 diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/1-new_task.md b/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/1-new_task.md index a60aeea32..d975fbd49 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/1-new_task.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/1-new_task.md @@ -20,8 +20,8 @@ `reset(env_indices)` 返回 `(obs_dict, info_dict)`，而 `step(actions)` 返回 `NpEnvState`。 7. 在相关 config 根目录下添加 owner YAML，例如 - `conf/ppo/task//.yaml` 或 - `conf/offpolicy/task///.yaml`。 + `conf/ppo/task//.yaml`，off-policy 算法则为 + `conf//task//.yaml`。 8. 把任务或场景的 keyframe 放进通过 `SceneCfg.fragment_files` 引用的 任务/场景 XML fragment；不要把 task-level keyframe 放进 `robot.xml`。 @@ -38,5 +38,5 @@ - Registry API：`src/unilab/base/registry.py` - Env 状态契约：`src/unilab/base/np_env.py` - 场景配置：`src/unilab/base/scene.py` -- 现有任务示例：`src/unilab/envs/locomotion/go2/joystick.py`、 - `src/unilab/envs/manipulation/allegro_inhand/rotation.py` +- 现有任务示例：`src/unilab/tasks/locomotion/go2/joystick.py`、 + `src/unilab/tasks/manipulation/allegro_inhand/rotation.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/3-new_algorithm.md b/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/3-new_algorithm.md index 7e90fc4b9..f31104623 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/3-new_algorithm.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/3-new_algorithm.md @@ -8,15 +8,15 @@ - 同步 on-policy 示例：`scripts/train_rsl_rl.py`。 - 异步 on-policy 示例：使用 `APPORunner` 的 `scripts/train_appo.py`。 -- Off-policy 示例：`scripts/train_offpolicy.py`，配合 `conf/offpolicy/` - 下的 SAC、TD3 与 FlashSAC 配置。 +- Off-policy 示例：`scripts/train_sac.py`、`scripts/train_td3.py` 与 + `scripts/train_flashsac.py`，各自在 `conf//` 下拥有独立的配置树。 ## 实现清单 1. 把可复用的 learner 或 runner 代码放在 `src/unilab/algos/` 下。 2. 在归属的 config 根目录下添加 Hydra config。一个新的 off-policy 变体 - 应当添加 `conf/offpolicy/algo/.yaml` 以及对应的 - `conf/offpolicy/task///.yaml` owner YAML。 + 应当建立自己的配置树：算法超参数内联在 `conf//config.yaml` 中， + 并添加对应的 `conf//task//.yaml` owner YAML。 3. 如果需要新的顶层训练脚本，请让它保持为组装层： compose Hydra、调用 `ensure_registries()`、通过 registry 路径构造 env， 然后把控制权交给 runner 或 trainer。 @@ -25,9 +25,8 @@ 5. 对于异步算法，复用 `AsyncRunner`、`ReplayBuffer` 或 `RolloutRingBuffer` 以及 `SharedWeightSync`，而不是新建一套 IPC 生命周期。 -6. 对于 off-policy 算法，保持 CLI 的 `--algo ` 选择与 owner YAML - 路径 `conf/offpolicy/task///.yaml` 对齐； - `assert_offpolicy_task_choice_matches_algo` 会强制这一约束。 +6. 对于 off-policy 算法，CLI 的 `--algo ` 选择映射到按算法划分的 + 配置树；owner YAML 位于 `conf//task//.yaml`。 ## 在风险点附近验证 @@ -41,4 +40,4 @@ - 结构化 config dataclass：`src/unilab/structured_configs.py` - 训练辅助工具：`src/unilab/training/common.py`、 `src/unilab/training/run.py` -- 现有算法包：`src/unilab/algos/torch/` +- 现有算法包：`src/unilab/algos/` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/4-new_terrain.md b/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/4-new_terrain.md index e330cd438..619a588b8 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/4-new_terrain.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/3-extending/4-new_terrain.md @@ -35,4 +35,4 @@ - 地形配置与 preset：`src/unilab/terrains/config.py` - 地形生成器：`src/unilab/terrains/terrain_generator.py` - Heightfield 地形类型：`src/unilab/terrains/heightfield_terrains.py` -- 高度扫描辅助工具：`src/unilab/envs/locomotion/common/height_scan.py` +- 高度扫描辅助工具：`src/unilab/tasks/locomotion/common/height_scan.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/6-agent_quick_reference.md b/docs/sphinx/source/zh_CN/4-developer_guide/6-agent_quick_reference.md index bf17e4e87..0eaa33671 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/6-agent_quick_reference.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/6-agent_quick_reference.md @@ -10,7 +10,8 @@ - 算法索引：{doc}`../2-user_guide/2-algorithms/0-index` - PPO 入口：`scripts/train_rsl_rl.py` - APPO 入口：`scripts/train_appo.py` -- SAC / TD3 / FlashSAC 入口：`scripts/train_offpolicy.py` +- SAC / TD3 / FlashSAC 入口：`scripts/train_sac.py` / + `scripts/train_td3.py` / `scripts/train_flashsac.py` - HIM-PPO 入口：`scripts/train_him_ppo.py` - HORA 蒸馏入口：`scripts/train_hora_distill.py` diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/7-motion_assets.md b/docs/sphinx/source/zh_CN/4-developer_guide/7-motion_assets.md index b8df1f175..e59cff65c 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/7-motion_assets.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/7-motion_assets.md @@ -75,38 +75,51 @@ env: 3. 在 env config 中引用新文件路径即可。 -## 机器人网格资产 +## 机器人二进制资产 -机器人二进制网格（`.STL`）采用相同方式外置，托管在 Hugging Face 数据集仓库 +机器人二进制网格和纹理（例如 `.STL`、`.obj`、`.png`）采用相同方式外置， +托管在 Hugging Face 数据集仓库 [unilabsim/unilab-robots](https://huggingface.co/datasets/unilabsim/unilab-robots)。 X2 网格在首次使用时按需下载，落盘到原始路径 `src/unilab/assets/robots/x2/meshes/`， -因此 XML 的 `meshdir` 引用保持有效。无需运行任务即可提前预拉取： +T800 的 OBJ 和纹理分别落盘到 `robots/t800/assets/` 和 `robots/t800/textures/`， +因此 XML 中的原始相对路径保持有效。无需运行任务即可提前预拉取： ```bash uv run unilab-pull-assets --robot x2 +uv run unilab-pull-assets --robot t800 ``` -新增某个机器人的网格： +新增某个机器人的二进制资产： -1. 上传到 HF 仓库，保持目录结构一致： +1. 按目录上传到 HF 仓库，保持目录结构一致。一个机器人有多个资产目录时， + 每个目录分别上传。例如 T800： ```bash - huggingface-cli upload unilabsim/unilab-robots \ - src/unilab/assets/robots//meshes robots//meshes \ + uv run hf upload unilabsim/unilab-robots \ + src/unilab/assets/robots/t800/assets robots/t800/assets \ + --repo-type dataset + uv run hf upload unilabsim/unilab-robots \ + src/unilab/assets/robots/t800/textures robots/t800/textures \ --repo-type dataset ``` -2. 在 `.gitignore` 中忽略本地 `*.STL`（保留 `.gitkeep` 以维持目录）。 -3. 在 env 的冷路径上调用一次目录 resolver，例如 - `resolve_robot_asset_dir("robots//meshes", marker="<某>.STL")`。 +2. 在 `.gitignore` 中忽略下载目录内容，并保留 `.gitkeep` 以维持目录。 +3. 在 backend 解析 XML 之前的冷路径解析每个引用目录。现有 API 一次只解析 + 一个目录，因此 T800 task 需要调用两次： + + ```python + resolve_robot_asset_dir("robots/t800/assets", marker="LINK_BASE.obj") + resolve_robot_asset_dir("robots/t800/textures", marker="LINK_BASE.png") + ``` ## 架构说明 - 资产解析模块：`src/unilab/assets/hub.py`（`resolve_motion_files`）。 -- 唯一集成点：`src/unilab/envs/motion_tracking/g1/motion_loader.py` 中的 +- 唯一集成点：`src/unilab/tasks/motion_tracking/common/motion_loader.py` 中的 `MotionLoader.__init__`，在冷路径上调用一次 resolver。 - 热路径（`step` / `reset`）**不会**触发任何文件下载或解析。 - `ASSETS_ROOT_PATH` 定义不变，下载落盘位置与原始本地路径完全一致。 -- 机器人网格使用同一目录 resolver（`resolve_robot_asset_dir`），集成点为 - `src/unilab/envs/motion_tracking/x2/flip_tracking.py` 中的 - `X2WallFlipTrackingEnv.__init__`，并通过 `unilab-pull-assets` CLI 暴露。 +- 机器人二进制资产使用同一目录 resolver（`resolve_robot_asset_dir`）。 + `src/unilab/tasks/motion_tracking/x2/__init__.py` 中的薄 + `make_x2_wall_flip_env` factory 会先在冷路径解析一次，再委托给共享 manager env + factory；同一 resolver 也通过 `unilab-pull-assets` CLI 暴露。 diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/8-motrix_contact_sensor.md b/docs/sphinx/source/zh_CN/4-developer_guide/8-motrix_contact_sensor.md index 2746b8918..656914a93 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/8-motrix_contact_sensor.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/8-motrix_contact_sensor.md @@ -80,7 +80,7 @@ shape = (num_envs, 1 + 4 * 12) = (num_envs, 49) ## 单一 norm 分支为何不够 -env 通过 `src/unilab/envs/manipulation/sharpa_inhand/base.py` 中的 +env 通过 `src/unilab/tasks/manipulation/sharpa_inhand/base.py` 中的 `_read_tactile_force()` → `_extract_sensor_scalar()` 读取触觉力。该 helper 目前对任意 `(N, >=3)` 数组都用 `np.linalg.norm(data[:, :3], axis=1)` 折叠。 如果 env 仍把两种后端形状都走这一个分支，MuJoCo 的 `(N, 3)` 是正确的（对真实力向量取 norm），但 Motrix 的 `(N, 4)` 会出错：`data[:, :3]` 取到的是 `[count, fx, fy]`——把接触点数当成了力分量，并且漏掉了 `fz`。正确做法不是在 env 里按形状特判，而是把每个后端的布局知识下沉到 backend 方法。 @@ -104,8 +104,8 @@ env 层的 `_read_tactile_force()` 对 contact sensor 走 `get_contact_force_mag | 文件 | 说明 | | --- | --- | -| `src/unilab/envs/manipulation/sharpa_inhand/base.py` | `_extract_sensor_scalar()`, `_read_tactile_force()` | -| `src/unilab/envs/manipulation/sharpa_inhand/rotation.py` | reward 计算，virtual torque | +| `src/unilab/tasks/manipulation/sharpa_inhand/base.py` | `_extract_sensor_scalar()`, `_read_tactile_force()` | +| `src/unilab/tasks/manipulation/sharpa_inhand/rotation.py` | reward 计算，virtual torque | | `src/unilab/assets/robots/sharpa_wave/right_sharpa_wave.xml` | contact sensor XML 定义 | | `src/unilab/base/backend/motrix/backend.py` | Motrix `get_sensor_data()` | | `src/unilab/base/backend/mujoco/backend.py` | MuJoCo `get_sensor_data()` | diff --git a/docs/sphinx/source/zh_CN/4-developer_guide/9-sim2sim_contract_status.md b/docs/sphinx/source/zh_CN/4-developer_guide/9-sim2sim_contract_status.md index 18459b02d..8c10efba9 100644 --- a/docs/sphinx/source/zh_CN/4-developer_guide/9-sim2sim_contract_status.md +++ b/docs/sphinx/source/zh_CN/4-developer_guide/9-sim2sim_contract_status.md @@ -28,12 +28,12 @@ uv run scripts/audit_sim2sim_contracts.py | Task | 判定 | 分歧 | |---|---|---| -| allegro_inhand · allegro_inhand_grasp · g1_climb_tracking · g1_motion_tracking · g1_wall_flip_tracking · go1_joystick_rough · go2_arm_manip_loco · go2_handstand · go2_joystick_flat · go2_joystick_rough · go2w_joystick_flat · go2w_joystick_rough · sharpa_inhand · sharpa_inhand_grasp | ✅ | 无 | +| allegro_inhand · allegro_inhand_grasp · g1_climb_tracking · g1_motion_tracking · g1_wall_flip_tracking · go1_joystick_rough · go2_arm_manip_loco · go2_footstand · go2_handstand · go2_joystick_flat · go2_joystick_rough · go2w_joystick_flat · go2w_joystick_rough · sharpa_inhand · sharpa_inhand_grasp | ✅ | 无 | | g1_box_tracking | ❌ | `empirical_normalization` false↔true；`obs_groups` critic 组差异 | | g1_flip_tracking | ❌ | `empirical_normalization` true↔false；`obs_groups`；`action_scale` 29 维↔默认 0.25；`sampling_mode` 两后端运行时同为 `start`（无害） | -| g1_walk_flat | ❌ | `action_scale` 0.25↔0.5；`empirical_normalization` false↔true；`obs_groups` | +| g1_walk_flat | ❌ | `env.actions.joint_pos.scale` 0.25↔0.5；`empirical_normalization` false↔true；`obs_groups` | | go1_joystick_flat | ❌ | `empirical_normalization` false↔true | -| g1_motion_tracking_deploy · go2_footstand | ⚪ | 仅 mujoco | +| g1_motion_tracking_deploy | ⚪ | 仅 mujoco | ## `conf/appo/task/` @@ -46,7 +46,8 @@ uv run scripts/audit_sim2sim_contracts.py ## 其它配置树 -`conf/ppo_him/task`、`conf/offpolicy/task`、`conf/hora_distill/task` 均无 mujoco↔motrix +`conf/ppo_him/task`、`conf/sac/task`、`conf/td3/task`、`conf/flashsac/task`、 +`conf/hora_distill/task` 均无 mujoco↔motrix 配对，sim2sim 不适用。 ## 字段语义速查 @@ -69,6 +70,7 @@ uv run scripts/audit_sim2sim_contracts.py | `action_scale` | **不可** | 改值即改训练动力学，必须 owner 决策 + 重训 | | `empirical_normalization` | **不可** | 改变网络结构，必须重训 | -试点示例：`conf/ppo/task/g1_walk_flat/{mujoco,motrix}.yaml`。每个后端 owner 自包含完整契约， -`motrix.yaml` 为单后端调参 override 了若干契约字段——这种 override +试点示例：`conf/ppo/task/g1_walk_flat/{base,mujoco,motrix}.yaml`。后端 owner 通过 Hydra +defaults 继承共享 base owner 的完整契约，`motrix.yaml` 为单后端调参 override +了若干契约字段——这种 override 即令该 task 在该后端不可 sim2sim 迁移，去掉 override 即可恢复。 diff --git a/docs/sphinx/source/zh_CN/5-reference/5-support_matrix.md b/docs/sphinx/source/zh_CN/5-reference/5-support_matrix.md index 10c7c6e0a..c9e2c0a15 100644 --- a/docs/sphinx/source/zh_CN/5-reference/5-support_matrix.md +++ b/docs/sphinx/source/zh_CN/5-reference/5-support_matrix.md @@ -38,14 +38,14 @@ uv run scripts/generate_support_matrix.py --write | 等级 | 仓库事实来源 | |------|--------------| | `Registered` | `ensure_registries()` 导入后的 `registry.list_registered_envs()` 中存在该 env/backend。 | -| `Configured` | 存在对应的 owner YAML：`conf/{ppo,appo,offpolicy}/task/...`。 | +| `Configured` | 存在对应的 owner YAML：`conf/{ppo,appo,sac,td3,flashsac}/task/...`。 | | `Tested` | `tests/` 中有自动化覆盖该 entrypoint/task owner/backend 组合，或存在显式 maintainer 完整训练验证并具备近风险自动化测试。这里的 `Tested` 不等同于默认推荐路径。 | | `Benchmarked` | 存在与该组合绑定的已提交 benchmark manifest。 | | `Recommended` | 仓库中存在显式 recommendation 元数据。 | `Tested` 只描述仓库中已有自动化覆盖或显式 maintainer 训练验证，不代表该组合具备同名 MuJoCo owner 的全部 backend capability；例如 phase-1 Motrix owner 可能只覆盖训练 smoke 和明确启用的 DR 子集。 -`mjwarp` 只支持 `g1_walk_flat` host adapter。PPO (torch) 与 SAC (torch) owner 已完成训练验证，并有 backend、contract 与 playback 自动化覆盖，因此标记为 `Tested`。mjwarp playback 仅支持显式、有限步数的 `record` 并复用 MuJoCo 离线 renderer，不支持 `auto`、interactive 或 native playback。其他 entrypoint 中出现的 `Registered` 只表示 env/backend registry identity，不代表对应算法、terrain、完整 DR 或 production training 支持。 +`mjwarp` 完成训练验证的只有 `g1_walk_flat` host adapter：PPO (torch) 与 SAC (torch) owner 已完成训练验证，并有 backend、contract 与 playback 自动化覆盖，因此标记为 `Tested`。SAC `t800_walk_flat` 的 mjwarp owner 只有 owner YAML 与 compose 覆盖，标记为 `Configured`，不代表训练验证。mjwarp playback 仅支持显式、有限步数的 `record` 并复用 MuJoCo 离线 renderer，不支持 `auto`、interactive 或 native playback。其他 entrypoint 中出现的 `Registered` 只表示 env/backend registry identity，不代表对应算法、terrain、完整 DR 或 production training 支持。 未检测到与这些组合绑定的已提交 benchmark manifest，因此当前不会自动提升到 `Benchmarked`。 仓库中目前也没有单独的 recommendation 元数据，因此当前不会自动提升到 `Recommended`。 @@ -84,6 +84,7 @@ uv run scripts/generate_support_matrix.py --write | PPO (torch) | `go2w_joystick_flat` (go2w joystick flat) | Tested | - | Tested | | PPO (torch) | `go2w_joystick_rough` (go2w joystick rough) | Tested | - | Tested | | PPO (torch) | `stewart_balance` (stewart balance) | Tested | - | Tested | +| PPO (torch) | `t800_walk_flat` (t800 walk flat) | Tested | Registered | - | | APPO (torch) | `go1_joystick_flat` (Go1 joystick) | Tested | - | Tested | | APPO (torch) | `go2_joystick_flat` (Go2 joystick) | Tested | - | Tested | | APPO (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Registered | Registered | @@ -100,7 +101,7 @@ uv run scripts/generate_support_matrix.py --write | APPO (torch) | `g1_climb_tracking` (g1 climb tracking) | Tested | - | Tested | | SAC (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Tested | Tested | | SAC (torch) | `g1_walk_rough` (G1 walk rough) | Tested | - | Tested | -| SAC (torch) | `g1_motion_tracking` (G1 motion tracking) | Tested | - | Tested | +| SAC (torch) | `g1_motion_tracking` (G1 motion tracking) | Tested | Configured | Tested | | SAC (torch) | `g1_flip_tracking` (G1 flip tracking) | Tested | - | Registered | | SAC (torch) | `g1_wall_flip_tracking` (G1 wall flip tracking) | Tested | - | Registered | | SAC (torch) | `g1_23dof_flip_tracking` (g1 23dof flip tracking) | Tested | - | Registered | @@ -110,6 +111,7 @@ uv run scripts/generate_support_matrix.py --write | SAC (torch) | `g1_23dof_wall_flip_tracking` (g1 23dof wall flip tracking) | Tested | - | Registered | | SAC (torch) | `g1_23dof_wbt_obs` (g1 23dof wbt obs) | Tested | - | Registered | | SAC (torch) | `g1_wbt_obs` (g1 wbt obs) | Tested | - | Registered | +| SAC (torch) | `t800_walk_flat` (t800 walk flat) | Tested | Configured | - | | TD3 (torch) | `go1_joystick_flat` (Go1 joystick) | Registered | - | Tested | | TD3 (torch) | `go2_joystick_flat` (Go2 joystick) | Registered | - | Tested | | TD3 (torch) | `g1_walk_flat` (G1 walk flat) | Tested | Registered | Registered | @@ -121,7 +123,7 @@ uv run scripts/generate_support_matrix.py --write ### Source Index - Registry bootstrap: `src/unilab/envs/**` decorators via `unilab.base.registry.ensure_registries()`. -- Owner YAML scan: `conf/ppo/task/**`, `conf/appo/task/**`, `conf/offpolicy/task/**`. +- Owner YAML scan: `conf/ppo/task/**`, `conf/appo/task/**`, `conf/sac/task/**`, `conf/td3/task/**`, `conf/flashsac/task/**`. - Generic compose coverage: `tests/config/test_config_system.py::test_supported_task_composes`. - Validated mjwarp entrypoints are explicitly recorded in `_MAINTAINER_VALIDATED_MJWARP_ENTRYPOINT_TASKS`; near-risk coverage lives in `tests/base/test_mjwarp_backend.py`, `tests/base/test_backend_conformance.py`, `tests/base/test_mjwarp_differential.py`, and `tests/base/test_mjwarp_playback.py`. diff --git a/pyproject.rocm.toml b/pyproject.rocm.toml index 5108794fd..eaee39c6e 100644 --- a/pyproject.rocm.toml +++ b/pyproject.rocm.toml @@ -40,19 +40,18 @@ dependencies = [ train = "unilab.cli:train_main" eval = "unilab.cli:eval_main" demo = "unilab.cli:demo_main" -unilab-complete = "unilab.tools.completion:main" -unilab-viz-nan = "unilab.tools.viz_nan:main" -unilab-export-scene = "unilab.tools.export_scene:main" -unilab-render-teaser = "unilab.tools.render_teaser:main" -unilab-import-robot = "unilab.tools.import_robot:main" -unilab-pull-assets = "unilab.tools.pull_assets:main" +unilab-complete = "unilab.cli_completion:main" +unilab-viz-nan = "unilab.utils.nan_viz:main" +unilab-export-scene = "unilab.base.backend.mujoco.export_scene:main" +unilab-render-teaser = "unilab.visualization.teaser:main" +unilab-pull-assets = "unilab.assets.pull:main" [project.optional-dependencies] mujoco = [ "mujoco>=3.5", # Bound stays loose; uv.rocm.lock pins the default solver version. The # tested window is >=3.5,<3.11 — switch via `make mujoco MJ=`. - "mujoco-uni-runtime==0.3.1", + "mujoco-uni-runtime==0.4.0", # pybind11/wheel are build requirements of mujoco-uni-runtime, which is # compiled in this environment (see no-build-isolation-package below). "pybind11>=2.12", @@ -128,7 +127,7 @@ warn_unused_configs = true ignore_missing_imports = true no_site_packages = true exclude = [ - "src/unilab/algos/torch/rsl_rl/", + "src/unilab/algos/rsl_rl/", ] [tool.pytest.ini_options] @@ -151,12 +150,12 @@ venvPath = "." venv = ".venv" include = ["src/unilab"] exclude = [ - "src/unilab/algos/torch/rsl_rl/", - "src/unilab/algos/torch/common/ane_*", + "src/unilab/algos/rsl_rl/", + "src/unilab/algos/common/ane_*", "src/unilab/base/backend/", "src/unilab/envs/", "src/unilab/terrains/", - "src/unilab/training/monitoring.py", + "src/unilab/utils/monitoring.py", "src/unilab/visualization/", ] reportMissingImports = "warning" diff --git a/pyproject.toml b/pyproject.toml index c9374d0fe..6ef219feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ license-files = ["LICENSE"] requires-python = ">=3.10,<3.14" dependencies = [ "numpy", + "numba>=0.67", + "prettytable>=3.10", "torch==2.9.0 ; sys_platform == 'linux' and platform_machine == 'aarch64'", "torch==2.7.0 ; sys_platform != 'linux' or platform_machine != 'aarch64'", "gymnasium", @@ -40,12 +42,11 @@ dependencies = [ train = "unilab.cli:train_main" eval = "unilab.cli:eval_main" demo = "unilab.cli:demo_main" -unilab-complete = "unilab.tools.completion:main" -unilab-viz-nan = "unilab.tools.viz_nan:main" -unilab-export-scene = "unilab.tools.export_scene:main" -unilab-render-teaser = "unilab.tools.render_teaser:main" -unilab-import-robot = "unilab.tools.import_robot:main" -unilab-pull-assets = "unilab.tools.pull_assets:main" +unilab-complete = "unilab.cli_completion:main" +unilab-viz-nan = "unilab.utils.nan_viz:main" +unilab-export-scene = "unilab.base.backend.mujoco.export_scene:main" +unilab-render-teaser = "unilab.visualization.teaser:main" +unilab-pull-assets = "unilab.assets.pull:main" [project.optional-dependencies] mujoco = [ @@ -53,7 +54,7 @@ mujoco = [ # (uv prefer-locked semantics). The tested window is >=3.5,<3.11 — switch # within it via `make mujoco MJ=`. "mujoco>=3.5", - "mujoco-uni-runtime==0.3.1", + "mujoco-uni-runtime==0.4.0", # pybind11/wheel are build requirements of mujoco-uni-runtime, which is # compiled in this environment (see no-build-isolation-package below). "pybind11>=2.12", @@ -139,7 +140,7 @@ warn_unused_configs = true ignore_missing_imports = true no_site_packages = true exclude = [ - "src/unilab/algos/torch/rsl_rl/", # vendored third-party, tracked in .gitignore + "src/unilab/algos/rsl_rl/", # vendored third-party, tracked in .gitignore ] [tool.pytest.ini_options] @@ -162,11 +163,11 @@ venvPath = "." venv = ".venv" include = ["src/unilab"] exclude = [ - "src/unilab/algos/torch/rsl_rl/", # vendored third-party + "src/unilab/algos/rsl_rl/", # vendored third-party "src/unilab/base/backend/", # mujoco-uni-runtime stubs mismatch; optional backends "src/unilab/envs/", # mujoco-uni-runtime internal API, stubs mismatch "src/unilab/terrains/", # mujoco-uni-runtime MjSpec API, stubs mismatch - "src/unilab/training/monitoring.py", # optional pynvml/psutil deps + "src/unilab/utils/monitoring.py", # optional pynvml/psutil deps "src/unilab/visualization/", # direct mujoco C bindings + optional viser deps ] reportMissingImports = "warning" diff --git a/scripts/audit_sim2sim_contracts.py b/scripts/audit_sim2sim_contracts.py index 012222af4..070869367 100644 --- a/scripts/audit_sim2sim_contracts.py +++ b/scripts/audit_sim2sim_contracts.py @@ -1,14 +1,14 @@ """Audit cross-backend sim2sim contract divergences across task owner YAMLs. For every task with >=2 backend YAMLs, hydra-composes each backend's effective config -and compares the DENYLIST / WARNING_LIST fields from ``unilab.training.sim2sim``. -Off-policy owners are grouped by algorithm so SAC, TD3, and FlashSAC are never -compared with one another. +and compares the DENYLIST / WARNING_LIST fields from ``unilab.utils.sim2sim``. +Off-policy owners now live in separate per-algorithm trees (``sac``, ``td3``, +``flashsac``), so SAC, TD3, and FlashSAC are never compared with one another. Read-only. uv run scripts/audit_sim2sim_contracts.py - uv run scripts/audit_sim2sim_contracts.py --trees ppo appo offpolicy + uv run scripts/audit_sim2sim_contracts.py --trees ppo appo sac td3 flashsac uv run scripts/audit_sim2sim_contracts.py --json """ @@ -23,7 +23,7 @@ from hydra.core.global_hydra import GlobalHydra from omegaconf import OmegaConf -from unilab.training.sim2sim import DENYLIST, ENV_STRUCTURAL_DENYLIST, WARNING_LIST, _normalize +from unilab.utils.sim2sim import DENYLIST, ENV_STRUCTURAL_DENYLIST, WARNING_LIST, _normalize REPO_ROOT = Path(__file__).resolve().parents[1] CONF_ROOT = REPO_ROOT / "conf" @@ -52,16 +52,7 @@ def _select(cfg: Any, path: str) -> Any: def _compose(tree: str, task_variant: str) -> Any: conf_dir = str(CONF_ROOT / tree) - overrides: list[str] = [] - if tree == "offpolicy": - variant_parts = task_variant.split("/") - if len(variant_parts) != 3: - raise ValueError( - f"offpolicy task variants must use '//', got {task_variant!r}" - ) - algo = variant_parts[0] - overrides.append(f"algo={algo}") - overrides.append(f"task={task_variant}") + overrides: list[str] = [f"task={task_variant}"] GlobalHydra.instance().clear() with initialize_config_dir(config_dir=conf_dir, version_base="1.3"): return compose("config", overrides=overrides) diff --git a/scripts/benchmark/benchmark_drake_performance.py b/scripts/benchmark/benchmark_drake_performance.py index 2886bebf6..5b7fcb059 100644 --- a/scripts/benchmark/benchmark_drake_performance.py +++ b/scripts/benchmark/benchmark_drake_performance.py @@ -1,10 +1,5 @@ #!/usr/bin/env python3 -"""Profile Drake vs MuJoCo env-step performance on selected UniLab tasks. - -This benchmark is intentionally task-level rather than raw-simulator-level. It -keeps UniLab's reset, observation, sensor-view, and body-query paths in the -loop so G1 motion tracking can expose the expensive integration points. -""" +"""Profile Drake vs MuJoCo env-step performance on selected UniLab tasks.""" from __future__ import annotations @@ -44,7 +39,7 @@ def _install_import_paths(drakeuni_src: Path | None) -> None: @dataclass(frozen=True) class TaskSpec: env_cfg_factory: Callable[[], Any] - env_cls_factory: Callable[[], type] + env_cls_factory: Callable[[], Callable[..., Any]] @dataclass @@ -101,39 +96,23 @@ def wrapped(*args: Any, **kwargs: Any) -> Any: def _task_specs() -> dict[str, TaskSpec]: def go1_cfg() -> Any: - from unilab.envs.locomotion.go1.joystick import Go1JoystickCfg + from unilab.envs import ManagerBasedRlEnvCfg - return Go1JoystickCfg() + return ManagerBasedRlEnvCfg() - def go1_env() -> type: - from unilab.envs.locomotion.go1.joystick import Go1WalkTask + def manager_env() -> Callable[..., Any]: + from unilab.envs import make_manager_based_rl_env - return Go1WalkTask + return make_manager_based_rl_env def go2_cfg() -> Any: - from unilab.envs.locomotion.go2.joystick import Go2JoystickCfg - - return Go2JoystickCfg() - - def go2_env() -> type: - from unilab.envs.locomotion.go2.joystick import Go2WalkTask - - return Go2WalkTask - - def g1_tracking_cfg() -> Any: - from unilab.envs.motion_tracking.g1.tracking import G1MotionTrackingEnvCfg - - return G1MotionTrackingEnvCfg() - - def g1_tracking_env() -> type: - from unilab.envs.motion_tracking.g1.tracking import G1MotionTrackingEnv + from unilab.envs import ManagerBasedRlEnvCfg - return G1MotionTrackingEnv + return ManagerBasedRlEnvCfg() return { - "g1_motion_tracking": TaskSpec(g1_tracking_cfg, g1_tracking_env), - "go1_joystick_flat": TaskSpec(go1_cfg, go1_env), - "go2_joystick_flat": TaskSpec(go2_cfg, go2_env), + "go1_joystick_flat": TaskSpec(go1_cfg, manager_env), + "go2_joystick_flat": TaskSpec(go2_cfg, manager_env), } @@ -141,8 +120,8 @@ def _compose_env_cfg(task: str, backend: str, spec: TaskSpec) -> Any: from hydra import compose, initialize_config_dir from hydra.core.global_hydra import GlobalHydra + from unilab.base.config_adapter import BackendAdapter from unilab.base.registry import apply_cfg_overrides - from unilab.training import BackendAdapter GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(ROOT_DIR / "conf" / "ppo"), version_base="1.3"): @@ -341,10 +320,7 @@ def main() -> None: parser.add_argument( "--tasks", default="go1_joystick_flat,go2_joystick_flat", - help=( - "Comma-separated task ids. Defaults stay within committed Drake task configs; " - "pass g1_motion_tracking explicitly when its Drake config is available." - ), + help="Comma-separated task ids with committed Drake YAML owners.", ) parser.add_argument("--backends", default="drake,mujoco", help="Comma-separated backends.") parser.add_argument("--num-envs", default="64,256,1024", help="Comma-separated env counts.") diff --git a/scripts/benchmark/core/device_info.py b/scripts/benchmark/core/device_info.py index 7503f19cb..21312bc67 100644 --- a/scripts/benchmark/core/device_info.py +++ b/scripts/benchmark/core/device_info.py @@ -1,294 +1,7 @@ -from __future__ import annotations - -import platform -import re -import subprocess -from functools import lru_cache -from typing import Dict - - -def _is_macos() -> bool: - return platform.system() == "Darwin" - - -def _is_linux() -> bool: - return platform.system() == "Linux" - - -def _is_windows() -> bool: - return platform.system() == "Windows" - - -def _get_device_info_macos() -> Dict[str, str]: - """Collect hardware info on macOS via system_profiler.""" - info: Dict[str, str] = { - "chip": "unknown", - "cpu_total_cores": "unknown", - "cpu_performance_cores": "unknown", - "cpu_efficiency_cores": "unknown", - "gpu_cores": "unknown", - "memory": "unknown", - } - try: - hw_text = subprocess.check_output( - ["system_profiler", "SPHardwareDataType"], text=True, stderr=subprocess.DEVNULL - ) - disp_text = subprocess.check_output( - ["system_profiler", "SPDisplaysDataType"], text=True, stderr=subprocess.DEVNULL - ) - except Exception: - return info - - chip_match = re.search(r"Chip:\s*(.+)", hw_text) - if chip_match: - info["chip"] = chip_match.group(1).strip() +"""Thin re-export of the library device-info helpers (moved in issue #1240).""" - mem_match = re.search(r"Memory:\s*(.+)", hw_text) - if mem_match: - info["memory"] = mem_match.group(1).strip() - - # Apple Silicon core descriptions vary by generation: - # M3/M4: "10 performance and 4 efficiency" - # M5 Pro/Max: "6 super and 12 performance" - cpu_match = re.search( - r"Total Number of Cores:\s*(\d+)\s*\(\s*(\d+)\s*(\w+)\s+and\s+(\d+)\s*(\w+)\s*\)", - hw_text, - ) - if cpu_match: - total, count1, type1, count2, type2 = cpu_match.groups() - info["cpu_total_cores"] = total - info["cpu_core_type_1"] = type1 - info["cpu_core_count_1"] = count1 - info["cpu_core_type_2"] = type2 - info["cpu_core_count_2"] = count2 - # Backward-compat keys for legacy P+E format - if type1 == "performance" and type2 == "efficiency": - info["cpu_performance_cores"] = count1 - info["cpu_efficiency_cores"] = count2 - elif type1 == "super" and type2 == "performance": - info["cpu_super_cores"] = count1 - info["cpu_performance_cores"] = count2 - else: - cpu_total_match = re.search(r"Total Number of Cores:\s*(\d+)", hw_text) - if cpu_total_match: - info["cpu_total_cores"] = cpu_total_match.group(1) - - gpu_match = re.search(r"Type:\s*GPU[\s\S]*?Total Number of Cores:\s*(\d+)", disp_text) - if gpu_match: - info["gpu_cores"] = gpu_match.group(1) - - return info - - -def _get_device_info_linux() -> Dict[str, str]: - """Collect hardware info on Linux via /proc and common CLI tools.""" - info: Dict[str, str] = { - "chip": "unknown", - "cpu_total_cores": "unknown", - "cpu_performance_cores": "unknown", - "cpu_efficiency_cores": "unknown", - "gpu_name": "unknown", - "memory": "unknown", - } - # CPU model - try: - with open("/proc/cpuinfo", encoding="utf-8") as f: - cpuinfo = f.read() - model_match = re.search(r"^model name\s*:\s*(.+)$", cpuinfo, re.MULTILINE) - if model_match: - info["chip"] = model_match.group(1).strip() - # Count physical cores (unique core id per physical id) - pairs = re.findall(r"physical id\s*:\s*(\d+).*?core id\s*:\s*(\d+)", cpuinfo, re.DOTALL) - if pairs: - info["cpu_total_cores"] = str(len(set(pairs))) - else: - processor_count = len(re.findall(r"^processor\s*:", cpuinfo, re.MULTILINE)) - if processor_count: - info["cpu_total_cores"] = str(processor_count) - except Exception: - pass - # Total memory - try: - with open("/proc/meminfo", encoding="utf-8") as f: - meminfo = f.read() - mem_match = re.search(r"MemTotal:\s*(\d+)\s*kB", meminfo) - if mem_match: - mem_gb = int(mem_match.group(1)) / 1024 / 1024 - info["memory"] = f"{mem_gb:.1f} GB" - except Exception: - pass - # GPU via nvidia-smi - try: - gpu_out = subprocess.check_output( - ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"], - text=True, - stderr=subprocess.DEVNULL, - ).strip() - if gpu_out: - # Take the first GPU line - first_line = gpu_out.splitlines()[0] - parts = [p.strip() for p in first_line.split(",")] - info["gpu_name"] = parts[0] - if len(parts) > 1: - info["gpu_memory"] = parts[1] - except Exception: - pass - # GPU via rocm-smi (AMD) - if info["gpu_name"] == "unknown": - try: - rocm_out = subprocess.check_output( - ["rocm-smi", "--showproductname"], - text=True, - stderr=subprocess.DEVNULL, - ) - gpu_match = re.search(r"Card series\s*:\s*(.+)", rocm_out, re.IGNORECASE) - if gpu_match: - info["gpu_name"] = gpu_match.group(1).strip() - except Exception: - pass - # GPU memory via amd-smi (AMD ROCm). On unified-memory APUs this reports - # the BIOS-allocated visible VRAM slice, e.g. 96 GB out of 128 GB. - try: - amd_smi_out = subprocess.check_output( - ["amd-smi", "metric"], - text=True, - stderr=subprocess.DEVNULL, - ) - vram_match = re.search(r"TOTAL_VISIBLE_VRAM:\s*(\d+)\s*MB", amd_smi_out) - if vram_match: - info["gpu_memory"] = f"{int(vram_match.group(1))} MB" - gtt_match = re.search(r"TOTAL_GTT:\s*(\d+)\s*MB", amd_smi_out) - if gtt_match: - info["gpu_gtt_memory"] = f"{int(gtt_match.group(1))} MB" - except Exception: - pass - # Fallback GPU via lspci (AMD/ATI, Intel iGPU/Arc, and others) - if info["gpu_name"] == "unknown": - try: - lspci_out = subprocess.check_output(["lspci"], text=True, stderr=subprocess.DEVNULL) - for line in lspci_out.splitlines(): - if "VGA" in line or "Display" in line or "3D" in line: - if "AMD" in line or "ATI" in line: - match = re.search(r"\[AMD/ATI\]\s*(.+)", line) - if match: - name = match.group(1).strip() - name = re.sub(r"\s*\(rev.*\)", "", name) - info["gpu_name"] = name - break - elif "Intel" in line: - # e.g. "Intel Corporation Meteor Lake-P [Intel Arc Graphics] (rev 08)" - match = re.search(r"\[([^\]]+)\]", line) - if match: - info["gpu_name"] = match.group(1).strip() - break - except Exception: - pass - # If GPU name is still generic/unknown, try to infer from CPU model (APUs) - if info["gpu_name"] in ("unknown", "AMD Radeon Graphics"): - chip = info.get("chip", "") - match = re.search(r"w(?:ith)?/\s*(Radeon\s+[\w\s\+]+)", chip, re.IGNORECASE) - if match: - info["gpu_name"] = match.group(1).strip() - return info - - -def _get_device_info_windows() -> Dict[str, str]: - """Collect hardware info on Windows via wmic.""" - info: Dict[str, str] = { - "chip": "unknown", - "cpu_total_cores": "unknown", - "cpu_performance_cores": "unknown", - "cpu_efficiency_cores": "unknown", - "gpu_name": "unknown", - "memory": "unknown", - } - try: - cpu_out = subprocess.check_output( - ["wmic", "cpu", "get", "Name,NumberOfCores", "/format:csv"], - text=True, - stderr=subprocess.DEVNULL, - ) - lines = [l for l in cpu_out.splitlines() if l.strip() and not l.strip().startswith("Node")] - if lines: - parts = lines[0].split(",") - if len(parts) >= 3: - info["cpu_total_cores"] = parts[1].strip() - info["chip"] = parts[2].strip() - except Exception: - pass - # Memory - try: - mem_out = subprocess.check_output( - ["wmic", "ComputerSystem", "get", "TotalPhysicalMemory", "/format:csv"], - text=True, - stderr=subprocess.DEVNULL, - ) - lines = [l for l in mem_out.splitlines() if l.strip() and not l.strip().startswith("Node")] - if lines: - parts = lines[0].split(",") - if len(parts) >= 2: - mem_gb = int(parts[1].strip()) / 1024**3 - info["memory"] = f"{mem_gb:.1f} GB" - except Exception: - pass - # GPU via nvidia-smi (also available on Windows) - try: - gpu_out = subprocess.check_output( - ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"], - text=True, - stderr=subprocess.DEVNULL, - ).strip() - if gpu_out: - first_line = gpu_out.splitlines()[0] - parts = [p.strip() for p in first_line.split(",")] - info["gpu_name"] = parts[0] - if len(parts) > 1: - info["gpu_memory"] = parts[1] - except Exception: - pass - return info - - -@lru_cache(maxsize=1) -def get_device_info_dict() -> Dict[str, str]: - base: Dict[str, str] = {"platform": platform.platform()} - if _is_macos(): - base.update(_get_device_info_macos()) - elif _is_linux(): - base.update(_get_device_info_linux()) - elif _is_windows(): - base.update(_get_device_info_windows()) - return base +from __future__ import annotations +from unilab.utils.device import get_device_info_dict, get_device_info_line -def get_device_info_line() -> str: - d = get_device_info_dict() - if _is_macos(): - # Build core-type summary dynamically so M5 (super+performance) is shown correctly - if d.get("cpu_core_type_1") and d.get("cpu_core_type_2"): - t1 = d["cpu_core_type_1"][0].upper() - t2 = d["cpu_core_type_2"][0].upper() - core_summary = f"{d['cpu_core_count_1']}{t1}+{d['cpu_core_count_2']}{t2}" - elif ( - d.get("cpu_performance_cores") != "unknown" - and d.get("cpu_efficiency_cores") != "unknown" - ): - core_summary = f"{d['cpu_performance_cores']}P+{d['cpu_efficiency_cores']}E" - else: - core_summary = "unknown" - return ( - f"Device: {d.get('chip', 'unknown')} | " - f"CPU: {d.get('cpu_total_cores', 'unknown')} cores " - f"({core_summary}) | " - f"GPU: {d.get('gpu_cores', 'unknown')} cores | " - f"Memory: {d.get('memory', 'unknown')}" - ) - else: - gpu_part = d.get("gpu_name", "unknown") - if "gpu_memory" in d: - gpu_part += f" ({d['gpu_memory']})" - return ( - f"CPU: {d.get('chip', 'unknown')} ({d.get('cpu_total_cores', 'unknown')} cores) | " - f"GPU: {gpu_part} | " - f"Memory: {d.get('memory', 'unknown')}" - ) +__all__ = ["get_device_info_dict", "get_device_info_line"] diff --git a/scripts/benchmark/core/task_names.py b/scripts/benchmark/core/task_names.py index b922c06f5..2bc007acc 100644 --- a/scripts/benchmark/core/task_names.py +++ b/scripts/benchmark/core/task_names.py @@ -2,10 +2,8 @@ from dataclasses import dataclass -from unilab.envs.locomotion.g1.joystick import G1WalkFlatCfg -from unilab.envs.locomotion.go1.joystick import Go1JoystickCfg -from unilab.envs.locomotion.go2.joystick import Go2JoystickCfg -from unilab.envs.manipulation.sharpa_inhand.rotation import SharpaInhandRotationCfg +from unilab.envs import ManagerBasedRlEnvCfg +from unilab.tasks.manipulation.sharpa_inhand.rotation import SharpaInhandRotationCfg @dataclass(frozen=True) @@ -14,6 +12,7 @@ class LocomotionTaskSpec: env_task_name: str display_name: str config_cls: type + model_file: str | None = None _TASK_SPECS = { @@ -21,19 +20,22 @@ class LocomotionTaskSpec: owner_task_id="go1_joystick_flat", env_task_name="Go1JoystickFlat", display_name="go1_joystick_flat", - config_cls=Go1JoystickCfg, + config_cls=ManagerBasedRlEnvCfg, + model_file="src/unilab/assets/robots/go1/scene_flat.xml", ), "go2_joystick_flat": LocomotionTaskSpec( owner_task_id="go2_joystick_flat", env_task_name="Go2JoystickFlat", display_name="go2_joystick_flat", - config_cls=Go2JoystickCfg, + config_cls=ManagerBasedRlEnvCfg, + model_file="src/unilab/assets/robots/go2/scene_flat.xml", ), "g1_walk_flat": LocomotionTaskSpec( owner_task_id="g1_walk_flat", env_task_name="G1WalkFlat", display_name="g1_walk_flat", - config_cls=G1WalkFlatCfg, + config_cls=ManagerBasedRlEnvCfg, + model_file="src/unilab/assets/robots/g1/scene_flat.xml", ), "sharpa_inhand": LocomotionTaskSpec( owner_task_id="sharpa_inhand", @@ -75,7 +77,10 @@ def locomotion_task_spec(task_name: str) -> LocomotionTaskSpec: def locomotion_task_model_file(task_name: str) -> str: - cfg = locomotion_task_spec(task_name).config_cls() + spec = locomotion_task_spec(task_name) + if spec.model_file is not None: + return spec.model_file + cfg = spec.config_cls() scene = getattr(cfg, "scene", None) model_file = getattr(scene, "model_file", None) if model_file: diff --git a/scripts/benchmark/env/benchmark_env_step.py b/scripts/benchmark/env/benchmark_env_step.py index 286514b81..6f8d59836 100644 --- a/scripts/benchmark/env/benchmark_env_step.py +++ b/scripts/benchmark/env/benchmark_env_step.py @@ -78,7 +78,7 @@ def _load_helper_module(module_name: str, relative_path: str): def _install_mjwarp_patch() -> bool: """Route ``backend_type == "mjwarp"`` to ``scripts/benchmark/mjwarp`` via factory patch. - Must run before any task env module (e.g. ``unilab.envs.locomotion.g1.joystick``) + Must run before any task env module (e.g. ``unilab.tasks.locomotion.g1``) is imported, because those modules bind ``create_backend`` at module load time via ``from unilab.base.backend import create_backend``. @@ -116,9 +116,26 @@ def _patched_create_backend(backend_type, scene, num_envs, sim_dt, **kwargs): _ub_backend.create_backend = _patched_create_backend _ub_backend._mjwarp_patched = True + _ub_backend._mjwarp_orig_create_backend = _orig_create_backend return True +def _uninstall_mjwarp_patch() -> None: + """Undo the import-time factory patch installed by :func:`_install_mjwarp_patch`. + + Importing this module inside a pytest session would otherwise leak the + mjwarp rerouting into unrelated tests that exercise the real factory. + """ + import unilab.base.backend as _ub_backend + + if not getattr(_ub_backend, "_mjwarp_patched", False): + return + orig = getattr(_ub_backend, "_mjwarp_orig_create_backend", None) + if orig is not None: + _ub_backend.create_backend = orig + _ub_backend._mjwarp_patched = False + + MJWARP_AVAILABLE = _install_mjwarp_patch() BACKENDS = ["mujoco", "motrix", "mjwarp"] @@ -129,7 +146,7 @@ class TaskConfig: task_id: str env_name: str cfg_factory: Callable[[str, list[str]], Any] - env_cls_factory: Callable[[], type] + env_cls_factory: Callable[[], Callable[..., Any]] backends: tuple[str, ...] = ("mujoco", "motrix") aliases: tuple[str, ...] = () cfg_finalizer: Callable[[Any, str], None] | None = None @@ -157,8 +174,8 @@ def _owner_yaml_cfg( from hydra import compose, initialize_config_dir from hydra.core.global_hydra import GlobalHydra + from unilab.base.config_adapter import BackendAdapter from unilab.base.registry import apply_cfg_overrides - from unilab.training import BackendAdapter GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(ROOT_DIR / "conf" / config_root), version_base="1.3"): @@ -219,9 +236,9 @@ def _sac_owner_yaml_cfg( ) -> Any: yaml_backend = _hydra_yaml_backend(backend) return _owner_yaml_cfg( - config_root="offpolicy", + config_root="sac", algo_name="sac", - overrides=["algo=sac", f"task=sac/{task_id}/{yaml_backend}"], + overrides=[f"task={task_id}/{yaml_backend}"], config_overrides=config_overrides, env_cfg_cls=env_cfg_cls, ) @@ -276,86 +293,70 @@ def _materialize_sharpa_motrix_scene() -> str: def _go1_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.go1.joystick import Go1JoystickCfg + from unilab.envs import ManagerBasedRlEnvCfg - return _ppo_owner_yaml_cfg("go1_joystick_flat", backend, Go1JoystickCfg, config_overrides) + return _ppo_owner_yaml_cfg("go1_joystick_flat", backend, ManagerBasedRlEnvCfg, config_overrides) -def _go1_env_cls() -> type: - from unilab.envs.locomotion.go1.joystick import Go1WalkTask +def _manager_env_cls() -> Callable[..., Any]: + from unilab.envs import make_manager_based_rl_env - return Go1WalkTask + return make_manager_based_rl_env def _go2_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.go2.joystick import Go2JoystickCfg + from unilab.envs import ManagerBasedRlEnvCfg - return _ppo_owner_yaml_cfg("go2_joystick_flat", backend, Go2JoystickCfg, config_overrides) - - -def _go2_env_cls() -> type: - from unilab.envs.locomotion.go2.joystick import Go2WalkTask - - return Go2WalkTask + return _ppo_owner_yaml_cfg("go2_joystick_flat", backend, ManagerBasedRlEnvCfg, config_overrides) def _go2_rough_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.go2.rough import Go2JoystickRoughCfg - - return _ppo_owner_yaml_cfg("go2_joystick_rough", backend, Go2JoystickRoughCfg, config_overrides) + from unilab.envs import ManagerBasedRlEnvCfg - -def _go2_rough_env_cls() -> type: - from unilab.envs.locomotion.go2.rough import Go2JoystickRoughEnv - - return Go2JoystickRoughEnv + return _ppo_owner_yaml_cfg( + "go2_joystick_rough", backend, ManagerBasedRlEnvCfg, config_overrides + ) def _go2w_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.go2w.joystick import Go2WJoystickCfg + from unilab.envs import ManagerBasedRlEnvCfg - return _ppo_owner_yaml_cfg("go2w_joystick_flat", backend, Go2WJoystickCfg, config_overrides) + return _ppo_owner_yaml_cfg( + "go2w_joystick_flat", backend, ManagerBasedRlEnvCfg, config_overrides + ) def _go2w_rough_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.go2w.rough import Go2WJoystickRoughCfg + from unilab.envs import ManagerBasedRlEnvCfg return _ppo_owner_yaml_cfg( - "go2w_joystick_rough", backend, Go2WJoystickRoughCfg, config_overrides + "go2w_joystick_rough", backend, ManagerBasedRlEnvCfg, config_overrides ) -def _go2w_env_cls() -> type: - from unilab.envs.locomotion.go2w.joystick import Go2WJoystickEnv - - return Go2WJoystickEnv - - -def _go2w_rough_env_cls() -> type: - from unilab.envs.locomotion.go2w.rough import Go2WJoystickRoughEnv - - return Go2WJoystickRoughEnv +def _go2w_env_cls() -> Callable[..., Any]: + return _manager_env_cls() def _g1_flat_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.g1.joystick import G1WalkFlatCfg + from unilab.envs import ManagerBasedRlEnvCfg - return _ppo_owner_yaml_cfg("g1_walk_flat", backend, G1WalkFlatCfg, config_overrides) + return _ppo_owner_yaml_cfg("g1_walk_flat", backend, ManagerBasedRlEnvCfg, config_overrides) def _g1_rough_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.locomotion.g1.joystick import G1WalkRoughCfg + from unilab.envs import ManagerBasedRlEnvCfg - return _sac_owner_yaml_cfg("g1_walk_rough", backend, G1WalkRoughCfg, config_overrides) + return _sac_owner_yaml_cfg("g1_walk_rough", backend, ManagerBasedRlEnvCfg, config_overrides) def _g1_motion_tracking_cfg(backend: str, config_overrides: list[str]) -> Any: - from unilab.envs.motion_tracking.g1.tracking import G1MotionTrackingEnvCfg + from unilab.envs import ManagerBasedRlEnvCfg return _ppo_owner_yaml_cfg( "g1_motion_tracking", backend, - G1MotionTrackingEnvCfg, + ManagerBasedRlEnvCfg, config_overrides, ) @@ -364,9 +365,9 @@ def _sharpa_inhand_cfg(backend: str, config_overrides: list[str]) -> Any: from hydra import compose, initialize_config_dir from hydra.core.global_hydra import GlobalHydra + from unilab.base.config_adapter import BackendAdapter from unilab.base.registry import apply_cfg_overrides - from unilab.envs.manipulation.sharpa_inhand.rotation import SharpaInhandRotationCfg - from unilab.training import BackendAdapter + from unilab.tasks.manipulation.sharpa_inhand.rotation import SharpaInhandRotationCfg yaml_backend = _hydra_yaml_backend(backend) GlobalHydra.instance().clear() @@ -396,7 +397,7 @@ def _sharpa_inhand_cfg(backend: str, config_overrides: list[str]) -> Any: def _ensure_sharpa_benchmark_grasp_cache(cfg: Any, _: str) -> None: - from unilab.envs.manipulation.sharpa_inhand.base import ( + from unilab.tasks.manipulation.sharpa_inhand.base import ( SOURCE_DEFAULT_HAND_JOINT_POS_DEG, resolve_grasp_cache_file, ) @@ -416,19 +417,13 @@ def _ensure_sharpa_benchmark_grasp_cache(cfg: Any, _: str) -> None: def _g1_walk_env_cls() -> type: - from unilab.envs.locomotion.g1.joystick import G1WalkEnv - - return G1WalkEnv + from unilab.tasks.locomotion.g1 import make_g1_walk_env - -def _g1_motion_tracking_env_cls() -> type: - from unilab.envs.motion_tracking.g1.tracking import G1MotionTrackingEnv - - return G1MotionTrackingEnv + return make_g1_walk_env def _sharpa_inhand_env_cls() -> type: - from unilab.envs.manipulation.sharpa_inhand.rotation import SharpaInhandRotationEnv + from unilab.tasks.manipulation.sharpa_inhand.rotation import SharpaInhandRotationEnv return SharpaInhandRotationEnv @@ -438,22 +433,22 @@ def _sharpa_inhand_env_cls() -> type: task_id="go1_joystick_flat", env_name="Go1JoystickFlat", cfg_factory=_go1_cfg, - env_cls_factory=_go1_env_cls, + env_cls_factory=_manager_env_cls, backends=("mujoco", "motrix", "mjwarp"), ), "go2": TaskConfig( task_id="go2_joystick_flat", env_name="Go2JoystickFlat", cfg_factory=_go2_cfg, - env_cls_factory=_go2_env_cls, + env_cls_factory=_manager_env_cls, backends=("mujoco", "motrix", "mjwarp"), ), "go2_rough": TaskConfig( task_id="go2_joystick_rough", env_name="Go2JoystickRough", cfg_factory=_go2_rough_cfg, - env_cls_factory=_go2_rough_env_cls, - backends=("mujoco", "motrix", "mjwarp"), + env_cls_factory=_manager_env_cls, + backends=("mujoco", "motrix"), ), "go2w": TaskConfig( task_id="go2w_joystick_flat", @@ -466,8 +461,8 @@ def _sharpa_inhand_env_cls() -> type: task_id="go2w_joystick_rough", env_name="Go2WJoystickRough", cfg_factory=_go2w_rough_cfg, - env_cls_factory=_go2w_rough_env_cls, - backends=("mujoco", "motrix", "mjwarp"), + env_cls_factory=_manager_env_cls, + backends=("mujoco", "motrix"), ), "g1": TaskConfig( task_id="g1_walk_flat", @@ -488,8 +483,8 @@ def _sharpa_inhand_env_cls() -> type: task_id="g1_motion_tracking", env_name="G1MotionTracking", cfg_factory=_g1_motion_tracking_cfg, - env_cls_factory=_g1_motion_tracking_env_cls, - backends=("mujoco", "motrix", "mjwarp"), + env_cls_factory=_manager_env_cls, + backends=("mujoco", "motrix"), ), "sharpa_inhand": TaskConfig( task_id="sharpa_inhand", @@ -502,7 +497,7 @@ def _sharpa_inhand_env_cls() -> type: } # Default benchmark parameters -DEFAULT_NUM_ENVS = 2048 +DEFAULT_NUM_ENVS = 4096 DEFAULT_NUM_STEPS = 20 DEFAULT_WARMUP_STEPS = 5 @@ -708,15 +703,21 @@ def _run_single(extra_args: list[str]) -> dict[str, Any]: env_cls = task_config.env_cls_factory() env = env_cls(cfg, num_envs=num_envs, backend_type=sim_backend) - nu = env._backend.num_actuators # type: ignore[reportAttributeAccessIssue] + action_shape = env.action_space.shape + if action_shape is None or len(action_shape) != 1: + raise ValueError( + f"Benchmark task {task_config.env_name!r} requires a flat action space, " + f"got {action_shape}" + ) + action_dim = int(action_shape[0]) env.init_state() for _ in range(warmup_steps): - actions = np.random.uniform(-1, 1, size=(num_envs, nu)).astype(np.float32) + actions = np.random.uniform(-1, 1, size=(num_envs, action_dim)).astype(np.float32) env.step(actions) for _ in range(num_steps): - actions = np.random.uniform(-1, 1, size=(num_envs, nu)).astype(np.float32) + actions = np.random.uniform(-1, 1, size=(num_envs, action_dim)).astype(np.float32) state = env.step(actions) timing = state.info.get("timing", {}) for k, v in timing.items(): diff --git a/scripts/benchmark/env/benchmark_env_step_phase_cpu.py b/scripts/benchmark/env/benchmark_env_step_phase_cpu.py new file mode 100644 index 000000000..904d91d1e --- /dev/null +++ b/scripts/benchmark/env/benchmark_env_step_phase_cpu.py @@ -0,0 +1,168 @@ +"""Per-phase wall/CPU attribution for a full task env step (issue #1328). + +Builds a real task env through the same Hydra compose + ``BackendAdapter`` +override path the off-policy collector uses, wraps ``backend.step`` / +``update_state`` / ``_reset_done_envs`` with process-wide CPU-time measurement +(``os.times``), and reports each phase's wall share and the average number of +cores it kept busy. This makes low-parallelism host phases visible next to the +thread-pool physics phase. + +``--cpu-ids 0-31`` additionally injects ``EnvCfg.cpu_ids`` into the env +override (the same key the multi-GPU DP collector path uses), which both pins +the MuJoCo pool workers and confines the process's host-side compute via +``apply_env_cpu_runtime`` — the A/B used in the issue. + +Run: + uv run scripts/benchmark/env/benchmark_env_step_phase_cpu.py + + # pinned A/B: + uv run scripts/benchmark/env/benchmark_env_step_phase_cpu.py --cpu-ids 0-31 + + # tuning: + uv run scripts/benchmark/env/benchmark_env_step_phase_cpu.py \ + --config-group sac --task g1_motion_tracking/mujoco \ + --num-envs 4096 --warmup 20 --iters 150 +""" + +from __future__ import annotations + +import argparse +import os +import time +from collections import defaultdict +from collections.abc import Sequence + +import numpy as np + +REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) + + +def _cpu_time() -> float: + t = os.times() + return t.user + t.system + + +def _parse_cpu_ids(spec: str) -> list[int]: + ids: list[int] = [] + for part in spec.split(","): + part = part.strip() + if "-" in part: + lo, hi = part.split("-", 1) + ids.extend(range(int(lo), int(hi) + 1)) + elif part: + ids.append(int(part)) + return ids + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--config-group", default="sac", help="conf/ used for compose") + parser.add_argument("--task", default="g1_motion_tracking/mujoco") + parser.add_argument("--num-envs", type=int, default=4096) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iters", type=int, default=150) + parser.add_argument( + "--cpu-ids", + default=None, + help="Optional env cpu_ids override, e.g. '0-31'; pins the MuJoCo pool " + "and confines host-side compute (sizes the pool to len(cpu_ids))", + ) + args = parser.parse_args(argv) + + import hydra + from omegaconf import OmegaConf + + from unilab.base.config_adapter import BackendAdapter, create_env + from unilab.training import ensure_registries + + ensure_registries() + with hydra.initialize_config_dir( + version_base="1.3", config_dir=os.path.join(REPO_ROOT, "conf", args.config_group) + ): + cfg = hydra.compose( + config_name="config", + overrides=[f"task={args.task}", f"algo.num_envs={args.num_envs}"], + ) + OmegaConf.resolve(cfg) + env_cfg_override = BackendAdapter( + cfg, root_dir=REPO_ROOT, algo_name=str(cfg.algo.algo) + ).build_task_env_cfg_override() + if args.cpu_ids is not None: + env_cfg_override = { + **(env_cfg_override or {}), + "cpu_ids": _parse_cpu_ids(args.cpu_ids), + } + env = create_env(cfg, num_envs=args.num_envs, env_cfg_override=env_cfg_override) + if env.state is None: + env.init_state() + + wall_ms: defaultdict[str, float] = defaultdict(float) + cpu_ms: defaultdict[str, float] = defaultdict(float) + counts: defaultdict[str, int] = defaultdict(int) + + def wrap(name, fn): + def wrapped(*a, **kw): + w0 = time.perf_counter() + c0 = _cpu_time() + out = fn(*a, **kw) + wall_ms[name] += (time.perf_counter() - w0) * 1000.0 + cpu_ms[name] += (_cpu_time() - c0) * 1000.0 + counts[name] += 1 + return out + + return wrapped + + env._backend.step = wrap("backend_step", env._backend.step) + env.update_state = wrap("update_state", env.update_state) + env._reset_done_envs = wrap("reset_done", env._reset_done_envs) + + action_dim = env.action_space.shape[-1] + rng = np.random.default_rng(0) + + def actions(): + return rng.uniform(-0.2, 0.2, size=(args.num_envs, action_dim)).astype(np.float32) + + for _ in range(args.warmup): + env.step(actions()) + wall_ms.clear() + cpu_ms.clear() + counts.clear() + + n_reset = 0 + wall0 = time.perf_counter() + cpu0 = _cpu_time() + for _ in range(args.iters): + state = env.step(actions()) + n_reset += int(np.count_nonzero(state.terminated | state.truncated)) + total_wall = (time.perf_counter() - wall0) * 1000.0 + total_cpu = (_cpu_time() - cpu0) * 1000.0 + + print( + f"pool nthread={env._backend._n_threads} num_envs={args.num_envs} " + f"cpu_ids={'None' if args.cpu_ids is None else args.cpu_ids}" + ) + print(f"iters={args.iters} total_resets={n_reset}") + print(f"{'phase':>16s} {'wall_ms':>9s} {'cpu_ms':>9s} {'cores':>6s} {'wall%':>6s}") + step_wall = total_wall / args.iters + for name in ("backend_step", "update_state", "reset_done"): + w = wall_ms[name] / args.iters + c = cpu_ms[name] / args.iters + print(f"{name:>16s} {w:9.2f} {c:9.2f} {c / w if w else 0:6.2f} {100 * w / step_wall:6.1f}") + other_w = total_wall - sum(wall_ms.values()) + other_c = total_cpu - sum(cpu_ms.values()) + print( + f"{'other(step glue)':>16s} {other_w / args.iters:9.2f} {other_c / args.iters:9.2f} " + f"{(other_c / other_w) if other_w > 0 else 0:6.2f} {100 * other_w / total_wall:6.1f}" + ) + print( + f"{'TOTAL step':>16s} {step_wall:9.2f} {total_cpu / args.iters:9.2f} " + f"{total_cpu / total_wall:6.2f} {100.0:6.1f}" + ) + print(f"steps/s={args.num_envs * args.iters / (total_wall / 1000.0):.0f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/benchmark/env/benchmark_mujoco_pool_thread_scaling.py b/scripts/benchmark/env/benchmark_mujoco_pool_thread_scaling.py new file mode 100644 index 000000000..65ff96343 --- /dev/null +++ b/scripts/benchmark/env/benchmark_mujoco_pool_thread_scaling.py @@ -0,0 +1,146 @@ +"""MuJoCo BatchEnvPool thread-count scaling probe (issue #1328). + +Steps a raw ``BatchEnvPool`` (no env semantics, no learner) on the G1 flat +scene with several ``nthread`` / ``cpu_ids`` configurations and reports, per +configuration, wall time per ``pool.step`` and the average number of cores the +process kept busy (process CPU time / wall time via ``os.times``). + +Used to separate two effects of the default +``nthread = min(num_envs, 2 * cpu_count)`` pool sizing: + +- thread count vs. pinning (``cpu_ids``): on the reference 16C/32T host the + 32-thread unpinned and pinned rows match, so the 2x-oversubscription loss + comes from the thread count itself; +- the physics scaling ceiling: throughput saturates near the physical core + count (memory-bandwidth bound), so extra threads mostly cost wall time. + +Run: + uv run scripts/benchmark/env/benchmark_mujoco_pool_thread_scaling.py + + # subset + tuning: + uv run scripts/benchmark/env/benchmark_mujoco_pool_thread_scaling.py \ + --num-envs 4096 --nstep 3 --chunk-size 6 \ + --configs 64:unpinned,32:unpinned,32:pinned,16:pinned +""" + +from __future__ import annotations + +import argparse +import os +import time +from typing import Sequence + +import numpy as np + +REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +DEFAULT_MODEL = os.path.join(REPO_ROOT, "src/unilab/assets/robots/g1/scene_flat.xml") + + +def _cpu_time() -> float: + t = os.times() + return t.user + t.system + + +def build_state(model, nenvs: int) -> np.ndarray: + """Tile the ``stand`` keyframe (or a plain forward) into a full-batch state.""" + import mujoco + + data = mujoco.MjData(model) + key_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_KEY, "stand") + if key_id >= 0: + mujoco.mj_resetDataKeyframe(model, data, key_id) + mujoco.mj_forward(model, data) + spec = int(mujoco.mjtState.mjSTATE_FULLPHYSICS) + row = np.empty(mujoco.mj_stateSize(model, spec), dtype=np.float64) + mujoco.mj_getState(model, data, row, spec) + return np.tile(row, (nenvs, 1)).copy() + + +def bench_config( + model, + state0: np.ndarray, + *, + nthread: int, + pinned: bool, + nstep: int, + chunk_size: int | None, + warmup: int, + iters: int, +) -> tuple[float, float]: + """Return (wall ms/step, busy cores) for one pool configuration.""" + from mujoco_uni.batch_env import BatchEnvPool + + cpu_ids = list(range(nthread)) if pinned else None + pool = BatchEnvPool(model, nbatch=state0.shape[0], nthread=nthread, cpu_ids=cpu_ids) + nenvs = state0.shape[0] + ctrl = np.zeros((nenvs, nstep, model.nu), dtype=np.float64) + st = state0.copy() + try: + for _ in range(warmup): + st = pool.step(st, nstep=nstep, control=ctrl, chunk_size=chunk_size) + t0 = time.perf_counter() + c0 = _cpu_time() + for _ in range(iters): + st = pool.step(st, nstep=nstep, control=ctrl, chunk_size=chunk_size) + wall_ms = (time.perf_counter() - t0) / iters * 1000.0 + cores = (_cpu_time() - c0) / iters * 1000.0 / wall_ms + return wall_ms, cores + finally: + pool.close() + + +def _parse_configs(spec: str) -> list[tuple[int, bool]]: + out = [] + for item in spec.split(","): + nthread_s, mode = item.strip().split(":") + if mode not in ("pinned", "unpinned"): + raise ValueError(f"unknown config mode {mode!r} in {item!r}") + out.append((int(nthread_s), mode == "pinned")) + return out + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", default=DEFAULT_MODEL, help="MuJoCo XML scene path") + parser.add_argument("--num-envs", type=int, default=4096) + parser.add_argument("--nstep", type=int, default=3, help="sim substeps per pool.step") + parser.add_argument("--chunk-size", type=int, default=6) + parser.add_argument("--warmup", type=int, default=5) + parser.add_argument("--iters", type=int, default=30) + parser.add_argument( + "--configs", + default="64:unpinned,32:unpinned,32:pinned,24:pinned,16:pinned,8:pinned", + help="Comma-separated nthread:pinned|unpinned entries", + ) + args = parser.parse_args(argv) + + import mujoco + + model = mujoco.MjModel.from_xml_path(args.model) + state0 = build_state(model, args.num_envs) + print( + f"model={os.path.basename(args.model)} nu={model.nu} nv={model.nv} " + f"nstate={state0.shape[1]} num_envs={args.num_envs} host_cpus={os.cpu_count()} " + f"nstep={args.nstep} chunk_size={args.chunk_size}" + ) + print(f"{'config':>18s} | {'ms/step':>8s} | {'cores':>6s}") + for nthread, pinned in _parse_configs(args.configs): + wall_ms, cores = bench_config( + model, + state0, + nthread=nthread, + pinned=pinned, + nstep=args.nstep, + chunk_size=args.chunk_size, + warmup=args.warmup, + iters=args.iters, + ) + label = f"{nthread}t {'pinned' if pinned else 'unpinned'}" + print(f"{label:>18s} | {wall_ms:8.2f} | {cores:6.1f}", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/benchmark/env/benchmark_sharpa_init_dr_construct.py b/scripts/benchmark/env/benchmark_sharpa_init_dr_construct.py index 31017047f..e080fbafa 100644 --- a/scripts/benchmark/env/benchmark_sharpa_init_dr_construct.py +++ b/scripts/benchmark/env/benchmark_sharpa_init_dr_construct.py @@ -140,7 +140,7 @@ def _compose_cfg(task: str, *, lower: float, upper: float, variant_count: int): @contextmanager def _init_dr_mode(enabled: bool) -> Iterator[None]: - from unilab.envs.manipulation.sharpa_inhand.rotation import SharpaInhandRotationDRProvider + from unilab.tasks.manipulation.sharpa_inhand.rotation import SharpaInhandRotationDRProvider original = SharpaInhandRotationDRProvider.build_init_randomization_plan if enabled: @@ -162,7 +162,7 @@ def disabled_build_init_randomization_plan(self: Any, env: Any) -> None: @contextmanager def _synthetic_grasp_cache_mode(enabled: bool) -> Iterator[None]: - from unilab.envs.manipulation.sharpa_inhand.rotation import SharpaInhandRotationDRProvider + from unilab.tasks.manipulation.sharpa_inhand.rotation import SharpaInhandRotationDRProvider original = SharpaInhandRotationDRProvider._load_grasp_cache if not enabled: @@ -208,7 +208,8 @@ def _construct_once( init_dr_enabled: bool, force_pool: bool, ) -> tuple[float, dict[str, Any]]: - from unilab.training import BackendAdapter, create_env, ensure_registries + from unilab.base.config_adapter import BackendAdapter, create_env + from unilab.training import ensure_registries ensure_registries() diff --git a/scripts/benchmark/mjwarp/backend.py b/scripts/benchmark/mjwarp/backend.py index 3287b8dbd..ea5ce32b8 100644 --- a/scripts/benchmark/mjwarp/backend.py +++ b/scripts/benchmark/mjwarp/backend.py @@ -200,7 +200,7 @@ def __init__( ) self._mj_model, self.terrain_origins, self.terrain_surface_sampler = result elif scene.fragment_files: - from unilab.base.backend.mujoco.xml import materialize_scene_fragments + from unilab.base.backend import materialize_scene_fragments xml_path = materialize_scene_fragments( scene.model_file, fragment_files=scene.fragment_files diff --git a/scripts/benchmark/physics/benchmark_mujoco_single_dispatch_callback.py b/scripts/benchmark/physics/benchmark_mujoco_single_dispatch_callback.py new file mode 100644 index 000000000..214c83084 --- /dev/null +++ b/scripts/benchmark/physics/benchmark_mujoco_single_dispatch_callback.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""One-off microbenchmark for issue #1262. + +Question: can one ``BatchEnvPool.step(nstep=N)`` dispatch reproduce the MBA +per-substep control semantics that today require N ``pool.step(nstep=1)`` +dispatches (``MuJoCoBackend._step_with_pre_step_control``)? + +Measured on the g1_walk_flat MuJoCo contract (scene_flat.xml + injected body +tracking sensors, sim_dt=1/150, sim_substeps=3): + +- ``multi``: current MBA path — N dispatches of nstep=1, with a Python-side + per-substep control recompute (JointPositionAction-style: + ``target = processed - encoder_bias``, constant within one + control step) between dispatches. +- ``traj``: single dispatch, nstep=N, control baked as an (nbatch, N, nu) + trajectory with identical rows. +- ``const``: single dispatch, nstep=N, control passed as one (nbatch, nu) + array (native ``control_is_constant`` fast path). + +All paths start from the same contact-rich state (standing keyframe settled +with foot contact) and use the same pool, model, sensors, and chunk size. +The numerical check compares final full-physics state and sensordata between +``multi`` and the single-dispatch paths. +""" + +from __future__ import annotations + +import argparse +import os +import time + +import mujoco +import numpy as np +from mujoco_uni import BatchEnvPool + +from unilab.base.backend.mujoco.xml import ( + create_discardvisual_xml, + inject_mujoco_tracking_sensors, +) + +SCENE_XML = "src/unilab/assets/robots/g1/scene_flat.xml" +BASE_BODY = "pelvis" +KEYFRAME = "stand" +SIM_DT = 1.0 / 150.0 +SUBSTEPS = 3 # ctrl_dt 0.02 / sim_dt 0.006667, matches g1_walk_flat +CTRL_SPEC = int(mujoco.mjtState.mjSTATE_CTRL) +FULLPHYSICS = mujoco.mjtState.mjSTATE_FULLPHYSICS + + +def build_model() -> mujoco.MjModel: + path = create_discardvisual_xml(SCENE_XML) + path, _, _ = inject_mujoco_tracking_sensors(path, baselink_name=BASE_BODY) + model = mujoco.MjModel.from_xml_path(path) + model.opt.timestep = SIM_DT + return model + + +def keyframe_state(model: mujoco.MjModel) -> np.ndarray: + data = mujoco.MjData(model) + kid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_KEY, KEYFRAME) + if kid < 0: + raise ValueError(f"keyframe '{KEYFRAME}' not found in {SCENE_XML}") + mujoco.mj_resetDataKeyframe(model, data, kid) + mujoco.mj_forward(model, data) + state = np.zeros(mujoco.mj_stateSize(model, FULLPHYSICS), dtype=np.float64) + mujoco.mj_getState(model, data, state, FULLPHYSICS) + return state + + +def make_ctrl(model: mujoco.MjModel, state0: np.ndarray, nbatch: int) -> np.ndarray: + """Stand-pose position targets plus per-env jitter (deterministic).""" + nq = model.nq + qpos = state0[1 : 1 + nq] # FULLPHYSICS: time, qpos, qvel, act, ... + stand = qpos[-model.nu :] # free root (7) precedes actuated joints + rng = np.random.default_rng(0) + ctrl = stand[None, :] + rng.uniform(-0.05, 0.05, size=(nbatch, model.nu)) + return np.ascontiguousarray(ctrl, dtype=np.float64) + + +def settle(pool: BatchEnvPool, state0: np.ndarray, ctrl: np.ndarray, chunk_size: int) -> np.ndarray: + """Roll out a few control steps so feet are in steady contact.""" + state = state0 + for _ in range(40): + state = pool.step( + state, + nstep=SUBSTEPS, + control=ctrl, + control_spec=CTRL_SPEC, + chunk_size=chunk_size, + ) + return state + + +def run_multi(pool, state, ctrl, bias, chunk_size): + """Current MBA path: SUBSTEPS dispatches of nstep=1 with callback between.""" + pool_ms = 0.0 + callback_ms = 0.0 + for _ in range(SUBSTEPS): + t0 = time.perf_counter() + native_ctrl = np.subtract(ctrl, bias) # JointPositionAction-style recompute + callback_ms += (time.perf_counter() - t0) * 1e3 + t0 = time.perf_counter() + state, _sensor = pool.step( + state, + nstep=1, + control=native_ctrl[:, None, :], + control_spec=CTRL_SPEC, + chunk_size=chunk_size, + return_sensor=True, + ) + pool_ms += (time.perf_counter() - t0) * 1e3 + return state, pool_ms, callback_ms + + +def run_traj(pool, state, ctrl, bias, chunk_size): + """Single dispatch, baked (nbatch, N, nu) trajectory with identical rows.""" + t0 = time.perf_counter() + native_ctrl = np.subtract(ctrl, bias) + callback_ms = (time.perf_counter() - t0) * 1e3 + t0 = time.perf_counter() + traj = np.broadcast_to(native_ctrl[:, None, :], (ctrl.shape[0], SUBSTEPS, ctrl.shape[1])) + state, _sensor = pool.step( + state, + nstep=SUBSTEPS, + control=traj, + control_spec=CTRL_SPEC, + chunk_size=chunk_size, + return_sensor=True, + ) + return state, (time.perf_counter() - t0) * 1e3, callback_ms + + +def run_const(pool, state, ctrl, bias, chunk_size): + """Single dispatch, baked (nbatch, nu) constant control (native fast path).""" + t0 = time.perf_counter() + native_ctrl = np.subtract(ctrl, bias) + callback_ms = (time.perf_counter() - t0) * 1e3 + t0 = time.perf_counter() + state, _sensor = pool.step( + state, + nstep=SUBSTEPS, + control=native_ctrl, + control_spec=CTRL_SPEC, + chunk_size=chunk_size, + return_sensor=True, + ) + return state, (time.perf_counter() - t0) * 1e3, callback_ms + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--num-envs", type=int, default=8192) + parser.add_argument("--chunk-sizes", type=int, nargs="+", default=[13, 41]) + parser.add_argument("--repeats", type=int, default=15) + parser.add_argument("--warmup", type=int, default=3) + args = parser.parse_args() + + model = build_model() + nthread = min(args.num_envs, (os.cpu_count() or 1) * 2) + print( + f"model: nu={model.nu} nq={model.nq} nv={model.nv} nsensor={model.nsensor} " + f"nsensordata={model.nsensordata}; nbatch={args.num_envs} nthread={nthread}" + ) + pool = BatchEnvPool(model, nbatch=args.num_envs, nthread=nthread) + try: + state0 = np.broadcast_to(keyframe_state(model)[None, :], (args.num_envs, pool.nstate)) + state0 = np.ascontiguousarray(state0, dtype=np.float64) + ctrl = make_ctrl(model, state0[0], args.num_envs) + bias = np.zeros_like(ctrl) # encoder_bias placeholder (zeros on g1_walk_flat) + contact_state = settle(pool, state0, ctrl, chunk_size=args.chunk_sizes[0]) + + paths = {"multi": run_multi, "traj": run_traj, "const": run_const} + + # Numerical comparison from the same contact-rich state. + finals = {} + for name, fn in paths.items(): + out = fn(pool, contact_state, ctrl, bias, args.chunk_sizes[0]) + finals[name] = out[0] + for name in ("traj", "const"): + diff = np.abs(finals[name] - finals["multi"]) + bitwise = float(np.mean(finals[name] == finals["multi"])) + print( + f"numerics {name} vs multi: max_abs_diff={diff.max():.3e} " + f"bitwise_equal={bitwise:.4f}" + ) + print( + f"numerics traj vs const: bitwise={bool(np.array_equal(finals['traj'], finals['const']))}" + ) + + # Timing. + for chunk_size in args.chunk_sizes: + print( + f"--- chunk_size={chunk_size} (ms per control step, median of {args.repeats}) ---" + ) + for name, fn in paths.items(): + for _ in range(args.warmup): + fn(pool, contact_state, ctrl, bias, chunk_size) + pool_ts, cb_ts = [], [] + for _ in range(args.repeats): + _s, pool_ms, cb_ms = fn(pool, contact_state, ctrl, bias, chunk_size) + pool_ts.append(pool_ms) + cb_ts.append(cb_ms) + print( + f" {name:>5}: pool={np.median(pool_ts):7.2f} " + f"callback={np.median(cb_ts):5.2f} " + f"total={np.median(pool_ts) + np.median(cb_ts):7.2f}" + ) + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark/rl/benchmark_offpolicy_collector_active.py b/scripts/benchmark/rl/benchmark_offpolicy_collector_active.py index 274f21d33..d611ec549 100644 --- a/scripts/benchmark/rl/benchmark_offpolicy_collector_active.py +++ b/scripts/benchmark/rl/benchmark_offpolicy_collector_active.py @@ -14,6 +14,9 @@ uv run scripts/benchmark/rl/benchmark_offpolicy_collector_active.py --cases auto --backend motrix uv run scripts/benchmark/rl/benchmark_offpolicy_collector_active.py --cases sac/g1_walk_flat/mujoco uv run scripts/benchmark/rl/benchmark_offpolicy_collector_active.py --cases sac/g1_walk_flat/motrixsim + # mjwarp (GPU) is opt-in and requires the optional mjwarp extra: + uv run --extra mjwarp scripts/benchmark/rl/benchmark_offpolicy_collector_active.py \ + --cases sac/g1_motion_tracking/mjwarp uv run scripts/benchmark/rl/benchmark_offpolicy_collector_active.py --num-envs 1024 --measure-steps 100 """ @@ -30,11 +33,13 @@ import sys import time from collections import defaultdict +from collections.abc import Sequence from dataclasses import asdict, dataclass, field from datetime import datetime, timezone +from importlib.util import find_spec from pathlib import Path from statistics import mean, median, pstdev -from typing import Any, Sequence, cast +from typing import Any, cast import numpy as np import torch @@ -61,6 +66,10 @@ DEFAULT_ALGOS = ("sac", "flashsac", "td3") DEFAULT_BACKEND = "motrix" BENCHMARK_BACKENDS = ("mujoco", "motrix") +# mjwarp (GPU) is opt-in only: it requires the optional ``mjwarp`` extra +# (mujoco-warp + warp-lang) and is never part of --all. Request it explicitly +# via --backend mjwarp or an explicit //mjwarp case. +OPTIONAL_BACKENDS = ("mjwarp",) DEFAULT_COLLECTOR_CPU_THREADS = 8 COLLECTOR_CPU_THREADS_ENV = "UNILAB_COLLECTOR_TORCH_THREADS" BACKEND_ALIASES = { @@ -119,6 +128,9 @@ "set_state_qpos_convert_ms", "set_state_pool_reset_ms", "set_state_state_scatter_ms", + "set_state_reset_upload_ms", + "set_state_reset_forward_ms", + "set_state_host_cache_refresh_ms", "set_state_internal_gap_ms", ) NP_ENV_STEP_COUNT_KEYS = ("reset_done_count",) @@ -175,6 +187,9 @@ ("set_state_qpos_convert_ms", "set_state_qpos_convert_ms"), ("set_state_pool_reset_ms", "set_state_pool_reset_ms"), ("set_state_state_scatter_ms", "set_state_state_scatter_ms"), + ("set_state_reset_upload_ms", "set_state_reset_upload_ms"), + ("set_state_reset_forward_ms", "set_state_reset_forward_ms"), + ("set_state_host_cache_refresh_ms", "set_state_host_cache_refresh_ms"), ("set_state_internal_gap_ms", "set_state_internal_gap_ms"), ) @@ -382,8 +397,7 @@ def _compose_offpolicy_cfg( ) -> DictConfig: owner_sim = _runtime_sim_backend(sim) overrides = [ - f"algo={algo}", - f"task={algo}/{task}/{owner_sim}", + f"task={task}/{owner_sim}", "hydra.run.dir=.", "hydra.output_subdir=null", "hydra/job_logging=disabled", @@ -395,21 +409,19 @@ def _compose_offpolicy_cfg( overrides.extend(extra_overrides) GlobalHydra.instance().clear() - with initialize_config_dir(config_dir=str(ROOT_DIR / "conf" / "offpolicy"), version_base="1.3"): + with initialize_config_dir(config_dir=str(ROOT_DIR / "conf" / algo), version_base="1.3"): return compose(config_name="config", overrides=overrides) def _owner_config_path(algo: str, task: str, sim: str) -> Path: - return ( - ROOT_DIR / "conf" / "offpolicy" / "task" / algo / task / f"{_runtime_sim_backend(sim)}.yaml" - ) + return ROOT_DIR / "conf" / algo / "task" / task / f"{_runtime_sim_backend(sim)}.yaml" def _discover_cases(*, algos: list[str], sim: str) -> list[str]: cases: list[str] = [] owner_sim = _runtime_sim_backend(sim) for algo in algos: - task_root = ROOT_DIR / "conf" / "offpolicy" / "task" / algo + task_root = ROOT_DIR / "conf" / algo / "task" if not task_root.is_dir(): continue for path in sorted(task_root.glob(f"*/{owner_sim}.yaml")): @@ -420,11 +432,25 @@ def _discover_cases(*, algos: list[str], sim: str) -> list[str]: def _resolve_backend_selection(*, backend: str, all_backends: bool) -> tuple[str, ...]: if all_backends: return BENCHMARK_BACKENDS + if backend in OPTIONAL_BACKENDS: + _check_optional_backend_deps(backend) + return (backend,) if backend not in BENCHMARK_BACKENDS: - raise ValueError(f"unsupported backend {backend!r}; expected one of {BENCHMARK_BACKENDS}") + raise ValueError( + f"unsupported backend {backend!r}; expected one of " + f"{BENCHMARK_BACKENDS + OPTIONAL_BACKENDS}" + ) return (backend,) +def _check_optional_backend_deps(backend: str) -> None: + if backend == "mjwarp" and (find_spec("mujoco_warp") is None or find_spec("warp") is None): + raise SystemExit( + "backend=mjwarp requires the mjwarp extra. Install it with `uv sync --extra mjwarp` " + "or run this benchmark with `uv run --extra mjwarp ...`." + ) + + def _default_case_specs(backends: Sequence[str]) -> list[str]: return [f"{template}/{backend}" for backend in backends for template in DEFAULT_CASE_TEMPLATES] @@ -500,8 +526,8 @@ def _make_env( *, env_cfg_override: dict[str, Any] | None, ): + from unilab.base.config_adapter import create_env from unilab.base.observations import get_obs_dims - from unilab.training import create_env env = create_env(cfg, num_envs=int(cfg.algo.num_envs), env_cfg_override=env_cfg_override) if env.state is None: @@ -610,7 +636,6 @@ def _run_active_window_case( ) else: env_step_timing_values["env_step_internal_gap_ms"] = None - phase_start_ns = time.perf_counter_ns() next_obs_np, next_critic_np = split_obs_dict(state.obs) next_obs_np = np.asarray(next_obs_np, dtype=np.float32) @@ -691,7 +716,7 @@ def _run_active_window_case( aux_samples["env_step_overhead_ms"].append(env_step_ms - physics_ms) for key, value in env_step_timing_values.items(): if value is not None: - env_step_timing_samples[key].append(value) + env_step_timing_samples.setdefault(key, []).append(value) finally: if random_profiler is not None: random_profiler.uninstall() @@ -750,8 +775,9 @@ def _build_and_run_case( variant: str = "default", profile_numpy_random: bool = False, ) -> CollectorResult: - from unilab.training import BackendAdapter, ensure_registries - from unilab.training.seed import apply_training_seed + from unilab.base.config_adapter import BackendAdapter + from unilab.training import ensure_registries + from unilab.utils.seed import apply_training_seed algo, task, sim = _parse_case(spec) owner_path = _owner_config_path(algo, task, sim) @@ -1296,27 +1322,28 @@ def _format_dr_reset_timing_table(results: list[CollectorResult]) -> str: ("set_state_internal_gap_ms", "Gap"), ) +_SET_STATE_MJWARP_KEYS = ( + ("set_state_reset_upload_ms", "Reset upload"), + ("set_state_reset_forward_ms", "Reset forward"), + ("set_state_host_cache_refresh_ms", "Host cache refresh"), + ("set_state_internal_gap_ms", "Gap"), +) -def _format_set_state_detail_table(results: list[CollectorResult]) -> str: - """Backend set_state sub-timing table (motrix keyset). - Renders the 14 motrix-oriented sub-keys next to the outer - ``dr_reset_set_state_ms``. Backends that don't populate a key emit 0.0 so - columns stay stable across backends. MuJoCo runs will show 0.0 for the - motrix-only sub-keys; use :func:`_format_set_state_mujoco_table` for the - MuJoCo-oriented view instead. - """ +def _format_set_state_backend_table( + results: list[CollectorResult], + keys: Sequence[tuple[str, str]], +) -> str: headers = ( "Algo", "Task", "Backend", "Set state ms (%env, %active)", - *(label for _, label in _SET_STATE_MOTRIX_KEYS), + *(label for _, label in keys), ) rows = [] for result in results: - env_step = result.phase_ms_per_vector_step.get("env_step_ms") - if env_step is None: + if result.phase_ms_per_vector_step.get("env_step_ms") is None: continue rows.append( ( @@ -1324,36 +1351,32 @@ def _format_set_state_detail_table(results: list[CollectorResult]) -> str: result.case.task, result.case.runtime_sim_backend, _format_np_env_timing(result, "dr_reset_set_state_ms"), - *(_format_set_state_sub_ms(result, key) for key, _ in _SET_STATE_MOTRIX_KEYS), + *(_format_set_state_sub_ms(result, key) for key, _ in keys), ) ) return _format_table(headers, rows) +def _format_set_state_detail_table(results: list[CollectorResult]) -> str: + """Backend set_state sub-timing table (motrix keyset). + + Renders the 14 motrix-oriented sub-keys next to the outer + ``dr_reset_set_state_ms``. Backends that don't populate a key emit 0.0 so + columns stay stable across backends. MuJoCo runs will show 0.0 for the + motrix-only sub-keys; use :func:`_format_set_state_mujoco_table` for the + MuJoCo-oriented view instead. + """ + return _format_set_state_backend_table(results, _SET_STATE_MOTRIX_KEYS) + + def _format_set_state_mujoco_table(results: list[CollectorResult]) -> str: """Backend set_state sub-timing table (mujoco keyset).""" - headers = ( - "Algo", - "Task", - "Backend", - "Set state ms (%env, %active)", - *(label for _, label in _SET_STATE_MUJOCO_KEYS), - ) - rows = [] - for result in results: - env_step = result.phase_ms_per_vector_step.get("env_step_ms") - if env_step is None: - continue - rows.append( - ( - result.case.algo, - result.case.task, - result.case.runtime_sim_backend, - _format_np_env_timing(result, "dr_reset_set_state_ms"), - *(_format_set_state_sub_ms(result, key) for key, _ in _SET_STATE_MUJOCO_KEYS), - ) - ) - return _format_table(headers, rows) + return _format_set_state_backend_table(results, _SET_STATE_MUJOCO_KEYS) + + +def _format_set_state_mjwarp_table(results: list[CollectorResult]) -> str: + """Backend set_state sub-timing table (mjwarp keyset).""" + return _format_set_state_backend_table(results, _SET_STATE_MJWARP_KEYS) def _format_np_env_step_timing_table(results: list[CollectorResult]) -> str: @@ -1565,9 +1588,12 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: ) parser.add_argument( "--backend", - choices=BENCHMARK_BACKENDS, + choices=(*BENCHMARK_BACKENDS, *OPTIONAL_BACKENDS), default=DEFAULT_BACKEND, - help="Backend to benchmark for --cases default/auto. Default: motrix.", + help=( + "Backend to benchmark for --cases default/auto. Default: motrix. " + "mjwarp is opt-in and requires the mjwarp extra." + ), ) parser.add_argument( "--all", @@ -1577,7 +1603,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: ) parser.add_argument( "--sim", - choices=(*BENCHMARK_BACKENDS, *BACKEND_ALIASES.keys()), + choices=(*BENCHMARK_BACKENDS, *OPTIONAL_BACKENDS, *BACKEND_ALIASES.keys()), default=None, help=argparse.SUPPRESS, ) @@ -1637,6 +1663,12 @@ def main() -> int: specs = _resolve_case_specs(args.cases, algos_arg=args.algos, backends=backends) if not specs: raise SystemExit("No benchmark cases resolved.") + # Explicit // cases bypass --backend, so check optional + # backend deps per resolved case as well. + for spec in specs: + _, _, spec_sim = _parse_case(spec) + if spec_sim in OPTIONAL_BACKENDS: + _check_optional_backend_deps(spec_sim) hardware_info = _get_benchmark_hardware_info() print(f"Device: {get_device_info_line()}") @@ -1722,6 +1754,8 @@ def main() -> int: print(_format_set_state_detail_table(results)) print("\nBackend set_state detail — mujoco keyset:") print(_format_set_state_mujoco_table(results)) + print("\nBackend set_state detail — mjwarp keyset:") + print(_format_set_state_mjwarp_table(results)) else: print("No successful benchmark cases.") return 0 if not errors else 1 diff --git a/scripts/benchmark/rl/benchmark_offpolicy_dp_scaling.py b/scripts/benchmark/rl/benchmark_offpolicy_dp_scaling.py index f89c5ff5e..f7fc028c2 100644 --- a/scripts/benchmark/rl/benchmark_offpolicy_dp_scaling.py +++ b/scripts/benchmark/rl/benchmark_offpolicy_dp_scaling.py @@ -1,6 +1,6 @@ """Off-policy multi-GPU data-parallel scaling benchmark (issue #968). -Runs real ``scripts/train_offpolicy.py`` training via subprocess (same Hydra +Runs real ``scripts/train_sac.py`` training via subprocess (same Hydra overrides as the production CLI entry, never importing training internals) and compares single-device N=1 (no ``training.devices``) against N-way data parallel (``training.devices=[d0..dN-1]``, default ``[0,1]``). Every config @@ -59,11 +59,11 @@ ) DEFAULT_RUNS_ROOT = ROOT_DIR / "scripts" / "benchmark" / "outputs" / "offpolicy_dp_scaling" / "runs" -TRAIN_SCRIPT = ROOT_DIR / "scripts" / "train_offpolicy.py" +TRAIN_SCRIPT = ROOT_DIR / "scripts" / "train_sac.py" # Route overrides equivalent to `uv run train --algo sac --task g1_walk_flat # --sim mujoco` (see src/unilab/cli.py build_route for off-policy algos). -ROUTE_OVERRIDES = ("algo=sac", "task=sac/g1_walk_flat/mujoco") +ROUTE_OVERRIDES = ("task=g1_walk_flat/mujoco",) STEPS_PER_SEC_TAG = "perf/steps_per_sec" SAMPLES_PER_SEC_TAG = "perf/effective_samples_per_sec" diff --git a/scripts/benchmark/rl/benchmark_replay_buffer_placement.py b/scripts/benchmark/rl/benchmark_replay_buffer_placement.py index 4e608a7fc..340d7475a 100644 --- a/scripts/benchmark/rl/benchmark_replay_buffer_placement.py +++ b/scripts/benchmark/rl/benchmark_replay_buffer_placement.py @@ -28,12 +28,12 @@ from datetime import datetime, timezone from pathlib import Path from statistics import mean, median, pstdev -from typing import Any, cast +from typing import Any import torch from hydra import compose, initialize_config_dir from hydra.core.global_hydra import GlobalHydra -from omegaconf import DictConfig, OmegaConf +from omegaconf import DictConfig ROOT_DIR = Path(__file__).resolve().parents[3] if str(ROOT_DIR) not in sys.path: @@ -108,7 +108,6 @@ class BenchmarkCase: benchmark_capacity_rows: int configured_batch_size: int learner_batch_size: int - symmetry_batch_multiplier: int updates_per_step: int sample_count: int learning_starts: int @@ -255,10 +254,9 @@ def _cleanup_device() -> None: def _compose_offpolicy_cfg(algo: str, task: str, sim: str) -> DictConfig: - config_dir = str(ROOT_DIR / "conf" / "offpolicy") + config_dir = str(ROOT_DIR / "conf" / algo) overrides = [ - f"algo={algo}", - f"task={algo}/{task}/{sim}", + f"task={task}/{sim}", "hydra.run.dir=.", "hydra.output_subdir=null", "hydra/job_logging=disabled", @@ -270,7 +268,7 @@ def _compose_offpolicy_cfg(algo: str, task: str, sim: str) -> DictConfig: def _owner_config_path(algo: str, task: str, sim: str) -> Path: - return ROOT_DIR / "conf" / "offpolicy" / "task" / algo / task / f"{sim}.yaml" + return ROOT_DIR / "conf" / algo / "task" / task / f"{sim}.yaml" def _owner_config_exists(algo: str, task: str, sim: str) -> bool: @@ -278,7 +276,7 @@ def _owner_config_exists(algo: str, task: str, sim: str) -> bool: def _discover_supported_tasks(algo: str, sim: str) -> list[str]: - task_root = ROOT_DIR / "conf" / "offpolicy" / "task" / algo + task_root = ROOT_DIR / "conf" / algo / "task" if not task_root.is_dir(): return [] return sorted(path.parent.name for path in task_root.glob(f"*/{sim}.yaml") if path.is_file()) @@ -317,9 +315,10 @@ def _resolve_targets( return targets, skipped -def _resolve_env_shape_and_symmetry(cfg: DictConfig, algo: str) -> tuple[ReplayShape, int]: +def _resolve_env_shape(cfg: DictConfig, algo: str) -> ReplayShape: + from unilab.base.config_adapter import BackendAdapter, create_env from unilab.base.observations import get_obs_dims - from unilab.training import BackendAdapter, create_env, ensure_registries + from unilab.training import ensure_registries ensure_registries() env_cfg_override = BackendAdapter( @@ -334,17 +333,6 @@ def _resolve_env_shape_and_symmetry(cfg: DictConfig, algo: str) -> tuple[ReplayS if action_shape is None: raise ValueError("env.action_space.shape must be defined") action_dim = int(action_shape[0]) - - symmetry_batch_multiplier = 1 - use_symmetry = bool(OmegaConf.select(cfg, "algo.use_symmetry", default=False)) - if algo == "sac" and use_symmetry: - symmetry_builder = getattr(env, "build_symmetry_augmentation", None) - if not callable(symmetry_builder): - raise ValueError(f"{cfg.training.task_name} does not provide symmetry augmentation") - symmetry = cast(Any, symmetry_builder(device="cpu")) - if symmetry is None: - raise ValueError(f"{cfg.training.task_name} does not provide symmetry augmentation") - symmetry_batch_multiplier = int(symmetry.batch_multiplier) finally: env.close() @@ -352,7 +340,7 @@ def _resolve_env_shape_and_symmetry(cfg: DictConfig, algo: str) -> tuple[ReplayS obs_dim=int(obs_dim), action_dim=action_dim, critic_dim=int(critic_dim), - ), symmetry_batch_multiplier + ) def _build_case( @@ -362,7 +350,6 @@ def _build_case( task: str, sim: str, shape: ReplayShape, - symmetry_batch_multiplier: int, max_capacity_rows: int | None, ) -> BenchmarkCase: num_envs = int(cfg.algo.num_envs) @@ -374,13 +361,6 @@ def _build_case( configured_batch_size = int(cfg.algo.batch_size) learner_batch_size = configured_batch_size - if algo == "sac" and bool(OmegaConf.select(cfg, "algo.use_symmetry", default=False)): - if configured_batch_size % symmetry_batch_multiplier != 0: - raise ValueError( - "SAC symmetry requires batch_size divisible by " - f"{symmetry_batch_multiplier}, got {configured_batch_size}" - ) - learner_batch_size = configured_batch_size // symmetry_batch_multiplier updates_per_step = int(cfg.algo.updates_per_step) env_steps_per_sync = int(cfg.training.env_steps_per_sync) @@ -399,7 +379,6 @@ def _build_case( benchmark_capacity_rows=benchmark_capacity_rows, configured_batch_size=configured_batch_size, learner_batch_size=learner_batch_size, - symmetry_batch_multiplier=symmetry_batch_multiplier, updates_per_step=updates_per_step, sample_count=learner_batch_size * updates_per_step, learning_starts=int(cfg.algo.learning_starts), @@ -1059,14 +1038,13 @@ def main(argv: list[str] | None = None) -> int: print(f"Skipping missing owner config: {Path(skipped['path']).relative_to(ROOT_DIR)}") for algo, task in targets: cfg = _compose_offpolicy_cfg(algo, task, args.sim) - shape, symmetry_batch_multiplier = _resolve_env_shape_and_symmetry(cfg, algo) + shape = _resolve_env_shape(cfg, algo) case = _build_case( cfg, algo=algo, task=task, sim=args.sim, shape=shape, - symmetry_batch_multiplier=symmetry_batch_multiplier, max_capacity_rows=max_capacity_rows, ) results.append( diff --git a/scripts/benchmark/rl/benchmark_sac_replay_buffer_sampling.py b/scripts/benchmark/rl/benchmark_sac_replay_buffer_sampling.py index 386d6e82e..d6d740d30 100644 --- a/scripts/benchmark/rl/benchmark_sac_replay_buffer_sampling.py +++ b/scripts/benchmark/rl/benchmark_sac_replay_buffer_sampling.py @@ -38,12 +38,12 @@ from datetime import datetime, timezone from pathlib import Path from statistics import mean, median, pstdev -from typing import Any, Callable, Iterable, cast +from typing import Any, Callable, Iterable import torch from hydra import compose, initialize_config_dir from hydra.core.global_hydra import GlobalHydra -from omegaconf import DictConfig, OmegaConf +from omegaconf import DictConfig ROOT_DIR = Path(__file__).resolve().parents[3] if str(ROOT_DIR) not in sys.path: @@ -98,7 +98,6 @@ class BenchmarkCase: config_capacity_rows: int configured_batch_size: int learner_batch_size: int - symmetry_batch_multiplier: int updates_per_step: int sample_count_per_rank: int learning_starts: int @@ -228,7 +227,7 @@ def _cleanup_device() -> None: def _owner_config_path(task: str, sim: str) -> Path: - return ROOT_DIR / "conf" / "offpolicy" / "task" / "sac" / task / f"{sim}.yaml" + return ROOT_DIR / "conf" / "sac" / "task" / task / f"{sim}.yaml" def _owner_config_exists(task: str, sim: str) -> bool: @@ -252,10 +251,9 @@ def _compose_offpolicy_cfg(task: str = DEFAULT_TASK, sim: str | None = None) -> raise FileNotFoundError( f"Missing SAC owner config for task={task_name!r}, sim={sim_name!r}: {owner_config}" ) - config_dir = str(ROOT_DIR / "conf" / "offpolicy") + config_dir = str(ROOT_DIR / "conf" / "sac") overrides = [ - "algo=sac", - f"task=sac/{task_name}/{sim_name}", + f"task={task_name}/{sim_name}", "hydra.run.dir=.", "hydra.output_subdir=null", "hydra/job_logging=disabled", @@ -266,9 +264,10 @@ def _compose_offpolicy_cfg(task: str = DEFAULT_TASK, sim: str | None = None) -> return compose(config_name="config", overrides=overrides) -def _resolve_env_shape_and_symmetry(cfg: DictConfig) -> tuple[ReplayShape, int]: +def _resolve_env_shape(cfg: DictConfig) -> ReplayShape: + from unilab.base.config_adapter import BackendAdapter, create_env from unilab.base.observations import get_obs_dims - from unilab.training import BackendAdapter, create_env, ensure_registries + from unilab.training import ensure_registries ensure_registries() env_cfg_override = BackendAdapter( @@ -283,23 +282,10 @@ def _resolve_env_shape_and_symmetry(cfg: DictConfig) -> tuple[ReplayShape, int]: if action_shape is None: raise ValueError("env.action_space.shape must be defined") action_dim = int(action_shape[0]) - - symmetry_batch_multiplier = 1 - if bool(OmegaConf.select(cfg, "algo.use_symmetry", default=False)): - symmetry_builder = getattr(env, "build_symmetry_augmentation", None) - if not callable(symmetry_builder): - raise ValueError(f"{cfg.training.task_name} does not provide symmetry augmentation") - symmetry = cast(Any, symmetry_builder(device="cpu")) - if symmetry is None: - raise ValueError(f"{cfg.training.task_name} does not provide symmetry augmentation") - symmetry_batch_multiplier = int(symmetry.batch_multiplier) finally: env.close() - return ( - ReplayShape(obs_dim=int(obs_dim), action_dim=action_dim, critic_dim=int(critic_dim)), - symmetry_batch_multiplier, - ) + return ReplayShape(obs_dim=int(obs_dim), action_dim=action_dim, critic_dim=int(critic_dim)) def _build_case( @@ -308,20 +294,12 @@ def _build_case( task: str = LEGACY_DEFAULT_TASK, sim: str = DEFAULT_SIM, shape: ReplayShape, - symmetry_batch_multiplier: int, ) -> BenchmarkCase: num_envs = int(cfg.algo.num_envs) env_steps_per_sync = int(cfg.training.env_steps_per_sync) replay_buffer_n = int(cfg.algo.replay_buffer_n) configured_batch_size = int(cfg.algo.batch_size) learner_batch_size = configured_batch_size - if bool(OmegaConf.select(cfg, "algo.use_symmetry", default=False)): - if configured_batch_size % symmetry_batch_multiplier != 0: - raise ValueError( - "SAC symmetry requires batch_size divisible by " - f"{symmetry_batch_multiplier}, got {configured_batch_size}" - ) - learner_batch_size = configured_batch_size // symmetry_batch_multiplier updates_per_step = int(cfg.algo.updates_per_step) return BenchmarkCase( @@ -336,7 +314,6 @@ def _build_case( config_capacity_rows=num_envs * replay_buffer_n, configured_batch_size=configured_batch_size, learner_batch_size=learner_batch_size, - symmetry_batch_multiplier=int(symmetry_batch_multiplier), updates_per_step=updates_per_step, sample_count_per_rank=learner_batch_size * updates_per_step, learning_starts=int(cfg.algo.learning_starts), @@ -1230,12 +1207,6 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--obs-dim", type=int, default=0) parser.add_argument("--action-dim", type=int, default=0) parser.add_argument("--critic-dim", type=int, default=0) - parser.add_argument( - "--symmetry-batch-multiplier", - type=int, - default=0, - help="Only used with manual --obs-dim/--action-dim/--critic-dim.", - ) parser.add_argument("--plot-dir", type=Path, default=None) parser.add_argument("--analysis-md", type=Path, default=None) parser.add_argument("--no-plots", action="store_true") @@ -1277,23 +1248,14 @@ def main(argv: list[str] | None = None) -> int: action_dim=int(args.action_dim), critic_dim=int(args.critic_dim), ) - if bool(OmegaConf.select(cfg, "algo.use_symmetry", default=False)): - if args.symmetry_batch_multiplier <= 0: - raise ValueError( - "Manual shape with SAC symmetry requires --symmetry-batch-multiplier" - ) - symmetry_batch_multiplier = int(args.symmetry_batch_multiplier) - else: - symmetry_batch_multiplier = 1 else: - shape, symmetry_batch_multiplier = _resolve_env_shape_and_symmetry(cfg) + shape = _resolve_env_shape(cfg) case = _build_case( cfg, task=args.task, sim=args.sim, shape=shape, - symmetry_batch_multiplier=symmetry_batch_multiplier, ) capacity_rows = _resolve_capacity_rows( config_capacity_rows=case.config_capacity_rows, diff --git a/scripts/benchmark/torch_env/motion_tracking.py b/scripts/benchmark/torch_env/motion_tracking.py index 029aed9a9..be0bae6cb 100644 --- a/scripts/benchmark/torch_env/motion_tracking.py +++ b/scripts/benchmark/torch_env/motion_tracking.py @@ -1,30 +1,23 @@ -"""G1MotionTrackingSAC (SAC/mujoco) update_state / reset_done workload. +"""G1MotionTrackingSAC (SAC/MuJoCo) numeric manager workload. -Faithful xp-port of the NumPy computation in the collector-timed sections of +Synthetic xp-port of the NumPy kernels in the collector-timed sections of `uv run train --algo sac --task g1_motion_tracking --sim mujoco` (num_envs=2048, 29-dof, 14 tracked bodies): -- `MotionTrackingEnv.update_state` - (src/unilab/envs/motion_tracking/common/tracking.py): - motion gather, relative transforms (transforms.py), terminations - (terminations.py), 9 active reward terms (rewards.py, incl. per-term logging - every 4 steps), observation build (observations.py, actor 160 / critic 289 - with the SAC +3 linvel tail), adaptive motion-sampler bookkeeping - (motion_loader.py). -- `MotionTrackingDomainRandomizationProvider.build_reset_plan` / - `build_reset_observation` + `build_motion_reference_state` (reset.py). -- `NpEnv._reset_done_envs` scatter/gather. +- `MotionCommand` gather, relative transforms, termination/reward terms, + observation-group assembly (actor 160 / critic 289), and adaptive sampler + bookkeeping from the Manager-Based motion runtime. +- Motion-command reset-state construction and `NpEnv._reset_done_envs` + scatter/gather. Excluded (identical across variants, not NumPy/Torch env math): `backend.step` physics, `backend.set_state`, sensor/body-state reads (replaced by persistent arrays), and the adaptive-sampler entropy metrics (3 scalar reductions per reset). -The real `build_motion_reference_state` samples pose/velocity randomization -with a per-element Python loop (num_reset x 6 draws, twice). The NumPy -workload reproduces that faithfully; pass ``vectorized_reset_rng=True`` for a -column-wise vectorized NumPy draw (used for cross-backend RNG replay during -validation, which is also the only mode Torch implements). +Pass ``vectorized_reset_rng=True`` for a column-wise vectorized NumPy draw; +validation uses that mode for cross-backend RNG replay, and Torch implements +that mode only. """ from __future__ import annotations diff --git a/scripts/benchmark/torch_env/run_benchmark.py b/scripts/benchmark/torch_env/run_benchmark.py index a9e3dae71..91d21a3c7 100644 --- a/scripts/benchmark/torch_env/run_benchmark.py +++ b/scripts/benchmark/torch_env/run_benchmark.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""NumPy vs Torch comparison for the collector-timed env computation. +"""NumPy vs Torch comparison for collector-timed numeric manager kernels. -Reproduces the `update_state` and `reset_done` sections of `NpEnv.step` (the -`env_step_update_state_ms` / `env_step_reset_done_ms` collector metrics) for the -two SAC/mujoco tasks, at identical scale and computation items: +Reproduces the numeric work represented by the `env_step_update_state_ms` and +`env_step_reset_done_ms` collector metrics for two SAC/MuJoCo tasks, at +identical scale and computation items: - g1_walk_flat (num_envs=2048, 29-dof, obs 98 / critic 101) - g1_motion_tracking (num_envs=2048, 29-dof, 14 bodies, obs 160 / critic 289) diff --git a/scripts/benchmark/torch_env/walk_flat.py b/scripts/benchmark/torch_env/walk_flat.py index d93ce437d..6762510aa 100644 --- a/scripts/benchmark/torch_env/walk_flat.py +++ b/scripts/benchmark/torch_env/walk_flat.py @@ -3,7 +3,8 @@ Faithful xp-port of the NumPy computation in the collector-timed sections of `uv run train --algo sac --task g1_walk_flat --sim mujoco` (num_envs=2048): -- `G1WalkEnv.update_state` (src/unilab/envs/locomotion/g1/joystick.py): +- the legacy `G1WalkEnv.update_state` computation (pre-Manager-Based migration; + src/unilab/tasks/locomotion/g1/manager_terms.py now owns the same math): termination, `_compute_reward` (9 active terms under the SAC scales incl. per-term logging every 4 steps), `_compute_obs` (noise + concat, walk profile), and the done-triggered curriculum bookkeeping. @@ -51,7 +52,7 @@ NOISE_SCALE_JOINT_ANGLE = 0.01 NOISE_SCALE_JOINT_VEL = 0.1 -# conf/offpolicy/task/sac/g1_walk_flat/mujoco.yaml pose_weights (29-dof) +# conf/sac/task/g1_walk_flat/mujoco.yaml pose_weights (29-dof) POSE_WEIGHTS = [0.01, 1.0, 5.0, 0.01, 5.0, 5.0] * 2 + [50.0] * 17 # Reward scales from the SAC owner YAML; penalty terms are multiplied by the diff --git a/scripts/completions/unilab.bash b/scripts/completions/unilab.bash index 024a569b8..b54177023 100644 --- a/scripts/completions/unilab.bash +++ b/scripts/completions/unilab.bash @@ -14,7 +14,7 @@ _unilab_uv_complete() { local candidates if ! mapfile -t candidates < <( uv run --no-sync unilab-complete --cword "$COMP_CWORD" -- "${COMP_WORDS[@]}" 2>/dev/null \ - || PYTHONPATH="$repo_root/src${PYTHONPATH:+:$PYTHONPATH}" uv run --no-sync python -m unilab.tools.completion --cword "$COMP_CWORD" -- "${COMP_WORDS[@]}" 2>/dev/null + || PYTHONPATH="$repo_root/src${PYTHONPATH:+:$PYTHONPATH}" uv run --no-sync python -m unilab.cli_completion --cword "$COMP_CWORD" -- "${COMP_WORDS[@]}" 2>/dev/null ); then return 0 fi diff --git a/scripts/completions/unilab.zsh b/scripts/completions/unilab.zsh index 3432470bf..53189f294 100644 --- a/scripts/completions/unilab.zsh +++ b/scripts/completions/unilab.zsh @@ -13,7 +13,7 @@ _unilab_uv_complete() { local output output="$( uv run --no-sync unilab-complete --cword "$((CURRENT - 1))" -- "${words[@]}" 2>/dev/null \ - || PYTHONPATH="$repo_root/src${PYTHONPATH:+:$PYTHONPATH}" uv run --no-sync python -m unilab.tools.completion --cword "$((CURRENT - 1))" -- "${words[@]}" 2>/dev/null + || PYTHONPATH="$repo_root/src${PYTHONPATH:+:$PYTHONPATH}" uv run --no-sync python -m unilab.cli_completion --cword "$((CURRENT - 1))" -- "${words[@]}" 2>/dev/null )" || return 0 if [[ -z "$output" ]]; then diff --git a/scripts/deploy/export_deploy_config.py b/scripts/deploy/export_deploy_config.py index b692e992e..dc6bbddf5 100644 --- a/scripts/deploy/export_deploy_config.py +++ b/scripts/deploy/export_deploy_config.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """Export deploy_config.yaml for the C++ G1-29DOF WBT deployment side. -Reads g1.xml + scene_flat.xml + tracking.py defaults to emit a single yaml +Reads the G1 scene plus the Manager-Based WBT owner contract to emit a single yaml that the deploy framework (~/deploy_ws/unitree_rl_lab/.../State_WBT) can load to drive the actor at runtime. This file is the SINGLE SOURCE OF TRUTH for the actor obs schema: - * Training side (tracking.py) assembles obs in the order documented here. + * Training side (ObservationManager) assembles terms in the order documented here. * Deploy side (State_WBT.cpp + ObservationManager) reads obs_layout from this yaml and assembles in matching order with per-term history buffers. * Alignment test (tests/test_obs_alignment_g1_wbt.py) verifies both code @@ -64,9 +64,9 @@ KEYFRAME_NAME = "stand" ROOT_QPOS_DIM = 7 # free joint: xyz + quat(wxyz) -# Default obs history length — matches g1_wbt_obs/mujoco.yaml's -# `noise_config.obs_history_length`. Override via --obs-history-length when -# exporting for other training profiles (e.g. g1_motion_tracking/mujoco.yaml uses 1). +# Default obs history length — matches the per-term ``history_length`` declarations +# in g1_wbt_obs/mujoco.yaml. Override via --obs-history-length when exporting for +# other training profiles (e.g. g1_motion_tracking/mujoco.yaml uses one step). DEFAULT_OBS_HISTORY_LENGTH = 5 @@ -77,7 +77,7 @@ def _round_list(arr, ndigits=6): def _build_obs_layout( num_action: int, hist_len: int, enable_zero_anchor_pos: bool, enable_zero_linvel: bool ): - """Build obs_layout in the exact order tracking.py:_compute_obs assembles. + """Build obs_layout in the exact order ObservationManager assembles it. Returns (layout_list, total_obs_dim). Order = single-step refs first, then per-term proprio history blocks, @@ -162,8 +162,8 @@ def main(): "--obs-history-length", type=int, default=DEFAULT_OBS_HISTORY_LENGTH, - help="Proprio history length H. Must match training-side " - "noise_config.obs_history_length. Default 5 = current " + help="Proprio history length H. Must match training-side per-term " + "history_length. Default 5 = current " "g1_wbt_obs/mujoco.yaml. Set 1 for the legacy 154-d schema.", ) ap.add_argument( @@ -171,14 +171,14 @@ def main(): action="store_true", default=True, help="Drop motion_anchor_pos_b from actor obs (mjlab parity). " - "Matches g1_wbt_obs/mujoco.yaml's noise_config flag.", + "Matches the null actor term in g1_wbt_obs/mujoco.yaml.", ) ap.add_argument( "--enable-zero-linvel", action="store_true", default=True, help="Drop base_lin_vel from actor obs (mjlab parity). " - "Matches g1_wbt_obs/mujoco.yaml's noise_config flag.", + "Matches the null actor term in g1_wbt_obs/mujoco.yaml.", ) args = ap.parse_args() @@ -299,7 +299,7 @@ def main(): # ---- obs layout (SINGLE SOURCE OF TRUTH for both ends) ---- # Each entry: name, dim (per-step), history_length, source. # Total obs_dim = sum(dim * history_length). - # Order MUST match tracking.py:_compute_obs assembly order. + # Order MUST match the WBT owner term order consumed by ObservationManager. # State_WBT.cpp:build_env_cfg translates names via its alias table. "obs_layout": obs_layout, } diff --git a/scripts/deploy/export_motion_bin.py b/scripts/deploy/export_motion_bin.py index 27905845c..058c82610 100644 --- a/scripts/deploy/export_motion_bin.py +++ b/scripts/deploy/export_motion_bin.py @@ -15,7 +15,7 @@ body_lin_vel_w [num_frames][num_bodies][3] body_ang_vel_w [num_frames][num_bodies][3] -NPZ source layout (per src/unilab/envs/motion_tracking/g1/motion_loader.py): +NPZ source layout (per src/unilab/tasks/motion_tracking/common/motion_loader.py): - 'fps' (int) - 'joint_pos' (N, 29) - 'joint_vel' (N, 29) diff --git a/scripts/deploy/sim_prototype.py b/scripts/deploy/sim_prototype.py index 7043b9bd2..d957058a5 100644 --- a/scripts/deploy/sim_prototype.py +++ b/scripts/deploy/sim_prototype.py @@ -26,6 +26,14 @@ - No noise injected on obs (matches deploy convention). - Robot anchor pos in world is locked to the first motion frame's torso pos (since real robot has no GPS/SLAM). + +Contract anchors (the numpy rewrite above is intentional — keep it standalone): + - Training-side obs contract owner: conf/offpolicy/task/sac/g1_wbt_obs/mujoco.yaml + (per-term obs layout + history as assembled by the training ObservationManager). + - Alignment test: tests/scripts/test_obs_alignment_g1_wbt.py checks this + file's ObsAssembler against training-side and deploy-side (C++) semantics + bit-for-bit. If the training obs terms change, that test must keep passing + WITHOUT this file importing env classes. """ from __future__ import annotations diff --git a/scripts/generate_support_matrix.py b/scripts/generate_support_matrix.py index 8947c28ce..931bf32cc 100644 --- a/scripts/generate_support_matrix.py +++ b/scripts/generate_support_matrix.py @@ -5,9 +5,14 @@ from __future__ import annotations import argparse +import sys from pathlib import Path -from unilab.utils.support_matrix import ( +ROOT_DIR = Path(__file__).resolve().parents[1] +if str(ROOT_DIR) not in sys.path: + sys.path.append(str(ROOT_DIR)) + +from scripts.tools.support_matrix import ( render_generated_block, render_support_matrix, replace_generated_block, diff --git a/scripts/manip_loco/benchmark_site_jacobian.py b/scripts/manip_loco/benchmark_site_jacobian.py index ab4f44be1..b12871b39 100644 --- a/scripts/manip_loco/benchmark_site_jacobian.py +++ b/scripts/manip_loco/benchmark_site_jacobian.py @@ -20,7 +20,11 @@ sys.path.insert(0, str(ROOT_DIR)) from unilab.base.backend import materialize_scene_visual_override -from unilab.training import BackendAdapter, create_env, ensure_registries +from unilab.base.config_adapter import ( + BackendAdapter, + create_env, +) +from unilab.training import ensure_registries def _coerce_str_list(value: Any, *, name: str) -> list[str]: diff --git a/scripts/manip_loco/diagnose_go2_arm_ik.py b/scripts/manip_loco/diagnose_go2_arm_ik.py index 601eb70b7..d859fba07 100644 --- a/scripts/manip_loco/diagnose_go2_arm_ik.py +++ b/scripts/manip_loco/diagnose_go2_arm_ik.py @@ -16,7 +16,7 @@ from unilab.base import registry from unilab.base.registry import ensure_registries -from unilab.envs.locomotion.go2_arm.manip_loco import RewardConfig +from unilab.tasks.locomotion.go2_arm.manip_loco import RewardConfig from unilab.utils.rotation import np_matrix_from_quat diff --git a/scripts/manip_loco/play_go2_arm_ik_only.py b/scripts/manip_loco/play_go2_arm_ik_only.py index dce252814..c1090ec84 100644 --- a/scripts/manip_loco/play_go2_arm_ik_only.py +++ b/scripts/manip_loco/play_go2_arm_ik_only.py @@ -18,8 +18,8 @@ if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) -from unilab.envs.locomotion.go2_arm.base import build_go2_arm_position_gains -from unilab.envs.locomotion.go2_arm.manip_loco import Go2ArmManipLocoCfg +from unilab.tasks.locomotion.go2_arm.base import build_go2_arm_position_gains +from unilab.tasks.locomotion.go2_arm.manip_loco import Go2ArmManipLocoCfg TARGET_BODY = "ik_mocap_target" diff --git a/src/unilab/tools/bones_seed_csv.py b/scripts/motion/bones_seed_csv.py similarity index 100% rename from src/unilab/tools/bones_seed_csv.py rename to scripts/motion/bones_seed_csv.py diff --git a/scripts/motion/bones_seed_csv_to_npz.py b/scripts/motion/bones_seed_csv_to_npz.py index ed21defdf..2d4f8e80f 100644 --- a/scripts/motion/bones_seed_csv_to_npz.py +++ b/scripts/motion/bones_seed_csv_to_npz.py @@ -21,57 +21,39 @@ - ``body_lin_vel_w`` - ``body_ang_vel_w`` +Interpolation and velocity estimation reuse the library implementation in +``unilab.tasks.motion_tracking.common.motion_loader``; forward kinematics reuse +``unilab.base.backend.compute_tracking_fk``. + Usage: uv run scripts/motion/bones_seed_csv_to_npz.py uv run scripts/motion/bones_seed_csv_to_npz.py --dry-run uv run scripts/motion/bones_seed_csv_to_npz.py --input path/to/flip_090_001__A304.csv """ -# pyright: reportAttributeAccessIssue=false - from __future__ import annotations import argparse -from dataclasses import dataclass from pathlib import Path -import mujoco import numpy as np -from tqdm import tqdm - -from unilab.assets import ASSETS_ROOT_PATH -from unilab.base.backend.mujoco.xml import inject_mujoco_tracking_sensors -from unilab.tools.bones_seed_csv import ( +from scripts.motion.bones_seed_csv import ( ROOT_COLUMNS, euler_deg_to_quat_wxyz, load_header, parse_joint_names, resolve_input_files, ) -from unilab.utils.rotation import np_quat_angular_velocity, np_quat_ensure_continuity + +from unilab.assets import ASSETS_ROOT_PATH +from unilab.base.backend import compute_tracking_fk +from unilab.tasks.motion_tracking.common.motion_loader import interpolate_motion +from unilab.utils.rotation import np_quat_ensure_continuity DEFAULT_INPUT = "src/unilab/assets/motions/g1/flip" DEFAULT_OUTPUT_DIR = "src/unilab/assets/motions/g1/flip_npz" -def quat_slerp(q1: np.ndarray, q2: np.ndarray, t: float) -> np.ndarray: - """Spherical linear interpolation between two quaternions (wxyz format).""" - dot = np.dot(q1, q2) - if dot < 0: - q2 = -q2 - dot = -dot - - if dot > 0.9995: - result = q1 + t * (q2 - q1) - return result / np.linalg.norm(result) - - theta = np.arccos(np.clip(dot, -1, 1)) - sin_theta = np.sin(theta) - w1 = np.sin((1 - t) * theta) / sin_theta - w2 = np.sin(t * theta) / sin_theta - return w1 * q1 + w2 * q2 - - def default_model_path() -> str: return str(ASSETS_ROOT_PATH / "robots" / "g1" / "scene_flat.xml") @@ -102,182 +84,34 @@ def resolve_output_targets( return [output_root / f"{csv_file.stem}.npz" for csv_file in csv_files] -@dataclass -class MotionLoader: - motion_file: Path - input_fps: int - output_fps: int - position_scale: float - euler_order: str - - def __post_init__(self) -> None: - self.input_dt = 1.0 / self.input_fps - self.output_dt = 1.0 / self.output_fps - self._load_motion() - self._interpolate_motion() - self._compute_velocities() - - def _load_motion(self) -> None: - header = load_header(self.motion_file) - self.joint_names = parse_joint_names(header, self.motion_file) - - motion = np.loadtxt(self.motion_file, delimiter=",", dtype=np.float32, skiprows=1) - motion = np.atleast_2d(motion) - if motion.shape[1] != len(header): +def load_csv_motion( + motion_file: Path, + *, + position_scale: float, + euler_order: str, +) -> tuple[list[str], np.ndarray, np.ndarray, np.ndarray]: + """Load a BONES-SEED CSV into (joint_names, base/dof trajectory arrays).""" + header = load_header(motion_file) + joint_names = parse_joint_names(header, motion_file) + + motion = np.loadtxt(motion_file, delimiter=",", dtype=np.float32, skiprows=1) + motion = np.atleast_2d(motion) + if motion.shape[1] != len(header): + raise ValueError(f"{motion_file} has {motion.shape[1]} columns, expected {len(header)}") + + frames = motion[:, 0].astype(np.int32) + if frames.shape[0] > 1: + frame_diffs = np.diff(frames) + if not np.all(frame_diffs == 1): raise ValueError( - f"{self.motion_file} has {motion.shape[1]} columns, expected {len(header)}" + f"{motion_file} has non-contiguous Frame values: {np.unique(frame_diffs)}" ) - self.frames = motion[:, 0].astype(np.int32) - if self.frames.shape[0] > 1: - frame_diffs = np.diff(self.frames) - if not np.all(frame_diffs == 1): - raise ValueError( - f"{self.motion_file} has non-contiguous Frame values: {np.unique(frame_diffs)}" - ) - - self.motion_base_poss_input = motion[:, 1:4] * self.position_scale - self.motion_base_rots_input = euler_deg_to_quat_wxyz(motion[:, 4:7], self.euler_order) - self.motion_base_rots_input = np_quat_ensure_continuity(self.motion_base_rots_input) - self.motion_dof_poss_input = np.deg2rad(motion[:, len(ROOT_COLUMNS) :]) - - self.input_frames = motion.shape[0] - self.duration = (self.input_frames - 1) * self.input_dt - - def _interpolate_motion(self) -> None: - times = np.arange(0, self.duration, self.output_dt, dtype=np.float32) - self.output_frames = times.shape[0] - index_0, index_1, blend = self._compute_frame_blend(times) - - self.motion_base_poss = ( - self.motion_base_poss_input[index_0] * (1 - blend[:, None]) - + self.motion_base_poss_input[index_1] * blend[:, None] - ) - - self.motion_base_rots = np.zeros((self.output_frames, 4), dtype=np.float32) - for i in range(self.output_frames): - self.motion_base_rots[i] = quat_slerp( - self.motion_base_rots_input[index_0[i]], - self.motion_base_rots_input[index_1[i]], - blend[i], - ) - self.motion_base_rots = np_quat_ensure_continuity(self.motion_base_rots) - - self.motion_dof_poss = ( - self.motion_dof_poss_input[index_0] * (1 - blend[:, None]) - + self.motion_dof_poss_input[index_1] * blend[:, None] - ) - - def _compute_frame_blend(self, times: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - phase = times / self.duration - index_0 = np.floor(phase * (self.input_frames - 1)).astype(np.int32) - index_1 = np.minimum(index_0 + 1, self.input_frames - 1) - blend = phase * (self.input_frames - 1) - index_0 - return index_0, index_1, blend - - def _compute_velocities(self) -> None: - self.motion_base_lin_vels = np.gradient(self.motion_base_poss, self.output_dt, axis=0) - self.motion_dof_vels = np.gradient(self.motion_dof_poss, self.output_dt, axis=0) - self.motion_base_ang_vels = np_quat_angular_velocity(self.motion_base_rots, self.output_dt) - - -def run_simulation( - motion_loader: MotionLoader, - model_file: str, - output_file: Path, -) -> None: - tmp_model_path, _, _ = inject_mujoco_tracking_sensors(model_file) - try: - model = mujoco.MjModel.from_xml_path(tmp_model_path) - finally: - Path(tmp_model_path).unlink(missing_ok=True) - data = mujoco.MjData(model) - - joint_indices = [] - for name in motion_loader.joint_names: - jnt_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, name) - if jnt_id < 0: - raise ValueError(f"Joint '{name}' not found in model") - joint_indices.append(jnt_id) - - num_frames = motion_loader.output_frames - num_joints = len(joint_indices) - num_bodies = model.nbody - - joint_pos = np.zeros((num_frames, num_joints), dtype=np.float32) - joint_vel = np.zeros((num_frames, num_joints), dtype=np.float32) - body_pos_w = np.zeros((num_frames, num_bodies, 3), dtype=np.float32) - body_quat_w = np.zeros((num_frames, num_bodies, 4), dtype=np.float32) - body_lin_vel_w = np.zeros((num_frames, num_bodies, 3), dtype=np.float32) - body_ang_vel_w = np.zeros((num_frames, num_bodies, 3), dtype=np.float32) - - sensor_adrs = np.full((num_bodies, 4), -1, dtype=np.int32) - sensor_dims = np.array([3, 4, 3, 3], dtype=np.int32) - sensor_prefixes = ( - "track_pos_w_", - "track_quat_w_", - "track_linvel_w_", - "track_angvel_w_", - ) - for body_id in range(num_bodies): - body_name = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_BODY, body_id) - if not body_name: - continue - for k, prefix in enumerate(sensor_prefixes): - sensor_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_SENSOR, f"{prefix}{body_name}") - if sensor_id >= 0: - sensor_adrs[body_id, k] = model.sensor_adr[sensor_id] - - for i in tqdm(range(num_frames), desc=output_file.stem, leave=False): - data.qpos[0:3] = motion_loader.motion_base_poss[i] - data.qpos[3:7] = motion_loader.motion_base_rots[i] - data.qvel[0:3] = motion_loader.motion_base_lin_vels[i] - data.qvel[3:6] = motion_loader.motion_base_ang_vels[i] - - for j, jnt_id in enumerate(joint_indices): - qpos_adr = model.jnt_qposadr[jnt_id] - qvel_adr = model.jnt_dofadr[jnt_id] - data.qpos[qpos_adr] = motion_loader.motion_dof_poss[i, j] - data.qvel[qvel_adr] = motion_loader.motion_dof_vels[i, j] - - mujoco.mj_forward(model, data) - - for j, jnt_id in enumerate(joint_indices): - qpos_adr = model.jnt_qposadr[jnt_id] - qvel_adr = model.jnt_dofadr[jnt_id] - joint_pos[i, j] = data.qpos[qpos_adr] - joint_vel[i, j] = data.qvel[qvel_adr] - - for body_id in range(num_bodies): - pos_adr, quat_adr, lin_adr, ang_adr = sensor_adrs[body_id] - - if pos_adr >= 0: - body_pos_w[i, body_id] = data.sensordata[pos_adr : pos_adr + sensor_dims[0]] - else: - body_pos_w[i, body_id] = data.xpos[body_id] - - if quat_adr >= 0: - body_quat_w[i, body_id] = data.sensordata[quat_adr : quat_adr + sensor_dims[1]] - else: - body_quat_w[i, body_id] = data.xquat[body_id] - - if lin_adr >= 0: - body_lin_vel_w[i, body_id] = data.sensordata[lin_adr : lin_adr + sensor_dims[2]] - - if ang_adr >= 0: - body_ang_vel_w[i, body_id] = data.sensordata[ang_adr : ang_adr + sensor_dims[3]] - - output_file.parent.mkdir(parents=True, exist_ok=True) - np.savez( - output_file, - fps=np.array([motion_loader.output_fps], dtype=np.int32), - joint_pos=joint_pos, - joint_vel=joint_vel, - body_pos_w=body_pos_w, - body_quat_w=body_quat_w, - body_lin_vel_w=body_lin_vel_w, - body_ang_vel_w=body_ang_vel_w, - ) + motion_base_poss_input = motion[:, 1:4] * position_scale + motion_base_rots_input = euler_deg_to_quat_wxyz(motion[:, 4:7], euler_order) + motion_base_rots_input = np_quat_ensure_continuity(motion_base_rots_input) + motion_dof_poss_input = np.deg2rad(motion[:, len(ROOT_COLUMNS) :]) + return joint_names, motion_base_poss_input, motion_base_rots_input, motion_dof_poss_input def print_plan( @@ -331,16 +165,40 @@ def convert(args: argparse.Namespace) -> None: run_dry_run(csv_files, output_files) return + input_fps = int(args.input_fps) + output_fps = int(args.output_fps) for csv_file, output_file in zip(csv_files, output_files, strict=True): print(f"[bones_seed_csv_to_npz] Converting {csv_file} -> {output_file}") - motion_loader = MotionLoader( - motion_file=csv_file, - input_fps=int(args.input_fps), - output_fps=int(args.output_fps), + joint_names, base_poss_input, base_rots_input, dof_poss_input = load_csv_motion( + csv_file, position_scale=args.position_scale, euler_order=args.euler_order, ) - run_simulation(motion_loader, model_file, output_file) + motion = interpolate_motion( + base_poss_input, + base_rots_input, + dof_poss_input, + input_fps=input_fps, + output_fps=output_fps, + ) + arrays = compute_tracking_fk( + model_file, + joint_names=joint_names, + base_poss=motion.base_poss, + base_rots=motion.base_rots, + base_lin_vels=motion.base_lin_vels, + base_ang_vels=motion.base_ang_vels, + dof_poss=motion.dof_poss, + dof_vels=motion.dof_vels, + progress=True, + progress_desc=output_file.stem, + ) + output_file.parent.mkdir(parents=True, exist_ok=True) + np.savez( + output_file, + fps=np.array([output_fps], dtype=np.int32), + **arrays, + ) def parse_args() -> argparse.Namespace: diff --git a/scripts/motion/csv_to_npz.py b/scripts/motion/csv_to_npz.py index 2794a7114..6a41fa83a 100644 --- a/scripts/motion/csv_to_npz.py +++ b/scripts/motion/csv_to_npz.py @@ -17,252 +17,44 @@ - body_quat_w: Body quaternions in world frame (N_frames × N_bodies × 4, wxyz) - body_lin_vel_w: Body linear velocities (N_frames × N_bodies × 3) - body_ang_vel_w: Body angular velocities (N_frames × N_bodies × 3) -""" -# pyright: reportAttributeAccessIssue=false +Interpolation and velocity estimation reuse the library implementation in +``unilab.tasks.motion_tracking.common.motion_loader``; forward kinematics reuse +``unilab.base.backend.compute_tracking_fk``. +""" import argparse from pathlib import Path -import mujoco import numpy as np -from tqdm import tqdm from unilab.assets import ASSETS_ROOT_PATH -from unilab.base.backend.mujoco.xml import inject_mujoco_tracking_sensors -from unilab.utils.rotation import np_quat_angular_velocity, np_quat_ensure_continuity - - -def quat_slerp(q1: np.ndarray, q2: np.ndarray, t: float) -> np.ndarray: - """Spherical linear interpolation between two quaternions (wxyz format).""" - # Ensure shortest path - dot = np.dot(q1, q2) - if dot < 0: - q2 = -q2 - dot = -dot - - # If quaternions are very close, use linear interpolation - if dot > 0.9995: - result = q1 + t * (q2 - q1) - return result / np.linalg.norm(result) - - # Compute angle - theta = np.arccos(np.clip(dot, -1, 1)) - sin_theta = np.sin(theta) - - # Compute interpolation weights - w1 = np.sin((1 - t) * theta) / sin_theta - w2 = np.sin(t * theta) / sin_theta - - return w1 * q1 + w2 * q2 - - -class MotionLoader: - """Load and interpolate motion from CSV file.""" - - def __init__( - self, - motion_file: str, - input_fps: int, - output_fps: int, - line_range: tuple[int, int] | None = None, - ): - self.motion_file = motion_file - self.input_fps = input_fps - self.output_fps = output_fps - self.input_dt = 1.0 / self.input_fps - self.output_dt = 1.0 / self.output_fps - self.line_range = line_range - self._load_motion() - self._interpolate_motion() - self._compute_velocities() - - def _load_motion(self): - """Load motion from CSV file.""" - if self.line_range is None: - motion = np.loadtxt(self.motion_file, delimiter=",", dtype=np.float32, skiprows=1) - else: - motion = np.loadtxt( - self.motion_file, - delimiter=",", - skiprows=max(1, self.line_range[0] - 1), - max_rows=self.line_range[1] - self.line_range[0] + 1, - dtype=np.float32, - ) - - self.motion_base_poss_input = motion[:, :3] - # Convert quaternion from xyzw to wxyz - self.motion_base_rots_input = motion[:, 3:7][:, [3, 0, 1, 2]] - self.motion_base_rots_input = np_quat_ensure_continuity(self.motion_base_rots_input) - self.motion_dof_poss_input = motion[:, 7:] - - self.input_frames = motion.shape[0] - self.duration = (self.input_frames - 1) * self.input_dt - - def _interpolate_motion(self): - """Interpolate motion to output FPS.""" - times = np.arange(0, self.duration, self.output_dt, dtype=np.float32) - self.output_frames = times.shape[0] - index_0, index_1, blend = self._compute_frame_blend(times) - - # Linear interpolation for positions - self.motion_base_poss = ( - self.motion_base_poss_input[index_0] * (1 - blend[:, None]) - + self.motion_base_poss_input[index_1] * blend[:, None] +from unilab.base.backend import compute_tracking_fk +from unilab.tasks.motion_tracking.common.motion_loader import interpolate_motion +from unilab.utils.rotation import np_quat_ensure_continuity + + +def load_csv_motion( + motion_file: str, line_range: tuple[int, int] | None = None +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Load a Unitree-convention CSV into base/dof trajectory arrays.""" + if line_range is None: + motion = np.loadtxt(motion_file, delimiter=",", dtype=np.float32, skiprows=1) + else: + motion = np.loadtxt( + motion_file, + delimiter=",", + skiprows=max(1, line_range[0] - 1), + max_rows=line_range[1] - line_range[0] + 1, + dtype=np.float32, ) - # Spherical linear interpolation for quaternions - self.motion_base_rots = np.zeros((self.output_frames, 4), dtype=np.float32) - for i in range(self.output_frames): - self.motion_base_rots[i] = quat_slerp( - self.motion_base_rots_input[index_0[i]], - self.motion_base_rots_input[index_1[i]], - blend[i], - ) - self.motion_base_rots = np_quat_ensure_continuity(self.motion_base_rots) - - # Linear interpolation for joint positions - self.motion_dof_poss = ( - self.motion_dof_poss_input[index_0] * (1 - blend[:, None]) - + self.motion_dof_poss_input[index_1] * blend[:, None] - ) - - print( - f"Motion interpolated: {self.input_frames} frames @ {self.input_fps} Hz " - f"→ {self.output_frames} frames @ {self.output_fps} Hz" - ) - - def _compute_frame_blend(self, times: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Compute frame indices and blend weights for interpolation.""" - phase = times / self.duration - index_0 = np.floor(phase * (self.input_frames - 1)).astype(np.int32) - index_1 = np.minimum(index_0 + 1, self.input_frames - 1) - blend = phase * (self.input_frames - 1) - index_0 - return index_0, index_1, blend - - def _compute_velocities(self): - """Compute velocities using numerical differentiation.""" - # Linear velocities - self.motion_base_lin_vels = np.gradient(self.motion_base_poss, self.output_dt, axis=0) - self.motion_dof_vels = np.gradient(self.motion_dof_poss, self.output_dt, axis=0) - - # Angular velocities from quaternion derivatives - self.motion_base_ang_vels = np_quat_angular_velocity(self.motion_base_rots, self.output_dt) - - -def run_simulation( - motion_loader: MotionLoader, - model_file: str, - joint_names: list[str], - output_file: str, -): - """Run MuJoCo simulation to compute forward kinematics for the export.""" - # Inject track_* sensors so exported body_* fields match training-time semantics. - tmp_model_path, _, _ = inject_mujoco_tracking_sensors(model_file) - try: - model = mujoco.MjModel.from_xml_path(tmp_model_path) - print(f"Model loaded from {tmp_model_path}") - finally: - Path(tmp_model_path).unlink(missing_ok=True) - data = mujoco.MjData(model) - - # Get joint indices - joint_indices = [] - for name in joint_names: - jnt_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, name) - if jnt_id < 0: - raise ValueError(f"Joint '{name}' not found in model") - joint_indices.append(jnt_id) - - # Prepare output arrays - num_frames = motion_loader.output_frames - num_joints = len(joint_indices) - num_bodies = model.nbody - - joint_pos = np.zeros((num_frames, num_joints), dtype=np.float32) - joint_vel = np.zeros((num_frames, num_joints), dtype=np.float32) - body_pos_w = np.zeros((num_frames, num_bodies, 3), dtype=np.float32) - body_quat_w = np.zeros((num_frames, num_bodies, 4), dtype=np.float32) - body_lin_vel_w = np.zeros((num_frames, num_bodies, 3), dtype=np.float32) - body_ang_vel_w = np.zeros((num_frames, num_bodies, 3), dtype=np.float32) - - # Keep NPZ in model body-id layout (nbody), but read from track_* sensors for - # named bodies to align with backend.get_body_*_w semantics used in training. - sensor_adrs = np.full((num_bodies, 4), -1, dtype=np.int32) - sensor_dims = np.array([3, 4, 3, 3], dtype=np.int32) - sensor_prefixes = ( - "track_pos_w_", - "track_quat_w_", - "track_linvel_w_", - "track_angvel_w_", - ) - for body_id in range(num_bodies): - body_name = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_BODY, body_id) - if not body_name: - continue - for k, prefix in enumerate(sensor_prefixes): - sensor_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_SENSOR, f"{prefix}{body_name}") - if sensor_id >= 0: - sensor_adrs[body_id, k] = model.sensor_adr[sensor_id] - - print(f"\nProcessing {num_frames} frames...") - for i in tqdm(range(num_frames)): - # Set root state - data.qpos[0:3] = motion_loader.motion_base_poss[i] - data.qpos[3:7] = motion_loader.motion_base_rots[i] - data.qvel[0:3] = motion_loader.motion_base_lin_vels[i] - data.qvel[3:6] = motion_loader.motion_base_ang_vels[i] - - # Set joint states - for j, jnt_id in enumerate(joint_indices): - qpos_adr = model.jnt_qposadr[jnt_id] - qvel_adr = model.jnt_dofadr[jnt_id] - data.qpos[qpos_adr] = motion_loader.motion_dof_poss[i, j] - data.qvel[qvel_adr] = motion_loader.motion_dof_vels[i, j] - - # Run forward pass so kinematics and sensors are up-to-date. - mujoco.mj_forward(model, data) - - # Extract joint states - for j, jnt_id in enumerate(joint_indices): - qpos_adr = model.jnt_qposadr[jnt_id] - qvel_adr = model.jnt_dofadr[jnt_id] - joint_pos[i, j] = data.qpos[qpos_adr] - joint_vel[i, j] = data.qvel[qvel_adr] - - # Extract body states - for body_id in range(num_bodies): - pos_adr, quat_adr, lin_adr, ang_adr = sensor_adrs[body_id] - - if pos_adr >= 0: - body_pos_w[i, body_id] = data.sensordata[pos_adr : pos_adr + sensor_dims[0]] - else: - body_pos_w[i, body_id] = data.xpos[body_id] - - if quat_adr >= 0: - body_quat_w[i, body_id] = data.sensordata[quat_adr : quat_adr + sensor_dims[1]] - else: - body_quat_w[i, body_id] = data.xquat[body_id] - - if lin_adr >= 0: - body_lin_vel_w[i, body_id] = data.sensordata[lin_adr : lin_adr + sensor_dims[2]] - - if ang_adr >= 0: - body_ang_vel_w[i, body_id] = data.sensordata[ang_adr : ang_adr + sensor_dims[3]] - - # Save to NPZ - print(f"\nSaving to {output_file}...") - np.savez( - output_file, - fps=np.array([motion_loader.output_fps], dtype=np.int32), - joint_pos=joint_pos, - joint_vel=joint_vel, - body_pos_w=body_pos_w, - body_quat_w=body_quat_w, - body_lin_vel_w=body_lin_vel_w, - body_ang_vel_w=body_ang_vel_w, - ) - print("Done!") + motion_base_poss_input = motion[:, :3] + # Convert quaternion from xyzw to wxyz + motion_base_rots_input = motion[:, 3:7][:, [3, 0, 1, 2]] + motion_base_rots_input = np_quat_ensure_continuity(motion_base_rots_input) + motion_dof_poss_input = motion[:, 7:] + return motion_base_poss_input, motion_base_rots_input, motion_dof_poss_input def main(): @@ -363,16 +155,47 @@ def main(): "right_wrist_yaw_joint", ] + input_fps = int(args.input_fps) + output_fps = int(args.output_fps) + # Load and interpolate motion - motion_loader = MotionLoader( + base_poss_input, base_rots_input, dof_poss_input = load_csv_motion( args.input_file, - int(args.input_fps), - int(args.output_fps), (args.line_range[0], args.line_range[1]) if args.line_range else None, ) + motion = interpolate_motion( + base_poss_input, + base_rots_input, + dof_poss_input, + input_fps=input_fps, + output_fps=output_fps, + ) + print( + f"Motion interpolated: {base_poss_input.shape[0]} frames @ {input_fps} Hz " + f"→ {motion.output_frames} frames @ {output_fps} Hz" + ) + + # Run forward kinematics and save to NPZ + print(f"\nProcessing {motion.output_frames} frames...") + arrays = compute_tracking_fk( + args.model_file, + joint_names=joint_names, + base_poss=motion.base_poss, + base_rots=motion.base_rots, + base_lin_vels=motion.base_lin_vels, + base_ang_vels=motion.base_ang_vels, + dof_poss=motion.dof_poss, + dof_vels=motion.dof_vels, + progress=True, + ) - # Run simulation - run_simulation(motion_loader, args.model_file, joint_names, args.output_file) + print(f"\nSaving to {args.output_file}...") + np.savez( + args.output_file, + fps=np.array([output_fps], dtype=np.int32), + **arrays, + ) + print("Done!") if __name__ == "__main__": diff --git a/scripts/motion/remap_fullbody_npz.py b/scripts/motion/remap_fullbody_npz.py index b3e2cafb7..e1a3ff4a9 100644 --- a/scripts/motion/remap_fullbody_npz.py +++ b/scripts/motion/remap_fullbody_npz.py @@ -22,7 +22,7 @@ import numpy as np from unilab.assets import ASSETS_ROOT_PATH -from unilab.base.backend.mujoco.xml import _get_named_bodies +from unilab.base.backend import get_named_bodies # MuJoCo root free-joint sizes _ROOT_QPOS_DIM = 7 # 3 pos + 4 quat @@ -57,7 +57,7 @@ def remap_npz(input_path: str, output_path: str, model_file: str, *, dry_run: bo source_body_names: list[str] = data["body_names"].tolist() # --- target body list from training model -------------------------------- - _, named_bodies = _get_named_bodies(model_file) + _, named_bodies = get_named_bodies(model_file) target_body_names = ["world"] + named_bodies # prepend MuJoCo implicit body 0 remap = _build_body_remap(source_body_names, target_body_names) diff --git a/scripts/motion/replay_bones_seed_csv.py b/scripts/motion/replay_bones_seed_csv.py index c791ff8f6..553230026 100644 --- a/scripts/motion/replay_bones_seed_csv.py +++ b/scripts/motion/replay_bones_seed_csv.py @@ -36,9 +36,7 @@ import mujoco import mujoco.viewer import numpy as np - -from unilab.assets import ASSETS_ROOT_PATH -from unilab.tools.bones_seed_csv import ( +from scripts.motion.bones_seed_csv import ( ROOT_COLUMNS, euler_deg_to_quat_wxyz, load_header, @@ -46,6 +44,8 @@ resolve_input_files, ) +from unilab.assets import ASSETS_ROOT_PATH + DEFAULT_INPUT = "src/unilab/assets/motions/g1/flip" diff --git a/scripts/motion/x2_csv_to_tracking_npz.py b/scripts/motion/x2_csv_to_tracking_npz.py index 3892eaba2..d32a08d27 100644 --- a/scripts/motion/x2_csv_to_tracking_npz.py +++ b/scripts/motion/x2_csv_to_tracking_npz.py @@ -8,6 +8,10 @@ The output layout matches the existing X2 ``*_g1format.npz`` assets consumed by the shared humanoid motion-tracking loader. + +Velocity estimation reuses the library implementation in +``unilab.tasks.motion_tracking.common.motion_loader``; forward kinematics reuse +``unilab.base.backend.compute_tracking_fk``. """ from __future__ import annotations @@ -19,8 +23,12 @@ import numpy as np from unilab.assets import ASSETS_ROOT_PATH -from unilab.base.backend.mujoco.xml import inject_mujoco_tracking_sensors -from unilab.utils.rotation import np_quat_angular_velocity, np_quat_ensure_continuity +from unilab.base.backend import compute_tracking_fk +from unilab.tasks.motion_tracking.common.motion_loader import ( + compute_motion_velocities, + quat_slerp, +) +from unilab.utils.rotation import np_quat_ensure_continuity ROOT_QPOS_DIM = 7 ROOT_QVEL_DIM = 6 @@ -29,24 +37,6 @@ DEFAULT_MODEL = ASSETS_ROOT_PATH / "robots" / "x2" / "x2_simple_collision.xml" -def _quat_slerp(q1: np.ndarray, q2: np.ndarray, t: float) -> np.ndarray: - q1 = q1.astype(np.float64, copy=False) - q2 = q2.astype(np.float64, copy=False) - dot = float(np.dot(q1, q2)) - if dot < 0.0: - q2 = -q2 - dot = -dot - if dot > 0.9995: - result = q1 + t * (q2 - q1) - return (result / np.linalg.norm(result)).astype(np.float32) - - theta = np.arccos(np.clip(dot, -1.0, 1.0)) - sin_theta = np.sin(theta) - w1 = np.sin((1.0 - t) * theta) / sin_theta - w2 = np.sin(t * theta) / sin_theta - return (w1 * q1 + w2 * q2).astype(np.float32) - - def _load_csv_qpos(input_path: Path, model_nq: int) -> np.ndarray: try: raw = np.loadtxt(input_path, delimiter=",", dtype=np.float32) @@ -92,11 +82,12 @@ def _resample_qpos(qpos: np.ndarray, input_fps: int, output_fps: int) -> np.ndar out = np.empty((output_times.shape[0], qpos.shape[1]), dtype=np.float32) out[:, :3] = qpos[index_0, :3] * (1.0 - blend[:, None]) + qpos[index_1, :3] * blend[:, None] for frame, t in enumerate(blend): - out[frame, 3:7] = _quat_slerp( - qpos[index_0[frame], 3:7], - qpos[index_1[frame], 3:7], + # Interpolate in float64, then cast back to float32. + out[frame, 3:7] = quat_slerp( + qpos[index_0[frame], 3:7].astype(np.float64), + qpos[index_1[frame], 3:7].astype(np.float64), float(t), - ) + ).astype(np.float32) out[:, 7:] = qpos[index_0, 7:] * (1.0 - blend[:, None]) + qpos[index_1, 7:] * blend[:, None] out[:, 3:7] = np_quat_ensure_continuity(out[:, 3:7]) return out @@ -104,10 +95,13 @@ def _resample_qpos(qpos: np.ndarray, input_fps: int, output_fps: int) -> np.ndar def _qvel_from_qpos(qpos: np.ndarray, fps: int) -> np.ndarray: dt = 1.0 / float(fps) + base_lin_vels, base_ang_vels, dof_vels = compute_motion_velocities( + qpos[:, :3], qpos[:, 3:7], qpos[:, 7:], dt + ) qvel = np.empty((qpos.shape[0], qpos.shape[1] - 1), dtype=np.float32) - qvel[:, :3] = np.gradient(qpos[:, :3], dt, axis=0).astype(np.float32) - qvel[:, 3:6] = np_quat_angular_velocity(qpos[:, 3:7], dt).astype(np.float32) - qvel[:, 6:] = np.gradient(qpos[:, 7:], dt, axis=0).astype(np.float32) + qvel[:, :3] = base_lin_vels.astype(np.float32) + qvel[:, 3:6] = base_ang_vels.astype(np.float32) + qvel[:, 6:] = dof_vels.astype(np.float32) return qvel @@ -123,75 +117,6 @@ def _target_joint_names(model: mujoco.MjModel) -> list[str]: return names -def _sensor_addresses(model: mujoco.MjModel) -> np.ndarray: - sensor_adrs = np.full((model.nbody, 4), -1, dtype=np.int32) - prefixes = ( - "track_pos_w_", - "track_quat_w_", - "track_linvel_w_", - "track_angvel_w_", - ) - for body_id in range(model.nbody): - body_name = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_BODY, body_id) - if not body_name: - continue - for slot, prefix in enumerate(prefixes): - sensor_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_SENSOR, prefix + body_name) - if sensor_id >= 0: - sensor_adrs[body_id, slot] = int(model.sensor_adr[sensor_id]) - return sensor_adrs - - -def _export_body_arrays( - model: mujoco.MjModel, - qpos: np.ndarray, - qvel: np.ndarray, - target_joint_names: list[str], -) -> dict[str, np.ndarray]: - data = mujoco.MjData(model) - frames = qpos.shape[0] - body_pos_w = np.zeros((frames, model.nbody, 3), dtype=np.float32) - body_quat_w = np.zeros((frames, model.nbody, 4), dtype=np.float32) - body_lin_vel_w = np.zeros((frames, model.nbody, 3), dtype=np.float32) - body_ang_vel_w = np.zeros((frames, model.nbody, 3), dtype=np.float32) - sensor_adrs = _sensor_addresses(model) - - joint_ids = [ - mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, name) for name in target_joint_names - ] - if any(joint_id < 0 for joint_id in joint_ids): - raise ValueError("Target joint list contains joints not found in model") - - for frame in range(frames): - data.qpos[:] = qpos[frame] - data.qvel[:] = qvel[frame] - mujoco.mj_forward(model, data) - - for body_id in range(model.nbody): - pos_adr, quat_adr, lin_adr, ang_adr = sensor_adrs[body_id] - if pos_adr >= 0: - body_pos_w[frame, body_id] = data.sensordata[pos_adr : pos_adr + 3] - else: - body_pos_w[frame, body_id] = data.xpos[body_id] - - if quat_adr >= 0: - body_quat_w[frame, body_id] = data.sensordata[quat_adr : quat_adr + 4] - else: - body_quat_w[frame, body_id] = data.xquat[body_id] - - if lin_adr >= 0: - body_lin_vel_w[frame, body_id] = data.sensordata[lin_adr : lin_adr + 3] - if ang_adr >= 0: - body_ang_vel_w[frame, body_id] = data.sensordata[ang_adr : ang_adr + 3] - - return { - "body_pos_w": body_pos_w, - "body_quat_w": body_quat_w, - "body_lin_vel_w": body_lin_vel_w, - "body_ang_vel_w": body_ang_vel_w, - } - - def convert_csv( input_path: Path, output_path: Path, @@ -200,22 +125,17 @@ def convert_csv( output_fps: int, dry_run: bool, ) -> None: - tmp_model_path, _, _ = inject_mujoco_tracking_sensors(str(model_file)) - try: - model = mujoco.MjModel.from_xml_path(tmp_model_path) - finally: - Path(tmp_model_path).unlink(missing_ok=True) + model = mujoco.MjModel.from_xml_path(str(model_file)) target_names = _target_joint_names(model) qpos_input = _load_csv_qpos(input_path, model.nq) qpos = _resample_qpos(qpos_input, input_fps, output_fps) qvel = _qvel_from_qpos(qpos, output_fps) - joint_pos = qpos[:, ROOT_QPOS_DIM:].astype(np.float32) - joint_vel = qvel[:, ROOT_QVEL_DIM:].astype(np.float32) - if joint_pos.shape[1] != len(target_names): + if qpos.shape[1] - ROOT_QPOS_DIM != len(target_names): raise ValueError( - f"CSV joint count {joint_pos.shape[1]} does not match model joints {len(target_names)}" + f"CSV joint count {qpos.shape[1] - ROOT_QPOS_DIM} does not match model joints " + f"{len(target_names)}" ) print(f"Source : {input_path}") @@ -223,7 +143,7 @@ def convert_csv( print(f"Output : {output_path}") print(f"frames : {qpos_input.shape[0]} -> {qpos.shape[0]}") print(f"fps : {input_fps} -> {output_fps}") - print(f"joints : {joint_pos.shape[1]}") + print(f"joints : {qpos.shape[1] - ROOT_QPOS_DIM}") print(f"bodies : {model.nbody} (MuJoCo body-id layout, including world)") if dry_run: @@ -231,13 +151,20 @@ def convert_csv( return output_path.parent.mkdir(parents=True, exist_ok=True) - body_arrays = _export_body_arrays(model, qpos, qvel, target_names) + arrays = compute_tracking_fk( + str(model_file), + joint_names=target_names, + base_poss=qpos[:, :3], + base_rots=qpos[:, 3:7], + base_lin_vels=qvel[:, :3], + base_ang_vels=qvel[:, 3:6], + dof_poss=qpos[:, ROOT_QPOS_DIM:], + dof_vels=qvel[:, ROOT_QVEL_DIM:], + ) np.savez( output_path, fps=np.array([output_fps], dtype=np.int32), - joint_pos=joint_pos, - joint_vel=joint_vel, - **body_arrays, + **arrays, ) diff --git a/scripts/play_a2arm_pos_force_interactive.py b/scripts/play_a2arm_pos_force_interactive.py new file mode 100644 index 000000000..e781c2b5b --- /dev/null +++ b/scripts/play_a2arm_pos_force_interactive.py @@ -0,0 +1,199 @@ +"""Interactive Manager-Based MuJoCo playback for A2Arm position-force CSE-PPO.""" + +from __future__ import annotations + +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, cast + +import hydra +import torch +from omegaconf import DictConfig, OmegaConf, open_dict + +ROOT_DIR = Path(__file__).parent.parent +SRC_DIR = ROOT_DIR / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +from unilab.algos.cse_ppo import CSEOnPolicyRunner +from unilab.algos.rsl_rl import RslRlVecEnvWrapper +from unilab.base.backend import materialize_scene_visual_override +from unilab.base.backend.mujoco.playback import resolve_render_play_model_files +from unilab.base.config_adapter import BackendAdapter, create_env +from unilab.tasks.locomotion.a2arm.state import A2ArmPosForceState +from unilab.training import algo_config_dict, ensure_registries, parse_checkpoint_path +from unilab.visualization.a2arm_pos_force import ( + clear_teleop_override, + draw_markers, + install_teleop_override, + make_key_callback, + make_teleop_from_state, + print_legend, +) + + +def _select_device(cfg: DictConfig) -> str: + configured = OmegaConf.select(cfg, "training.device") + if configured: + return str(configured) + if torch.cuda.is_available(): + return "cuda" + if torch.backends.mps.is_available(): + return "mps" + return "cpu" + + +def _backend_adapter(cfg: DictConfig) -> BackendAdapter: + return BackendAdapter( + cfg, + root_dir=ROOT_DIR, + algo_name="ppo_cse", + scene_materializer=materialize_scene_visual_override, + ) + + +def _interactive_env_cfg_override(cfg: DictConfig) -> dict[str, Any]: + play_cfg = cast( + DictConfig, + OmegaConf.merge(cfg, {"training": {"play_only": True}}), + ) + with open_dict(play_cfg): + play_cfg.play_profile = OmegaConf.to_container( + cfg.interactive_play_profile, + resolve=False, + ) + return cast(dict[str, Any], _backend_adapter(play_cfg).build_play_env_cfg_override()) + + +def _load_checkpoint(cfg: DictConfig) -> Path | None: + path, _directory = parse_checkpoint_path(cfg, root_dir=ROOT_DIR) + if path is None or not path.is_file(): + print( + "[play] checkpoint not found; pass algo.load_run= " + "and optionally algo.checkpoint=" + ) + return None + keys = set(torch.load(path, map_location="cpu", weights_only=True).keys()) + if "actor_state_dict" not in keys: + print(f"[play] {path} is not a CSE-PPO checkpoint (keys={sorted(keys)}).") + return None + return path + + +def _print_force_estimate(runner: CSEOnPolicyRunner, env: Any, obs: torch.Tensor) -> None: + state: A2ArmPosForceState = env.command_manager.get_term("task_state") + pred = runner.actor_critic.estimator.predict(obs).detach().cpu().numpy()[0] + # CSE target layout is [base velocity, EE sphere, EE force, base force]. + ee_est = pred[6:9] / 0.01 + base_est = pred[9:12] / 0.01 + print( + f"[force] EE est={ee_est.round(1)} true={state.force_ee_world[0].round(1)} | " + f"base est={base_est.round(1)} true={state.force_base_world[0].round(1)}" + ) + + +def _load_viewer_model(env: Any) -> Any: + import mujoco + + with tempfile.TemporaryDirectory(prefix="unilab-a2arm-interactive-") as tmp_dir: + model_files = resolve_render_play_model_files(env, num_envs=1, tmp_dir=tmp_dir) + model_file = model_files[0] if isinstance(model_files, list) else model_files + if Path(model_file).suffix.lower() == ".mjb": + return mujoco.MjModel.from_binary_path(str(model_file)) + return mujoco.MjModel.from_xml_path(str(model_file)) + + +def play_interactive(cfg: DictConfig, device: str) -> None: + import mujoco + import mujoco.viewer + + checkpoint = _load_checkpoint(cfg) + if checkpoint is None: + return + override = _interactive_env_cfg_override(cfg) + env = create_env( + cfg, + num_envs=1, + env_cfg_override=override, + sim_backend="mujoco", + ) + wrapped = RslRlVecEnvWrapper(env, device=device) + runner = CSEOnPolicyRunner(wrapped, algo_config_dict(cfg), log_dir=None, device=device) + runner.load(str(checkpoint)) + policy = runner.get_inference_policy(device=device) + + state: A2ArmPosForceState = env.command_manager.get_term("task_state") + teleop = make_teleop_from_state(state) + paused = {"value": False} + reset_requested = {"value": False} + show_range = {"value": True} + + def toggle_pause() -> None: + paused["value"] = not paused["value"] + print(f"[play] {'paused' if paused['value'] else 'resumed'}") + + def request_reset() -> None: + reset_requested["value"] = True + + def toggle_range() -> None: + show_range["value"] = not show_range["value"] + print(f"[play] sample range {'on' if show_range['value'] else 'off'}") + + callback = make_key_callback( + teleop, + on_pause=toggle_pause, + on_reset=request_reset, + on_toggle_range=toggle_range, + ) + + obs_td, _info = wrapped.reset() + obs = obs_td["actor"] + install_teleop_override(env, teleop) + model = _load_viewer_model(env) + data = mujoco.MjData(model) + print_legend() + print(f"[play] loading {checkpoint}; close the viewer or press Escape to quit") + diagnostics = 0 + with mujoco.viewer.launch_passive(model, data, key_callback=callback) as viewer: + viewer.cam.distance = 2.5 + with torch.inference_mode(): + while viewer.is_running(): + started = time.perf_counter() + if reset_requested["value"]: + reset_requested["value"] = False + obs = wrapped.reset()[0]["actor"] + teleop.reset() + install_teleop_override(env, teleop) + print("[play] reset") + if not paused["value"]: + teleop.advance_forces() + install_teleop_override(env, teleop) + obs = wrapped.step(policy(obs))[0]["actor"] + diagnostics += 1 + if diagnostics % 25 == 0: + _print_force_estimate(runner, env, obs) + + physics = env.get_physics_state_snapshot()[0] + mujoco.mj_setState(model, data, physics, mujoco.mjtState.mjSTATE_FULLPHYSICS) + mujoco.mj_forward(model, data) + draw_markers(viewer, env, show_range=show_range["value"]) + viewer.sync() + remaining = float(env.step_dt) - (time.perf_counter() - started) + if remaining > 0.0: + time.sleep(remaining) + clear_teleop_override(env) + env.close() + + +@hydra.main(version_base="1.3", config_path="../conf/ppo_cse", config_name="config") +def main(cfg: DictConfig) -> None: + ensure_registries() + play_interactive(cfg, _select_device(cfg)) + + +if __name__ == "__main__": + main() diff --git a/scripts/play_interactive.py b/scripts/play_interactive.py index e812c11e1..f5e673740 100644 --- a/scripts/play_interactive.py +++ b/scripts/play_interactive.py @@ -49,27 +49,36 @@ if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) +from unilab.algos.rsl_rl import ( + RslRlVecEnvWrapper, + get_policy_obs_dims, + normalize_ppo_train_cfg, +) from unilab.training import ( + algo_config_dict, ensure_registries, +) +from unilab.utils.checkpoint import ( get_entrypoint_log_root, resolve_task_checkpoint_path, ) -from unilab.training.offpolicy import build_offpolicy_env_cfg_override -from unilab.training.rsl_rl import ( - RslRlVecEnvWrapper, - get_policy_obs_dims, - normalize_ppo_train_cfg, -) +from unilab.utils.rotation import np_matrix_from_quat from unilab.visualization.interactive_playback import ( _HORA_DISTILL_CHECKPOINT_UNAVAILABLE, KeyboardCommander, PlaybackControls, - RslRlPlaybackConfig, + PlayInteractiveArgs, + available_backends_for_task, + build_offpolicy_env_cfg_override, + build_play_backend_adapter, + build_playback_config, create_appo_playback_session, create_hora_distill_playback_session, create_rsl_rl_playback_session, create_sac_playback_session, + infer_checkpoint_actor_input_dim, make_sim2sim_preflight, + normalize_checkpoint_value, prepare_motion_overlay_selection, select_torch_device, ) @@ -107,74 +116,6 @@ import mujoco.viewer -@dataclass -class PlayInteractiveArgs: - task: str - load_run: str - checkpoint: str | None - action_mode: str - policy_obs_mode: str - algo_log_name: str - log_root: str | None - show_target_bodies: bool - show_reward_debug: bool - target_show_axes: bool - target_body_names: str - target_max_bodies: int - target_marker_radius: float - target_axis_length: float - target_marker_alpha: float - reward_debug_show_velocity: bool - reward_debug_lin_vel_scale: float - reward_debug_ang_vel_scale: float - reward_debug_show_connectors: bool - reward_debug_show_global_anchor: bool - camera_follow_body: bool - camera_focus_body_name: str - camera_height_offset: float - camera_distance: float | None - camera_elevation: float | None - camera_azimuth: float | None - use_env_visual_model: bool - speed: float - start_paused: bool - keyboard: bool = False - keyboard_step_lin: float = 0.1 - keyboard_step_ang: float = 0.2 - require_keyboard_command_obs: bool = True - algo: str = "ppo" - - -def _infer_checkpoint_actor_input_dim(ckpt_path: str) -> int | None: - loaded = torch.load(ckpt_path, map_location="cpu", weights_only=True) - state_dict = loaded.get("actor_state_dict") - if not isinstance(state_dict, dict): - return None - - # Common rsl-rl naming: "mlp.0.weight" or nested prefixes ending with ".0.weight". - for key in ("mlp.0.weight", "actor.mlp.0.weight"): - w = state_dict.get(key) - if isinstance(w, torch.Tensor) and w.ndim == 2: - return int(w.shape[1]) - - for key, w in state_dict.items(): - if key.endswith(".0.weight") and isinstance(w, torch.Tensor) and w.ndim == 2: - return int(w.shape[1]) - return None - - -def _backend_adapter(cfg: DictConfig, *, algo_name: str = "ppo"): - from unilab.base.backend import materialize_scene_visual_override - from unilab.training import BackendAdapter - - return BackendAdapter( - cfg, - root_dir=ROOT_DIR, - algo_name=algo_name, - scene_materializer=materialize_scene_visual_override, - ) - - def _algo_config_dict(cfg: DictConfig | None) -> dict[str, Any]: """Return the composed PPO algo config as a plain dict. @@ -187,18 +128,15 @@ def _algo_config_dict(cfg: DictConfig | None) -> dict[str, Any]: """ if cfg is None: return cast(dict[str, Any], PPOConfig().to_dict()) - train_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) - if not isinstance(train_cfg_raw, dict): - raise TypeError("cfg.algo must resolve to a dict") - return cast(dict[str, Any], train_cfg_raw) + return algo_config_dict(cfg) SUPPORTED_INTERACTIVE_ALGOS = ("ppo", "appo", "sac", "flashsac", "hora_distill") _CONFIG_ROOT_BY_ALGO = { "ppo": "ppo", "appo": "appo", - "sac": "offpolicy", - "flashsac": "offpolicy", + "sac": "sac", + "flashsac": "flashsac", "hora_distill": "hora_distill", } _OFFPOLICY_INTERACTIVE_ALGOS = {"sac", "flashsac"} @@ -262,27 +200,9 @@ def _interactive_overrides_from_cli( def _normalize_interactive_overrides(algo: str, overrides: list[str]) -> list[str]: - normalized: list[str] = [] - has_algo_group = False - - for override in overrides: - key = _override_key(override) - if algo in _OFFPOLICY_INTERACTIVE_ALGOS and key == "algo": - value = override.split("=", 1)[1] if "=" in override else "" - if value != algo: - raise SystemExit( - f"--algo {algo} cannot be combined with a non-{algo} Hydra algo group." - ) - has_algo_group = True - if algo in _OFFPOLICY_INTERACTIVE_ALGOS and key == "task" and "=" in override: - value = override.split("=", 1)[1] - if not value.startswith(f"{algo}/"): - override = f"task={algo}/{value}" - normalized.append(override) - - if algo in _OFFPOLICY_INTERACTIVE_ALGOS and not has_algo_group: - normalized.insert(0, f"algo={algo}") - return normalized + # All algos now compose uniformly from their own tree with + # `task=/`; no per-algo override rewriting is needed. + return list(overrides) def _compose_interactive_config(algo: str, overrides: list[str]) -> DictConfig: @@ -351,15 +271,7 @@ def _quat_to_rotmat_wxyz(quat: np.ndarray) -> np.ndarray: n = np.linalg.norm(q) if n < 1e-12: return np.eye(3, dtype=np.float64) - w, x, y, z = q / n - return np.array( - [ - [1.0 - 2.0 * (y * y + z * z), 2.0 * (x * y - z * w), 2.0 * (x * z + y * w)], - [2.0 * (x * y + z * w), 1.0 - 2.0 * (x * x + z * z), 2.0 * (y * z - x * w)], - [2.0 * (x * z - y * w), 2.0 * (y * z + x * w), 1.0 - 2.0 * (x * x + y * y)], - ], - dtype=np.float64, - ) + return np_matrix_from_quat(q / n) def _add_sphere_marker(scene, pos: np.ndarray, radius: float, rgba: np.ndarray) -> bool: @@ -482,15 +394,6 @@ def _default_viewer_camera_distance(mj_model, env: Any, *, follow_body: bool) -> return min(extent_distance, _FOLLOW_CAMERA_MAX_DISTANCE) -def _available_backends_for_task(task_name: str) -> tuple[str, ...]: - envs = registry.list_registered_envs() - task_meta = envs.get(task_name, {}) - backends = task_meta.get("available_backends", ()) - if not isinstance(backends, list): - return () - return tuple(str(backend) for backend in backends) - - def _can_launch_glfw_viewer() -> bool: try: import glfw @@ -843,21 +746,6 @@ def _load_viewer_model(env: Any, *, use_env_visual_model: bool): return playback_model -def _build_playback_config(args, *, num_envs: int = 1) -> RslRlPlaybackConfig: - return RslRlPlaybackConfig( - task=str(args.task), - load_run=str(args.load_run), - checkpoint=getattr(args, "checkpoint", None), - action_mode=str(args.action_mode), - policy_obs_mode=str(args.policy_obs_mode), - algo_log_name=str(getattr(args, "algo_log_name", "rsl_rl_ppo")), - log_root=getattr(args, "log_root", None), - num_envs=num_envs, - speed=float(getattr(args, "speed", 1.0)), - start_paused=bool(getattr(args, "start_paused", False)), - ) - - def _build_keyboard_commander(env: Any, args) -> KeyboardCommander | None: """Set up keyboard velocity teleop, or return None when unsupported/disabled.""" if not bool(getattr(args, "keyboard", False)): @@ -894,12 +782,14 @@ def _state_has_velocity_commands(env: Any) -> bool: ) -def _is_locomotion_env(env: Any) -> bool: - return type(env).__module__.startswith("unilab.envs.locomotion") +def _has_velocity_command_config(env: Any) -> bool: + cfg = getattr(env, "cfg", None) + commands_cfg = getattr(cfg, "commands", None) if cfg is not None else None + return getattr(commands_cfg, "vel_limit", None) is not None def _is_velocity_command_locomotion_task(env: Any) -> bool: - if not _is_locomotion_env(env): + if not _has_velocity_command_config(env): return False cfg = getattr(env, "cfg", None) candidate_names = [ @@ -1007,7 +897,7 @@ def play_interactive(args, cfg: DictConfig | None = None, *, algo: str | None = algo = str(algo or getattr(args, "algo", "ppo")) # Always use a single env for interactive view - available_backends = _available_backends_for_task(args.task) + available_backends = available_backends_for_task(args.task) if available_backends and "mujoco" not in available_backends: print( "[play_interactive] Task does not support MuJoCo backend: " @@ -1019,12 +909,14 @@ def play_interactive(args, cfg: DictConfig | None = None, *, algo: str | None = def _create_env(num_envs: int): if cfg is None: return registry.make(args.task, num_envs=num_envs, sim_backend="mujoco") - from unilab.training import create_env + from unilab.base.config_adapter import create_env if algo in _OFFPOLICY_INTERACTIVE_ALGOS: env_cfg_override = build_offpolicy_env_cfg_override(algo, cfg, root_dir=ROOT_DIR) else: - env_cfg_override = _backend_adapter(cfg, algo_name=algo).build_task_env_cfg_override() + env_cfg_override = build_play_backend_adapter( + cfg, root_dir=ROOT_DIR, algo_name=algo + ).build_task_env_cfg_override() try: return create_env( cfg, @@ -1044,11 +936,11 @@ def _create_env(num_envs: int): raise try: - playback_cfg = _build_playback_config(args, num_envs=1) + playback_cfg = build_playback_config(args, num_envs=1) if algo == "ppo": wrapper_cls = RslRlVecEnvWrapper if cfg is not None: - from unilab.algos.torch.rsl_rl_runtime import resolve_rsl_rl_ppo_runtime + from unilab.algos.rsl_rl_runtime import resolve_rsl_rl_ppo_runtime wrapper_cls = resolve_rsl_rl_ppo_runtime( _algo_config_dict(cfg), @@ -1061,7 +953,7 @@ def _create_env(num_envs: int): root_dir=ROOT_DIR, device=device, checkpoint_resolver=resolve_checkpoint, - checkpoint_input_dim_reader=_infer_checkpoint_actor_input_dim, + checkpoint_input_dim_reader=infer_checkpoint_actor_input_dim, entrypoint_log_root=get_entrypoint_log_root, wrapper_cls=wrapper_cls, runner_cls=OnPolicyRunner, @@ -1306,18 +1198,11 @@ def _on_key(keycode: int) -> None: print("[play_interactive] Done.") -def _normalize_checkpoint_value(value: object) -> str | None: - if value is None: - return None - text = str(value) - return None if text in {"-1", "None", "null"} else text - - def _build_play_args(cfg: DictConfig, *, algo: str = "ppo") -> PlayInteractiveArgs: return PlayInteractiveArgs( task=str(cfg.training.task_name), load_run=str(cfg.algo.load_run), - checkpoint=_normalize_checkpoint_value(OmegaConf.select(cfg, "algo.checkpoint")), + checkpoint=normalize_checkpoint_value(OmegaConf.select(cfg, "algo.checkpoint")), action_mode=str(cfg.interactive.action_mode), policy_obs_mode=str(cfg.interactive.policy_obs_mode), algo_log_name=str(cfg.algo.algo_log_name), diff --git a/scripts/play_viser.py b/scripts/play_viser.py index 222d9dc6c..4b464d91b 100644 --- a/scripts/play_viser.py +++ b/scripts/play_viser.py @@ -32,7 +32,7 @@ import sys import time from pathlib import Path -from typing import Any, cast +from typing import Any import hydra import numpy as np @@ -46,18 +46,21 @@ if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) -from unilab.training import ( - ensure_registries, - get_entrypoint_log_root, -) -from unilab.training.rsl_rl import ( +from unilab.algos.rsl_rl import ( RslRlVecEnvWrapper, get_policy_obs_dims, normalize_ppo_train_cfg, ) +from unilab.training import ensure_registries +from unilab.utils.checkpoint import get_entrypoint_log_root from unilab.visualization.interactive_playback import ( PlaybackControls, + PlayInteractiveArgs, + available_backends_for_task, + build_play_backend_adapter, + build_playback_config, create_rsl_rl_playback_session, + infer_checkpoint_actor_input_dim, make_sim2sim_preflight, select_torch_device, ) @@ -84,35 +87,15 @@ import mujoco import viser # noqa: E402 -from play_interactive import ( # noqa: E402 - PlayInteractiveArgs, - _available_backends_for_task, - _backend_adapter, - _build_playback_config, - _infer_checkpoint_actor_input_dim, - resolve_checkpoint, -) +from play_interactive import resolve_checkpoint # noqa: E402 + +from unilab.training import algo_config_dict # noqa: E402 # --------------------------------------------------------------------------- # # Core viewer # # --------------------------------------------------------------------------- # -def _algo_config_dict(cfg: DictConfig) -> dict[str, Any]: - """Return the composed PPO algo config as a plain dict. - - Args: - cfg: Hydra config for the current playback run. - - Returns: - The resolved ``cfg.algo`` subtree as a mutable dict for rsl_rl. - """ - train_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) - if not isinstance(train_cfg_raw, dict): - raise TypeError("cfg.algo must resolve to a dict") - return cast(dict[str, Any], train_cfg_raw) - - def _load_env_playback_model(env: Any, env_index: int) -> mujoco.MjModel: """Resolve the exact MuJoCo model for one playback env. @@ -219,7 +202,7 @@ def play_viser(args: PlayInteractiveArgs, cfg: DictConfig) -> None: print(f"[play_viser] Device: {device}") # --- Validate backend --------------------------------------------------- - available_backends = _available_backends_for_task(args.task) + available_backends = available_backends_for_task(args.task) if available_backends and "mujoco" not in available_backends: print( f"[play_viser] Task {args.task} does not support MuJoCo backend. " @@ -233,9 +216,11 @@ def play_viser(args: PlayInteractiveArgs, cfg: DictConfig) -> None: def _create_env(env_count: int): if cfg is None: return registry.make(args.task, num_envs=env_count, sim_backend="mujoco") - from unilab.training import create_env + from unilab.base.config_adapter import create_env - env_cfg_override = _backend_adapter(cfg).build_task_env_cfg_override() + env_cfg_override = build_play_backend_adapter( + cfg, root_dir=ROOT_DIR + ).build_task_env_cfg_override() return create_env( cfg, num_envs=env_count, @@ -245,13 +230,13 @@ def _create_env(env_count: int): ) playback_session, _policy_obs_mode, _checkpoint_path = create_rsl_rl_playback_session( - playback_cfg=_build_playback_config(args, num_envs=num_envs), + playback_cfg=build_playback_config(args, num_envs=num_envs), env_factory=_create_env, - algo_config=_algo_config_dict(cfg), + algo_config=algo_config_dict(cfg), root_dir=ROOT_DIR, device=device, checkpoint_resolver=resolve_checkpoint, - checkpoint_input_dim_reader=_infer_checkpoint_actor_input_dim, + checkpoint_input_dim_reader=infer_checkpoint_actor_input_dim, entrypoint_log_root=get_entrypoint_log_root, wrapper_cls=RslRlVecEnvWrapper, runner_cls=OnPolicyRunner, diff --git a/src/unilab/algos/torch/__init__.py b/scripts/tools/__init__.py similarity index 100% rename from src/unilab/algos/torch/__init__.py rename to scripts/tools/__init__.py diff --git a/src/unilab/tools/import_robot.py b/scripts/tools/import_robot.py similarity index 99% rename from src/unilab/tools/import_robot.py rename to scripts/tools/import_robot.py index 5a46ffe21..4c923d83d 100644 --- a/src/unilab/tools/import_robot.py +++ b/scripts/tools/import_robot.py @@ -2,7 +2,7 @@ """Convert a URDF robot to a UniLab robot MJCF asset directory. Usage: - uv run unilab-import-robot [robot_name] + uv run scripts/tools/import_robot.py [robot_name] """ @@ -16,7 +16,7 @@ from pathlib import Path from typing import Any, Iterable, Sequence, cast -REPO_ROOT = Path(__file__).resolve().parents[3] +REPO_ROOT = Path(__file__).resolve().parents[2] ROBOT_ASSET_ROOT = REPO_ROOT / "src" / "unilab" / "assets" / "robots" TEMP_MESH_PREFIX = "meshes/meshes/" DEFAULT_MATERIAL = "default_material" @@ -486,7 +486,7 @@ def _materialize_tuning_scene( add_height_joint: bool = True, height_range: tuple[float, float] = (-1.0, 1.0), ) -> Path: - from unilab.base.backend.mujoco.xml import materialize_scene_fragments + from unilab.base.backend import materialize_scene_fragments merged = materialize_scene_fragments(str(robot_xml), fragment_files=[str(scene_xml)]) tree = ET.parse(merged) diff --git a/src/unilab/utils/support_matrix.py b/scripts/tools/support_matrix.py similarity index 95% rename from src/unilab/utils/support_matrix.py rename to scripts/tools/support_matrix.py index a879fc4e6..8cc0c12dd 100644 --- a/src/unilab/utils/support_matrix.py +++ b/scripts/tools/support_matrix.py @@ -118,21 +118,21 @@ class SupportRow: EntrypointSpec( entrypoint_id="sac_torch", label="SAC (torch)", - config_dir="conf/offpolicy/task/sac", + config_dir="conf/sac/task", task_glob="*/*.yaml", generic_tested=True, ), EntrypointSpec( entrypoint_id="td3_torch", label="TD3 (torch)", - config_dir="conf/offpolicy/task/td3", + config_dir="conf/td3/task", task_glob="*/*.yaml", generic_tested=True, ), EntrypointSpec( entrypoint_id="flashsac_torch", label="FlashSAC (torch)", - config_dir="conf/offpolicy/task/flashsac", + config_dir="conf/flashsac/task", task_glob="*/*.yaml", generic_tested=True, ), @@ -286,7 +286,7 @@ def render_support_matrix(root: Path | None = None) -> str: "| 等级 | 仓库事实来源 |", "|------|--------------|", "| `Registered` | `ensure_registries()` 导入后的 `registry.list_registered_envs()` 中存在该 env/backend。 |", - "| `Configured` | 存在对应的 owner YAML：`conf/{ppo,appo,offpolicy}/task/...`。 |", + "| `Configured` | 存在对应的 owner YAML：`conf/{ppo,appo,sac,td3,flashsac}/task/...`。 |", "| `Tested` | `tests/` 中有自动化覆盖该 entrypoint/task owner/backend 组合，或存在显式 maintainer 完整训练验证并具备近风险自动化测试。这里的 `Tested` 不等同于默认推荐路径。 |", "| `Benchmarked` | 存在与该组合绑定的已提交 benchmark manifest。 |", "| `Recommended` | 仓库中存在显式 recommendation 元数据。 |", @@ -294,8 +294,10 @@ def render_support_matrix(root: Path | None = None) -> str: "`Tested` 只描述仓库中已有自动化覆盖或显式 maintainer 训练验证，不代表该组合具备同名 MuJoCo " "owner 的全部 backend capability；例如 phase-1 Motrix owner 可能只覆盖训练 smoke 和明确启用的 DR 子集。", "", - "`mjwarp` 只支持 `g1_walk_flat` host adapter。PPO (torch) 与 SAC (torch) owner 已完成训练验证，并有 " - "backend、contract 与 playback 自动化覆盖，因此标记为 `Tested`。" + "`mjwarp` 完成训练验证的只有 `g1_walk_flat` host adapter：PPO (torch) 与 SAC (torch) owner " + "已完成训练验证，并有 backend、contract 与 playback 自动化覆盖，因此标记为 `Tested`。" + "SAC `t800_walk_flat` 的 mjwarp owner 只有 owner YAML 与 compose 覆盖，标记为 `Configured`，" + "不代表训练验证。" "mjwarp playback 仅支持显式、有限步数的 `record` 并复用 MuJoCo 离线 renderer，不支持 `auto`、" "interactive 或 native playback。其他 entrypoint 中出现的 `Registered` 只表示 env/backend registry " "identity，不代表对应算法、terrain、完整 DR 或 production training 支持。", @@ -322,7 +324,7 @@ def render_support_matrix(root: Path | None = None) -> str: "### Source Index", "", "- Registry bootstrap: `src/unilab/envs/**` decorators via `unilab.base.registry.ensure_registries()`.", - "- Owner YAML scan: `conf/ppo/task/**`, `conf/appo/task/**`, `conf/offpolicy/task/**`.", + "- Owner YAML scan: `conf/ppo/task/**`, `conf/appo/task/**`, `conf/sac/task/**`, `conf/td3/task/**`, `conf/flashsac/task/**`.", "- Generic compose coverage: `tests/config/test_config_system.py::test_supported_task_composes`.", "- Validated mjwarp entrypoints are explicitly recorded in `_MAINTAINER_VALIDATED_MJWARP_ENTRYPOINT_TASKS`; near-risk coverage lives in `tests/base/test_mjwarp_backend.py`, `tests/base/test_backend_conformance.py`, `tests/base/test_mjwarp_differential.py`, and `tests/base/test_mjwarp_playback.py`.", ] diff --git a/scripts/train_appo.py b/scripts/train_appo.py index 93e193429..8e29b8d8a 100644 --- a/scripts/train_appo.py +++ b/scripts/train_appo.py @@ -7,7 +7,7 @@ import sys from collections.abc import Callable from pathlib import Path -from typing import Any, cast +from typing import Any import hydra import torch @@ -16,20 +16,30 @@ ROOT_DIR = Path(__file__).parent.parent sys.path.append(str(ROOT_DIR)) -from unilab.algos.torch.appo.runtime import resolve_appo_runtime -from unilab.training import ( +from unilab.algos.appo.runtime import resolve_appo_runtime +from unilab.algos.rsl_rl import RslRlVecEnvWrapper +from unilab.base.backend.base import log_playback_plan +from unilab.base.config_adapter import ( BackendAdapter, - apply_configured_training_seed, create_env, +) +from unilab.training import ( + algo_config_dict, + build_run_dir_name, ensure_registries, get_log_root, - log_playback_plan, - resolve_appo_checkpoint_path, + resolve_nan_guard_cfg, should_run_playback, ) from unilab.training.experiment import ExperimentTracker from unilab.training.onnx_export import export_policy_onnx, verify_policy_onnx -from unilab.training.sim2sim import policy_load_dim_guard, resolve_sim2sim_config +from unilab.utils.checkpoint import resolve_appo_checkpoint_path +from unilab.utils.seed import apply_configured_training_seed +from unilab.visualization.interactive_playback import ( + RslRlPlaybackConfig, + create_appo_playback_session, + normalize_checkpoint_value, +) def _training_resume_requested(load_run: Any) -> bool: @@ -45,10 +55,7 @@ def build_appo_runner_kwargs( rl_cfg: dict[str, Any] | None = None, ) -> dict: if rl_cfg is None: - rl_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) - if not isinstance(rl_cfg_raw, dict): - raise TypeError("cfg.algo must resolve to a dict") - rl_cfg = cast(dict[str, Any], rl_cfg_raw) + rl_cfg = algo_config_dict(cfg) runner_kwargs = { "env_name": cfg.training.task_name, @@ -73,16 +80,9 @@ def build_appo_runner_kwargs( raise FileNotFoundError(f"Could not resolve APPO resume checkpoint: {load_run}") runner_kwargs["resume_path"] = resume_path - nan_guard_cfg = getattr(cfg.training, "nan_guard", None) - if nan_guard_cfg is not None and getattr(nan_guard_cfg, "enabled", False): - from unilab.utils.nan_guard import NanGuardCfg - - runner_kwargs["nan_guard_cfg"] = NanGuardCfg( - enabled=True, - buffer_size=int(getattr(nan_guard_cfg, "buffer_size", 100)), - max_envs_to_dump=int(getattr(nan_guard_cfg, "max_envs_to_dump", 5)), - output_dir=getattr(nan_guard_cfg, "output_dir", None), - ) + nan_guard_cfg = resolve_nan_guard_cfg(cfg.training) + if nan_guard_cfg is not None: + runner_kwargs["nan_guard_cfg"] = nan_guard_cfg return runner_kwargs @@ -162,9 +162,6 @@ def play_appo( native Motrix viewer or when no checkpoint could be resolved. """ del root_dir - import numpy as np - from rsl_rl.utils import resolve_callable - from tensordict import TensorDict if resolve_checkpoint_path is not None: load_path, load_path_dir = resolve_checkpoint_path(cfg) @@ -177,20 +174,6 @@ def play_appo( print(f"Could not find run to load. load_path={load_path}") return None - cfg = ( - resolve_sim2sim_config( - load_path_dir, - cfg, - algo_name="appo", - strict=bool(getattr(cfg.training, "sim2sim_strict", True)), - ) - or cfg - ) - - env_cfg_override = BackendAdapter( - cfg, root_dir=ROOT_DIR, algo_name="appo" - ).build_task_env_cfg_override() - device = cfg.training.device or ( "cuda" if torch.cuda.is_available() @@ -200,58 +183,37 @@ def play_appo( ) print(f"Using device for play: {device}") - env = cast( - Any, - create_env( + playback_cfg = RslRlPlaybackConfig( + task=str(cfg.training.task_name), + load_run=str(cfg.algo.load_run), + checkpoint=normalize_checkpoint_value( + OmegaConf.select(cfg, "algo.checkpoint", default=None) + ), + action_mode="policy", + policy_obs_mode="flat", + algo_log_name=str(cfg.algo.algo_log_name), + log_root=None, + num_envs=cfg.training.play_env_num, + ) + session, _policy_obs_mode, _checkpoint_path = create_appo_playback_session( + playback_cfg=playback_cfg, + cfg=cfg, + rl_cfg=rl_cfg, + env_factory=lambda n: create_env( cfg, - num_envs=cfg.training.play_env_num, - env_cfg_override=env_cfg_override, + num_envs=n, + env_cfg_override=BackendAdapter( + cfg, root_dir=ROOT_DIR, algo_name="appo" + ).build_task_env_cfg_override(), ), + root_dir=ROOT_DIR, + device=device, + wrapper_cls=RslRlVecEnvWrapper, ) - from unilab.base.observations import get_obs_dims - - obs_dim, critic_dim = get_obs_dims(env.obs_groups_spec) - action_shape = env.action_space.shape - if action_shape is None: - raise ValueError("env.action_space.shape must be defined") - action_dim = int(action_shape[0]) - - rl_cfg_dict = dict(rl_cfg) - if "obs_groups" not in rl_cfg_dict: - rl_cfg_dict["obs_groups"] = { - "actor": {"policy": obs_dim}, - "critic": {"policy": critic_dim if critic_dim > 0 else obs_dim}, - } - else: - actor_group = rl_cfg_dict["obs_groups"].get( - "actor", rl_cfg_dict["obs_groups"].get("policy", {}) - ) - if isinstance(actor_group, dict) and "policy" in actor_group: - actor_group["policy"] = obs_dim - critic_group = rl_cfg_dict["obs_groups"].get("critic") - if critic_group is None: - rl_cfg_dict["obs_groups"]["critic"] = { - "policy": critic_dim if critic_dim > 0 else obs_dim - } - elif isinstance(critic_group, dict) and "policy" in critic_group: - critic_group["policy"] = critic_dim if critic_dim > 0 else obs_dim - - from copy import deepcopy - - obs_example = torch.zeros((cfg.training.play_env_num, obs_dim), device=device) - td_example = TensorDict({"policy": obs_example}, batch_size=cfg.training.play_env_num) - - actor_cfg = deepcopy(rl_cfg_dict["actor"]) - actor_cls = resolve_callable(actor_cfg.pop("class_name")) - actor_cfg.pop("num_actions", None) - actor = actor_cls(td_example, rl_cfg_dict["obs_groups"], "actor", action_dim, **actor_cfg) - actor = actor.to(device) - actor.eval() - - print(f"Loading model: {load_path}") - checkpoint = torch.load(load_path, map_location=device, weights_only=True) - with policy_load_dim_guard(env_obs_dim=obs_dim, env_action_dim=action_dim, algo_name="appo"): - actor.load_state_dict(checkpoint["actor"]) + env = session.env + actor = session.actor + # The checkpoint early-return above guarantees a loaded actor here. + assert actor is not None # Export actor to ONNX if load_path_dir is not None: @@ -267,6 +229,7 @@ def forward(self, obs: torch.Tensor) -> torch.Tensor: export_module = _DeterministicAPPOActor(actor.mlp) onnx_path = os.path.join(load_path_dir, "policy.onnx") + obs_dim = int(session.wrapped_env.num_obs) dummy_input = torch.randn(1, obs_dim, device=device) export_policy_onnx(export_module, onnx_path, (dummy_input,), input_names=["obs"]) @@ -274,9 +237,6 @@ def forward(self, obs: torch.Tensor) -> torch.Tensor: verify_input = torch.randn(1, obs_dim, device=device) verify_policy_onnx(export_module, onnx_path, (verify_input,), input_names=["obs"]) - if env.state is None: - env.init_state() - with torch.inference_mode(): play_video_path = env.run_playback_mode( play_render_mode=getattr(cfg.training, "play_render_mode", "auto"), @@ -285,24 +245,8 @@ def forward(self, obs: torch.Tensor) -> torch.Tensor: render_spacing=float( getattr(cfg.training, "render_spacing", getattr(env.cfg, "render_spacing", 1.0)) ), - initialize=lambda: np.asarray( - env.reset(np.arange(cfg.training.play_env_num, dtype=np.int32))[0]["obs"], - dtype=np.float32, - ), - step=lambda obs_np: np.asarray( - env.step( - actor( - TensorDict( - {"policy": torch.from_numpy(obs_np).to(device)}, - batch_size=cfg.training.play_env_num, - ) - ) - .cpu() - .numpy() - .astype(np.float32) - ).obs["obs"], - dtype=np.float32, - ), + initialize=session.reset, + step=lambda _obs: session.step_once(), camera_kwargs={ "cam_distance": cfg.training.cam_distance, "cam_elevation": cfg.training.cam_elevation, @@ -330,10 +274,7 @@ def main(cfg: DictConfig) -> None: ).build_task_env_cfg_override() # Convert algo config to plain dict for APPORunner / RSL-RL internals - rl_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) - if not isinstance(rl_cfg_raw, dict): - raise TypeError("cfg.algo must resolve to a dict") - rl_cfg = cast(dict[str, Any], rl_cfg_raw) + rl_cfg = algo_config_dict(cfg) apply_appo_runtime_flags(rl_cfg, cfg, training_enabled=not cfg.training.play_only) appo_runtime = resolve_appo_runtime(rl_cfg, default_play_fn=play_appo) @@ -343,7 +284,7 @@ def main(cfg: DictConfig) -> None: log_dir = os.path.join( log_root, cfg.training.task_name, - f"{timestamp}_{cfg.training.sim_backend}", + build_run_dir_name(timestamp, str(cfg.training.sim_backend)), ) else: log_dir = cfg.training.log_dir diff --git a/scripts/train_cse_ppo.py b/scripts/train_cse_ppo.py new file mode 100644 index 000000000..0781a41e6 --- /dev/null +++ b/scripts/train_cse_ppo.py @@ -0,0 +1,232 @@ +"""Train or play the CSE-PPO A2Arm policy.""" + +from __future__ import annotations + +import datetime +import statistics +import sys +import time +from pathlib import Path +from typing import Any, cast + +import hydra +import torch +from omegaconf import DictConfig, OmegaConf + +ROOT_DIR = Path(__file__).parent.parent +SRC_DIR = ROOT_DIR / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +from unilab.algos.cse_ppo import CSEOnPolicyRunner +from unilab.algos.rsl_rl import RslRlVecEnvWrapper, get_policy_obs_dims +from unilab.base.backend import materialize_scene_visual_override +from unilab.base.config_adapter import BackendAdapter, create_env +from unilab.training import ( + algo_config_dict, + apply_env_nan_guard, + build_run_dir_name, + ensure_registries, + format_play_checkpoint_error, + get_log_root, + parse_checkpoint_path, +) +from unilab.training.experiment import ExperimentTracker +from unilab.utils.checkpoint import get_entrypoint_log_root +from unilab.visualization import render_play_mode +from unilab.visualization.interactive_playback import ( + RslRlPlaybackConfig, + create_rsl_rl_playback_session, + make_sim2sim_preflight, + normalize_checkpoint_value, +) + +EXPORT_POLICY = False + + +def _backend_adapter(cfg: DictConfig) -> BackendAdapter: + return BackendAdapter( + cfg, + root_dir=ROOT_DIR, + algo_name="ppo_cse", + scene_materializer=materialize_scene_visual_override, + ) + + +def _get_log_root(cfg: DictConfig) -> str: + return str(get_log_root(ROOT_DIR, cfg)) + + +def play_cse_ppo(cfg: DictConfig, device: str) -> str | None: + """Resolve a checkpoint and render a CSE policy using the shared playback session.""" + rl_cfg = algo_config_dict(cfg) + task_log_root = get_log_root(ROOT_DIR, cfg) / str(cfg.training.task_name) + load_path, load_path_dir = parse_checkpoint_path(cfg, root_dir=ROOT_DIR) + if load_path is None or load_path_dir is None or not load_path.exists(): + print( + format_play_checkpoint_error( + cfg, task_log_root=task_log_root, load_path=load_path, load_path_dir=load_path_dir + ) + ) + return None + keys = set(torch.load(load_path, map_location="cpu", weights_only=True).keys()) + if "actor_state_dict" not in keys: + print( + f"Checkpoint at {load_path} is not a CSE-PPO checkpoint (found keys: {keys}). Aborting play." + ) + return None + + def env_factory(num_envs: int): + override = cast(dict[str, Any], _backend_adapter(cfg).build_play_env_cfg_override()) + return create_env(cfg, num_envs=num_envs, env_cfg_override=override) + + session, _mode, _checkpoint = create_rsl_rl_playback_session( + playback_cfg=RslRlPlaybackConfig( + task=str(cfg.training.task_name), + load_run=str(cfg.algo.load_run), + checkpoint=normalize_checkpoint_value( + OmegaConf.select(cfg, "algo.checkpoint", default=None) + ), + action_mode="policy", + policy_obs_mode="flat", + algo_log_name=str(cfg.algo.algo_log_name), + log_root=getattr(cfg.training, "log_root", None), + num_envs=int(cfg.training.play_env_num), + ), + env_factory=env_factory, + algo_config=rl_cfg, + root_dir=ROOT_DIR, + device=device, + checkpoint_resolver=lambda *_args: str(load_path), + # The first CSE actor layer consumes ``one_step_obs + latent`` rather + # than the flattened history, so its state-dict width is not the + # environment policy-input width used by the generic RSL guard. + checkpoint_input_dim_reader=lambda _path: None, + entrypoint_log_root=get_entrypoint_log_root, + wrapper_cls=RslRlVecEnvWrapper, + runner_cls=CSEOnPolicyRunner, + runner_loader=lambda runner, path: runner.load(path), + policy_obs_dims_getter=get_policy_obs_dims, + train_cfg_normalizer=lambda train_cfg: train_cfg, + sim2sim_preflight=make_sim2sim_preflight(cfg, algo_name="ppo_cse"), + guard_algo_name="ppo_cse", + ) + env = session.env + assert session.runner is not None and session.policy is not None + cse_policy = session.policy + session.policy = lambda obs: cse_policy(obs["actor"]) + if EXPORT_POLICY: + session.runner.export_policy_to_jit(path=str(load_path_dir)) + output_video = Path(load_path_dir) / "play_video.mp4" + with torch.inference_mode(): + render_play_mode( + env, + sim_backend=cfg.training.sim_backend, + render_spacing=float( + getattr(cfg.training, "render_spacing", getattr(env.cfg, "render_spacing", 1.0)) + ), + num_steps=cfg.training.play_steps, + output_video=output_video, + initialize=lambda: session.reset()["actor"], + step=lambda _obs: session.step_once()["actor"], + camera_kwargs={ + "cam_distance": cfg.training.cam_distance, + "cam_elevation": cfg.training.cam_elevation, + "cam_azimuth": cfg.training.cam_azimuth, + "cam_lookat": getattr(cfg.training, "cam_lookat", None), + "cam_tracking": getattr(cfg.training, "cam_tracking", False), + "cam_tracking_env_idx": getattr(cfg.training, "cam_tracking_env_idx", 0), + "cam_tracking_extra_envs": getattr(cfg.training, "cam_tracking_extra_envs", 2), + }, + ) + return str(output_video) + + +@hydra.main(version_base="1.3", config_path="../conf/ppo_cse", config_name="config") +def main(cfg: DictConfig) -> None: + ensure_registries() + override = cast(dict[str, Any], _backend_adapter(cfg).build_task_env_cfg_override()) + device = ( + "cuda" + if torch.cuda.is_available() + else ("mps" if torch.backends.mps.is_available() else "cpu") + ) + max_iterations = int(cfg.algo.max_iterations) + if cfg.training.num_timesteps: + max_iterations = max( + 1, int(cfg.training.num_timesteps / (cfg.algo.num_steps_per_env * cfg.algo.num_envs)) + ) + log_dir: str | None = None + if not cfg.training.play_only: + timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + log_dir = str( + Path(_get_log_root(cfg)) + / str(cfg.training.task_name) + / build_run_dir_name(timestamp, str(cfg.training.sim_backend)) + ) + tracker = None + if log_dir is not None: + tracker = ExperimentTracker( + root_dir=ROOT_DIR, + log_dir=log_dir, + algo_name="ppo_cse", + task_name=cfg.training.task_name, + sim_backend=cfg.training.sim_backend, + training_cfg=cfg.training, + full_cfg=cfg, + device=device, + ) + tracker.start() + try: + if not cfg.training.play_only: + env = create_env(cfg, num_envs=cfg.algo.num_envs, env_cfg_override=override) + apply_env_nan_guard(env, cfg.training) + runner = CSEOnPolicyRunner( + RslRlVecEnvWrapper(env, device=device), + algo_config_dict(cfg), + log_dir=log_dir, + device=device, + ) + if cfg.algo.load_run != "-1": + resume_path, _ = parse_checkpoint_path(cfg, root_dir=ROOT_DIR) + if resume_path: + runner.load(str(resume_path)) + started = time.time() + runner.learn(num_learning_iterations=max_iterations, init_at_random_ep_len=True) + assert log_dir is not None + if tracker is not None: + tracker.update_summary( + { + "status": "completed", + "completed_iterations": int(runner.current_learning_iteration), + "total_env_steps": int(runner.logger.tot_timesteps), + "final_mean_reward": float(statistics.mean(runner.logger.rewbuffer)) + if runner.logger.rewbuffer + else None, + "best_mean_reward": float(max(runner.logger.rewbuffer)) + if runner.logger.rewbuffer + else None, + "mean_episode_length": float(statistics.mean(runner.logger.lenbuffer)) + if runner.logger.lenbuffer + else None, + "last_checkpoint": str( + Path(log_dir) / f"model_{runner.current_learning_iteration}.pt" + ), + "training_wall_time_sec": time.time() - started, + } + ) + env.close() + if cfg.training.play_only or not cfg.training.no_play: + output = play_cse_ppo(cfg, device) + if tracker is not None: + tracker.log_video(output) + finally: + if tracker is not None: + tracker.finish() + + +if __name__ == "__main__": + EXPORT_POLICY = True + main() diff --git a/scripts/train_flashsac.py b/scripts/train_flashsac.py new file mode 100644 index 000000000..654091596 --- /dev/null +++ b/scripts/train_flashsac.py @@ -0,0 +1,14 @@ +"""FlashSAC training/playback entrypoint (shared implementation in train_offpolicy.py).""" + +import hydra +from omegaconf import DictConfig +from train_offpolicy import main as _offpolicy_main + + +@hydra.main(version_base="1.3", config_path="../conf/flashsac", config_name="config") +def main(cfg: DictConfig) -> None: + _offpolicy_main(cfg) + + +if __name__ == "__main__": + main() diff --git a/scripts/train_him_ppo.py b/scripts/train_him_ppo.py index e104aa42f..76caeb4e7 100644 --- a/scripts/train_him_ppo.py +++ b/scripts/train_him_ppo.py @@ -7,7 +7,7 @@ import hydra import torch -from omegaconf import DictConfig, OmegaConf +from omegaconf import DictConfig EXPORT_POLICY = False # set to True in __main__ block @@ -18,20 +18,32 @@ if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) -from unilab.algos.torch.him_ppo.runner import HIMOnPolicyRunner +from unilab.algos.him_ppo.runner import HIMOnPolicyRunner +from unilab.algos.rsl_rl import RslRlVecEnvWrapper, get_policy_obs_dims from unilab.base.backend import materialize_scene_visual_override -from unilab.training import ( +from unilab.base.config_adapter import ( BackendAdapter, create_env, +) +from unilab.training import ( + algo_config_dict, + apply_env_nan_guard, + build_run_dir_name, ensure_registries, - get_latest_checkpoint, - get_latest_run, + format_play_checkpoint_error, get_log_root, parse_checkpoint_path, ) from unilab.training.experiment import ExperimentTracker -from unilab.training.sim2sim import policy_load_dim_guard, resolve_sim2sim_config +from unilab.utils.checkpoint import get_entrypoint_log_root from unilab.visualization import render_play_mode +from unilab.visualization.interactive_playback import ( + RslRlPlaybackConfig, + create_rsl_rl_playback_session, + infer_checkpoint_actor_input_dim, + make_sim2sim_preflight, + normalize_checkpoint_value, +) def _backend_adapter(cfg: DictConfig) -> BackendAdapter: @@ -47,57 +59,15 @@ def _get_log_root(cfg: DictConfig) -> str: return str(get_log_root(ROOT_DIR, cfg)) -def _algo_config_dict(cfg: DictConfig) -> dict[str, Any]: - raw = OmegaConf.to_container(cfg.algo, resolve=True) - if not isinstance(raw, dict): - raise TypeError("cfg.algo must resolve to a dict") - return cast(dict[str, Any], raw) - - -def _format_play_checkpoint_error( - cfg: DictConfig, - *, - task_log_root: Path, - load_path: Path | None, - load_path_dir: Path | None, -) -> str: - selected_checkpoint = OmegaConf.select(cfg, "algo.checkpoint", default=-1) - checkpoint_hint = ( - f" algo.checkpoint={selected_checkpoint!r}" - if selected_checkpoint not in (None, "", -1, "-1") - else "" - ) - if load_path_dir is not None and load_path is None and checkpoint_hint: - reason = f"Requested checkpoint was not found under resolved_run={load_path_dir}." - elif not task_log_root.exists(): - reason = "Task log root does not exist." - else: - latest_run = get_latest_run(task_log_root) - if latest_run is None: - reason = "No run directories were found under the task log root." - elif get_latest_checkpoint(latest_run) is None: - reason = f"Resolved latest run has no model_*.pt checkpoint files: {latest_run}." - else: - reason = "Requested run or checkpoint could not be resolved." - - return ( - "Could not resolve a checkpoint for play mode. " - f"{reason} task={cfg.training.task_name} task_log_root={task_log_root} " - f"algo.load_run={cfg.algo.load_run!r}{checkpoint_hint}." - " Use algo.load_run= " - "and optionally algo.checkpoint=." - ) - - def play_him_ppo(cfg: DictConfig, device: str) -> str | None: """Play mode for HIM-PPO.""" - rl_cfg = _algo_config_dict(cfg) + rl_cfg = algo_config_dict(cfg) task_log_root = get_log_root(ROOT_DIR, cfg) / str(cfg.training.task_name) load_path, load_path_dir = parse_checkpoint_path(cfg, root_dir=ROOT_DIR) if load_path is None or load_path_dir is None or not load_path.exists(): print( - _format_play_checkpoint_error( + format_play_checkpoint_error( cfg, task_log_root=task_log_root, load_path=load_path, @@ -115,31 +85,49 @@ def play_him_ppo(cfg: DictConfig, device: str) -> str | None: ) return None - cfg = ( - resolve_sim2sim_config( - load_path_dir, - cfg, - algo_name="ppo", - strict=bool(getattr(cfg.training, "sim2sim_strict", True)), - ) - or cfg + def _create_env(num_envs: int): + env_cfg_override = cast(dict[str, Any], _backend_adapter(cfg).build_play_env_cfg_override()) + return create_env(cfg, num_envs=num_envs, env_cfg_override=env_cfg_override) + + session, _policy_obs_mode, _checkpoint_path = create_rsl_rl_playback_session( + playback_cfg=RslRlPlaybackConfig( + task=str(cfg.training.task_name), + load_run=str(getattr(cfg.algo, "load_run", "-1")), + checkpoint=normalize_checkpoint_value(getattr(cfg.algo, "checkpoint", None)), + action_mode="policy", + policy_obs_mode="flat", + algo_log_name=str(cfg.algo.algo_log_name), + log_root=getattr(cfg.training, "log_root", None), + num_envs=int(cfg.training.play_env_num), + ), + env_factory=_create_env, + algo_config=rl_cfg, + root_dir=ROOT_DIR, + device=device, + # The checkpoint was already resolved above for the friendly early exit. + checkpoint_resolver=lambda *_args: str(load_path), + checkpoint_input_dim_reader=infer_checkpoint_actor_input_dim, + entrypoint_log_root=get_entrypoint_log_root, + wrapper_cls=RslRlVecEnvWrapper, + runner_cls=HIMOnPolicyRunner, + # HIMOnPolicyRunner.load does not accept a load_cfg argument. + runner_loader=lambda runner, path: runner.load(path), + policy_obs_dims_getter=get_policy_obs_dims, + train_cfg_normalizer=lambda train_cfg: train_cfg, + sim2sim_preflight=make_sim2sim_preflight(cfg, algo_name="ppo"), + guard_algo_name="him_ppo", ) - env_cfg_override = cast(dict[str, Any], _backend_adapter(cfg).build_play_env_cfg_override()) - env = create_env(cfg, num_envs=cfg.training.play_env_num, env_cfg_override=env_cfg_override) - from unilab.training.rsl_rl import RslRlVecEnvWrapper - - wrapped_env = RslRlVecEnvWrapper(env, device=device) - runner = HIMOnPolicyRunner(wrapped_env, rl_cfg, log_dir=None, device=device) - with policy_load_dim_guard( - env_obs_dim=getattr(wrapped_env, "num_obs", None), - env_action_dim=getattr(wrapped_env, "num_actions", None), - algo_name="him_ppo", - ): - runner.load(str(load_path)) - policy = runner.get_inference_policy(device=device) + env = session.env + assert session.runner is not None and session.policy is not None + + # HIM's inference policy consumes the flat actor tensor, not the full obs + # TensorDict the session hands to ``policy``. + him_policy = session.policy + session.policy = lambda obs: him_policy(obs["actor"]) + if EXPORT_POLICY: - runner.export_policy_to_onnx(path=str(load_path_dir)) - runner.export_policy_to_jit(path=str(load_path_dir)) + session.runner.export_policy_to_onnx(path=str(load_path_dir)) + session.runner.export_policy_to_jit(path=str(load_path_dir)) output_video = Path(load_path_dir) / "play_video.mp4" print(f"Rendering video to {output_video}...") @@ -153,8 +141,8 @@ def play_him_ppo(cfg: DictConfig, device: str) -> str | None: ), num_steps=cfg.training.play_steps, output_video=output_video, - initialize=lambda: wrapped_env.reset()[0]["actor"], - step=lambda obs: wrapped_env.step(policy(obs))[0]["actor"], + initialize=lambda: session.reset()["actor"], + step=lambda _obs: session.step_once()["actor"], camera_kwargs={ "cam_distance": cfg.training.cam_distance, "cam_elevation": cfg.training.cam_elevation, @@ -202,7 +190,9 @@ def main(cfg: DictConfig) -> None: timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") log_root = _get_log_root(cfg) log_dir = str( - Path(log_root) / cfg.training.task_name / f"{timestamp}_{cfg.training.sim_backend}" + Path(log_root) + / cfg.training.task_name + / build_run_dir_name(timestamp, str(cfg.training.sim_backend)) ) else: log_dir = None @@ -224,26 +214,11 @@ def main(cfg: DictConfig) -> None: try: if not cfg.training.play_only: env = create_env(cfg, num_envs=cfg.algo.num_envs, env_cfg_override=env_cfg_override) - from unilab.training.rsl_rl import RslRlVecEnvWrapper - - nan_guard_cfg = getattr(cfg.training, "nan_guard", None) - if nan_guard_cfg is not None and getattr(nan_guard_cfg, "enabled", False): - from unilab.utils.nan_guard import NanGuard, NanGuardCfg - - guard = NanGuard( - NanGuardCfg( - enabled=True, - buffer_size=int(getattr(nan_guard_cfg, "buffer_size", 100)), - max_envs_to_dump=int(getattr(nan_guard_cfg, "max_envs_to_dump", 5)), - output_dir=getattr(nan_guard_cfg, "output_dir", None), - ), - num_envs=env.num_envs, - supports_state_playback=env.play_capabilities.supports_physics_state_playback, - ) - env.set_nan_guard(guard) + + apply_env_nan_guard(env, cfg.training) wrapped_env = RslRlVecEnvWrapper(env, device=device) - rl_cfg = _algo_config_dict(cfg) + rl_cfg = algo_config_dict(cfg) runner = HIMOnPolicyRunner(wrapped_env, rl_cfg, log_dir=log_dir, device=device) if cfg.algo.load_run != "-1": diff --git a/scripts/train_hora_distill.py b/scripts/train_hora_distill.py index d289424e9..cb74547d8 100644 --- a/scripts/train_hora_distill.py +++ b/scripts/train_hora_distill.py @@ -1,5 +1,4 @@ import datetime -import json import sys from pathlib import Path from typing import Any, cast @@ -15,42 +14,44 @@ if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) -from unilab.algos.torch.hora import HoraDistillationTrainer -from unilab.algos.torch.hora.distill import ( +from unilab.algos.hora import HoraDistillationTrainer +from unilab.algos.hora.distill import ( build_student_actor_and_normalizer, cfg_with_checkpoint_runtime, load_distilled_checkpoint, student_policy, ) -from unilab.algos.torch.hora.distill_config import ( +from unilab.algos.hora.distill_config import ( apply_teacher_defaults as _apply_teacher_defaults, ) -from unilab.algos.torch.hora.distill_config import ( +from unilab.algos.hora.distill_config import ( get_teacher_owner_spec as _get_teacher_owner_spec, ) -from unilab.algos.torch.hora.distill_config import ( +from unilab.algos.hora.distill_config import ( resolve_teacher_checkpoint_path as _resolve_teacher_checkpoint_path, ) -from unilab.algos.torch.hora.distill_config import ( +from unilab.algos.hora.distill_config import ( resolved_distill_runtime_cfg as _resolved_distill_runtime_cfg, ) -from unilab.algos.torch.hora.distill_config import ( +from unilab.algos.hora.distill_config import ( teacher_run_metadata as _teacher_run_metadata, ) -from unilab.algos.torch.hora.rsl_rl import HoraRslRlVecEnvWrapper as RslRlVecEnvWrapper +from unilab.algos.hora.rsl_rl import HoraRslRlVecEnvWrapper as RslRlVecEnvWrapper from unilab.base.backend import materialize_scene_visual_override -from unilab.training import ( +from unilab.base.backend.base import log_playback_plan +from unilab.base.config_adapter import ( BackendAdapter, create_env, +) +from unilab.training import ( ensure_registries, format_hora_stage2_checkpoint_error, get_log_root, - log_playback_plan, resolve_hora_stage2_checkpoint_path, setup_logger, should_run_playback, ) -from unilab.training.experiment import get_device_info_dict +from unilab.training.experiment import get_device_info_dict, write_run_config_snapshot def _write_distill_run_config( @@ -69,8 +70,9 @@ def _write_distill_run_config( Returns: None. Writes `distill_run_config.json` into `log_dir`. """ - payload = { - "run": { + write_run_config_snapshot( + log_dir, + run_metadata={ "algo": "hora_distill", "task": str(OmegaConf.select(cfg, "training.task_name")), "sim_backend": str(OmegaConf.select(cfg, "training.sim_backend")), @@ -78,11 +80,10 @@ def _write_distill_run_config( "hardware": get_device_info_dict(), "teacher": teacher_metadata, }, - "config": OmegaConf.to_container(cfg, resolve=True), - } - with (log_dir / "distill_run_config.json").open("w", encoding="utf-8") as f: - json.dump(payload, f, indent=2, ensure_ascii=True) - f.write("\n") + full_cfg=cfg, + filename="distill_run_config.json", + trailing_newline=True, + ) def _build_env_cfg_override(cfg: DictConfig) -> dict[str, Any]: diff --git a/scripts/train_offpolicy.py b/scripts/train_offpolicy.py index e490d64d8..46a732b01 100644 --- a/scripts/train_offpolicy.py +++ b/scripts/train_offpolicy.py @@ -1,4 +1,9 @@ -"""Unified off-policy training entry for SAC and TD3.""" +"""Shared off-policy (SAC/TD3/FlashSAC) train/play implementation. + +This module is no longer runnable directly; use the per-algorithm entry +scripts instead: ``scripts/train_sac.py``, ``scripts/train_td3.py``, and +``scripts/train_flashsac.py``. +""" from __future__ import annotations @@ -9,12 +14,14 @@ from pathlib import Path from typing import Any, cast -import hydra from omegaconf import DictConfig, OmegaConf ROOT_DIR = Path(__file__).parent.parent sys.path.append(str(ROOT_DIR)) +from unilab.base.backend.base import log_playback_plan +from unilab.base.backend.process_device import configure_backend_process_device +from unilab.base.config_adapter import create_env from unilab.ipc.dp_launcher import ( UNILAB_DP_LOG_DIR, DpRankSupervisor, @@ -27,33 +34,29 @@ validate_dp_launchable, ) from unilab.training import ( - apply_configured_training_seed, assert_offpolicy_task_choice_matches_algo, - create_env, + build_run_dir_name, ensure_registries, get_log_root, - log_playback_plan, + resolve_nan_guard_cfg, should_run_playback, ) from unilab.training.experiment import ExperimentTracker -from unilab.training.offpolicy import ( - build_offpolicy_env_cfg_override as _build_offpolicy_env_cfg_override, +from unilab.training.onnx_export import export_policy_onnx, verify_policy_onnx +from unilab.utils.checkpoint import ( + resolve_offpolicy_checkpoint_path as resolve_checkpoint_path, ) -from unilab.training.offpolicy import ( - build_play_actor, +from unilab.utils.seed import apply_configured_training_seed +from unilab.visualization.interactive_playback import ( + RslRlPlaybackConfig, + create_sac_playback_session, default_device, - extract_play_obs, - extract_reset_obs, - load_play_actor, - resolve_play_obs_dim, + resolve_play_actor_spec, resolve_play_obs_dims, ) -from unilab.training.onnx_export import export_policy_onnx, verify_policy_onnx -from unilab.training.run import ( - resolve_offpolicy_checkpoint_path as resolve_checkpoint_path, +from unilab.visualization.interactive_playback import ( + build_offpolicy_env_cfg_override as _build_offpolicy_env_cfg_override, ) -from unilab.training.sim2sim import policy_load_dim_guard, resolve_sim2sim_config -from unilab.utils.nan_guard import NanGuardCfg def enable_faulthandler() -> None: @@ -80,11 +83,6 @@ def build_failure_summary(exc: BaseException, run_summary: Any | None = None) -> return summary -def build_run_dir_name(timestamp: str, sim_backend: str, *, world_size: int = 1) -> str: - gpu_suffix = f"_gpux{world_size}" if world_size > 1 else "" - return f"{timestamp}_{sim_backend}{gpu_suffix}" - - def build_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict[str, Any] | None: return _build_offpolicy_env_cfg_override(algo_name, cfg, root_dir=ROOT_DIR) @@ -92,7 +90,7 @@ def build_offpolicy_env_cfg_override(algo_name: str, cfg: DictConfig) -> dict[st def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): """Build algorithm runner from unified Hydra config.""" env_cfg_override = build_offpolicy_env_cfg_override(algo_name, cfg) - from unilab.algos.torch.offpolicy.thread_budget import ( + from unilab.algos.offpolicy.thread_budget import ( apply_torch_thread_runtime, resolve_torch_thread_runtime, ) @@ -109,6 +107,10 @@ def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): from unilab.utils.device import get_default_device rank_device = resolve_dp_rank_device(dp_devices, dp_rank) or get_default_device() + # Bind backend-global device state before algorithm builders materialize + # their probe envs. The spawned collector repeats this binding in its own + # process using the same rank-local device. + configure_backend_process_device(str(cfg.training.sim_backend), rank_device) host_cpu_count = os.cpu_count() or 1 explicit_cpu_ids = getattr(cfg.training, "dp_collector_cpu_ids", None) if explicit_cpu_ids is not None: @@ -145,15 +147,7 @@ def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): ) apply_torch_thread_runtime(torch_thread_runtime, role="learner") - nan_guard_cfg = getattr(cfg.training, "nan_guard", None) - _nan_guard_cfg: NanGuardCfg | None = None - if nan_guard_cfg is not None and getattr(nan_guard_cfg, "enabled", False): - _nan_guard_cfg = NanGuardCfg( - enabled=True, - buffer_size=int(getattr(nan_guard_cfg, "buffer_size", 100)), - max_envs_to_dump=int(getattr(nan_guard_cfg, "max_envs_to_dump", 5)), - output_dir=getattr(nan_guard_cfg, "output_dir", None), - ) + _nan_guard_cfg = resolve_nan_guard_cfg(cfg.training) replay_prefetch_mode = getattr(cfg.training, "replay_prefetch_mode", "one_tick") if replay_prefetch_mode != "one_tick": @@ -165,7 +159,7 @@ def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): replay_device = require_offpolicy_replay_device(rank_device) if algo_name == "sac": - from unilab.algos.torch.fast_sac.double_buffer import ( + from unilab.algos.fast_sac.double_buffer import ( build_sac_double_buffer_runner, ) @@ -181,7 +175,7 @@ def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): ) if algo_name == "td3": - from unilab.algos.torch.fast_td3.double_buffer import ( + from unilab.algos.fast_td3.double_buffer import ( build_td3_double_buffer_runner, ) @@ -197,7 +191,7 @@ def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): ) if algo_name == "flashsac": - from unilab.algos.torch.flash_sac.double_buffer import ( + from unilab.algos.flash_sac.double_buffer import ( build_flashsac_double_buffer_runner, ) @@ -217,11 +211,8 @@ def build_runner(algo_name: str, cfg: DictConfig, log_dir: str | None = None): def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: """Play pipeline for off-policy algorithms.""" - import numpy as np import torch - from unilab.algos.torch.offpolicy.worker import resolve_offpolicy_actor_priv_info - load_path, load_path_dir = resolve_checkpoint_path( ROOT_DIR, cfg.algo.algo_log_name, @@ -232,55 +223,46 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: print(f"Could not find checkpoint. load_path={load_path}") return None - cfg = ( - resolve_sim2sim_config( - load_path_dir, - cfg, - algo_name=algo_name, - strict=bool(getattr(cfg.training, "sim2sim_strict", True)), - ) - or cfg - ) - - env_cfg_override = build_offpolicy_env_cfg_override(algo_name, cfg) - devices = resolve_dp_topology(cfg.training.devices) device = default_device(torch, resolve_dp_rank_device(devices, current_dp_rank())) print(f"Using device for play: {device}") - env = cast( - Any, - create_env( + playback_cfg = RslRlPlaybackConfig( + task=str(cfg.training.task_name), + load_run=str(cfg.algo.load_run), + checkpoint=None, + action_mode="policy", + policy_obs_mode="actor", + algo_log_name=str(cfg.algo.algo_log_name), + log_root=None, + num_envs=int(cfg.training.play_env_num), + ) + session, _policy_obs_mode, _checkpoint_path = create_sac_playback_session( + playback_cfg=playback_cfg, + cfg=cfg, + env_factory=lambda n: create_env( cfg, - num_envs=cfg.training.play_env_num, - env_cfg_override=env_cfg_override, + num_envs=n, + env_cfg_override=build_offpolicy_env_cfg_override(algo_name, cfg), ), - ) - obs_dim, critic_obs_dim = resolve_play_obs_dims(env.obs_groups_spec) - action_shape = env.action_space.shape - if action_shape is None: - raise ValueError("env.action_space.shape must be defined") - action_dim = int(action_shape[0]) - actor, normalizer, actor_algo_type, actor_kwargs = build_play_actor( - algo_name, - cfg, - obs_dim=obs_dim, - critic_obs_dim=critic_obs_dim, - action_dim=action_dim, + root_dir=ROOT_DIR, device=device, + algo_name=algo_name, ) - print(f"Loading model: {load_path}") - checkpoint = torch.load(load_path, map_location=device, weights_only=True) - with policy_load_dim_guard(env_obs_dim=obs_dim, env_action_dim=action_dim, algo_name=algo_name): - load_play_actor( - algo_name, - actor, - normalizer, - checkpoint, - ) + env = cast(Any, session.env) + actor = session.actor + normalizer = session.normalizer + actor_algo_type = session.actor_algo_type # Export actor to ONNX if load_path_dir is not None and bool(getattr(cfg.training, "export_onnx", True)): + obs_dim, critic_obs_dim = resolve_play_obs_dims(env.obs_groups_spec) + _, actor_kwargs = resolve_play_actor_spec( + algo_name, + cfg, + obs_dim=obs_dim, + critic_obs_dim=critic_obs_dim, + ) onnx_path = os.path.join(load_path_dir, "policy.onnx") dummy_input = torch.randn(1, obs_dim, device=device) dummy_priv_info = ( @@ -321,72 +303,13 @@ def play_offpolicy(algo_name: str, cfg: DictConfig) -> str | None: elif load_path_dir is not None: print("Skipping ONNX export because training.export_onnx=false.") - if env.state is None: - env.init_state() - - current_priv_info: np.ndarray | None = None - - def _resolve_play_priv_info(obs_dict: dict[str, np.ndarray], info: dict | None) -> np.ndarray: - if actor_algo_type != "hora_sac": - raise ValueError("Privileged play info was requested for a non-HORA actor.") - from unilab.base.observations import split_obs_dict - - actor_obs_np, critic_np = split_obs_dict(obs_dict) - priv_info = resolve_offpolicy_actor_priv_info( - algo_type=actor_algo_type, - obs_np=np.asarray(actor_obs_np, dtype=np.float32), - critic_np=np.asarray(critic_np, dtype=np.float32), - info=info, - ) - if priv_info is None: - raise ValueError("HORA-SAC play step is missing privileged info.") - return priv_info - - def _extract_reset_play_obs(reset_result) -> np.ndarray: - nonlocal current_priv_info - if not isinstance(reset_result, tuple) or len(reset_result) != 2: - raise ValueError(f"Unexpected env.reset return format: {type(reset_result)!r}") - obs_out, info_out = reset_result - if actor_algo_type == "hora_sac": - current_priv_info = _resolve_play_priv_info(obs_out, info_out) - return np.asarray(extract_play_obs(obs_out), dtype=np.float32) - - def _policy_step(obs_np: np.ndarray) -> np.ndarray: - nonlocal current_priv_info - obs_torch = torch.from_numpy(obs_np).to(device) - if normalizer: - obs_torch = normalizer(obs_torch, update=False) - if actor_algo_type == "hora_sac": - if current_priv_info is None: - raise ValueError("HORA-SAC play step is missing privileged info.") - priv_info_torch = torch.from_numpy(current_priv_info).to(device) - actions_np = ( - actor.explore( - obs_torch, - priv_info_torch, - deterministic=True, - ) - .cpu() - .numpy() - ) - elif algo_name in ("sac", "flashsac"): - actions_np = actor.explore(obs_torch, deterministic=True).cpu().numpy() - else: - actions_np = actor(obs_torch).cpu().numpy() - state = env.step(actions_np) - if actor_algo_type == "hora_sac": - current_priv_info = _resolve_play_priv_info(state.obs, state.info) - return np.asarray(extract_play_obs(state.obs), dtype=np.float32) - with torch.inference_mode(): play_video_path = env.run_playback_mode( play_render_mode=getattr(cfg.training, "play_render_mode", "auto"), play_steps=getattr(cfg.training, "play_steps", None), output_video=os.path.join(load_path_dir, "play_video.mp4") if load_path_dir else None, - initialize=lambda: _extract_reset_play_obs( - env.reset(np.arange(cfg.training.play_env_num, dtype=np.int32)) - ), - step=_policy_step, + initialize=session.reset, + step=lambda _obs: session.step_once(), camera_kwargs={ "cam_distance": cfg.training.cam_distance, "cam_elevation": cfg.training.cam_elevation, @@ -400,7 +323,6 @@ def _policy_step(obs_np: np.ndarray) -> np.ndarray: return play_video_path -@hydra.main(version_base="1.3", config_path="../conf/offpolicy", config_name="config") def main(cfg: DictConfig) -> None: enable_faulthandler() ensure_registries() @@ -498,4 +420,8 @@ def main(cfg: DictConfig) -> None: if __name__ == "__main__": - main() + raise SystemExit( + "scripts/train_offpolicy.py is a shared implementation module and is no longer " + "runnable directly. Use scripts/train_sac.py, scripts/train_td3.py, or " + "scripts/train_flashsac.py instead." + ) diff --git a/scripts/train_rsl_rl.py b/scripts/train_rsl_rl.py index feea13c9c..842465fe2 100644 --- a/scripts/train_rsl_rl.py +++ b/scripts/train_rsl_rl.py @@ -17,8 +17,20 @@ if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) -from unilab.algos.torch.rsl_rl_runtime import resolve_rsl_rl_ppo_runtime +from unilab.algos.rsl_rl import ( + RslRlVecEnvWrapper, + apply_rsl_rl_rank_seed, + finish_rsl_rl_distributed, + get_policy_obs_dims, + normalize_ppo_train_cfg, + ppo_samples_per_iteration, + resolve_rsl_rl_device, + rsl_rl_single_process_topology, +) +from unilab.algos.rsl_rl_runtime import resolve_rsl_rl_ppo_runtime from unilab.base.backend import RenderClosedError, materialize_scene_visual_override +from unilab.base.backend.base import log_playback_plan +from unilab.base.config_adapter import BackendAdapter, create_env from unilab.base.run_control import RunComplete from unilab.ipc.dp_launcher import ( UNILAB_DP_LOG_DIR, @@ -30,14 +42,12 @@ validate_dp_launchable, ) from unilab.training import ( - BackendAdapter, - apply_configured_training_seed, - create_env, + algo_config_dict, + apply_env_nan_guard, + build_run_dir_name, ensure_registries, - get_latest_checkpoint, - get_latest_run, + format_play_checkpoint_error, get_log_root, - log_playback_plan, parse_checkpoint_path, should_run_playback, ) @@ -47,17 +57,16 @@ patch_rsl_rl_resume_state, patch_rsl_rl_wandb_writer, ) -from unilab.training.rsl_rl import ( - RslRlVecEnvWrapper, - apply_rsl_rl_rank_seed, - finish_rsl_rl_distributed, - normalize_ppo_train_cfg, - ppo_samples_per_iteration, - resolve_rsl_rl_device, - rsl_rl_single_process_topology, -) -from unilab.training.sim2sim import policy_load_dim_guard, resolve_sim2sim_config +from unilab.utils.checkpoint import get_entrypoint_log_root from unilab.utils.device import get_default_device +from unilab.utils.seed import apply_configured_training_seed +from unilab.visualization.interactive_playback import ( + RslRlPlaybackConfig, + create_rsl_rl_playback_session, + infer_checkpoint_actor_input_dim, + make_sim2sim_preflight, + normalize_checkpoint_value, +) try: from rsl_rl.runners import OnPolicyRunner @@ -107,11 +116,6 @@ def _get_log_root(cfg: DictConfig) -> str: return str(get_log_root(ROOT_DIR, cfg)) -def build_ppo_run_dir_name(timestamp: str, sim_backend: str, *, world_size: int = 1) -> str: - gpu_suffix = f"_gpux{world_size}" if world_size > 1 else "" - return f"{timestamp}_{sim_backend}{gpu_suffix}" - - def resolve_ppo_log_dir( cfg: DictConfig, *, @@ -129,7 +133,7 @@ def resolve_ppo_log_dir( return str( Path(_get_log_root(cfg)) / str(cfg.training.task_name) - / build_ppo_run_dir_name( + / build_run_dir_name( timestamp, str(cfg.training.sim_backend), world_size=world_size, @@ -137,13 +141,6 @@ def resolve_ppo_log_dir( ) -def _algo_config_dict(cfg: DictConfig) -> dict[str, Any]: - train_cfg_raw = OmegaConf.to_container(cfg.algo, resolve=True) - if not isinstance(train_cfg_raw, dict): - raise TypeError("cfg.algo must resolve to a dict") - return cast(dict[str, Any], train_cfg_raw) - - def _resolve_ppo_wrapper_cls(rl_cfg: dict[str, Any]) -> type[RslRlVecEnvWrapper]: """Resolve the VecEnv wrapper class from the owner-selected PPO runtime. @@ -192,42 +189,6 @@ def validate_ppo_run_completion_topology( ) -def _format_play_checkpoint_error( - cfg: DictConfig, - *, - task_log_root: Path, - load_path: Path | None, - load_path_dir: Path | None, -) -> str: - selected_checkpoint = OmegaConf.select(cfg, "algo.checkpoint", default=-1) - checkpoint_hint = ( - f" algo.checkpoint={selected_checkpoint!r}" - if selected_checkpoint not in (None, "", -1, "-1") - else "" - ) - - if load_path_dir is not None and load_path is None and checkpoint_hint: - reason = f"Requested checkpoint was not found under resolved_run={load_path_dir}." - elif not task_log_root.exists(): - reason = "Task log root does not exist." - else: - latest_run = get_latest_run(task_log_root) - if latest_run is None: - reason = "No run directories were found under the task log root." - elif get_latest_checkpoint(latest_run) is None: - reason = f"Resolved latest run has no model_*.pt checkpoint files: {latest_run}." - else: - reason = "Requested run or checkpoint could not be resolved." - - return ( - "Could not resolve a checkpoint for play mode. " - f"{reason} task={cfg.training.task_name} task_log_root={task_log_root} " - f"algo.load_run={cfg.algo.load_run!r}{checkpoint_hint}." - " Use algo.load_run= " - "and optionally algo.checkpoint=." - ) - - def _resolve_play_num_steps(cfg: DictConfig) -> int | None: play_steps = OmegaConf.select(cfg, "training.play_steps", default=None) if play_steps is None: @@ -237,14 +198,13 @@ def _resolve_play_num_steps(cfg: DictConfig) -> int | None: def play_rsl_rl(cfg: DictConfig, device: str) -> str | None: """Play mode for RSL-RL.""" - rl_cfg = _algo_config_dict(cfg) - wrapper_cls = _resolve_ppo_wrapper_cls(rl_cfg) + rl_cfg = algo_config_dict(cfg) task_log_root = get_log_root(ROOT_DIR, cfg) / str(cfg.training.task_name) load_path, load_path_dir = parse_checkpoint_path(cfg, root_dir=ROOT_DIR) if load_path is None or load_path_dir is None or not load_path.exists(): print( - _format_play_checkpoint_error( + format_play_checkpoint_error( cfg, task_log_root=task_log_root, load_path=load_path, @@ -262,42 +222,48 @@ def play_rsl_rl(cfg: DictConfig, device: str) -> str | None: ) return None - cfg = ( - resolve_sim2sim_config( - load_path_dir, - cfg, - algo_name="ppo", - strict=bool(getattr(cfg.training, "sim2sim_strict", True)), - ) - or cfg - ) - - env_cfg_override = build_ppo_play_env_cfg_override(cfg) - - env = create_env( - cfg, + def _normalize_play_train_cfg(train_cfg: dict[str, Any]) -> dict[str, Any]: + normalized = normalize_ppo_train_cfg(train_cfg) + apply_ppo_runtime_flags(normalized, cfg, training_enabled=False) + return normalized + + playback_cfg = RslRlPlaybackConfig( + task=str(cfg.training.task_name), + load_run=str(cfg.algo.load_run), + checkpoint=normalize_checkpoint_value( + OmegaConf.select(cfg, "algo.checkpoint", default=None) + ), + action_mode="policy", + policy_obs_mode="flat", + algo_log_name=str(cfg.algo.algo_log_name), + log_root=None, num_envs=cfg.training.play_env_num, - env_cfg_override=env_cfg_override, ) - wrapped_env = wrapper_cls(env, device=device) - train_cfg = normalize_ppo_train_cfg(rl_cfg) - apply_ppo_runtime_flags(train_cfg, cfg, training_enabled=False) - if "runner" not in train_cfg: - train_cfg["runner"] = {} - train_cfg["runner"]["logger"] = "none" - - runner = cast( - Any, - OnPolicyRunner(cast(Any, wrapped_env), train_cfg, log_dir=None, device=device), + session, _policy_obs_mode, _checkpoint_path = create_rsl_rl_playback_session( + playback_cfg=playback_cfg, + env_factory=lambda n: create_env( + cfg, + num_envs=n, + env_cfg_override=build_ppo_play_env_cfg_override(cfg), + ), + algo_config=rl_cfg, + root_dir=ROOT_DIR, + device=device, + checkpoint_resolver=lambda *_args: str(load_path), + checkpoint_input_dim_reader=infer_checkpoint_actor_input_dim, + entrypoint_log_root=get_entrypoint_log_root, + wrapper_cls=_resolve_ppo_wrapper_cls(rl_cfg), + runner_cls=OnPolicyRunner, + policy_obs_dims_getter=get_policy_obs_dims, + train_cfg_normalizer=_normalize_play_train_cfg, + sim2sim_preflight=make_sim2sim_preflight(cfg, algo_name="ppo"), + guard_algo_name="ppo", ) - with policy_load_dim_guard( - env_obs_dim=getattr(wrapped_env, "num_obs", None), - env_action_dim=getattr(wrapped_env, "num_actions", None), - algo_name="ppo", - ): - runner.load(str(load_path), map_location=device) - policy = runner.get_inference_policy(device=device) + env = session.env + runner = session.runner if EXPORT_POLICY: + # The checkpoint early-returns above guarantee a loaded runner here. + assert runner is not None runner.export_policy_to_onnx(path=str(load_path_dir)) runner.export_policy_to_jit(path=str(load_path_dir)) num_steps = _resolve_play_num_steps(cfg) @@ -319,8 +285,8 @@ def _log_plan(plan) -> None: getattr(cfg.training, "render_spacing", getattr(env.cfg, "render_spacing", 1.0)) ), render_offset_mode=str(getattr(env.cfg, "render_offset_mode", "grid")), - initialize=lambda: wrapped_env.reset()[0], - step=lambda obs: wrapped_env.step(policy(obs))[0], + initialize=session.reset, + step=lambda _obs: session.step_once(), camera_kwargs={ "cam_distance": cfg.training.cam_distance, "cam_elevation": cfg.training.cam_elevation, @@ -450,26 +416,10 @@ def main(cfg: DictConfig) -> None: env_cfg_override=env_cfg_override, ) try: - rl_cfg = _algo_config_dict(cfg) + rl_cfg = algo_config_dict(cfg) wrapper_cls = _resolve_ppo_wrapper_cls(rl_cfg) - nan_guard_cfg = getattr(cfg.training, "nan_guard", None) - if nan_guard_cfg is not None and getattr(nan_guard_cfg, "enabled", False): - from unilab.utils.nan_guard import NanGuard, NanGuardCfg - - guard = NanGuard( - NanGuardCfg( - enabled=True, - buffer_size=int(getattr(nan_guard_cfg, "buffer_size", 100)), - max_envs_to_dump=int(getattr(nan_guard_cfg, "max_envs_to_dump", 5)), - output_dir=getattr(nan_guard_cfg, "output_dir", None), - ), - num_envs=env.num_envs, - supports_state_playback=( - env.play_capabilities.supports_physics_state_playback - ), - ) - env.set_nan_guard(guard) + apply_env_nan_guard(env, cfg.training) wrapped_env = wrapper_cls(env, device=device) diff --git a/scripts/train_sac.py b/scripts/train_sac.py new file mode 100644 index 000000000..0a1bf96c3 --- /dev/null +++ b/scripts/train_sac.py @@ -0,0 +1,14 @@ +"""SAC training/playback entrypoint (shared implementation in train_offpolicy.py).""" + +import hydra +from omegaconf import DictConfig +from train_offpolicy import main as _offpolicy_main + + +@hydra.main(version_base="1.3", config_path="../conf/sac", config_name="config") +def main(cfg: DictConfig) -> None: + _offpolicy_main(cfg) + + +if __name__ == "__main__": + main() diff --git a/scripts/train_td3.py b/scripts/train_td3.py new file mode 100644 index 000000000..afd1cb239 --- /dev/null +++ b/scripts/train_td3.py @@ -0,0 +1,14 @@ +"""TD3 training/playback entrypoint (shared implementation in train_offpolicy.py).""" + +import hydra +from omegaconf import DictConfig +from train_offpolicy import main as _offpolicy_main + + +@hydra.main(version_base="1.3", config_path="../conf/td3", config_name="config") +def main(cfg: DictConfig) -> None: + _offpolicy_main(cfg) + + +if __name__ == "__main__": + main() diff --git a/scripts/visualize_task_env.py b/scripts/visualize_task_env.py index 60bfd2124..12a98a533 100644 --- a/scripts/visualize_task_env.py +++ b/scripts/visualize_task_env.py @@ -284,13 +284,14 @@ def _run_mujoco(env, num_envs: int) -> None: def _build_env_cfg_override(task_name: str) -> dict[str, Any]: """Build the env_cfg_override dict from CLI args alone — no Hydra.""" - if task_name not in registry._envs: + if not registry.contains(task_name): raise SystemExit( - f"Task '{task_name}' is not registered. Available: {sorted(registry._envs.keys())}" + f"Task '{task_name}' is not registered. " + f"Available: {sorted(registry.list_registered_envs())}" ) - env_cfg_cls = registry._envs[task_name].env_cfg_cls + env_cfg = registry.materialize_env_config(task_name) override: dict[str, Any] = {} - reward_stub = _build_reward_stub(env_cfg_cls) + reward_stub = _build_reward_stub(type(env_cfg)) if reward_stub is not None: override["reward_config"] = reward_stub return override diff --git a/src/unilab/algos/torch/appo/__init__.py b/src/unilab/algos/appo/__init__.py similarity index 100% rename from src/unilab/algos/torch/appo/__init__.py rename to src/unilab/algos/appo/__init__.py diff --git a/src/unilab/algos/torch/appo/learner.py b/src/unilab/algos/appo/learner.py similarity index 99% rename from src/unilab/algos/torch/appo/learner.py rename to src/unilab/algos/appo/learner.py index 966ad2aaf..71305b917 100644 --- a/src/unilab/algos/torch/appo/learner.py +++ b/src/unilab/algos/appo/learner.py @@ -21,7 +21,7 @@ from rsl_rl.utils import resolve_optimizer from tensordict import TensorDict -from unilab.algos.torch.common.compile import get_torch_compile_for_cuda +from unilab.algos.common.compile import get_torch_compile_for_cuda _LOG_2_PI = math.log(2.0 * math.pi) _NORMAL_ENTROPY_OFFSET = 0.5 * (1.0 + _LOG_2_PI) diff --git a/src/unilab/algos/torch/appo/runner.py b/src/unilab/algos/appo/runner.py similarity index 97% rename from src/unilab/algos/torch/appo/runner.py rename to src/unilab/algos/appo/runner.py index 22800aaff..370ad4b2b 100644 --- a/src/unilab/algos/torch/appo/runner.py +++ b/src/unilab/algos/appo/runner.py @@ -17,13 +17,13 @@ import torch from rsl_rl.utils import resolve_callable -from unilab.algos.torch.appo.learner import APPOLearner -from unilab.algos.torch.appo.staging import RolloutStagingPool -from unilab.algos.torch.appo.worker import appo_collector_fn +from unilab.algos.appo.learner import APPOLearner +from unilab.algos.appo.staging import RolloutStagingPool +from unilab.algos.appo.worker import appo_collector_fn from unilab.ipc import AsyncRunner, RolloutRingBuffer, SharedWeightSync from unilab.logging import OffPolicyLogger -from unilab.training.seed import apply_training_seed, derive_worker_seed from unilab.utils.nan_guard import NanGuardCfg +from unilab.utils.seed import apply_training_seed, derive_worker_seed def _optimizer_lr_from_state(optimizer: torch.optim.Optimizer) -> float: @@ -312,7 +312,10 @@ def learn( ) logger_started = False - reward_history: deque = deque(maxlen=200) + # Recent collector reports; each entry is already the collector's + # rolling 100-episode mean, so a short window keeps the logged + # reward timely without losing smoothing. + reward_history: deque = deque(maxlen=10) latest_reward_components: dict = {} staging_pool = RolloutStagingPool( @@ -396,9 +399,7 @@ def learn( logger.update_staging_pool(staging_pool.active_count, staging_pool.capacity) mean_reward = ( - sum(list(reward_history)[-50:]) / max(len(list(reward_history)[-50:]), 1) - if reward_history - else 0.0 + sum(reward_history) / max(len(reward_history), 1) if reward_history else 0.0 ) last_mean_reward = float(mean_reward) best_mean_reward = max(best_mean_reward, last_mean_reward) diff --git a/src/unilab/algos/torch/appo/runtime.py b/src/unilab/algos/appo/runtime.py similarity index 97% rename from src/unilab/algos/torch/appo/runtime.py rename to src/unilab/algos/appo/runtime.py index ad5b58bd8..d48131f18 100644 --- a/src/unilab/algos/torch/appo/runtime.py +++ b/src/unilab/algos/appo/runtime.py @@ -46,7 +46,7 @@ def resolve_appo_runtime( """ runtime_resolver = rl_cfg.get("runtime_resolver") if runtime_resolver in (None, ""): - from unilab.algos.torch.appo.runner import APPORunner + from unilab.algos.appo.runner import APPORunner return APPORuntime(runner_cls=APPORunner, play_fn=default_play_fn) diff --git a/src/unilab/algos/torch/appo/staging.py b/src/unilab/algos/appo/staging.py similarity index 100% rename from src/unilab/algos/torch/appo/staging.py rename to src/unilab/algos/appo/staging.py diff --git a/src/unilab/algos/torch/appo/worker.py b/src/unilab/algos/appo/worker.py similarity index 95% rename from src/unilab/algos/torch/appo/worker.py rename to src/unilab/algos/appo/worker.py index 3058c3bca..35eed27d3 100644 --- a/src/unilab/algos/torch/appo/worker.py +++ b/src/unilab/algos/appo/worker.py @@ -8,7 +8,7 @@ import statistics import sys import time -from collections import defaultdict +from collections import defaultdict, deque from queue import Empty, Full from typing import Any, Dict @@ -16,11 +16,11 @@ import torch from rsl_rl.utils import resolve_callable -from unilab.algos.torch.common.collector_timing import extract_env_step_breakdown_timing_ms +from unilab.algos.common.collector_timing import extract_env_step_breakdown_timing_ms from unilab.base.final_observation import resolve_terminal_observation_contract from unilab.base.observations import split_obs_dict from unilab.base.registry import ensure_registries -from unilab.training.seed import apply_training_seed +from unilab.utils.seed import apply_training_seed def put_latest_metrics(metrics_queue: Any, msg: dict[str, Any], *, worker_name: str) -> None: @@ -235,8 +235,10 @@ def to_float32_np(x): obs_td = TensorDict({"policy": obs_torch}, batch_size=num_envs, device=collector_device) total_steps = 0 - ep_rewards = [] - ep_lengths = [] + # Bounded rolling window of the most recent completed episodes; an + # unbounded list here grows for the entire run. + ep_rewards: deque[float] = deque(maxlen=100) + ep_lengths: deque[int] = deque(maxlen=100) current_ep_rewards = np.zeros(num_envs, dtype=np.float32) current_ep_lengths = np.zeros(num_envs, dtype=np.int32) ep_reward_components = defaultdict(list) @@ -353,15 +355,17 @@ def to_float32_np(x): if k.startswith("reward/"): ep_reward_components[k].append(v) - if metrics_queue is not None and total_steps % (num_envs * 10) == 0: + # Report every env step so learner-side reward and throughput + # displays track the current policy without extra lag. + if metrics_queue is not None: try: msg: dict[str, Any] = { "total_steps": total_steps, } if ep_rewards: - msg["mean_ep_reward"] = statistics.mean(ep_rewards[-100:]) + msg["mean_ep_reward"] = statistics.mean(ep_rewards) msg["mean_ep_length"] = ( - statistics.mean(ep_lengths[-100:]) if ep_lengths else 0.0 + statistics.mean(ep_lengths) if ep_lengths else 0.0 ) if ep_completions > 0: msg["timeout_rate"] = ep_timeouts / ep_completions diff --git a/src/unilab/algos/common/__init__.py b/src/unilab/algos/common/__init__.py new file mode 100644 index 000000000..8863fd9f4 --- /dev/null +++ b/src/unilab/algos/common/__init__.py @@ -0,0 +1,18 @@ +from unilab.algos.common.actor_factory import build_actor +from unilab.algos.common.device import get_env_dims +from unilab.algos.common.networks import Critic, DistributionalQNetwork +from unilab.algos.common.normalization import EmpiricalNormalization +from unilab.algos.common.stability import check_nan_loss, clip_gradients, safe_tensor +from unilab.base.registry import ensure_registries + +__all__ = [ + "EmpiricalNormalization", + "DistributionalQNetwork", + "Critic", + "get_env_dims", + "check_nan_loss", + "clip_gradients", + "safe_tensor", + "ensure_registries", + "build_actor", +] diff --git a/src/unilab/algos/torch/common/actor_factory.py b/src/unilab/algos/common/actor_factory.py similarity index 88% rename from src/unilab/algos/torch/common/actor_factory.py rename to src/unilab/algos/common/actor_factory.py index af1656a54..2c6eead1a 100644 --- a/src/unilab/algos/torch/common/actor_factory.py +++ b/src/unilab/algos/common/actor_factory.py @@ -21,7 +21,7 @@ def build_actor( ): """Build the correct actor model based on algorithm type.""" if algo_type == "sac": - from unilab.algos.torch.fast_sac.learner import SACActor + from unilab.algos.fast_sac.learner import SACActor return SACActor( obs_dim=obs_dim, @@ -33,7 +33,7 @@ def build_actor( if algo_type == "hora_sac": if priv_info_dim is None: raise ValueError("build_actor(algo_type='hora_sac') requires priv_info_dim.") - from unilab.algos.torch.hora.sac_models import HoraSACActor + from unilab.algos.hora.sac_models import HoraSACActor return HoraSACActor( obs_dim=obs_dim, @@ -46,7 +46,7 @@ def build_actor( device=device, ) if algo_type == "td3": - from unilab.algos.torch.fast_td3.learner import TD3Actor + from unilab.algos.fast_td3.learner import TD3Actor return TD3Actor( obs_dim=obs_dim, @@ -59,7 +59,7 @@ def build_actor( device=device, ) if algo_type == "flashsac": - from unilab.algos.torch.flash_sac.network import FlashSACActor + from unilab.algos.flash_sac.network import FlashSACActor return FlashSACActor( num_blocks=actor_num_blocks, diff --git a/src/unilab/algos/torch/common/collector_timing.py b/src/unilab/algos/common/collector_timing.py similarity index 100% rename from src/unilab/algos/torch/common/collector_timing.py rename to src/unilab/algos/common/collector_timing.py diff --git a/src/unilab/algos/torch/common/compile.py b/src/unilab/algos/common/compile.py similarity index 100% rename from src/unilab/algos/torch/common/compile.py rename to src/unilab/algos/common/compile.py diff --git a/src/unilab/algos/torch/common/device.py b/src/unilab/algos/common/device.py similarity index 100% rename from src/unilab/algos/torch/common/device.py rename to src/unilab/algos/common/device.py diff --git a/src/unilab/algos/torch/common/networks.py b/src/unilab/algos/common/networks.py similarity index 100% rename from src/unilab/algos/torch/common/networks.py rename to src/unilab/algos/common/networks.py diff --git a/src/unilab/algos/torch/common/normalization.py b/src/unilab/algos/common/normalization.py similarity index 100% rename from src/unilab/algos/torch/common/normalization.py rename to src/unilab/algos/common/normalization.py diff --git a/src/unilab/algos/torch/common/stability.py b/src/unilab/algos/common/stability.py similarity index 100% rename from src/unilab/algos/torch/common/stability.py rename to src/unilab/algos/common/stability.py diff --git a/src/unilab/algos/cse_ppo/__init__.py b/src/unilab/algos/cse_ppo/__init__.py new file mode 100644 index 000000000..5cd9dfd6c --- /dev/null +++ b/src/unilab/algos/cse_ppo/__init__.py @@ -0,0 +1,9 @@ +"""Concurrent state-estimator PPO.""" + +from .actor_critic import CSEActorCritic +from .algorithm import CSEPPO +from .estimator import CSEEstimator +from .runner import CSEOnPolicyRunner +from .storage import CSERolloutStorage + +__all__ = ["CSEPPO", "CSEActorCritic", "CSEEstimator", "CSEOnPolicyRunner", "CSERolloutStorage"] diff --git a/src/unilab/algos/cse_ppo/actor_critic.py b/src/unilab/algos/cse_ppo/actor_critic.py new file mode 100644 index 000000000..51b311a90 --- /dev/null +++ b/src/unilab/algos/cse_ppo/actor_critic.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""CSE-PPO actor-critic network.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import torch +from torch import nn +from torch.distributions import Normal + +from .estimator import CSEEstimator, _mlp + + +class CSEActorCritic(nn.Module): + is_recurrent = False + + def __init__( + self, + num_actor_obs: int, + num_critic_obs: int, + num_one_step_obs: int, + num_actions: int, + actor_hidden_dims: Sequence[int] = (512, 256, 128), + critic_hidden_dims: Sequence[int] = (512, 256, 128), + activation: str = "elu", + init_noise_std: float = 1.0, + estimator: dict | None = None, + ) -> None: + super().__init__() + if num_one_step_obs <= 0: + raise ValueError("num_one_step_obs must be positive") + if num_actor_obs % num_one_step_obs: + raise ValueError( + "num_actor_obs must be an integer multiple of num_one_step_obs, " + f"got {num_actor_obs} and {num_one_step_obs}" + ) + if len(actor_hidden_dims) == 0 or len(critic_hidden_dims) == 0: + raise ValueError("actor_hidden_dims and critic_hidden_dims must not be empty") + self.history_size = int(num_actor_obs // num_one_step_obs) + self.num_actor_obs = int(num_actor_obs) + self.num_critic_obs = int(num_critic_obs) + self.num_actions = int(num_actions) + self.num_one_step_obs = int(num_one_step_obs) + self.estimator = CSEEstimator( + temporal_steps=self.history_size, + num_one_step_obs=self.num_one_step_obs, + activation=activation, + **dict(estimator or {}), + ) + self.actor = _mlp( + self.num_one_step_obs + self.estimator.num_latent, + self.num_actions, + actor_hidden_dims, + activation, + ) + self.critic = _mlp(self.num_critic_obs, 1, critic_hidden_dims, activation) + self.std = nn.Parameter(float(init_noise_std) * torch.ones(self.num_actions)) + self.distribution: Normal | None = None + Normal.set_default_validate_args(False) + + @property + def action_mean(self) -> torch.Tensor: + assert self.distribution is not None + return self.distribution.mean + + @property + def action_std(self) -> torch.Tensor: + assert self.distribution is not None + return self.distribution.stddev + + @property + def entropy(self) -> torch.Tensor: + assert self.distribution is not None + return self.distribution.entropy().sum(dim=-1) + + def reset(self, dones: torch.Tensor | None = None) -> None: + del dones + + def forward(self) -> torch.Tensor: + raise NotImplementedError + + def _actor_input(self, obs_history: torch.Tensor) -> torch.Tensor: + latent = self.estimator.encode(obs_history) + return torch.cat((obs_history[:, -self.num_one_step_obs :], latent), dim=-1) + + def update_distribution(self, obs_history: torch.Tensor) -> None: + mean = self.actor(self._actor_input(obs_history)) + self.distribution = Normal(mean, mean * 0.0 + self.std) + + def act(self, obs_history: torch.Tensor, **kwargs) -> torch.Tensor: + del kwargs + self.update_distribution(obs_history) + assert self.distribution is not None + return self.distribution.sample() + + def get_actions_log_prob(self, actions: torch.Tensor) -> torch.Tensor: + assert self.distribution is not None + return self.distribution.log_prob(actions).sum(dim=-1) + + def act_inference(self, obs_history: torch.Tensor, observations=None) -> torch.Tensor: + del observations + return self.actor(self._actor_input(obs_history)) + + def evaluate(self, critic_observations: torch.Tensor, **kwargs) -> torch.Tensor: + del kwargs + return self.critic(critic_observations) diff --git a/src/unilab/algos/cse_ppo/algorithm.py b/src/unilab/algos/cse_ppo/algorithm.py new file mode 100644 index 000000000..aa769ab17 --- /dev/null +++ b/src/unilab/algos/cse_ppo/algorithm.py @@ -0,0 +1,232 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""PPO update with a concurrently trained supervised state estimator.""" + +from __future__ import annotations + +import contextlib +from typing import Any + +import torch +from tensordict import TensorDict +from torch import nn, optim + +from .actor_critic import CSEActorCritic +from .storage import CSERolloutStorage + + +class CSEPPO: + actor_critic: CSEActorCritic + + def __init__( + self, + actor_critic: CSEActorCritic, + num_learning_epochs: int = 1, + num_mini_batches: int = 1, + clip_param: float = 0.2, + gamma: float = 0.998, + lam: float = 0.95, + value_loss_coef: float = 1.0, + entropy_coef: float = 0.0, + learning_rate: float = 1e-3, + max_grad_norm: float = 1.0, + use_clipped_value_loss: bool = True, + schedule: str = "fixed", + desired_kl: float | None = 0.01, + min_learning_rate: float = 1e-5, + max_learning_rate: float = 1e-2, + min_policy_std: float = 1e-2, + max_policy_std: float | None = None, + use_amp: bool = False, + amp_dtype: str = "bfloat16", + device: str = "cpu", + **kwargs: Any, + ) -> None: + del kwargs + self.device = device + self.actor_critic = actor_critic.to(device) + self.optimizer = optim.Adam(self.actor_critic.parameters(), lr=float(learning_rate)) + self.storage: CSERolloutStorage | None = None + self.transition = CSERolloutStorage.Transition() + self.num_learning_epochs = int(num_learning_epochs) + self.num_mini_batches = int(num_mini_batches) + self.clip_param = float(clip_param) + self.gamma = float(gamma) + self.lam = float(lam) + self.value_loss_coef = float(value_loss_coef) + self.entropy_coef = float(entropy_coef) + self.max_grad_norm = float(max_grad_norm) + self.use_clipped_value_loss = bool(use_clipped_value_loss) + self.learning_rate = float(learning_rate) + self.schedule = schedule + self.desired_kl = desired_kl + self.min_learning_rate = float(min_learning_rate) + self.max_learning_rate = float(max_learning_rate) + self.min_policy_std = float(min_policy_std) + self.max_policy_std = None if max_policy_std is None else float(max_policy_std) + self._amp_enabled = bool(use_amp) and "cuda" in str(device) + self._amp_dtype = torch.bfloat16 if amp_dtype == "bfloat16" else torch.float16 + + def init_storage( + self, + num_envs: int, + num_transitions_per_env: int, + actor_obs_shape, + critic_obs_shape, + action_shape, + ) -> None: + self.storage = CSERolloutStorage( + num_envs, + num_transitions_per_env, + actor_obs_shape, + critic_obs_shape, + action_shape, + self.device, + ) + + def test_mode(self) -> None: + self.actor_critic.eval() + + def train_mode(self) -> None: + self.actor_critic.train() + + def act(self, obs: torch.Tensor, critic_obs: torch.Tensor) -> torch.Tensor: + self.transition.actions = self.actor_critic.act(obs).detach() + self.transition.values = self.actor_critic.evaluate(critic_obs).detach() + self.transition.actions_log_prob = self.actor_critic.get_actions_log_prob( + self.transition.actions + ).detach() + self.transition.action_mean = self.actor_critic.action_mean.detach() + self.transition.action_sigma = self.actor_critic.action_std.detach() + self.transition.observations = obs + self.transition.critic_observations = critic_obs + return self.transition.actions + + def process_env_step( + self, + next_obs: TensorDict | torch.Tensor, + rewards: torch.Tensor, + dones: torch.Tensor, + extras: dict[str, Any], + ) -> None: + del next_obs + self.transition.rewards = rewards.clone() + self.transition.dones = dones + timeouts = extras.get("time_outs") + bootstrap = extras.get("time_out_bootstrap_obs") + if isinstance(timeouts, torch.Tensor): + mask = timeouts.to(self.device).bool().view(-1).float() + if bootstrap is not None and torch.count_nonzero(mask) > 0: + values = self.actor_critic.evaluate(_critic_obs(bootstrap.to(self.device))).detach() + else: + values = self.transition.values + assert values is not None + correction = self.gamma * torch.squeeze(values * mask.unsqueeze(1), 1) + rewards = self.transition.rewards + assert rewards is not None + if rewards.ndim == 2 and rewards.shape[-1] == 1: + correction = correction.unsqueeze(1) + self.transition.rewards = rewards + correction + assert self.storage is not None + self.storage.add_transition(self.transition) + self.transition.clear() + self.actor_critic.reset(dones) + + def compute_returns(self, last_critic_obs: torch.Tensor) -> None: + assert self.storage is not None + self.storage.compute_returns( + self.actor_critic.evaluate(last_critic_obs).detach(), self.gamma, self.lam + ) + + def _amp_ctx(self): + return ( + torch.autocast(device_type="cuda", dtype=self._amp_dtype) + if self._amp_enabled + else contextlib.nullcontext() + ) + + def _adapt_learning_rate(self, kl_mean: float) -> None: + if self.desired_kl is None or self.schedule != "adaptive": + return + if kl_mean > self.desired_kl * 2: + self.learning_rate = max(self.min_learning_rate, self.learning_rate / 1.5) + elif 0 < kl_mean < self.desired_kl / 2: + self.learning_rate = min(self.max_learning_rate, self.learning_rate * 1.5) + for group in self.optimizer.param_groups: + group["lr"] = self.learning_rate + + def update(self) -> tuple[float, float, float]: + assert self.storage is not None + value_total = policy_total = estimator_total = 0.0 + for batch in self.storage.mini_batch_generator( + self.num_mini_batches, self.num_learning_epochs + ): + ( + obs, + critic_obs, + actions, + old_values, + advantages, + returns, + old_log_prob, + old_mu, + old_sigma, + ) = batch + with self._amp_ctx(): + self.actor_critic.act(obs) + log_prob = self.actor_critic.get_actions_log_prob(actions) + values = self.actor_critic.evaluate(critic_obs) + log_prob, values = log_prob.float(), values.float() + mu, sigma, entropy = ( + self.actor_critic.action_mean.float(), + self.actor_critic.action_std, + self.actor_critic.entropy, + ) + if self.desired_kl is not None and self.schedule == "adaptive": + with torch.inference_mode(): + kl = torch.sum( + torch.log(sigma / old_sigma + 1e-5) + + (old_sigma.square() + (old_mu - mu).square()) / (2 * sigma.square()) + - 0.5, + dim=-1, + ) + self._adapt_learning_rate(float(kl.mean())) + ratio = torch.exp(log_prob - old_log_prob.squeeze()) + surrogate = torch.max( + -advantages.squeeze() * ratio, + -advantages.squeeze() * ratio.clamp(1 - self.clip_param, 1 + self.clip_param), + ).mean() + if self.use_clipped_value_loss: + value_clipped = old_values + (values - old_values).clamp( + -self.clip_param, self.clip_param + ) + value_loss = torch.max( + (values - returns).square(), (value_clipped - returns).square() + ).mean() + else: + value_loss = (returns - values).square().mean() + loss = ( + surrogate + self.value_loss_coef * value_loss - self.entropy_coef * entropy.mean() + ) + self.optimizer.zero_grad() + loss.backward() + nn.utils.clip_grad_norm_(self.actor_critic.parameters(), self.max_grad_norm) + self.optimizer.step() + self.actor_critic.std.data.clamp_(min=self.min_policy_std, max=self.max_policy_std) + estimator_loss = self.actor_critic.estimator.update( + obs, critic_obs, autocast_enabled=self._amp_enabled, autocast_dtype=self._amp_dtype + ) + value_total += float(value_loss.item()) + policy_total += float(surrogate.item()) + estimator_total += estimator_loss + updates = self.num_learning_epochs * self.num_mini_batches + self.storage.clear() + return value_total / updates, policy_total / updates, estimator_total / updates + + +def _critic_obs(obs: TensorDict | torch.Tensor) -> torch.Tensor: + if isinstance(obs, TensorDict): + for key in ("critic", "policy", "actor"): + if key in obs: + return obs[key] + raise KeyError("CSE-PPO TensorDict obs must contain critic, policy, or actor") + return obs diff --git a/src/unilab/algos/cse_ppo/estimator.py b/src/unilab/algos/cse_ppo/estimator.py new file mode 100644 index 000000000..6fda2aa6d --- /dev/null +++ b/src/unilab/algos/cse_ppo/estimator.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Concurrent state estimator used by the CSE-PPO actor.""" + +from __future__ import annotations + +import contextlib +from collections.abc import Sequence +from typing import cast + +import torch +from torch import nn, optim +from torch.nn import functional as F + + +def get_activation(name: str) -> nn.Module: + activations = { + "elu": nn.ELU, + "selu": nn.SELU, + "relu": nn.ReLU, + "crelu": nn.ReLU, + "silu": nn.SiLU, + "lrelu": nn.LeakyReLU, + "tanh": nn.Tanh, + "sigmoid": nn.Sigmoid, + } + try: + return activations[name]() + except KeyError as exc: + raise ValueError(f"Unsupported activation: {name}") from exc + + +def _mlp( + input_dim: int, output_dim: int, hidden_dims: Sequence[int], activation: str +) -> nn.Sequential: + layers: list[nn.Module] = [] + last = int(input_dim) + for dim in hidden_dims: + layers.extend((nn.Linear(last, int(dim)), get_activation(activation))) + last = int(dim) + layers.append(nn.Linear(last, int(output_dim))) + return nn.Sequential(*layers) + + +class CSEEstimator(nn.Module): + """Supervised encoder/decoder for the privileged current-state target.""" + + def __init__( + self, + temporal_steps: int, + num_one_step_obs: int, + num_pred: int = 12, + enc_hidden_dims: Sequence[int] = (256, 128), + latent_dim: int = 19, + dec_hidden_dims: Sequence[int] = (64,), + activation: str = "elu", + learning_rate: float = 1e-5, + max_grad_norm: float = 10.0, + target_weights: Sequence[float] | None = None, + target_start: int = 0, + target_group_sizes: Sequence[int] | None = None, + ) -> None: + super().__init__() + if temporal_steps <= 0: + raise ValueError("temporal_steps must be positive") + if num_one_step_obs <= 0: + raise ValueError("num_one_step_obs must be positive") + if num_pred <= 0: + raise ValueError("num_pred must be positive") + self.temporal_steps = int(temporal_steps) + self.num_one_step_obs = int(num_one_step_obs) + self.num_pred = int(num_pred) + self.num_latent = int(latent_dim) + self.target_start = int(target_start) + self.max_grad_norm = float(max_grad_norm) + self.target_group_sizes = ( + tuple(int(size) for size in target_group_sizes) + if target_group_sizes is not None + else None + ) + if self.target_group_sizes is not None and sum(self.target_group_sizes) != self.num_pred: + raise ValueError( + f"target_group_sizes {self.target_group_sizes} must sum to num_pred {self.num_pred}" + ) + weight_count = ( + len(self.target_group_sizes) if self.target_group_sizes is not None else self.num_pred + ) + weights = ( + torch.ones(weight_count) + if target_weights is None + else torch.as_tensor(list(target_weights), dtype=torch.float32) + ) + if weights.numel() != weight_count: + kind = "groups" if self.target_group_sizes is not None else "num_pred" + raise ValueError(f"target_weights length {weights.numel()} != {weight_count} ({kind})") + self.register_buffer("target_weights", weights) + self.encoder = _mlp( + self.temporal_steps * self.num_one_step_obs, + self.num_latent, + enc_hidden_dims, + activation, + ) + self.decoder = _mlp(self.num_latent, self.num_pred, dec_hidden_dims, activation) + self.learning_rate = float(learning_rate) + self.optimizer = optim.Adam(self.parameters(), lr=self.learning_rate) + + def encode(self, obs_history: torch.Tensor) -> torch.Tensor: + return self.encoder(obs_history) + + def get_latent(self, obs_history: torch.Tensor) -> torch.Tensor: + return self.encoder(obs_history.detach()).detach() + + def forward(self, obs_history: torch.Tensor) -> torch.Tensor: + return self.get_latent(obs_history) + + def predict(self, obs_history: torch.Tensor) -> torch.Tensor: + return self.decoder(self.encoder(obs_history.detach())).detach() + + def _regression_loss(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + weights = cast(torch.Tensor, self.target_weights).to(pred.device) + if self.target_group_sizes is None: + return (weights * F.mse_loss(pred, target, reduction="none")).mean() + loss = pred.new_zeros(()) + offset = 0 + for size, weight in zip(self.target_group_sizes, weights, strict=True): + part = slice(offset, offset + size) + loss = loss + F.mse_loss(pred[:, part] * weight, target[:, part] * weight) + offset += size + return loss + + def update( + self, + obs_history: torch.Tensor, + critic_obs: torch.Tensor, + lr: float | None = None, + autocast_enabled: bool = False, + autocast_dtype: torch.dtype | None = None, + ) -> float: + if lr is not None: + self.learning_rate = float(lr) + for group in self.optimizer.param_groups: + group["lr"] = self.learning_rate + end = self.target_start + self.num_pred + if critic_obs.shape[-1] < end: + raise ValueError( + "critic_obs is too small for the CSE estimator target slice: " + f"shape={tuple(critic_obs.shape)}, target=[{self.target_start}:{end}]" + ) + target = critic_obs[:, self.target_start : end].detach() + amp_ctx = ( + torch.autocast(device_type="cuda", dtype=autocast_dtype or torch.bfloat16) + if autocast_enabled + else contextlib.nullcontext() + ) + with amp_ctx: + loss = self._regression_loss(self.decoder(self.encoder(obs_history)), target) + self.optimizer.zero_grad() + loss.backward() + nn.utils.clip_grad_norm_(self.parameters(), self.max_grad_norm) + self.optimizer.step() + return float(loss.item()) diff --git a/src/unilab/algos/cse_ppo/runner.py b/src/unilab/algos/cse_ppo/runner.py new file mode 100644 index 000000000..7239f0c96 --- /dev/null +++ b/src/unilab/algos/cse_ppo/runner.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""On-policy runner for CSE-PPO.""" + +from __future__ import annotations + +import os +import time +from collections import deque +from collections.abc import Callable +from typing import Any, cast + +import torch + +from .actor_critic import CSEActorCritic +from .algorithm import CSEPPO + + +class _CSELogger: + def __init__(self) -> None: + self.rewbuffer: deque[float] = deque(maxlen=100) + self.lenbuffer: deque[float] = deque(maxlen=100) + self.tot_timesteps = 0 + + +class CSEOnPolicyRunner: + """Train and serve CSE-PPO against the ManagerBased VecEnv wrapper.""" + + def __init__( + self, env: Any, train_cfg: dict[str, Any], log_dir: str | None = None, device: str = "cpu" + ) -> None: + self.env, self.device, self.log_dir = env, device, log_dir + self.current_learning_iteration = 0 + self.logger = _CSELogger() + cfg = dict(train_cfg) + one_step = int(cfg["num_one_step_obs"]) + num_actor_obs = int(env.num_obs) + if num_actor_obs % one_step: + raise ValueError( + "Manager observation dimension must be divisible by num_one_step_obs; " + f"got num_obs={num_actor_obs}, num_one_step_obs={one_step}" + ) + num_critic_obs = int(getattr(env, "num_privileged_obs", None) or env.num_obs) + policy_cfg, estimator_cfg, algorithm_cfg = ( + dict(cfg.get(name) or {}) for name in ("policy", "estimator", "algorithm") + ) + self.actor_critic = CSEActorCritic( + num_actor_obs=num_actor_obs, + num_critic_obs=num_critic_obs, + num_one_step_obs=one_step, + num_actions=int(env.num_actions), + actor_hidden_dims=policy_cfg.get("actor_hidden_dims", [512, 256, 128]), + critic_hidden_dims=policy_cfg.get("critic_hidden_dims", [512, 256, 128]), + activation=str(policy_cfg.get("activation", "elu")), + init_noise_std=float(policy_cfg.get("init_noise_std", 1.0)), + estimator=estimator_cfg, + ).to(device) + self.alg = CSEPPO(self.actor_critic, device=device, **algorithm_cfg) + self.num_steps_per_env = int(cfg.get("num_steps_per_env", 24)) + self.save_interval = int(cfg.get("save_interval", 100)) + self.alg.init_storage( + env.num_envs, + self.num_steps_per_env, + [num_actor_obs], + [num_critic_obs], + [int(env.num_actions)], + ) + self._ep_returns = torch.zeros(env.num_envs, device=device) + self._ep_lengths = torch.zeros(env.num_envs, device=device) + self._writer: Any = None + if log_dir is not None: + try: + from torch.utils.tensorboard import SummaryWriter + + self._writer = SummaryWriter(log_dir=log_dir) + except ImportError: + pass + + def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = True) -> None: + obs_td, _ = self.env.reset() + obs, critic_obs = ( + obs_td["actor"].to(self.device), + obs_td.get("critic", obs_td["actor"]).to(self.device), + ) + if init_at_random_ep_len: + self._initialize_random_episode_lengths() + self.alg.train_mode() + start = self.current_learning_iteration + total = start + int(num_learning_iterations) + run_start = time.perf_counter() + for iteration in range(start, total): + infos: dict[str, Any] = {} + with torch.inference_mode(): + for _ in range(self.num_steps_per_env): + actions = self.alg.act(obs, critic_obs) + obs_td, rewards, dones, infos = self.env.step(actions) + next_obs, next_critic_obs = ( + obs_td["actor"].to(self.device), + obs_td.get("critic", obs_td["actor"]).to(self.device), + ) + self._ep_returns += rewards.to(self.device) + self._ep_lengths += 1 + done_ids = dones.nonzero(as_tuple=False).flatten() + if done_ids.numel(): + self.logger.rewbuffer.extend(self._ep_returns[done_ids].tolist()) + self.logger.lenbuffer.extend(self._ep_lengths[done_ids].tolist()) + self._ep_returns[done_ids] = 0 + self._ep_lengths[done_ids] = 0 + self.alg.process_env_step(obs_td, rewards, dones, infos) + obs, critic_obs = next_obs, next_critic_obs + self.alg.compute_returns(critic_obs) + value_loss, surrogate_loss, estimation_loss = self.alg.update() + iteration_end = time.perf_counter() + self.current_learning_iteration = iteration + 1 + self.logger.tot_timesteps += self.num_steps_per_env * self.env.num_envs + elapsed = iteration_end - run_start + completed = self.current_learning_iteration - start + remaining = total - self.current_learning_iteration + eta = elapsed / completed * remaining if completed else 0.0 + stats = { + "value_loss": value_loss, + "surrogate_loss": surrogate_loss, + "estimation_loss": estimation_loss, + "learning_rate": self.alg.learning_rate, + "mean_noise_std": float(self.actor_critic.std.mean().detach()), + } + self._print_iter( + self.current_learning_iteration, + total, + stats, + elapsed, + eta, + infos, + ) + if self._writer is not None: + step = self.current_learning_iteration + for key, value in ( + ("Loss/value", value_loss), + ("Loss/surrogate", surrogate_loss), + ("Loss/estimation", estimation_loss), + ("Loss/learning_rate", self.alg.learning_rate), + ("Policy/mean_noise_std", float(self.actor_critic.std.mean())), + ): + self._writer.add_scalar(key, value, step) + for key, value in (infos.get("log") or {}).items(): + self._writer.add_scalar(key, value, step) + if ( + self.log_dir is not None + and self.current_learning_iteration % self.save_interval == 0 + ): + self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) + if self.log_dir is not None: + self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) + + def _initialize_random_episode_lengths(self) -> None: + values = torch.randint( + low=0, + high=int(self.env.max_episode_length), + size=(self.env.num_envs,), + device=self.device, + ) + self.env.set_episode_length_buf(values) + + def save(self, path: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + torch.save( + { + "actor_state_dict": self.actor_critic.state_dict(), + "optimizer_state_dict": self.alg.optimizer.state_dict(), + "iteration": self.current_learning_iteration, + }, + path, + ) + + def load(self, path: str) -> None: + checkpoint = torch.load(path, map_location=self.device, weights_only=True) + self.actor_critic.load_state_dict(checkpoint["actor_state_dict"]) + if "optimizer_state_dict" in checkpoint: + self.alg.optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) + if "iteration" in checkpoint: + self.current_learning_iteration = int(checkpoint["iteration"]) + + def get_inference_policy(self, device: str | None = None) -> Callable[..., Any]: + self.actor_critic.eval() + if device is not None: + self.actor_critic.to(device) + return cast(Callable[..., Any], self.actor_critic.act_inference) + + def export_policy_to_jit(self, path: str, filename: str = "policy.pt") -> None: + original_device = next(self.actor_critic.parameters()).device + ac = self.actor_critic.cpu().eval() + one_step = ac.num_one_step_obs + + class PolicyExport(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.estimator, self.actor_mlp = ac.estimator, ac.actor + + def forward(self, obs_history: torch.Tensor) -> torch.Tensor: + latent = self.estimator.get_latent(obs_history) + return self.actor_mlp(torch.cat((obs_history[:, -one_step:], latent), dim=-1)) + + os.makedirs(path, exist_ok=True) + with torch.inference_mode(): + traced = torch.jit.trace(PolicyExport(), (torch.zeros(1, ac.num_actor_obs),)) + cast(Any, traced).save(os.path.join(path, filename)) + self.actor_critic.to(original_device) + + def _print_iter( + self, + it: int, + tot: int, + stats: dict[str, float], + elapsed: float, + eta: float, + infos: dict[str, Any], + ) -> None: + """Print the established CSE-PPO iteration summary.""" + sep = "-" * 80 + mean_rew = ( + sum(self.logger.rewbuffer) / len(self.logger.rewbuffer) + if self.logger.rewbuffer + else 0.0 + ) + mean_len = ( + sum(self.logger.lenbuffer) / len(self.logger.lenbuffer) + if self.logger.lenbuffer + else 0.0 + ) + elapsed_str = time.strftime("%H:%M:%S", time.gmtime(elapsed)) + eta_str = time.strftime("%H:%M:%S", time.gmtime(eta)) + print(sep) + print(f"{'Iteration':>40}: {it}/{tot}") + print(f"{'Mean value loss':>40}: {stats['value_loss']:.4f}") + print(f"{'Mean surrogate loss':>40}: {stats['surrogate_loss']:.4f}") + print(f"{'Mean estimation loss':>40}: {stats['estimation_loss']:.4f}") + print(f"{'Learning rate':>40}: {stats['learning_rate']:.2e}") + print(f"{'Mean action noise std':>40}: {stats['mean_noise_std']:.3f}") + if mean_rew: + print(f"{'Mean episode reward':>40}: {mean_rew:.4f}") + if mean_len: + print(f"{'Mean episode length':>40}: {mean_len:.1f}") + for key, value in sorted((infos.get("log") or {}).items()): + print(f"{key:>40}: {value:.4f}") + print(f"{'Total timesteps':>40}: {self.logger.tot_timesteps}") + print(f"{'Time elapsed':>40}: {elapsed_str}") + print(f"{'ETA':>40}: {eta_str}") + print(sep) diff --git a/src/unilab/algos/cse_ppo/storage.py b/src/unilab/algos/cse_ppo/storage.py new file mode 100644 index 000000000..a8c1278d5 --- /dev/null +++ b/src/unilab/algos/cse_ppo/storage.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Rollout storage for CSE-PPO.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import cast + +import torch + + +class CSERolloutStorage: + class Transition: + observations: torch.Tensor | None + critic_observations: torch.Tensor | None + actions: torch.Tensor | None + rewards: torch.Tensor | None + dones: torch.Tensor | None + values: torch.Tensor | None + actions_log_prob: torch.Tensor | None + action_mean: torch.Tensor | None + action_sigma: torch.Tensor | None + + def __init__(self) -> None: + self.observations = self.critic_observations = None + self.actions = self.rewards = self.dones = self.values = None + self.actions_log_prob = self.action_mean = self.action_sigma = None + + def clear(self) -> None: + self.observations = None + self.critic_observations = None + self.actions = None + self.rewards = None + self.dones = None + self.values = None + self.actions_log_prob = None + self.action_mean = None + self.action_sigma = None + + def __init__( + self, + num_envs: int, + num_transitions_per_env: int, + obs_shape: Sequence[int], + privileged_obs_shape: Sequence[int | None], + actions_shape: Sequence[int], + device: str = "cpu", + ) -> None: + self.device = device + self.num_transitions_per_env = int(num_transitions_per_env) + self.num_envs = int(num_envs) + self.obs_shape = tuple(obs_shape) + self.privileged_obs_shape = tuple(privileged_obs_shape) + self.actions_shape = tuple(actions_shape) + self.step = 0 + self.observations = torch.zeros( + self.num_transitions_per_env, self.num_envs, *self.obs_shape, device=device + ) + self.privileged_observations = None + if self.privileged_obs_shape and self.privileged_obs_shape[0] is not None: + if any(dim is None for dim in self.privileged_obs_shape): + raise ValueError("privileged_obs_shape cannot contain None values") + privileged_shape = cast(tuple[int, ...], self.privileged_obs_shape) + self.privileged_observations = torch.zeros( + self.num_transitions_per_env, + self.num_envs, + *privileged_shape, + device=device, + ) + self.rewards = torch.zeros(self.num_transitions_per_env, self.num_envs, 1, device=device) + self.actions = torch.zeros( + self.num_transitions_per_env, self.num_envs, *self.actions_shape, device=device + ) + self.dones = torch.zeros( + self.num_transitions_per_env, self.num_envs, 1, dtype=torch.bool, device=device + ) + self.actions_log_prob = torch.zeros_like(self.rewards) + self.values = torch.zeros_like(self.rewards) + self.returns = torch.zeros_like(self.rewards) + self.advantages = torch.zeros_like(self.rewards) + self.mu = torch.zeros_like(self.actions) + self.sigma = torch.zeros_like(self.actions) + + def add_transition(self, transition: Transition) -> None: + if self.step >= self.num_transitions_per_env: + raise AssertionError("Rollout buffer overflow") + required = ( + "observations", + "actions", + "rewards", + "dones", + "values", + "actions_log_prob", + "action_mean", + "action_sigma", + ) + if any(getattr(transition, name) is None for name in required): + raise ValueError("incomplete CSE-PPO transition") + observations = transition.observations + actions = transition.actions + rewards = transition.rewards + dones = transition.dones + values = transition.values + actions_log_prob = transition.actions_log_prob + action_mean = transition.action_mean + action_sigma = transition.action_sigma + assert ( + observations is not None + and actions is not None + and rewards is not None + and dones is not None + and values is not None + and actions_log_prob is not None + and action_mean is not None + and action_sigma is not None + ) + self.observations[self.step].copy_(observations) + if self.privileged_observations is not None: + critic_observations = transition.critic_observations + if critic_observations is None: + raise ValueError("transition.critic_observations is required") + self.privileged_observations[self.step].copy_(critic_observations) + self.actions[self.step].copy_(actions) + self.rewards[self.step].copy_(rewards.view(-1, 1)) + self.dones[self.step].copy_(dones.view(-1, 1).bool()) + self.values[self.step].copy_(values) + self.actions_log_prob[self.step].copy_(actions_log_prob.view(-1, 1)) + self.mu[self.step].copy_(action_mean) + self.sigma[self.step].copy_(action_sigma) + self.step += 1 + + def clear(self) -> None: + self.step = 0 + + def compute_returns(self, last_values: torch.Tensor, gamma: float, lam: float) -> None: + advantage = torch.zeros_like(last_values) + for step in reversed(range(self.num_transitions_per_env)): + next_values = ( + last_values if step == self.num_transitions_per_env - 1 else self.values[step + 1] + ) + not_terminal = 1.0 - self.dones[step].float() + delta = self.rewards[step] + not_terminal * gamma * next_values - self.values[step] + advantage = delta + not_terminal * gamma * lam * advantage + self.returns[step] = advantage + self.values[step] + self.advantages = self.returns - self.values + self.advantages = (self.advantages - self.advantages.mean()) / ( + self.advantages.std() + 1e-8 + ) + + def mini_batch_generator(self, num_mini_batches: int, num_epochs: int = 8): + batch_size = self.num_envs * self.num_transitions_per_env + mini_batch_size = batch_size // int(num_mini_batches) + if mini_batch_size <= 0: + raise ValueError("num_mini_batches is too large for the rollout batch") + indices = torch.randperm(int(num_mini_batches) * mini_batch_size, device=self.device) + arrays = [ + self.observations.flatten(0, 1), + self.privileged_observations.flatten(0, 1) + if self.privileged_observations is not None + else self.observations.flatten(0, 1), + self.actions.flatten(0, 1), + self.values.flatten(0, 1), + self.advantages.flatten(0, 1), + self.returns.flatten(0, 1), + self.actions_log_prob.flatten(0, 1), + self.mu.flatten(0, 1), + self.sigma.flatten(0, 1), + ] + for _ in range(int(num_epochs)): + for i in range(int(num_mini_batches)): + idx = indices[i * mini_batch_size : (i + 1) * mini_batch_size] + yield tuple(array[idx] for array in arrays) diff --git a/src/unilab/algos/torch/fast_sac/__init__.py b/src/unilab/algos/fast_sac/__init__.py similarity index 100% rename from src/unilab/algos/torch/fast_sac/__init__.py rename to src/unilab/algos/fast_sac/__init__.py diff --git a/src/unilab/algos/torch/fast_sac/double_buffer.py b/src/unilab/algos/fast_sac/double_buffer.py similarity index 78% rename from src/unilab/algos/torch/fast_sac/double_buffer.py rename to src/unilab/algos/fast_sac/double_buffer.py index 88b813366..607f41277 100644 --- a/src/unilab/algos/torch/fast_sac/double_buffer.py +++ b/src/unilab/algos/fast_sac/double_buffer.py @@ -6,11 +6,12 @@ from omegaconf import DictConfig, OmegaConf -from unilab.algos.torch.fast_sac.learner import FastSACLearner -from unilab.algos.torch.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner -from unilab.algos.torch.offpolicy.runtime import resolve_custom_offpolicy_runtime +from unilab.algos.fast_sac.learner import FastSACLearner +from unilab.algos.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner +from unilab.algos.offpolicy.runtime import resolve_custom_offpolicy_runtime +from unilab.base.config_adapter import create_env from unilab.base.np_env import NpEnv -from unilab.training import create_env, ensure_registries +from unilab.base.registry import ensure_registries from unilab.utils.nan_guard import NanGuardCfg if TYPE_CHECKING: @@ -41,28 +42,10 @@ def build_sac_double_buffer_runner( assert action_shape obs_dim, critic_obs_dim = get_obs_dims(env.obs_groups_spec) action_dim = int(action_shape[0]) - symmetry_augmentation = None - if ( - custom_runtime is not None - and cfg.algo.use_symmetry - and not custom_runtime.supports_symmetry - ): - raise ValueError("Selected SAC off-policy runtime does not support symmetry.") - if cfg.algo.use_symmetry: - symmetry_augmentation = env.build_symmetry_augmentation(device=device) - if symmetry_augmentation is None: - raise ValueError(f"{cfg.training.task_name} does not provide symmetry augmentation") finally: env.close() batch_size = cfg.algo.batch_size - if symmetry_augmentation is not None: - if batch_size % symmetry_augmentation.batch_multiplier != 0: - raise ValueError( - "Symmetry augmentation requires batch_size divisible by " - f"{symmetry_augmentation.batch_multiplier}, got {batch_size}" - ) - batch_size = batch_size // symmetry_augmentation.batch_multiplier learner_cls: type[Any] = FastSACLearner algo_type = "sac" @@ -110,8 +93,6 @@ def build_sac_double_buffer_runner( getattr(cfg.algo.algo_params, "use_cuda_graph_actor_packed_staging", False) ), "nvtx_profile_ranges": bool(getattr(cfg.training, "nvtx_profile_ranges", False)), - "use_symmetry": cfg.algo.use_symmetry, - "symmetry_augmentation": symmetry_augmentation, "critic_obs_dim": critic_obs_dim, } learner_kwargs.update(learner_extra_kwargs) diff --git a/src/unilab/algos/torch/fast_sac/learner.py b/src/unilab/algos/fast_sac/learner.py similarity index 95% rename from src/unilab/algos/torch/fast_sac/learner.py rename to src/unilab/algos/fast_sac/learner.py index 5d714a37c..7f47e9547 100644 --- a/src/unilab/algos/torch/fast_sac/learner.py +++ b/src/unilab/algos/fast_sac/learner.py @@ -20,9 +20,8 @@ import torch.nn.functional as F import torch.optim as optim -from unilab.algos.torch.common.compile import get_torch_compile_for_cuda -from unilab.algos.torch.common.normalization import EmpiricalNormalization -from unilab.base.augmentation import SymmetryAugmentation +from unilab.algos.common.compile import get_torch_compile_for_cuda +from unilab.algos.common.normalization import EmpiricalNormalization @contextmanager @@ -418,7 +417,6 @@ def __init__( weight_decay: float = 0.001, max_grad_norm: float = 0.0, use_autotune: bool = True, - use_symmetry: bool = False, use_amp: bool = False, amp_dtype: str = "auto", use_compile: bool = False, @@ -428,7 +426,6 @@ def __init__( use_cuda_graph_critic_packed_staging: bool = False, use_cuda_graph_actor_packed_staging: bool = False, nvtx_profile_ranges: bool = False, - symmetry_augmentation: SymmetryAugmentation | None = None, ): self.device = device self._device_type = torch.device(device).type @@ -550,15 +547,8 @@ def __init__( requested_cuda_graph_actor_packed_staging and self.use_cuda_graph_actor and self.scaler is None - and not use_symmetry ) - self.symmetry = symmetry_augmentation - if use_symmetry and symmetry_augmentation is None: - raise ValueError( - "FastSACLearner use_symmetry=True requires a symmetry_augmentation contract" - ) - self.use_symmetry = use_symmetry self._cuda_graph_critic: torch.cuda.CUDAGraph | None = None self._cuda_graph_critic_static_inputs: dict[str, torch.Tensor] | None = None self._cuda_graph_critic_static_packed_input: torch.Tensor | None = None @@ -1326,7 +1316,7 @@ def update_actor_cuda_graph( return self.update_actor(batch) if self._device_type != "cuda": return self.update_actor(batch) - if self.scaler is not None or self.use_symmetry: + if self.scaler is not None: return self.update_actor(batch) if self._cuda_graph_actor_shapes != self._actor_graph_input_shapes(batch): self._reset_actor_cuda_graph() @@ -1357,39 +1347,6 @@ def update_critic(self, batch: Dict[str, torch.Tensor]) -> Dict[str, float]: dones = batch["dones"] truncated = batch["truncated"] - # Apply symmetry augmentation - if self.use_symmetry: - with _cuda_nvtx_range("critic/symmetry_augment", self.nvtx_profile_ranges): - assert self.symmetry is not None - with _cuda_nvtx_range("critic/symmetry_obs_actions", self.nvtx_profile_ranges): - obs, actions = self.symmetry.augment_obs_and_actions( - obs, - actions, - obs_group="obs", - ) - with _cuda_nvtx_range("critic/symmetry_next_obs", self.nvtx_profile_ranges): - next_obs = self.symmetry.augment_obs( - next_obs, - obs_group="obs", - ) - - with _cuda_nvtx_range("critic/symmetry_critic_obs", self.nvtx_profile_ranges): - critic_obs = self.symmetry.augment_obs( - critic_obs, - obs_group="critic", - ) - with _cuda_nvtx_range("critic/symmetry_critic_next_obs", self.nvtx_profile_ranges): - critic_next_obs = self.symmetry.augment_obs( - critic_next_obs, - obs_group="critic", - ) - - # Double the batch size for other tensors - with _cuda_nvtx_range("critic/symmetry_aux_repeat", self.nvtx_profile_ranges): - rewards = rewards.repeat(2) - dones = dones.repeat(2) - truncated = truncated.repeat(2) - self.normalize_obs(obs, update=True) next_obs = self.normalize_obs(next_obs, update=False) @@ -1466,15 +1423,6 @@ def update_actor(self, batch: Dict[str, torch.Tensor]) -> Dict[str, float]: obs = batch["obs"] critic_obs = batch["critic"] - # Apply symmetry augmentation - if self.use_symmetry: - with _cuda_nvtx_range("actor/symmetry_augment", self.nvtx_profile_ranges): - assert self.symmetry is not None - with _cuda_nvtx_range("actor/symmetry_obs", self.nvtx_profile_ranges): - obs = self.symmetry.augment_obs(obs, obs_group="obs") - with _cuda_nvtx_range("actor/symmetry_critic_obs", self.nvtx_profile_ranges): - critic_obs = self.symmetry.augment_obs(critic_obs, obs_group="critic") - obs = self.normalize_obs(obs, update=False) with _cuda_nvtx_range("actor/loss_compiled", self.nvtx_profile_ranges): actor_loss, policy_entropy, action_std = self._actor_loss_tensors(obs, critic_obs) @@ -1551,7 +1499,7 @@ def set_gradient_sync( sync is not None and ( (self.use_cuda_graph_critic and self.scaler is None) - or (self.use_cuda_graph_actor and self.scaler is None and not self.use_symmetry) + or (self.use_cuda_graph_actor and self.scaler is None) ) ) diff --git a/src/unilab/algos/torch/fast_sac/runner.py b/src/unilab/algos/fast_sac/runner.py similarity index 73% rename from src/unilab/algos/torch/fast_sac/runner.py rename to src/unilab/algos/fast_sac/runner.py index f6a72a8c8..ea5856c66 100644 --- a/src/unilab/algos/torch/fast_sac/runner.py +++ b/src/unilab/algos/fast_sac/runner.py @@ -1,15 +1,12 @@ """FastSAC runner using unified OffPolicyRunner.""" -import logging from typing import Any -from unilab.algos.torch.fast_sac.learner import FastSACLearner -from unilab.algos.torch.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner +from unilab.algos.fast_sac.learner import FastSACLearner +from unilab.algos.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner from unilab.ipc.replay_pipelines.gpu_resident import require_offpolicy_replay_device from unilab.utils.device import get_default_device -logger = logging.getLogger(__name__) - class FastSACRunner(DoubleBufferOffPolicyRunner): """FastSAC using the single device-authoritative replay path.""" @@ -44,7 +41,6 @@ def __init__( use_cuda_graph_critic: bool = False, use_cuda_graph_actor: bool = False, sim_backend: str = "mujoco", - use_symmetry: bool = False, seed: int | None = None, trace_enabled: bool = False, trace_output_dir: str | None = None, @@ -53,7 +49,7 @@ def __init__( ): from unilab.base import registry from unilab.base.registry import ensure_registries - from unilab.training.seed import apply_training_seed + from unilab.utils.seed import apply_training_seed device = require_offpolicy_replay_device(device or get_default_device()) ensure_registries() @@ -67,14 +63,6 @@ def __init__( act_space_shape = env.action_space.shape assert act_space_shape is not None action_dim = act_space_shape[0] - symmetry_augmentation = None - if use_symmetry: - symmetry_augmentation = env.build_symmetry_augmentation(device=device) - if symmetry_augmentation is None: - env.close() - raise ValueError( - f"{env_name} with backend={sim_backend} does not provide symmetry augmentation" - ) env.close() learner = FastSACLearner( @@ -98,24 +86,9 @@ def __init__( obs_normalization=obs_normalization, use_cuda_graph_critic=use_cuda_graph_critic, use_cuda_graph_actor=use_cuda_graph_actor, - use_symmetry=use_symmetry, - symmetry_augmentation=symmetry_augmentation, critic_obs_dim=critic_obs_dim, ) - if symmetry_augmentation is not None: - if batch_size % symmetry_augmentation.batch_multiplier != 0: - raise ValueError( - "Symmetry augmentation requires algo.batch_size to be divisible by " - f"{symmetry_augmentation.batch_multiplier}, got {batch_size}" - ) - batch_size = batch_size // symmetry_augmentation.batch_multiplier - logger.info( - "[FastSAC] Symmetry enabled: batch_size adjusted to %d (effective: %d)", - batch_size, - batch_size * symmetry_augmentation.batch_multiplier, - ) - super().__init__( learner=learner, env_name=env_name, diff --git a/src/unilab/algos/torch/fast_td3/__init__.py b/src/unilab/algos/fast_td3/__init__.py similarity index 100% rename from src/unilab/algos/torch/fast_td3/__init__.py rename to src/unilab/algos/fast_td3/__init__.py diff --git a/src/unilab/algos/torch/fast_td3/double_buffer.py b/src/unilab/algos/fast_td3/double_buffer.py similarity index 93% rename from src/unilab/algos/torch/fast_td3/double_buffer.py rename to src/unilab/algos/fast_td3/double_buffer.py index 34f5df0bc..c260e1c2a 100644 --- a/src/unilab/algos/torch/fast_td3/double_buffer.py +++ b/src/unilab/algos/fast_td3/double_buffer.py @@ -6,9 +6,9 @@ from omegaconf import DictConfig -from unilab.algos.torch.common.device import get_env_dims -from unilab.algos.torch.fast_td3.learner import FastTD3Learner -from unilab.algos.torch.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner +from unilab.algos.common.device import get_env_dims +from unilab.algos.fast_td3.learner import FastTD3Learner +from unilab.algos.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner from unilab.utils.nan_guard import NanGuardCfg if TYPE_CHECKING: diff --git a/src/unilab/algos/torch/fast_td3/learner.py b/src/unilab/algos/fast_td3/learner.py similarity index 98% rename from src/unilab/algos/torch/fast_td3/learner.py rename to src/unilab/algos/fast_td3/learner.py index 772cce22e..6d7804714 100644 --- a/src/unilab/algos/torch/fast_td3/learner.py +++ b/src/unilab/algos/fast_td3/learner.py @@ -23,9 +23,9 @@ import torch.nn.functional as F import torch.optim as optim -from unilab.algos.torch.common.networks import Critic -from unilab.algos.torch.common.normalization import EmpiricalNormalization -from unilab.algos.torch.common.stability import check_nan_loss, clip_gradients +from unilab.algos.common.networks import Critic +from unilab.algos.common.normalization import EmpiricalNormalization +from unilab.algos.common.stability import check_nan_loss, clip_gradients # --------------------------------------------------------------------------- # Actor (deterministic, ReLU, per-env noise) diff --git a/src/unilab/algos/flash_sac/__init__.py b/src/unilab/algos/flash_sac/__init__.py new file mode 100644 index 000000000..7cadda2ed --- /dev/null +++ b/src/unilab/algos/flash_sac/__init__.py @@ -0,0 +1,12 @@ +"""FlashSAC algorithm package.""" + +from unilab.algos.flash_sac.learner import FlashSACLearner +from unilab.algos.flash_sac.network import FlashSACActor, FlashSACDoubleCritic +from unilab.algos.flash_sac.runner import FlashSACRunner + +__all__ = [ + "FlashSACActor", + "FlashSACDoubleCritic", + "FlashSACLearner", + "FlashSACRunner", +] diff --git a/src/unilab/algos/torch/flash_sac/double_buffer.py b/src/unilab/algos/flash_sac/double_buffer.py similarity index 94% rename from src/unilab/algos/torch/flash_sac/double_buffer.py rename to src/unilab/algos/flash_sac/double_buffer.py index 26d21869b..96bd18301 100644 --- a/src/unilab/algos/torch/flash_sac/double_buffer.py +++ b/src/unilab/algos/flash_sac/double_buffer.py @@ -6,13 +6,14 @@ from omegaconf import DictConfig -from unilab.algos.torch.flash_sac.learner import FlashSACLearner -from unilab.algos.torch.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner +from unilab.algos.flash_sac.learner import FlashSACLearner +from unilab.algos.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner +from unilab.base.config_adapter import create_env +from unilab.base.registry import ensure_registries from unilab.ipc.replay_pipelines.gpu_resident import require_offpolicy_replay_device -from unilab.training import create_env, ensure_registries -from unilab.training.seed import apply_training_seed from unilab.utils.device import get_default_device from unilab.utils.nan_guard import NanGuardCfg +from unilab.utils.seed import apply_training_seed if TYPE_CHECKING: from unilab.ipc.dp_sync import DpParameterSync diff --git a/src/unilab/algos/torch/flash_sac/layers.py b/src/unilab/algos/flash_sac/layers.py similarity index 100% rename from src/unilab/algos/torch/flash_sac/layers.py rename to src/unilab/algos/flash_sac/layers.py diff --git a/src/unilab/algos/torch/flash_sac/learner.py b/src/unilab/algos/flash_sac/learner.py similarity index 99% rename from src/unilab/algos/torch/flash_sac/learner.py rename to src/unilab/algos/flash_sac/learner.py index 3f0285838..dc150f478 100644 --- a/src/unilab/algos/torch/flash_sac/learner.py +++ b/src/unilab/algos/flash_sac/learner.py @@ -11,14 +11,14 @@ import torch.nn as nn import torch.optim as optim -from unilab.algos.torch.common.compile import get_torch_compile_for_cuda -from unilab.algos.torch.common.normalization import EmpiricalNormalization -from unilab.algos.torch.flash_sac.network import ( +from unilab.algos.common.compile import get_torch_compile_for_cuda +from unilab.algos.common.normalization import EmpiricalNormalization +from unilab.algos.flash_sac.network import ( FlashSACActor, FlashSACDoubleCritic, FlashSACTemperature, ) -from unilab.algos.torch.flash_sac.update import ( +from unilab.algos.flash_sac.update import ( build_lr_lambda, resolve_target_entropy, select_min_q_log_probs, diff --git a/src/unilab/algos/torch/flash_sac/network.py b/src/unilab/algos/flash_sac/network.py similarity index 99% rename from src/unilab/algos/torch/flash_sac/network.py rename to src/unilab/algos/flash_sac/network.py index e810513fd..37acff9bf 100644 --- a/src/unilab/algos/torch/flash_sac/network.py +++ b/src/unilab/algos/flash_sac/network.py @@ -8,7 +8,7 @@ import torch import torch.nn as nn -from unilab.algos.torch.flash_sac.layers import ( +from unilab.algos.flash_sac.layers import ( EnsembleCategoricalValue, EnsembleFlashSACBlock, EnsembleFlashSACEmbedder, diff --git a/src/unilab/algos/torch/flash_sac/runner.py b/src/unilab/algos/flash_sac/runner.py similarity index 96% rename from src/unilab/algos/torch/flash_sac/runner.py rename to src/unilab/algos/flash_sac/runner.py index 729f49de9..c1bd53a90 100644 --- a/src/unilab/algos/torch/flash_sac/runner.py +++ b/src/unilab/algos/flash_sac/runner.py @@ -4,8 +4,8 @@ from typing import Any -from unilab.algos.torch.flash_sac.learner import FlashSACLearner -from unilab.algos.torch.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner +from unilab.algos.flash_sac.learner import FlashSACLearner +from unilab.algos.offpolicy.double_buffer_runner import DoubleBufferOffPolicyRunner from unilab.ipc.replay_pipelines.gpu_resident import require_offpolicy_replay_device from unilab.utils.device import get_default_device @@ -66,7 +66,7 @@ def __init__( from unilab.base import registry from unilab.base.observations import get_obs_dims from unilab.base.registry import ensure_registries - from unilab.training.seed import apply_training_seed + from unilab.utils.seed import apply_training_seed runtime_device = require_offpolicy_replay_device(device or get_default_device()) ensure_registries() diff --git a/src/unilab/algos/torch/flash_sac/update.py b/src/unilab/algos/flash_sac/update.py similarity index 100% rename from src/unilab/algos/torch/flash_sac/update.py rename to src/unilab/algos/flash_sac/update.py diff --git a/src/unilab/algos/him_ppo/__init__.py b/src/unilab/algos/him_ppo/__init__.py new file mode 100644 index 000000000..2ecfec5cd --- /dev/null +++ b/src/unilab/algos/him_ppo/__init__.py @@ -0,0 +1,11 @@ +from unilab.algos.him_ppo.actor_critic import HIMActorCritic +from unilab.algos.him_ppo.algorithm import HIMPPO +from unilab.algos.him_ppo.estimator import HIMEstimator +from unilab.algos.him_ppo.storage import HIMRolloutStorage + +__all__ = [ + "HIMActorCritic", + "HIMPPO", + "HIMEstimator", + "HIMRolloutStorage", +] diff --git a/src/unilab/algos/torch/him_ppo/actor_critic.py b/src/unilab/algos/him_ppo/actor_critic.py similarity index 98% rename from src/unilab/algos/torch/him_ppo/actor_critic.py rename to src/unilab/algos/him_ppo/actor_critic.py index 47ac66e01..dc62a4074 100644 --- a/src/unilab/algos/torch/him_ppo/actor_critic.py +++ b/src/unilab/algos/him_ppo/actor_critic.py @@ -8,7 +8,7 @@ import torch.nn as nn from torch.distributions import Normal -from unilab.algos.torch.him_ppo.estimator import HIMEstimator, get_activation +from unilab.algos.him_ppo.estimator import HIMEstimator, get_activation class HIMActorCritic(nn.Module): diff --git a/src/unilab/algos/torch/him_ppo/algorithm.py b/src/unilab/algos/him_ppo/algorithm.py similarity index 98% rename from src/unilab/algos/torch/him_ppo/algorithm.py rename to src/unilab/algos/him_ppo/algorithm.py index eba081cf0..5892479c5 100644 --- a/src/unilab/algos/torch/him_ppo/algorithm.py +++ b/src/unilab/algos/him_ppo/algorithm.py @@ -11,8 +11,8 @@ import torch.optim as optim from tensordict import TensorDict -from unilab.algos.torch.him_ppo.actor_critic import HIMActorCritic -from unilab.algos.torch.him_ppo.storage import HIMRolloutStorage +from unilab.algos.him_ppo.actor_critic import HIMActorCritic +from unilab.algos.him_ppo.storage import HIMRolloutStorage class HIMPPO: diff --git a/src/unilab/algos/torch/him_ppo/estimator.py b/src/unilab/algos/him_ppo/estimator.py similarity index 100% rename from src/unilab/algos/torch/him_ppo/estimator.py rename to src/unilab/algos/him_ppo/estimator.py diff --git a/src/unilab/algos/torch/him_ppo/runner.py b/src/unilab/algos/him_ppo/runner.py similarity index 99% rename from src/unilab/algos/torch/him_ppo/runner.py rename to src/unilab/algos/him_ppo/runner.py index bab234d42..6a8d7ab16 100644 --- a/src/unilab/algos/torch/him_ppo/runner.py +++ b/src/unilab/algos/him_ppo/runner.py @@ -12,8 +12,8 @@ import torch -from unilab.algos.torch.him_ppo.actor_critic import HIMActorCritic -from unilab.algos.torch.him_ppo.algorithm import HIMPPO +from unilab.algos.him_ppo.actor_critic import HIMActorCritic +from unilab.algos.him_ppo.algorithm import HIMPPO logger = logging.getLogger(__name__) diff --git a/src/unilab/algos/torch/him_ppo/storage.py b/src/unilab/algos/him_ppo/storage.py similarity index 100% rename from src/unilab/algos/torch/him_ppo/storage.py rename to src/unilab/algos/him_ppo/storage.py diff --git a/src/unilab/algos/torch/hora/__init__.py b/src/unilab/algos/hora/__init__.py similarity index 100% rename from src/unilab/algos/torch/hora/__init__.py rename to src/unilab/algos/hora/__init__.py diff --git a/src/unilab/algos/torch/hora/appo.py b/src/unilab/algos/hora/appo.py similarity index 96% rename from src/unilab/algos/torch/hora/appo.py rename to src/unilab/algos/hora/appo.py index e8b9835f1..785dca0d4 100644 --- a/src/unilab/algos/torch/hora/appo.py +++ b/src/unilab/algos/hora/appo.py @@ -11,15 +11,16 @@ import torch from omegaconf import DictConfig -from unilab.algos.torch.hora.appo_runner import HoraAPPORunner -from unilab.algos.torch.hora.rsl_rl_compat import ( +from unilab.algos.hora.appo_runner import HoraAPPORunner +from unilab.algos.hora.rsl_rl_compat import ( convert_config_v3_to_v4, is_rsl_rl_v4, is_rsl_rl_v5, ) +from unilab.base.backend.base import log_playback_plan +from unilab.base.config_adapter import BackendAdapter, create_env from unilab.base.observations import get_obs_dims -from unilab.training import BackendAdapter, create_env, log_playback_plan -from unilab.training.sim2sim import policy_load_dim_guard, resolve_sim2sim_config +from unilab.utils.sim2sim import policy_load_dim_guard, resolve_sim2sim_config from .models import build_hora_shared_actor_critic from .observations import build_hora_actor_tensordict, split_hora_obs_with_priv_info diff --git a/src/unilab/algos/torch/hora/appo_learner.py b/src/unilab/algos/hora/appo_learner.py similarity index 97% rename from src/unilab/algos/torch/hora/appo_learner.py rename to src/unilab/algos/hora/appo_learner.py index ed9578fed..be77ab29c 100644 --- a/src/unilab/algos/torch/hora/appo_learner.py +++ b/src/unilab/algos/hora/appo_learner.py @@ -7,13 +7,13 @@ import torch from tensordict import TensorDict -from unilab.algos.torch.appo.learner import ( +from unilab.algos.appo.learner import ( APPOLearner, _distribution_std, _sample_tensor_for_metric, vtrace_advantages, ) -from unilab.algos.torch.hora.models import HoraActorModel, HoraCriticModel +from unilab.algos.hora.models import HoraActorModel, HoraCriticModel def _build_hora_obs_td( diff --git a/src/unilab/algos/torch/hora/appo_runner.py b/src/unilab/algos/hora/appo_runner.py similarity index 95% rename from src/unilab/algos/torch/hora/appo_runner.py rename to src/unilab/algos/hora/appo_runner.py index d3d0d40c6..3bfb1b965 100644 --- a/src/unilab/algos/torch/hora/appo_runner.py +++ b/src/unilab/algos/hora/appo_runner.py @@ -13,16 +13,16 @@ import torch from rsl_rl.utils import resolve_callable -from unilab.algos.torch.appo.runner import ( +from unilab.algos.appo.runner import ( APPORunner, _optimizer_lr_from_state, _sync_resume_target_actor, ) -from unilab.algos.torch.appo.staging import RolloutStagingPool -from unilab.algos.torch.hora.appo_learner import HoraAPPOLearner -from unilab.algos.torch.hora.appo_worker import hora_appo_collector_fn -from unilab.algos.torch.hora.models import build_hora_shared_actor_critic -from unilab.algos.torch.hora.rsl_rl_compat import ( +from unilab.algos.appo.staging import RolloutStagingPool +from unilab.algos.hora.appo_learner import HoraAPPOLearner +from unilab.algos.hora.appo_worker import hora_appo_collector_fn +from unilab.algos.hora.models import build_hora_shared_actor_critic +from unilab.algos.hora.rsl_rl_compat import ( convert_config_v3_to_v4, is_rsl_rl_v4, is_rsl_rl_v5, @@ -31,7 +31,7 @@ from unilab.base.registry import ensure_registries from unilab.ipc import RolloutRingBuffer, SharedWeightSync from unilab.logging import OffPolicyLogger -from unilab.training.seed import apply_training_seed, derive_worker_seed +from unilab.utils.seed import apply_training_seed, derive_worker_seed def _validate_hora_shared_checkpoint(checkpoint: dict[str, Any]) -> None: @@ -303,7 +303,10 @@ def learn( f"epochs={learner.num_learning_epochs})" ) - reward_history: deque = deque(maxlen=200) + # Recent collector reports; each entry is already the collector's + # rolling 100-episode mean, so a short window keeps the logged + # reward timely without losing smoothing. + reward_history: deque = deque(maxlen=10) latest_reward_components: dict = {} staging_pool = RolloutStagingPool( capacity=self.staging_pool_size, @@ -379,9 +382,7 @@ def learn( logger.update_staging_pool(staging_pool.active_count, staging_pool.capacity) mean_reward = ( - sum(list(reward_history)[-50:]) / max(len(list(reward_history)[-50:]), 1) - if reward_history - else 0.0 + sum(reward_history) / max(len(reward_history), 1) if reward_history else 0.0 ) last_mean_reward = float(mean_reward) best_mean_reward = max(best_mean_reward, last_mean_reward) diff --git a/src/unilab/algos/torch/hora/appo_worker.py b/src/unilab/algos/hora/appo_worker.py similarity index 95% rename from src/unilab/algos/torch/hora/appo_worker.py rename to src/unilab/algos/hora/appo_worker.py index b303452aa..0d5ec8002 100644 --- a/src/unilab/algos/torch/hora/appo_worker.py +++ b/src/unilab/algos/hora/appo_worker.py @@ -5,21 +5,21 @@ import statistics import sys import time -from collections import defaultdict +from collections import defaultdict, deque from typing import Any, Dict import numpy as np import torch from rsl_rl.utils import resolve_callable -from unilab.algos.torch.appo.worker import ( +from unilab.algos.appo.worker import ( compute_rollout_active_steps_per_sec, put_latest_metrics, ) -from unilab.algos.torch.common.collector_timing import extract_env_step_breakdown_timing_ms +from unilab.algos.common.collector_timing import extract_env_step_breakdown_timing_ms from unilab.base.final_observation import resolve_terminal_observation_contract from unilab.base.registry import ensure_registries -from unilab.training.seed import apply_training_seed +from unilab.utils.seed import apply_training_seed from .observations import split_hora_obs_with_priv_info @@ -89,8 +89,8 @@ def hora_appo_collector_fn( from tensordict import TensorDict - from unilab.algos.torch.hora.models import build_hora_shared_actor_critic - from unilab.algos.torch.hora.rsl_rl_compat import ( + from unilab.algos.hora.models import build_hora_shared_actor_critic + from unilab.algos.hora.rsl_rl_compat import ( convert_config_v3_to_v4, is_rsl_rl_v4, is_rsl_rl_v5, @@ -227,8 +227,10 @@ def to_float32_np(x): ) total_steps = 0 - ep_rewards = [] - ep_lengths = [] + # Bounded rolling window of the most recent completed episodes; an + # unbounded list here grows for the entire run. + ep_rewards: deque[float] = deque(maxlen=100) + ep_lengths: deque[int] = deque(maxlen=100) current_ep_rewards = np.zeros(num_envs, dtype=np.float32) current_ep_lengths = np.zeros(num_envs, dtype=np.int32) ep_reward_components = defaultdict(list) @@ -365,15 +367,17 @@ def to_float32_np(x): if k.startswith("reward/"): ep_reward_components[k].append(v) - if metrics_queue is not None and total_steps % (num_envs * 10) == 0: + # Report every env step so learner-side reward and throughput + # displays track the current policy without extra lag. + if metrics_queue is not None: try: msg: dict[str, Any] = { "total_steps": total_steps, } if ep_rewards: - msg["mean_ep_reward"] = statistics.mean(ep_rewards[-100:]) + msg["mean_ep_reward"] = statistics.mean(ep_rewards) msg["mean_ep_length"] = ( - statistics.mean(ep_lengths[-100:]) if ep_lengths else 0.0 + statistics.mean(ep_lengths) if ep_lengths else 0.0 ) if ep_completions > 0: msg["timeout_rate"] = ep_timeouts / ep_completions diff --git a/src/unilab/algos/torch/hora/distill.py b/src/unilab/algos/hora/distill.py similarity index 98% rename from src/unilab/algos/torch/hora/distill.py rename to src/unilab/algos/hora/distill.py index d98bd93ea..b1f1a84be 100644 --- a/src/unilab/algos/torch/hora/distill.py +++ b/src/unilab/algos/hora/distill.py @@ -13,14 +13,14 @@ from omegaconf import DictConfig, OmegaConf from tensordict import TensorDict -from unilab.algos.torch.common.normalization import EmpiricalNormalization -from unilab.algos.torch.hora.models import ( +from unilab.algos.common.normalization import EmpiricalNormalization +from unilab.algos.hora.models import ( HoraActorModel, HoraCoreOutput, HoraSharedActorCritic, ProprioAdaptTConv, ) -from unilab.algos.torch.hora.sac_models import HoraSACActor +from unilab.algos.hora.sac_models import HoraSACActor class HoraSACDistillShared(nn.Module): @@ -306,7 +306,7 @@ def cfg_with_checkpoint_runtime(cfg: DictConfig, checkpoint: dict[str, Any]) -> Config using the current owner env/reward settings and checkpoint model construction fields. """ - from unilab.algos.torch.hora.distill_config import apply_teacher_defaults + from unilab.algos.hora.distill_config import apply_teacher_defaults cfg_with_owner_defaults = apply_teacher_defaults(cfg) cfg_clone = OmegaConf.create(OmegaConf.to_container(cfg_with_owner_defaults, resolve=False)) diff --git a/src/unilab/algos/torch/hora/distill_config.py b/src/unilab/algos/hora/distill_config.py similarity index 96% rename from src/unilab/algos/torch/hora/distill_config.py rename to src/unilab/algos/hora/distill_config.py index 7e7b4c9fb..c40ba7236 100644 --- a/src/unilab/algos/torch/hora/distill_config.py +++ b/src/unilab/algos/hora/distill_config.py @@ -10,13 +10,13 @@ from hydra.core.global_hydra import GlobalHydra from omegaconf import DictConfig, OmegaConf -from unilab.training.run import resolve_task_checkpoint_path +from unilab.utils.checkpoint import resolve_task_checkpoint_path -_REPO_ROOT = Path(__file__).resolve().parents[5] +_REPO_ROOT = Path(__file__).resolve().parents[4] # Teacher owner configs are Hydra-composed from their family config tree. -# SAC teachers live in the shared offpolicy tree behind the `algo` group. -_TEACHER_TREE_BY_FAMILY = {"sac": "offpolicy"} +# SAC teachers live in their own per-algo tree; there is no `algo` group anymore. +_TEACHER_TREE_BY_FAMILY = {"sac": "sac"} # Teacher -> student `algo.model` mappings expressed in YAML; see # conf/hora_distill/student_model/*.yaml. The mapping files interpolate against @@ -56,8 +56,6 @@ def load_teacher_owner_config( algo_family = str(algo_family) conf_dir = root / "conf" / _TEACHER_TREE_BY_FAMILY.get(algo_family, algo_family) overrides = [f"task={task}"] - if algo_family == "sac": - overrides.insert(0, "algo=sac") GlobalHydra.instance().clear() with initialize_config_dir(config_dir=str(conf_dir.absolute()), version_base="1.3"): return compose("config", overrides=overrides) diff --git a/src/unilab/algos/torch/hora/models.py b/src/unilab/algos/hora/models.py similarity index 100% rename from src/unilab/algos/torch/hora/models.py rename to src/unilab/algos/hora/models.py diff --git a/src/unilab/algos/torch/hora/observations.py b/src/unilab/algos/hora/observations.py similarity index 100% rename from src/unilab/algos/torch/hora/observations.py rename to src/unilab/algos/hora/observations.py diff --git a/src/unilab/algos/torch/hora/ppo.py b/src/unilab/algos/hora/ppo.py similarity index 98% rename from src/unilab/algos/torch/hora/ppo.py rename to src/unilab/algos/hora/ppo.py index 899de6fb0..56c501da8 100644 --- a/src/unilab/algos/torch/hora/ppo.py +++ b/src/unilab/algos/hora/ppo.py @@ -14,8 +14,8 @@ from rsl_rl.utils import resolve_obs_groups, resolve_optimizer from tensordict import TensorDict -from unilab.algos.torch.hora.models import HoraActorModel, HoraCriticModel, HoraSharedActorCritic -from unilab.algos.torch.rsl_rl_ppo import FinalObservationAwarePPO +from unilab.algos.hora.models import HoraActorModel, HoraCriticModel, HoraSharedActorCritic +from unilab.algos.rsl_rl_ppo import FinalObservationAwarePPO logger = logging.getLogger(__name__) diff --git a/src/unilab/algos/torch/hora/rsl_rl.py b/src/unilab/algos/hora/rsl_rl.py similarity index 98% rename from src/unilab/algos/torch/hora/rsl_rl.py rename to src/unilab/algos/hora/rsl_rl.py index 2f301a412..53b67db31 100644 --- a/src/unilab/algos/torch/hora/rsl_rl.py +++ b/src/unilab/algos/hora/rsl_rl.py @@ -9,8 +9,8 @@ import torch from tensordict import TensorDict +from unilab.algos.rsl_rl import RslRlVecEnvWrapper from unilab.base.final_observation import resolve_terminal_observation_contract -from unilab.training.rsl_rl import RslRlVecEnvWrapper from unilab.utils.tensor import to_numpy, to_torch from .observations import build_hora_obs_tensordict diff --git a/src/unilab/algos/torch/hora/rsl_rl_compat.py b/src/unilab/algos/hora/rsl_rl_compat.py similarity index 100% rename from src/unilab/algos/torch/hora/rsl_rl_compat.py rename to src/unilab/algos/hora/rsl_rl_compat.py diff --git a/src/unilab/algos/torch/hora/runtime.py b/src/unilab/algos/hora/runtime.py similarity index 100% rename from src/unilab/algos/torch/hora/runtime.py rename to src/unilab/algos/hora/runtime.py diff --git a/src/unilab/algos/torch/hora/sac.py b/src/unilab/algos/hora/sac.py similarity index 86% rename from src/unilab/algos/torch/hora/sac.py rename to src/unilab/algos/hora/sac.py index 4d0524716..73e3c0edc 100644 --- a/src/unilab/algos/torch/hora/sac.py +++ b/src/unilab/algos/hora/sac.py @@ -5,9 +5,9 @@ from dataclasses import dataclass, field from typing import Any -from unilab.algos.torch.hora.runtime import HORA_SAC_RUNTIME_IMPL, is_hora_sac_runtime -from unilab.algos.torch.hora.sac_learner import HoraSACLearner -from unilab.algos.torch.offpolicy.runtime import OffPolicyRuntime +from unilab.algos.hora.runtime import HORA_SAC_RUNTIME_IMPL, is_hora_sac_runtime +from unilab.algos.hora.sac_learner import HoraSACLearner +from unilab.algos.offpolicy.runtime import OffPolicyRuntime @dataclass(frozen=True) @@ -16,7 +16,6 @@ class HoraSACRuntime(OffPolicyRuntime): learner_cls: type[Any] | None = HoraSACLearner algo_type: str | None = HORA_SAC_RUNTIME_IMPL - supports_symmetry: bool = False actor_cfg: dict[str, Any] = field(default_factory=dict) def build_model_kwargs(self, *, obs_dim: int, critic_obs_dim: int) -> dict[str, Any]: diff --git a/src/unilab/algos/torch/hora/sac_learner.py b/src/unilab/algos/hora/sac_learner.py similarity index 90% rename from src/unilab/algos/torch/hora/sac_learner.py rename to src/unilab/algos/hora/sac_learner.py index 029783ec7..0acdd477a 100644 --- a/src/unilab/algos/torch/hora/sac_learner.py +++ b/src/unilab/algos/hora/sac_learner.py @@ -8,8 +8,8 @@ import torch import torch.optim as optim -from unilab.algos.torch.fast_sac.learner import FastSACLearner -from unilab.algos.torch.hora.sac_models import HoraSACActor +from unilab.algos.fast_sac.learner import FastSACLearner +from unilab.algos.hora.sac_models import HoraSACActor def derive_priv_info_from_critic_obs( @@ -49,12 +49,8 @@ def __init__( use_layer_norm: bool = True, actor_lr: float = 3e-4, weight_decay: float = 0.001, - use_symmetry: bool = False, - symmetry_augmentation: Any | None = None, **kwargs: Any, ) -> None: - if use_symmetry or symmetry_augmentation is not None: - raise ValueError("HORA-SAC does not support symmetry augmentation.") if int(priv_info_dim) <= 0: raise ValueError(f"HORA-SAC requires positive priv_info_dim, got {priv_info_dim}.") @@ -70,8 +66,6 @@ def __init__( use_layer_norm=use_layer_norm, actor_lr=actor_lr, weight_decay=weight_decay, - use_symmetry=False, - symmetry_augmentation=None, **kwargs, ) self.use_cuda_graph_critic = False diff --git a/src/unilab/algos/torch/hora/sac_models.py b/src/unilab/algos/hora/sac_models.py similarity index 100% rename from src/unilab/algos/torch/hora/sac_models.py rename to src/unilab/algos/hora/sac_models.py diff --git a/src/unilab/algos/torch/offpolicy/__init__.py b/src/unilab/algos/offpolicy/__init__.py similarity index 56% rename from src/unilab/algos/torch/offpolicy/__init__.py rename to src/unilab/algos/offpolicy/__init__.py index 60749cd92..d578222bd 100644 --- a/src/unilab/algos/torch/offpolicy/__init__.py +++ b/src/unilab/algos/offpolicy/__init__.py @@ -1,7 +1,7 @@ """Off-policy RL unified infrastructure.""" -from unilab.algos.torch.offpolicy.runner import OffPolicyRunner -from unilab.algos.torch.offpolicy.worker import off_policy_collector_fn +from unilab.algos.offpolicy.runner import OffPolicyRunner +from unilab.algos.offpolicy.worker import off_policy_collector_fn from unilab.logging import OffPolicyLogger __all__ = [ diff --git a/src/unilab/algos/torch/offpolicy/double_buffer_runner.py b/src/unilab/algos/offpolicy/double_buffer_runner.py similarity index 98% rename from src/unilab/algos/torch/offpolicy/double_buffer_runner.py rename to src/unilab/algos/offpolicy/double_buffer_runner.py index 114f9d927..ce506b23b 100644 --- a/src/unilab/algos/torch/offpolicy/double_buffer_runner.py +++ b/src/unilab/algos/offpolicy/double_buffer_runner.py @@ -16,17 +16,18 @@ if TYPE_CHECKING: from unilab.ipc.dp_sync import DpParameterSync -from unilab.algos.torch.offpolicy.runner import ( +from unilab.algos.offpolicy.runner import ( OffPolicyRunner, build_offpolicy_sample_info, build_reward_comparison_metrics, replay_buffer_ready_for_learning, ) -from unilab.algos.torch.offpolicy.thread_budget import ( +from unilab.algos.offpolicy.thread_budget import ( format_torch_thread_runtime, torch_thread_env, ) -from unilab.algos.torch.offpolicy.worker import off_policy_collector_fn, sample_offpolicy_actions +from unilab.algos.offpolicy.worker import off_policy_collector_fn, sample_offpolicy_actions +from unilab.base.backend.process_device import resolve_backend_process_device from unilab.ipc.async_runner import _SPAWN_CTX from unilab.ipc.inference_slot import SharedInferenceSlot from unilab.ipc.replay_buffer import DEFAULT_REPLAY_INGRESS_DEPTH, ReplayBuffer @@ -35,7 +36,7 @@ require_offpolicy_replay_device, ) from unilab.logging import OffPolicyLogger, TraceRecorder -from unilab.training.seed import derive_worker_seed +from unilab.utils.seed import derive_worker_seed # Terminal/W&B display names for the off-policy algo types. Keep these # user-facing (no internal "Fast*" implementation prefixes). @@ -159,6 +160,10 @@ def __init__( **kwargs, ): kwargs["device"] = require_offpolicy_replay_device(kwargs.get("device")) + collector_backend_device = resolve_backend_process_device( + str(kwargs.get("sim_backend", "mujoco")), + kwargs["device"], + ) super().__init__(**kwargs) if replay_prefetch_mode != "one_tick": raise ValueError( @@ -168,6 +173,7 @@ def __init__( # Per-rank CPU block owned by this rank's collector (multi-GPU DP); # merged into the collector-only env override at collector startup. self.collector_cpu_ids = list(collector_cpu_ids) if collector_cpu_ids is not None else None + self.collector_backend_device = collector_backend_device # Multi-GPU synchronous data parallelism (None = the bit-identical # single-rank path): startup model broadcast, then gradient averaging # before every actor/critic/temperature optimizer step. @@ -181,7 +187,8 @@ def __init__( self.runtime_manifest = { "inference_owner": "learner", "collector_actor": False, - "collector_accelerator_context": False, + "collector_accelerator_context": self.collector_backend_device is not None, + "collector_backend_device": self.collector_backend_device, "collector_torch_inference": False, "learner_actor_reused": True, "logger_owner_rank": 0, @@ -905,8 +912,6 @@ def learn( log_backend=self._logger_backend(logger_type), ) logger.update_runtime_manifest(self.runtime_manifest) - if hasattr(self.learner, "use_symmetry") and self.learner.use_symmetry: - logger.log_status("Symmetry augmentation: enabled") logger.log_status(format_torch_thread_runtime(self.torch_thread_runtime)) logger.log_status("Replay storage: device-authoritative bounded ingress") logger.log_status(f"Replay prefetch mode: {self.replay_prefetch_mode}") @@ -920,7 +925,9 @@ def learn( f"({self.replay_transfer_backend.get('device_family')})" ) logger.log_status(f"Inference owner: learner.actor ({self.device})") - logger.log_status("Collector model/accelerator ownership: none") + if self.collector_backend_device is not None: + logger.log_status(f"Collector backend device: {self.collector_backend_device}") + logger.log_status("Collector actor/inference ownership: none") logger.log_status("Replay learner lightweight: fixed (log_interval=1)") self._active_logger = logger logger.start() @@ -945,6 +952,7 @@ def learn( "inference_request_queue": inference_request_queue, "inference_response_queue": inference_response_queue, "sim_backend": self.sim_backend, + "backend_device": self.collector_backend_device, "env_cfg_override": self._collector_env_cfg_override(), "inference_slot": inference_slot, "seed": derive_worker_seed(self.seed, worker_index=0), @@ -961,7 +969,10 @@ def learn( time.sleep(0.5) - reward_history: deque = deque(maxlen=100) + # Recent collector reports; each entry is already the collector's + # rolling 100-episode mean, so a short window keeps the logged + # reward timely without losing smoothing. + reward_history: deque = deque(maxlen=10) latest_reward_components: dict[str, float] = {} has_logged_reward = False last_buf_log = 0 @@ -1310,7 +1321,6 @@ def learn( **build_offpolicy_sample_info( replay_batch_size_per_rank=self.batch_size, updates_per_step=self.updates_per_step, - learner=self.learner, ), }, ) diff --git a/src/unilab/algos/torch/offpolicy/runner.py b/src/unilab/algos/offpolicy/runner.py similarity index 92% rename from src/unilab/algos/torch/offpolicy/runner.py rename to src/unilab/algos/offpolicy/runner.py index 09fbed4bd..9f3cdf18d 100644 --- a/src/unilab/algos/torch/offpolicy/runner.py +++ b/src/unilab/algos/offpolicy/runner.py @@ -6,12 +6,12 @@ from collections import deque from typing import Any -from unilab.algos.torch.common.device import get_env_dims +from unilab.algos.common.device import get_env_dims from unilab.ipc.async_runner import AsyncRunner from unilab.logging import OffPolicyLogger -from unilab.training.seed import apply_training_seed from unilab.utils.device import get_default_device from unilab.utils.nan_guard import NanGuardCfg +from unilab.utils.seed import apply_training_seed def compute_train_start_threshold(batch_size: int, learning_starts: int, num_envs: int) -> int: @@ -34,30 +34,18 @@ def replay_buffer_ready_for_learning( ) -def get_learner_batch_multiplier(learner: Any) -> int: - """Return the effective learner batch multiplier for one replay row.""" - if not bool(getattr(learner, "use_symmetry", False)): - return 1 - symmetry = getattr(learner, "symmetry", None) - multiplier = int(getattr(symmetry, "batch_multiplier", 1) or 1) - return max(multiplier, 1) - - def build_offpolicy_sample_info( *, replay_batch_size_per_rank: int, updates_per_step: int, - learner: Any, ) -> dict[str, int]: """Describe replay rows and effective learner samples for logging.""" updates_per_step = max(int(updates_per_step), 0) - replay_batch_size_per_rank = max(int(replay_batch_size_per_rank), 0) - batch_multiplier = get_learner_batch_multiplier(learner) - batch_size_per_rank = replay_batch_size_per_rank * batch_multiplier + batch_size_per_rank = max(int(replay_batch_size_per_rank), 0) return { "batch_size_per_rank": batch_size_per_rank, "effective_batch_size": batch_size_per_rank, - "replay_samples_per_iter": replay_batch_size_per_rank * updates_per_step, + "replay_samples_per_iter": batch_size_per_rank * updates_per_step, "learner_samples_per_iter": batch_size_per_rank * updates_per_step, } diff --git a/src/unilab/algos/torch/offpolicy/runtime.py b/src/unilab/algos/offpolicy/runtime.py similarity index 98% rename from src/unilab/algos/torch/offpolicy/runtime.py rename to src/unilab/algos/offpolicy/runtime.py index d6d007ba0..e9722eb52 100644 --- a/src/unilab/algos/torch/offpolicy/runtime.py +++ b/src/unilab/algos/offpolicy/runtime.py @@ -18,7 +18,6 @@ class OffPolicyRuntime: learner_cls: type[Any] | None = None algo_type: str | None = None actor_kwargs: dict[str, Any] = field(default_factory=dict) - supports_symmetry: bool = True def build_model_kwargs(self, *, obs_dim: int, critic_obs_dim: int) -> dict[str, Any]: """Build learner model kwargs from the environment observation contract.""" diff --git a/src/unilab/algos/torch/offpolicy/thread_budget.py b/src/unilab/algos/offpolicy/thread_budget.py similarity index 100% rename from src/unilab/algos/torch/offpolicy/thread_budget.py rename to src/unilab/algos/offpolicy/thread_budget.py diff --git a/src/unilab/algos/torch/offpolicy/worker.py b/src/unilab/algos/offpolicy/worker.py similarity index 93% rename from src/unilab/algos/torch/offpolicy/worker.py rename to src/unilab/algos/offpolicy/worker.py index f00fcc16b..07fa38fca 100644 --- a/src/unilab/algos/torch/offpolicy/worker.py +++ b/src/unilab/algos/offpolicy/worker.py @@ -8,12 +8,13 @@ import numpy as np import torch -from unilab.algos.torch.common.collector_timing import extract_env_step_breakdown_timing_ms -from unilab.algos.torch.offpolicy.thread_budget import apply_torch_thread_runtime +from unilab.algos.common.collector_timing import extract_env_step_breakdown_timing_ms +from unilab.algos.offpolicy.thread_budget import apply_torch_thread_runtime +from unilab.base.backend.process_device import configure_backend_process_device from unilab.base.final_observation import resolve_terminal_observation_contract from unilab.base.observations import split_obs_dict from unilab.base.registry import ensure_registries -from unilab.training.seed import apply_training_seed +from unilab.utils.seed import apply_training_seed # Exclusive phases for one collector loop iteration (one vectorized env.step). # Every key is recorded once per iteration so the reported averages share one @@ -64,7 +65,7 @@ def resolve_offpolicy_actor_priv_info( if algo_type != "hora_sac": return None - from unilab.algos.torch.hora.observations import split_hora_obs_with_priv_info + from unilab.algos.hora.observations import split_hora_obs_with_priv_info _, _, priv_info_np = split_hora_obs_with_priv_info( {"obs": obs_np, "critic": critic_np}, @@ -156,6 +157,7 @@ def off_policy_collector_fn( algo_type: str = "sac", metrics_queue=None, sim_backend: str = "mujoco", + backend_device: str | None = None, env_cfg_override: dict | None = None, seed: int | None = None, trace_enabled: bool = False, @@ -179,6 +181,7 @@ def off_policy_collector_fn( algo_type=algo_type, metrics_queue=metrics_queue, sim_backend=sim_backend, + backend_device=backend_device, env_cfg_override=env_cfg_override, seed=seed, trace_enabled=trace_enabled, @@ -199,6 +202,7 @@ def _run_collector( algo_type, metrics_queue, sim_backend, + backend_device, env_cfg_override, seed, trace_enabled, @@ -209,6 +213,7 @@ def _run_collector( from unilab.base import registry apply_torch_thread_runtime(torch_thread_runtime, role="collector", torch_module=torch) + configured_backend_device = configure_backend_process_device(sim_backend, backend_device) ensure_registries() apply_training_seed(seed, torch_runtime=False, cuda=False) @@ -238,12 +243,15 @@ def _run_collector( replay_buffer.trace_recorder = trace_recorder replay_buffer.trace_thread_time = trace_thread_time replay_buffer.attach_stop_event(stop_event) + from collections import defaultdict, deque + total_steps = 0 - ep_rewards = [] - ep_lengths = [] + # Bounded rolling window of the most recent completed episodes; an + # unbounded list here grows for the entire run. + ep_rewards: deque[float] = deque(maxlen=100) + ep_lengths: deque[int] = deque(maxlen=100) current_ep_rewards = np.zeros(num_envs, dtype=np.float32) current_ep_lengths = np.zeros(num_envs, dtype=np.int32) - from collections import defaultdict ep_reward_components = defaultdict(list) timing_accum_ms: defaultdict[str, float] = defaultdict(float) @@ -265,6 +273,8 @@ def _run_collector( "actor_owned": False, "weight_sync_attached": False, "torch_inference": False, + "collector_accelerator_context": configured_backend_device is not None, + "collector_backend_device": configured_backend_device, "cuda_context_initialized": bool(torch.cuda.is_initialized()), } if trace_recorder: @@ -447,8 +457,9 @@ def _run_collector( if k.startswith("reward/"): ep_reward_components[k].append(v) - # Send metrics periodically - if metrics_queue is not None and total_steps % (num_envs * 10) == 0: + # Send metrics every collector cycle so learner-side reward and + # throughput displays track the current policy without extra lag. + if metrics_queue is not None: import statistics try: @@ -457,10 +468,8 @@ def _run_collector( "buffer_size": int(replay_buffer.size[0]), } if ep_rewards: - msg["mean_ep_reward"] = statistics.mean(ep_rewards[-100:]) - msg["mean_ep_length"] = ( - statistics.mean(ep_lengths[-100:]) if ep_lengths else 0.0 - ) + msg["mean_ep_reward"] = statistics.mean(ep_rewards) + msg["mean_ep_length"] = statistics.mean(ep_lengths) if ep_lengths else 0.0 # Add mean reward components if ep_reward_components: components_mean = {} diff --git a/src/unilab/training/rsl_rl.py b/src/unilab/algos/rsl_rl.py similarity index 95% rename from src/unilab/training/rsl_rl.py rename to src/unilab/algos/rsl_rl.py index 0ac431771..d1849698e 100644 --- a/src/unilab/training/rsl_rl.py +++ b/src/unilab/algos/rsl_rl.py @@ -306,6 +306,16 @@ def reset(self) -> tuple[TensorDict, dict[str, Any]]: self.episode_lengths[:] = 0 return self._obs_to_tensordict(obs_out, info), info + def set_episode_length_buf(self, values: torch.Tensor | np.ndarray) -> None: + """Set episode counters in both the wrapper and manager environment.""" + values_np = to_numpy(values) + if values_np.shape != (self.num_envs,): + raise ValueError( + f"episode length values must have shape ({self.num_envs},), got {values_np.shape}" + ) + self.env.set_episode_length_buf(np.asarray(values_np, dtype=np.int64)) + self.episode_length_buf[:] = torch.as_tensor(values_np, device=self.device) + def get_observations(self) -> TensorDict: assert self.env.state is not None return self._obs_to_tensordict(self.env.state.obs, self.env.state.info) diff --git a/src/unilab/algos/torch/rsl_rl_ppo.py b/src/unilab/algos/rsl_rl_ppo.py similarity index 99% rename from src/unilab/algos/torch/rsl_rl_ppo.py rename to src/unilab/algos/rsl_rl_ppo.py index 67ca1b17d..5f8836db5 100644 --- a/src/unilab/algos/torch/rsl_rl_ppo.py +++ b/src/unilab/algos/rsl_rl_ppo.py @@ -7,7 +7,7 @@ from rsl_rl.algorithms import PPO from tensordict import TensorDict -from unilab.algos.torch.common.compile import get_torch_compile_for_cuda +from unilab.algos.common.compile import get_torch_compile_for_cuda _LOG_2_PI = math.log(2.0 * math.pi) _NORMAL_ENTROPY_OFFSET = 0.5 * (1.0 + _LOG_2_PI) diff --git a/src/unilab/algos/torch/rsl_rl_runtime.py b/src/unilab/algos/rsl_rl_runtime.py similarity index 96% rename from src/unilab/algos/torch/rsl_rl_runtime.py rename to src/unilab/algos/rsl_rl_runtime.py index 8afa405d6..d4134601a 100644 --- a/src/unilab/algos/torch/rsl_rl_runtime.py +++ b/src/unilab/algos/rsl_rl_runtime.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import Any -from unilab.training.rsl_rl import RslRlVecEnvWrapper +from unilab.algos.rsl_rl import RslRlVecEnvWrapper @dataclass(frozen=True) diff --git a/src/unilab/algos/torch/common/__init__.py b/src/unilab/algos/torch/common/__init__.py deleted file mode 100644 index 75b190f0b..000000000 --- a/src/unilab/algos/torch/common/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from unilab.algos.torch.common.actor_factory import build_actor -from unilab.algos.torch.common.device import get_env_dims -from unilab.algos.torch.common.networks import Critic, DistributionalQNetwork -from unilab.algos.torch.common.normalization import EmpiricalNormalization -from unilab.algos.torch.common.stability import check_nan_loss, clip_gradients, safe_tensor -from unilab.base.registry import ensure_registries - -__all__ = [ - "EmpiricalNormalization", - "DistributionalQNetwork", - "Critic", - "get_env_dims", - "check_nan_loss", - "clip_gradients", - "safe_tensor", - "ensure_registries", - "build_actor", -] diff --git a/src/unilab/algos/torch/flash_sac/__init__.py b/src/unilab/algos/torch/flash_sac/__init__.py deleted file mode 100644 index 2219b855b..000000000 --- a/src/unilab/algos/torch/flash_sac/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""FlashSAC algorithm package.""" - -from unilab.algos.torch.flash_sac.learner import FlashSACLearner -from unilab.algos.torch.flash_sac.network import FlashSACActor, FlashSACDoubleCritic -from unilab.algos.torch.flash_sac.runner import FlashSACRunner - -__all__ = [ - "FlashSACActor", - "FlashSACDoubleCritic", - "FlashSACLearner", - "FlashSACRunner", -] diff --git a/src/unilab/algos/torch/him_ppo/__init__.py b/src/unilab/algos/torch/him_ppo/__init__.py deleted file mode 100644 index 225f04629..000000000 --- a/src/unilab/algos/torch/him_ppo/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from unilab.algos.torch.him_ppo.actor_critic import HIMActorCritic -from unilab.algos.torch.him_ppo.algorithm import HIMPPO -from unilab.algos.torch.him_ppo.estimator import HIMEstimator -from unilab.algos.torch.him_ppo.storage import HIMRolloutStorage - -__all__ = [ - "HIMActorCritic", - "HIMPPO", - "HIMEstimator", - "HIMRolloutStorage", -] diff --git a/src/unilab/assets/pull.py b/src/unilab/assets/pull.py new file mode 100644 index 000000000..571be13b3 --- /dev/null +++ b/src/unilab/assets/pull.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Pre-fetch robot binary assets from Hugging Face into their project paths. + +Robot meshes and textures are hosted on Hugging Face rather than committed to git. +They are also downloaded automatically on first use, but this command lets you pull +them ahead of time (e.g. for CI or offline prep) with a single invocation. Files land +under ``src/unilab/assets/robots//`` — no manual file moving needed. + +Usage: + uv run unilab-pull-assets # pull the default robot (x2) + uv run unilab-pull-assets --robot x2 + uv run unilab-pull-assets --robot a2arm + uv run unilab-pull-assets --robot t800 +""" + +from __future__ import annotations + +import argparse +import logging +from collections.abc import Sequence + +from unilab.assets.hub import resolve_robot_asset_dir + +# robot name -> ((ASSETS_ROOT_PATH-relative dir, marker, glob, label), ...) +_ROBOT_ASSETS: dict[str, tuple[tuple[str, str, str, str], ...]] = { + "x2": (("robots/x2/meshes", "pelvis.STL", "*.STL", "STL"),), + "a2arm": (("robots/a2arm/meshes", "adapter_plate.STL", "*.STL", "STL"),), + "t800": ( + ("robots/t800/assets", "LINK_BASE.obj", "*.obj", "OBJ"), + ("robots/t800/textures", "LINK_BASE.png", "*.png", "PNG"), + ), +} + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--robot", + default="x2", + choices=sorted(_ROBOT_ASSETS), + help="Robot whose binary assets to download (default: x2).", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + args = _parse_args(argv) + + for directory, marker, pattern, label in _ROBOT_ASSETS[args.robot]: + target = resolve_robot_asset_dir(directory, marker=marker) + count = len(list(target.rglob(pattern))) + print(f"{args.robot} assets ready at {target} ({count} {label} files)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/unilab/assets/robots/a2/a2.xml b/src/unilab/assets/robots/a2/a2.xml index debefe71e..92a6ccb15 100644 --- a/src/unilab/assets/robots/a2/a2.xml +++ b/src/unilab/assets/robots/a2/a2.xml @@ -18,20 +18,18 @@ - + - + - + - + diff --git a/src/unilab/assets/robots/a2/scene_flat.xml b/src/unilab/assets/robots/a2/scene_flat.xml index e378a4668..11e1fc5f4 100644 --- a/src/unilab/assets/robots/a2/scene_flat.xml +++ b/src/unilab/assets/robots/a2/scene_flat.xml @@ -4,20 +4,21 @@ - - + + + - - - + + + - + + + - - + + + - - - - + + + + - + diff --git a/src/unilab/assets/robots/stewart/motphys-ground.png b/src/unilab/assets/robots/stewart/motphys-ground.png deleted file mode 100644 index 04967dcfb..000000000 Binary files a/src/unilab/assets/robots/stewart/motphys-ground.png and /dev/null differ diff --git a/src/unilab/assets/robots/stewart/scene.xml b/src/unilab/assets/robots/stewart/scene.xml index 2887522a6..a3a8b7f41 100644 --- a/src/unilab/assets/robots/stewart/scene.xml +++ b/src/unilab/assets/robots/stewart/scene.xml @@ -1,12 +1,19 @@