[update] update README and postion retargeting examples

README.md

@@ -45,13 +45,15 @@ pip install -e ".[example]"
 
 ### Retargeting from human hand video
 
-This type of retargeting can be used for applications like teleoperation, 
+This type of retargeting can be used for applications like teleoperation,
 e.g. [AnyTeleop](https://yzqin.github.io/anyteleop/).
 
 [Tutorial on retargeting from human hand video](example/vector_retargeting/README.md)
 
 ### Retarget from hand object pose dataset
 
+![teaser](example/position_retargeting/hand_object.webp)
+
 This type of retargeting can be used post-process human data for robot imitation,
 e.g. [DexMV](https://yzqin.github.io/dexmv/).
 
@@ -73,10 +75,10 @@ enhancements. If you utilize this work, please cite it as follows:
 
 ## Acknowledgments
 
-This repository builds on the foundational work from [pinocchio](https://github.com/stack-of-tasks/pinocchio). Examples
-within utilize a renderer derived from [SAPIEN](https://github.com/haosulab/SAPIEN).
-The [PositionOptimizer](https://github.com/dexsuite/dex-retargeting/blob/main/dex_retargeting/optimizer.py) leverages
-methodologies from our earlier
-project, [From One Hand to Multiple Hands](https://yzqin.github.io/dex-teleop-imitation/). Additionally,
-the [DexPilotOptimizer](https://github.com/dexsuite/dex-retargeting/blob/main/dex_retargeting/optimizer.py) is crafted
-using insights from [DexPilot](https://sites.google.com/view/dex-pilot).
+The robot hand models in this repository are sourced directly from [dex-urdf](https://github.com/dexsuite/dex-urdf).
+The robot kinematics in this repo are based on [pinocchio](https://github.com/stack-of-tasks/pinocchio).
+Examples use [SAPIEN](https://github.com/haosulab/SAPIEN) for rendering and visualization.
+
+The `PositionOptimizer` leverages methodologies from our earlier
+project, [From One Hand to Multiple Hands](https://yzqin.github.io/dex-teleop-imitation/).
+Additionally, the `DexPilotOptimizer`is crafted using insights from [DexPilot](https://sites.google.com/view/dex-pilot).

example/position_retargeting/.gitignore

@@ -1 +1,2 @@
 /manopth/
+/data/

example/position_retargeting/README.md

@@ -1,5 +1,7 @@
 ## Retarget Robot Motion from Hand Object Pose Dataset
 
+![teaser](hand_object.webp)
+
 ### Setting up DexYCB Dataset
 
 This example illustrates how you can utilize the impressive DexYCB dataset to create a robot motion trajectory.

example/position_retargeting/hand_robot_viewer.py

@@ -3,6 +3,8 @@
 from typing import Dict, List
 
 import numpy as np
+import cv2
+from tqdm import trange
 import sapien
 import transforms3d.quaternions
 
@@ -38,6 +40,7 @@ class RobotHandDatasetSAPIENViewer(HandDatasetSAPIENViewer):
     def __init__(self, robot_names: List[RobotName], hand_type: HandType, headless=False, use_ray_tracing=False):
         super().__init__(headless=headless, use_ray_tracing=use_ray_tracing)
 
+        self.robot_names = robot_names
         self.robots: List[sapien.Articulation] = []
         self.robot_file_names: List[str] = []
         self.retargetings: List[SeqRetargeting] = []
@@ -62,7 +65,6 @@ def __init__(self, robot_names: List[RobotName], hand_type: HandType, headless=F
             urdf_path = Path(config.urdf_path)
             if "glb" not in urdf_path.stem:
                 urdf_path = str(urdf_path).replace(".urdf", "_glb.urdf")
-
             robot_urdf = urdf.URDF.load(str(urdf_path), add_dummy_free_joints=True, build_scene_graph=False)
             urdf_name = urdf_path.split("/")[-1]
             temp_dir = tempfile.mkdtemp(prefix="dex_retargeting-")
@@ -90,8 +92,12 @@ def render_dexycb_data(self, data: Dict, fps=5, y_offset=0.8):
         # Set table and viewer pose for better visual effect only
         global_y_offset = -y_offset * len(self.robots) / 2
         self.table.set_pose(sapien.Pose([0.5, global_y_offset + 0.2, 0]))
-        if self.viewer is not None:
+        if not self.headless:
             self.viewer.set_camera_xyz(1.5, global_y_offset, 1)
+        else:
+            local_pose = self.camera.get_local_pose()
+            local_pose.set_p(np.array([1.5, global_y_offset, 1]))
+            self.camera.set_local_pose(local_pose)
 
         hand_pose = data["hand_pose"]
         object_pose = data["object_pose"]
@@ -115,8 +121,17 @@ def render_dexycb_data(self, data: Dict, fps=5, y_offset=0.8):
                 start_frame = i
                 break
 
+        if self.headless:
+            robot_names = [robot.name for robot in self.robot_names]
+            robot_names = "_".join(robot_names)
+            video_path = Path(__file__).parent.resolve() / f"data/{robot_names}_video.mp4"
+            writer = cv2.VideoWriter(
+                str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (self.camera.get_width(), self.camera.get_height())
+            )
+
         # Loop rendering
-        for i in range(start_frame, num_frame):
+        step_per_frame = int(60 / fps)
+        for i in trange(start_frame, num_frame):
             object_pose_frame = object_pose[i]
             hand_pose_frame = hand_pose[i]
             vertex, joint = self._compute_hand_geometry(hand_pose_frame)
@@ -141,8 +156,18 @@ def render_dexycb_data(self, data: Dict, fps=5, y_offset=0.8):
                 qpos = retargeting.retarget(ref_value)[retarget2sapien]
                 robot.set_qpos(qpos)
 
-            for k in range(60 // fps):
-                self.viewer.render()
-
-        self.viewer.paused = True
-        self.viewer.render()
+            self.scene.update_render()
+            if self.headless:
+                self.camera.take_picture()
+                rgb = self.camera.get_picture("Color")[..., :3]
+                rgb = (np.clip(rgb, 0, 1) * 255).astype(np.uint8)
+                writer.write(rgb[..., ::-1])
+            else:
+                for k in range(start_frame):
+                    self.viewer.render()
+
+        if self.headless:
+            writer.release()
+        else:
+            self.viewer.paused = True
+            self.viewer.render()

example/position_retargeting/hand_viewer.py

@@ -1,8 +1,10 @@
+from pathlib import Path
 from typing import Dict, List, Optional
 
+import cv2
+from tqdm import trange
 import numpy as np
 import sapien
-import sapien.core as sapien
 import torch
 from pytransform3d import transformations as pt
 from sapien import internal_renderer as R
@@ -47,7 +49,7 @@ def __init__(self, headless=False, use_ray_tracing=False):
         else:
             sapien.render.set_viewer_shader_dir("rt")
             sapien.render.set_camera_shader_dir("rt")
-            sapien.render.set_ray_tracing_samples_per_pixel(16)
+            sapien.render.set_ray_tracing_samples_per_pixel(64)
             sapien.render.set_ray_tracing_path_depth(8)
             sapien.render.set_ray_tracing_denoiser("oidn")
 
@@ -57,17 +59,9 @@ def __init__(self, headless=False, use_ray_tracing=False):
 
         # Lighting
         scene.set_environment_map(create_dome_envmap(sky_color=[0.2, 0.2, 0.2], ground_color=[0.2, 0.2, 0.2]))
-        scene.add_directional_light(np.array([1, -1, -1]), np.array([1, 1, 1]), shadow=True)
-        scene.add_directional_light([0, 0, -1], [0.9, 0.8, 0.8], shadow=False)
+        scene.add_directional_light(np.array([1, -1, -1]), np.array([2, 2, 2]), shadow=True)
+        scene.add_directional_light([0, 0, -1], [1.8, 1.6, 1.6], shadow=False)
         scene.set_ambient_light(np.array([0.2, 0.2, 0.2]))
-        scene.add_spot_light(
-            np.array([0, 0, 1.5]),
-            direction=np.array([0, 0, -1]),
-            inner_fov=0.3,
-            outer_fov=1.0,
-            color=np.array([0.5, 0.5, 0.5]),
-            shadow=False,
-        )
 
         # Add ground
         visual_material = sapien.render.RenderMaterial()
@@ -82,10 +76,14 @@ def __init__(self, headless=False, use_ray_tracing=False):
             viewer = Viewer()
             viewer.set_scene(scene)
             viewer.set_camera_xyz(1.5, 0, 1)
-            viewer.set_camera_rpy(0, -0.6, 3.14)
+            viewer.set_camera_rpy(0, -0.8, 3.14)
             viewer.control_window.toggle_origin_frame(False)
             self.viewer = viewer
-        self.gui = not headless
+        else:
+            self.camera = scene.add_camera("cam", 1920, 640, 0.9, 0.01, 100)
+            self.camera.set_local_pose(sapien.Pose([1.5, 0, 1], [0, 0.389418, 0, -0.921061]))
+
+        self.headless = headless
 
         # Create table
         white_diffuse = sapien.render.RenderMaterial()
@@ -187,14 +185,18 @@ def _update_hand(self, vertex):
         self.nodes.append(node)
 
     def render_dexycb_data(self, data: Dict, fps=10):
-        if not self.gui:
-            raise RuntimeError(f"Could not render data frames when the gui is disabled.")
         hand_pose = data["hand_pose"]
         object_pose = data["object_pose"]
         frame_num = hand_pose.shape[0]
 
+        if self.headless:
+            video_path = Path(__file__).parent.resolve() / "data/human_hand_video.mp4"
+            writer = cv2.VideoWriter(
+                str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (self.camera.get_width(), self.camera.get_height())
+            )
+
         step_per_frame = int(60 / fps)
-        for i in range(frame_num):
+        for i in trange(frame_num):
             object_pose_frame = object_pose[i]
             hand_pose_frame = hand_pose[i]
             vertex, _ = self._compute_hand_geometry(hand_pose_frame)
@@ -205,8 +207,17 @@ def render_dexycb_data(self, data: Dict, fps=10):
                 pose = self.camera_pose * sapien.Pose(pos_quat[4:], np.concatenate([pos_quat[3:4], pos_quat[:3]]))
                 self.objects[k].set_pose(pose)
             self.scene.update_render()
-            for _ in range(step_per_frame):
-                self.viewer.render()
-
-        self.viewer.paused = True
-        self.viewer.render()
+            if self.headless:
+                self.camera.take_picture()
+                rgb = self.camera.get_picture("Color")[..., :3]
+                rgb = (np.clip(rgb, 0, 1) * 255).astype(np.uint8)
+                writer.write(rgb[..., ::-1])
+            else:
+                for _ in range(step_per_frame):
+                    self.viewer.render()
+
+        if not self.headless:
+            self.viewer.paused = True
+            self.viewer.render()
+        else:
+            writer.release()

example/position_retargeting/render_hand_object.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+from typing import Optional, Tuple, List
+
+import numpy as np
+import tyro
+
+from dataset import DexYCBVideoDataset
+from dex_retargeting.constants import RobotName, HandType
+from dex_retargeting.retargeting_config import RetargetingConfig
+from hand_robot_viewer import RobotHandDatasetSAPIENViewer
+from hand_viewer import HandDatasetSAPIENViewer
+
+# For numpy version compatibility
+np.bool = bool
+np.int = int
+np.float = float
+np.str = str
+np.complex = complex
+np.object = object
+np.unicode = np.unicode_
+
+
+def viz_hand_object(robots: Optional[Tuple[RobotName]], data_root: Path, fps: int):
+    dataset = DexYCBVideoDataset(data_root, hand_type="right")
+    if robots is None:
+        viewer = HandDatasetSAPIENViewer(headless=True, use_ray_tracing=True)
+    else:
+        viewer = RobotHandDatasetSAPIENViewer(list(robots), HandType.right, headless=True, use_ray_tracing=True)
+
+    # Data ID, feel free to change it to visualize different trajectory
+    data_id = 4
+
+    sampled_data = dataset[data_id]
+    for key, value in sampled_data.items():
+        if "pose" not in key:
+            print(f"{key}: {value}")
+    viewer.load_object_hand(sampled_data)
+    viewer.render_dexycb_data(sampled_data, fps)
+
+
+def main(dexycb_dir: str, robots: Optional[List[RobotName]] = None, fps: int = 10):
+    """
+    Render the human and robot trajectories for grasping object inside DexYCB dataset.
+    The human trajectory is visualized as provided, while the robot trajectory is generated from position retargeting
+
+    Args:
+        dexycb_dir: Data root path to the dexycb dataset
+        robots: The names of robots to render, if None, render human hand trajectory only
+        fps: frequency to render hand-object trajectory
+
+    """
+    data_root = Path(dexycb_dir).absolute()
+    robot_dir = Path(__file__).absolute().parent.parent.parent / "assets" / "robots" / "hands"
+    RetargetingConfig.set_default_urdf_dir(robot_dir)
+    if not data_root.exists():
+        raise ValueError(f"Path to DexYCB dir: {data_root} does not exist.")
+    else:
+        print(f"Using DexYCB dir: {data_root}")
+
+    viz_hand_object(robots, data_root, fps)
+
+
+if __name__ == "__main__":
+    tyro.cli(main)