From 25e5a2753b13621e7157f634cebcf00281629b14 Mon Sep 17 00:00:00 2001 From: Seongjin Bien Date: Thu, 9 Apr 2026 15:52:39 +0200 Subject: [PATCH 1/3] add missing requirements in build_deps and tqdm --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f9f56376..1a2c1772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "simplejpeg", "mujoco==3.2.6", "pin==3.7.0", + "tqdm", ] readme = "README.md" maintainers = [{ name = "Tobias Jülg", email = "tobias.juelg@utn.de" }] @@ -71,6 +72,7 @@ dev = [ build_deps = [ "mujoco==3.2.6", "pin==3.7.0", + "scikit-build-core>=0.3.3", ] [tool.cibuildwheel] From ac7e6581b25f32d570d6ad19f9aa463977a5f4d8 Mon Sep 17 00:00:00 2001 From: Seongjin Bien Date: Thu, 9 Apr 2026 15:53:12 +0200 Subject: [PATCH 2/3] add taxim extension with updated example code --- .../scenes/fr3_digit_simple_pick_up/fr3_0.xml | 23 +--- .../fr3_digit_simple_pick_up/fr3_common.xml | 1 - .../scenes/fr3_digit_simple_pick_up/scene.xml | 21 ++- examples/fr3/grasp_digit_demo.py | 6 +- extensions/rcs_taxim/pyproject.toml | 27 ++++ .../rcs_taxim/src/rcs_taxim/__init__.py | 1 + .../rcs_taxim/src/rcs_taxim/creators.py | 124 ++++++++++++++++++ .../rcs_taxim/src/rcs_taxim/taxim_wrapper.py | 110 ++++++++++++++++ python/rcs/envs/sim.py | 4 +- 9 files changed, 278 insertions(+), 39 deletions(-) create mode 100644 extensions/rcs_taxim/pyproject.toml create mode 100644 extensions/rcs_taxim/src/rcs_taxim/__init__.py create mode 100644 extensions/rcs_taxim/src/rcs_taxim/creators.py create mode 100644 extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py diff --git a/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml b/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml index 1b45e7f5..e16d09f8 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml @@ -1,7 +1,4 @@ - - - @@ -102,10 +99,10 @@ - + - + @@ -123,7 +120,7 @@ - + @@ -162,18 +159,4 @@ - - - - - - - - - - - - - - diff --git a/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml b/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml index a2c836dd..e6569313 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml @@ -61,7 +61,6 @@ - diff --git a/assets/scenes/fr3_digit_simple_pick_up/scene.xml b/assets/scenes/fr3_digit_simple_pick_up/scene.xml index 564fe576..26998a54 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/scene.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/scene.xml @@ -1,4 +1,4 @@ - + @@ -12,19 +12,14 @@ - - - - + + - - - + @@ -32,9 +27,9 @@ - - - + + + diff --git a/examples/fr3/grasp_digit_demo.py b/examples/fr3/grasp_digit_demo.py index 1f746b66..2dc80e5d 100644 --- a/examples/fr3/grasp_digit_demo.py +++ b/examples/fr3/grasp_digit_demo.py @@ -6,7 +6,7 @@ import numpy as np from rcs._core.common import Pose from rcs.envs.base import GripperWrapper, RobotEnv -from rcs_tacto.creators import FR3TactoSimplePickUpSimEnvCreator +from rcs_taxim.creators import FR3TaximSimplePickUpSimEnvCreator from tqdm import tqdm logger = logging.getLogger(__name__) @@ -79,7 +79,7 @@ def pickup(self, geom_name: str): def main(): - env_fact = FR3TactoSimplePickUpSimEnvCreator() + env_fact = FR3TaximSimplePickUpSimEnvCreator() env = env_fact( render_mode="human", delta_actions=False, @@ -89,7 +89,7 @@ def main(): # reset the environment env.reset() controller = PickUpDemo(env) - controller.pickup("yellow_box_geom") + controller.pickup("box_geom") env.close() diff --git a/extensions/rcs_taxim/pyproject.toml b/extensions/rcs_taxim/pyproject.toml new file mode 100644 index 00000000..3d533373 --- /dev/null +++ b/extensions/rcs_taxim/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "rcs_taxim" +version = "0.6.3" +description = "RCS integration of mujoco-taxim" +dependencies = [ + "rcs>=0.6.3", + "omegaconf", + "mujoco-taxim@git+https://github.com/utn-air/mujoco-taxim.git@main", +] +readme = "README.md" +maintainers = [ + { name = "Seongjin Bien", email = "seongjin.bien@utn.de" }, + { name = "Tobias Jülg", email = "tobias.juelg@utn.de" }, +] +authors = [{ name = "Seongjin Bien", email = "seongjin.bien@utn.de" }] +requires-python = ">=3.10" + +[tool.black] +line-length = 120 +target-version = ["py310"] + +[tool.isort] +profile = "black" diff --git a/extensions/rcs_taxim/src/rcs_taxim/__init__.py b/extensions/rcs_taxim/src/rcs_taxim/__init__.py new file mode 100644 index 00000000..63af8876 --- /dev/null +++ b/extensions/rcs_taxim/src/rcs_taxim/__init__.py @@ -0,0 +1 @@ +__version__ = "0.6.3" diff --git a/extensions/rcs_taxim/src/rcs_taxim/creators.py b/extensions/rcs_taxim/src/rcs_taxim/creators.py new file mode 100644 index 00000000..70ee3232 --- /dev/null +++ b/extensions/rcs_taxim/src/rcs_taxim/creators.py @@ -0,0 +1,124 @@ +import logging +import typing + +import gymnasium as gym +import numpy as np +from gymnasium.envs.registration import EnvCreator +from rcs._core.common import Pose +from rcs._core.sim import CameraType +from rcs.camera.sim import SimCameraConfig +from rcs.envs.base import ControlMode +from rcs.envs.creators import SimTaskEnvCreator +from rcs.envs.utils import default_sim_robot_cfg +from rcs.sim import SimGripperConfig +from rcs_taxim.taxim_wrapper import TaximSimWrapper + +import rcs + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class FR3TaximSimplePickUpSimEnvCreator(EnvCreator): + def __call__( # type: ignore + self, + render_mode: str = "human", + control_mode: ControlMode = ControlMode.CARTESIAN_TRPY, + resolution: tuple[int, int] | None = None, + frame_rate: int = 0, + delta_actions: bool = True, + cam_list: tuple[str, ...] = ( + "wrist_0", + "bird_eye_cam", + "openvla_view", + "right_side", + "front", + "left_side", + "side_view", + ), + taxim_kwargs: dict[str, typing.Any] | None = None, + **kwargs, + ) -> gym.Env: + if resolution is None: + resolution = (256, 256) + cameras = { + cam: SimCameraConfig( + identifier=cam, + type=CameraType.fixed, + resolution_height=resolution[1], + resolution_width=resolution[0], + frame_rate=frame_rate, + ) + for cam in cam_list + } + robot_cfg = default_sim_robot_cfg(scene="fr3_digit_simple_pick_up") # id = 0 by default + # TODO: Figure out why feeding it the default doesn't work. + # Probably because Pinocchio freaks out over all the weird tags? + robot_cfg.kinematic_model_path = rcs.scenes["fr3_empty_world"].mjcf_robot + robot_cfg.tcp_offset = Pose( + translation=np.array([0.0, 0.0, 0.15]), # type: ignore + rotation=np.array([[0.707, 0.707, 0], [-0.707, 0.707, 0], [0, 0, 1]]), # type: ignore + ) + gripper_cfg = SimGripperConfig() + + # the digit gripper has some custom finger collisions + # not seen in the defaults. These need to be configured properly. + gripper_cfg.collision_geoms = [ + "hand_c", + "d435i_collision", + "finger_a_left", + "finger_b_left", + "finger_c_left", + "finger_a_right", + "finger_b_right", + "finger_c_right", + ] + gripper_cfg.collision_geoms_fingers = [ + "finger_a_left", + "finger_b_left", + "finger_c_left", + "finger_a_right", + "finger_b_right", + "finger_c_right", + ] + + # Append the id to keep it consistent with the model + gripper_cfg.add_id("0") + random_pos_args = {"joint_name": "box_joint"} + + env = SimTaskEnvCreator()( + robot_cfg, + render_mode, + control_mode, + delta_actions, + cameras, + gripper_cfg=gripper_cfg, + random_pos_args=random_pos_args, + **kwargs, + ) + + ''' + env: gym.Env, + taxim_sites: list[str], + taxim_pad_geoms: list[str], + target_geom_mesh_dict: dict[str, str], + taxim_sensor_type: str = "digit", + taxim_bg_idx: int = 0, + taxim_bg_randomize: bool = False, + enable_depth: bool = False, + taxim_fps: int = 60, + visualize: bool = False, + ''' + # Here, we feed some default values for the taxim wrapper + # that aligns with what we have in the fr3_digit_simple_pick_up + if taxim_kwargs is None: + taxim_kwargs = {} + taxim_kwargs["taxim_sites"] = ["left_taxim_pad_0", "right_taxim_pad_0"] + taxim_kwargs["taxim_pad_geoms"] = ["finger_1_left_0", "finger_1_right_0"] + taxim_kwargs["target_geom_mesh_dict"] = {"box_geom": "box_geom"} + taxim_kwargs["taxim_sensor_type"] = "digit" + taxim_kwargs["taxim_fps"] = 60 + taxim_kwargs["enable_depth"] = True + taxim_kwargs["visualize"] = True + + return TaximSimWrapper(env, **taxim_kwargs) diff --git a/extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py b/extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py new file mode 100644 index 00000000..ea6a0657 --- /dev/null +++ b/extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py @@ -0,0 +1,110 @@ +import logging +import os +from importlib.resources import files +from typing import Any + +import cv2 +import gymnasium as gym +from TaximSensor import TaximSensor +from omegaconf import OmegaConf + +logger = logging.getLogger(__name__) + + +class TaximSimWrapper(gym.Wrapper): + """Wrapper to use Taxim with RCS Sim.""" + + def __init__( + self, + env: gym.Env, + taxim_sites: list[str], + taxim_pad_geoms: list[str], + target_geom_mesh_dict: dict[str, str], + taxim_sensor_type: str = "digit", + taxim_bg_idx: int = 0, + taxim_bg_randomize: bool = False, + enable_depth: bool = False, + taxim_fps: int = 60, + visualize: bool = False, + ): + """ + Initialize Taxim sensor with the given configuration. + Args: + env (gym.Env): The environment to wrap. + simulation (sim.Sim): The simulation instance. + taxim_sites (list[str]): List of sites to mount Taxim cameras. + taxim_pad_geoms (list[str]): List of tactile sensor pad geoms which should act as the contact surfaces. + target_geom_mesh_dict (dict[str, str]): Dictionary mapping mjGeom names to mjMesh names. + taxim_sensor_type (str)="digit": The type of Taxim sensor to use. either 'digit' or 'gelsight_r1.5'. + taxim_bg_idx (int)=0: The index of the background image to use. + taxim_bg_randomize (bool)=False: Whether to randomize the background image for every contact. + enable_depth (bool)=False: Whether to enable depth rendering. + taxim_fps (int)=60: Frames per second for Taxim rendering. + visualize (bool)=False: Whether to visualize Taxim rendering in a separate window. + """ + super().__init__(env) + self.env = env + self.taxim_sensors = [] + + self.model = self.env.get_wrapper_attr("sim").model + self.data = self.env.get_wrapper_attr("sim").data + + self.taxim_sites = taxim_sites + self.taxim_pad_geoms = taxim_pad_geoms + self.target_geom_mesh_dict = target_geom_mesh_dict + self.taxim_sensor_type = taxim_sensor_type + self.taxim_bg_idx = taxim_bg_idx + self.taxim_bg_randomize = taxim_bg_randomize + + self.colors = [] + self.depths = [] + + self.taxim_fps = taxim_fps + self.taxim_last_render = -1 + self.enable_depth = enable_depth + + self.initialized = False + self.visualize = visualize + + def reset( + self, seed: int | None = None, options: dict[str, Any] | None = None + ) -> tuple[dict[str, Any], dict[str, Any]]: + obs, info = super().reset(seed=seed, options=options) + if not self.initialized: + # Create taxim sensors for each specified site + print(self.taxim_sites) + for i, site in enumerate(self.taxim_sites): + sensor = TaximSensor(resize=(240,320), sensor_type=self.taxim_sensor_type, preprocess_bg=False) + sensor.add_camera_mujoco(site, self.model, self.data) + sensor.change_bg(self.taxim_bg_idx) + # Add the target geoms to the sensor + for geom, mesh in self.target_geom_mesh_dict.items(): + sensor.add_geom_mujoco(geom, self.model, self.data, mesh) + + sensor.set_sensor_pad_geom(self.taxim_pad_geoms[i]) + self.taxim_sensors.append(sensor) + + self.initialized = True + + self.taxim_last_render = -1 # Reset last render time + + for i, sensor in enumerate(self.taxim_sensors): + rgb, depth, _ = sensor.render_taxim(self.model, self.data, visualize=False) + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("rgb", {})["data"] = rgb + if self.enable_depth: + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("depth", {})["data"] = depth + return obs, info + + def step(self, action: dict[str, Any]): + obs, reward, done, truncated, info = super().step(action) + if self.taxim_last_render + (1 / self.taxim_fps) > self.data.time: + return obs, reward, done, truncated, info + + for i, sensor in enumerate(self.taxim_sensors): + rgb, depth, _ = sensor.render_taxim(self.model, self.data, visualize=self.visualize) + self.taxim_last_render = self.data.time + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("rgb", {})["data"] = rgb + if self.enable_depth: + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("depth", {})["data"] = depth + + return obs, reward, done, truncated, info diff --git a/python/rcs/envs/sim.py b/python/rcs/envs/sim.py index 589b49fd..1a1cf60f 100644 --- a/python/rcs/envs/sim.py +++ b/python/rcs/envs/sim.py @@ -410,12 +410,12 @@ class PickCubeSuccessWrapper(gym.Wrapper): - whether the arm is standing still once the task is solved. """ - def __init__(self, env, cube_joint_name="box_joint"): + def __init__(self, env, cube_joint_name="box_joint", cube_geom_name="box_geom"): super().__init__(env) self.unwrapped: RobotEnv assert isinstance(self.unwrapped.robot, sim.SimRobot), "Robot must be a sim.SimRobot instance." self.sim = env.get_wrapper_attr("sim") - self.cube_geom_name = "box_geom" + self.cube_geom_name = cube_geom_name self.home_pose = self.unwrapped.robot.get_cartesian_position() self._gripper_closing = 0 self._gripper = self.get_wrapper_attr("_gripper") From 6af3a6c3a1ad7af0fa61e709a4df27d467a25068 Mon Sep 17 00:00:00 2001 From: Seongjin Bien Date: Thu, 9 Apr 2026 15:56:44 +0200 Subject: [PATCH 3/3] ! remove references to deprecated rcs_tacto --- .github/workflows/ci.yaml | 2 +- .../scenes/fr3_digit_simple_pick_up/fr3_0.xml | 8 +- .../fr3_digit_simple_pick_up/fr3_common.xml | 4 +- docs/extensions/index.md | 2 +- docs/extensions/overview.md | 2 +- docs/extensions/rcs_tacto.md | 6 +- extensions/rcs_tacto/README.md | 8 -- extensions/rcs_tacto/pyproject.toml | 27 ----- .../rcs_tacto/src/rcs_tacto/__init__.py | 1 - .../rcs_tacto/src/rcs_tacto/creators.py | 110 ------------------ .../rcs_tacto/src/rcs_tacto/tacto_wrapper.py | 102 ---------------- pyproject.toml | 6 +- python/rcs/__init__.py | 1 - 13 files changed, 15 insertions(+), 264 deletions(-) delete mode 100644 extensions/rcs_tacto/README.md delete mode 100644 extensions/rcs_tacto/pyproject.toml delete mode 100644 extensions/rcs_tacto/src/rcs_tacto/__init__.py delete mode 100644 extensions/rcs_tacto/src/rcs_tacto/creators.py delete mode 100644 extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 609b2596..0a5ef362 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,7 +72,7 @@ jobs: - rcs_xarm7 - rcs_realsense - rcs_robotiq2f85 - - rcs_tacto + - rcs_taxim - rcs_usb_cam runs-on: ubuntu-latest steps: diff --git a/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml b/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml index e16d09f8..ea5d5bd4 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml @@ -92,14 +92,14 @@ - + - + @@ -113,14 +113,14 @@ - + - + diff --git a/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml b/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml index e6569313..a266ad31 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml @@ -18,10 +18,10 @@ - + - + diff --git a/docs/extensions/index.md b/docs/extensions/index.md index 58c0391e..79936223 100644 --- a/docs/extensions/index.md +++ b/docs/extensions/index.md @@ -10,7 +10,7 @@ rcs_xarm7 rcs_so101 rcs_realsense rcs_usb_cam -rcs_tacto +rcs_taxim rcs_robotics_library rcs_robotiq2f85 ``` diff --git a/docs/extensions/overview.md b/docs/extensions/overview.md index 84334d4e..89ec0269 100644 --- a/docs/extensions/overview.md +++ b/docs/extensions/overview.md @@ -28,7 +28,7 @@ RCS comes with several supported extensions: - **rcs_so101**: Support for the SO101 robot. - **rcs_realsense**: Support for Intel RealSense cameras. - **rcs_usb_cam**: Support for generic USB webcams. -- **rcs_tacto**: Integration with the Tacto tactile sensor simulator. +- **rcs_taxim**: Integration with the Taxim tactile sensor simulator. - **rcs_robotics_library**: Integration with the Robotics Library (RL). - **rcs_robotiq2f85**: Integration with the Robotiq 2F-85 Gripper. diff --git a/docs/extensions/rcs_tacto.md b/docs/extensions/rcs_tacto.md index a1b94112..8d67a520 100644 --- a/docs/extensions/rcs_tacto.md +++ b/docs/extensions/rcs_tacto.md @@ -1,9 +1,9 @@ -# RCS Tacto Extension +# RCS Taxim Extension -This extension provides integration with the [Tacto](https://github.com/facebookresearch/tacto) tactile sensor simulator. +This extension provides integration with the [Taxim](https://github.com/Robo-Touch/Taxim) tactile sensor simulator. ## Installation ```shell -pip install -ve extensions/rcs_tacto +pip install -ve extensions/rcs_taxim ``` diff --git a/extensions/rcs_tacto/README.md b/extensions/rcs_tacto/README.md deleted file mode 100644 index 3951f76e..00000000 --- a/extensions/rcs_tacto/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TACTO integration for RCS -This package can be installed by running `pip install -e extensions/rcs_tacto` from the RCS repository root. - -An example on how to create the environment is found in `{REPO_ROOT}/examples/grasp_digit_demo.py`. - -Particularly, take a look at `FR3TactoSimplePickUpSimEnvCreator` to understand how Tacto is inserted into the RCS stack. - -Note that it is rather tricky to get the correct contact simulation settings to allow for a robust grasping of objects, so when using it, you will need to play around with the simulation settings. We recommend taking a look at the corresponding [MuJoCo documentation](https://mujoco.readthedocs.io/en/stable/computation/index.html) for more tips. \ No newline at end of file diff --git a/extensions/rcs_tacto/pyproject.toml b/extensions/rcs_tacto/pyproject.toml deleted file mode 100644 index f69dd2cb..00000000 --- a/extensions/rcs_tacto/pyproject.toml +++ /dev/null @@ -1,27 +0,0 @@ -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - -[project] -name = "rcs_tacto" -version = "0.6.3" -description = "RCS integration of tacto" -dependencies = [ - "rcs>=0.6.3", - "omegaconf", - "mujoco-tacto@git+https://github.com/utn-air/mujoco-tacto.git@main", -] -readme = "README.md" -maintainers = [ - { name = "Tobias Jülg", email = "tobias.juelg@utn.de" }, - { name = "Seongjin Bien", email = "seongjin.bien@utn.de" }, -] -authors = [{ name = "Seongjin Bien", email = "seongjin.bien@utn.de" }] -requires-python = ">=3.10" - -[tool.black] -line-length = 120 -target-version = ["py310"] - -[tool.isort] -profile = "black" diff --git a/extensions/rcs_tacto/src/rcs_tacto/__init__.py b/extensions/rcs_tacto/src/rcs_tacto/__init__.py deleted file mode 100644 index 63af8876..00000000 --- a/extensions/rcs_tacto/src/rcs_tacto/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.6.3" diff --git a/extensions/rcs_tacto/src/rcs_tacto/creators.py b/extensions/rcs_tacto/src/rcs_tacto/creators.py deleted file mode 100644 index e78581ee..00000000 --- a/extensions/rcs_tacto/src/rcs_tacto/creators.py +++ /dev/null @@ -1,110 +0,0 @@ -import logging -import typing - -import gymnasium as gym -import numpy as np -from gymnasium.envs.registration import EnvCreator -from rcs._core.common import Pose -from rcs._core.sim import CameraType -from rcs.camera.sim import SimCameraConfig -from rcs.envs.base import ControlMode -from rcs.envs.creators import SimTaskEnvCreator -from rcs.envs.utils import default_sim_robot_cfg -from rcs.sim import SimGripperConfig -from rcs_tacto.tacto_wrapper import TactoSimWrapper - -import rcs - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - -class FR3TactoSimplePickUpSimEnvCreator(EnvCreator): - def __call__( # type: ignore - self, - render_mode: str = "human", - control_mode: ControlMode = ControlMode.CARTESIAN_TRPY, - resolution: tuple[int, int] | None = None, - frame_rate: int = 0, - delta_actions: bool = True, - cam_list: tuple[str, ...] = ( - "wrist_0", - "bird_eye_cam", - "openvla_view", - "right_side", - "front", - "left_side", - "side_view", - ), - tacto_kwargs: dict[str, typing.Any] | None = None, - **kwargs, - ) -> gym.Env: - if resolution is None: - resolution = (256, 256) - cameras = { - cam: SimCameraConfig( - identifier=cam, - type=CameraType.fixed, - resolution_height=resolution[1], - resolution_width=resolution[0], - frame_rate=frame_rate, - ) - for cam in cam_list - } - robot_cfg = default_sim_robot_cfg(scene="fr3_digit_simple_pick_up") # id = 0 by default - # TODO: Figure out why feeding it the default doesn't work. - # Probably because Pinocchio freaks out over all the weird tags? - robot_cfg.kinematic_model_path = rcs.scenes["fr3_empty_world"].mjcf_robot - robot_cfg.tcp_offset = Pose( - translation=np.array([0.0, 0.0, 0.15]), # type: ignore - rotation=np.array([[0.707, 0.707, 0], [-0.707, 0.707, 0], [0, 0, 1]]), # type: ignore - ) - gripper_cfg = SimGripperConfig() - - # the digit gripper has some custom finger collisions - # not seen in the defaults. These need to be configured properly. - gripper_cfg.collision_geoms = [ - "hand_c", - "d435i_collision", - "finger_a_left", - "finger_b_left", - "finger_c_left", - "finger_a_right", - "finger_b_right", - "finger_c_right", - ] - gripper_cfg.collision_geoms_fingers = [ - "finger_a_left", - "finger_b_left", - "finger_c_left", - "finger_a_right", - "finger_b_right", - "finger_c_right", - ] - - # Append the id to keep it consistent with the model - gripper_cfg.add_id("0") - random_pos_args = {"joint_name": "yellow-box-joint"} - - env = SimTaskEnvCreator()( - robot_cfg, - render_mode, - control_mode, - delta_actions, - cameras, - gripper_cfg=gripper_cfg, - random_pos_args=random_pos_args, - **kwargs, - ) - - # Here, we feed some default values for the tacto wrapper - # that aligns with what we have in the fr3_digit_simple_pick_up - if tacto_kwargs is None: - tacto_kwargs = {} - tacto_kwargs["tacto_sites"] = ["left_tacto_pad_0", "right_tacto_pad_0"] - tacto_kwargs["tacto_geoms"] = ["yellow_box_geom"] - tacto_kwargs["tacto_fps"] = 60 - tacto_kwargs["enable_depth"] = True - tacto_kwargs["visualize"] = True - - return TactoSimWrapper(env, **tacto_kwargs) diff --git a/extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py b/extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py deleted file mode 100644 index 89e30956..00000000 --- a/extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py +++ /dev/null @@ -1,102 +0,0 @@ -import logging -import os -from importlib.resources import files -from typing import Any - -import cv2 -import gymnasium as gym -import tacto -from omegaconf import OmegaConf - -logger = logging.getLogger(__name__) - - -class TactoSimWrapper(gym.Wrapper): - """Wrapper to use Tacto with RCS Sim.""" - - def __init__( - self, - env: gym.Env, - tacto_sites: list[str], - tacto_geoms: list[str], - tacto_meshes: dict[str, str] | None = None, - tacto_config: str | None = None, - tacto_bg: str | None = None, - enable_depth: bool = False, - tacto_fps: int = 60, - visualize: bool = False, - ): - """ - Initialize Tacto sensor with the given configuration. - Args: - env (gym.Env): The environment to wrap. - simulation (sim.Sim): The simulation instance. - tacto_sites (list[str]): List of sites to mount Tacto cameras. - tacto_geoms (list[str]): List of mjOBJ_GEOM names to add. - tacto_meshes (dict[str, str] | None): Dictionary mapping geom names to mesh names. - Needed when geom names are not the same as the mesh name in the XML. - tacto_config (str)=None: Absolute path to the Tacto configuration folder containing "digit.yaml". - If None, package default is used. - tacto_bg (str)=None: Absolute path to the background image for Tacto, ending with ".jpg". - If None, package default is used. - enable_depth (bool)=False: Whether to enable depth rendering. - tacto_fps (int)=60: Frames per second for Tacto rendering. - visualize (bool)=False: Whether to visualize Tacto rendering in a separate window. - """ - super().__init__(env) - self.env = env - if tacto_config is None: - tacto_config = os.path.dirname(str(files("tacto") / "cfg" / "digit.yaml")) - logger.warning(f"No tacto_config provided, using default from package: {tacto_config}/digit.yaml") - if tacto_bg is None: - tacto_bg = str(files("tacto") / "assets" / "bg_digit_240_320.jpg") - logger.warning(f"No tacto_bg provided, using default from package: {tacto_bg}") - config_path = os.path.join(tacto_config, "digit.yaml") - t_config = OmegaConf.load(config_path) - self.tacto_sensor = tacto.Sensor(**t_config.tacto, background=cv2.imread(tacto_bg)) - self.tacto_fps = tacto_fps - self.tacto_last_render = -1 - self.tacto_sites = tacto_sites - self.tacto_geoms = tacto_geoms - self.tacto_meshes = tacto_meshes if tacto_meshes is not None else {} - self.enable_depth = enable_depth - self.model = self.env.get_wrapper_attr("sim").model - self.data = self.env.get_wrapper_attr("sim").data - self.initialized = False - self.visualize = visualize - - def reset( - self, seed: int | None = None, options: dict[str, Any] | None = None - ) -> tuple[dict[str, Any], dict[str, Any]]: - obs, info = super().reset(seed=seed, options=options) - if not self.initialized: - # Set up Tacto sensor with the simulation - for site in self.tacto_sites: - self.tacto_sensor.add_camera_mujoco(site, self.model, self.data) - for geom in self.tacto_geoms: - if geom in self.tacto_meshes: - self.tacto_sensor.add_geom_mujoco(geom, self.model, self.data, self.tacto_meshes[geom]) - else: - self.tacto_sensor.add_geom_mujoco(geom, self.model, self.data) - self.initialized = True - self.tacto_last_render = -1 # Reset last render time - colors, depths = self.tacto_sensor.render(self.model, self.data) - for site, color, depth in zip(self.tacto_sites, colors, depths, strict=False): - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("rgb", {})["data"] = color - if self.enable_depth: - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("depth", {})["data"] = depth - return obs, info - - def step(self, action: dict[str, Any]): - obs, reward, done, truncated, info = super().step(action) - if self.tacto_last_render + (1 / self.tacto_fps) < self.data.time: - colors, depths = self.tacto_sensor.render(self.model, self.data) - self.tacto_sensor.updateGUI(colors, depths) if self.visualize else None - self.tacto_last_render = self.data.time - for site, color, depth in zip(self.tacto_sites, colors, depths, strict=False): - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("rgb", {})["data"] = color - if self.enable_depth: - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("depth", {})[ - "data" - ] = depth - return obs, reward, done, truncated, info diff --git a/pyproject.toml b/pyproject.toml index 1a2c1772..6cdcb7f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -213,9 +213,9 @@ version_files = [ "extensions/rcs_ur5e/src/rcs_ur5e/__init__.py:__version__", "extensions/rcs_ur5e/pyproject.toml:\"rcs>=(.*)\"", - "extensions/rcs_tacto/pyproject.toml:version", - "extensions/rcs_tacto/src/rcs_tacto/__init__.py:__version__", - "extensions/rcs_tacto/pyproject.toml:\"rcs>=(.*)\"", + "extensions/rcs_taxim/pyproject.toml:version", + "extensions/rcs_taxim/src/rcs_taxim/__init__.py:__version__", + "extensions/rcs_taxim/pyproject.toml:\"rcs>=(.*)\"", "extensions/rcs_robotiq2f85/pyproject.toml:version", "extensions/rcs_robotiq2f85/src/rcs_robotiq2f85/__init__.py:__version__", diff --git a/python/rcs/__init__.py b/python/rcs/__init__.py index 8c4efd3a..d47a0076 100644 --- a/python/rcs/__init__.py +++ b/python/rcs/__init__.py @@ -102,7 +102,6 @@ def get_scene_urdf(scene_name: str) -> str | None: entry_point=FR3LabDigitGripperPickUpSimEnvCreator(), ) -# Genius TODO: Add the tacto version of the SimEnvCreator # TODO: gym.make("rcs/FR3SimEnv-v0") results in a pickling error: # TypeError: cannot pickle 'rcs._core.sim.SimRobotConfig' object # cf. https://pybind11.readthedocs.io/en/stable/advanced/classes.html#deepcopy-support