diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 609b2596..0a5ef362 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,7 +72,7 @@ jobs: - rcs_xarm7 - rcs_realsense - rcs_robotiq2f85 - - rcs_tacto + - rcs_taxim - rcs_usb_cam runs-on: ubuntu-latest steps: diff --git a/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml b/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml index 1b45e7f5..ea5d5bd4 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/fr3_0.xml @@ -1,7 +1,4 @@ - - - @@ -95,17 +92,17 @@ - + - + - + @@ -116,14 +113,14 @@ - + - + @@ -162,18 +159,4 @@ - - - - - - - - - - - - - - diff --git a/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml b/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml index a2c836dd..a266ad31 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/fr3_common.xml @@ -18,10 +18,10 @@ - + - + @@ -61,7 +61,6 @@ - diff --git a/assets/scenes/fr3_digit_simple_pick_up/scene.xml b/assets/scenes/fr3_digit_simple_pick_up/scene.xml index 564fe576..26998a54 100644 --- a/assets/scenes/fr3_digit_simple_pick_up/scene.xml +++ b/assets/scenes/fr3_digit_simple_pick_up/scene.xml @@ -1,4 +1,4 @@ - + @@ -12,19 +12,14 @@ - - - - + + - - - + @@ -32,9 +27,9 @@ - - - + + + diff --git a/docs/extensions/index.md b/docs/extensions/index.md index 58c0391e..79936223 100644 --- a/docs/extensions/index.md +++ b/docs/extensions/index.md @@ -10,7 +10,7 @@ rcs_xarm7 rcs_so101 rcs_realsense rcs_usb_cam -rcs_tacto +rcs_taxim rcs_robotics_library rcs_robotiq2f85 ``` diff --git a/docs/extensions/overview.md b/docs/extensions/overview.md index 84334d4e..89ec0269 100644 --- a/docs/extensions/overview.md +++ b/docs/extensions/overview.md @@ -28,7 +28,7 @@ RCS comes with several supported extensions: - **rcs_so101**: Support for the SO101 robot. - **rcs_realsense**: Support for Intel RealSense cameras. - **rcs_usb_cam**: Support for generic USB webcams. -- **rcs_tacto**: Integration with the Tacto tactile sensor simulator. +- **rcs_taxim**: Integration with the Taxim tactile sensor simulator. - **rcs_robotics_library**: Integration with the Robotics Library (RL). - **rcs_robotiq2f85**: Integration with the Robotiq 2F-85 Gripper. diff --git a/docs/extensions/rcs_tacto.md b/docs/extensions/rcs_tacto.md index a1b94112..8d67a520 100644 --- a/docs/extensions/rcs_tacto.md +++ b/docs/extensions/rcs_tacto.md @@ -1,9 +1,9 @@ -# RCS Tacto Extension +# RCS Taxim Extension -This extension provides integration with the [Tacto](https://github.com/facebookresearch/tacto) tactile sensor simulator. +This extension provides integration with the [Taxim](https://github.com/Robo-Touch/Taxim) tactile sensor simulator. ## Installation ```shell -pip install -ve extensions/rcs_tacto +pip install -ve extensions/rcs_taxim ``` diff --git a/examples/fr3/grasp_digit_demo.py b/examples/fr3/grasp_digit_demo.py index 1f746b66..2dc80e5d 100644 --- a/examples/fr3/grasp_digit_demo.py +++ b/examples/fr3/grasp_digit_demo.py @@ -6,7 +6,7 @@ import numpy as np from rcs._core.common import Pose from rcs.envs.base import GripperWrapper, RobotEnv -from rcs_tacto.creators import FR3TactoSimplePickUpSimEnvCreator +from rcs_taxim.creators import FR3TaximSimplePickUpSimEnvCreator from tqdm import tqdm logger = logging.getLogger(__name__) @@ -79,7 +79,7 @@ def pickup(self, geom_name: str): def main(): - env_fact = FR3TactoSimplePickUpSimEnvCreator() + env_fact = FR3TaximSimplePickUpSimEnvCreator() env = env_fact( render_mode="human", delta_actions=False, @@ -89,7 +89,7 @@ def main(): # reset the environment env.reset() controller = PickUpDemo(env) - controller.pickup("yellow_box_geom") + controller.pickup("box_geom") env.close() diff --git a/extensions/rcs_tacto/README.md b/extensions/rcs_tacto/README.md deleted file mode 100644 index 3951f76e..00000000 --- a/extensions/rcs_tacto/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TACTO integration for RCS -This package can be installed by running `pip install -e extensions/rcs_tacto` from the RCS repository root. - -An example on how to create the environment is found in `{REPO_ROOT}/examples/grasp_digit_demo.py`. - -Particularly, take a look at `FR3TactoSimplePickUpSimEnvCreator` to understand how Tacto is inserted into the RCS stack. - -Note that it is rather tricky to get the correct contact simulation settings to allow for a robust grasping of objects, so when using it, you will need to play around with the simulation settings. We recommend taking a look at the corresponding [MuJoCo documentation](https://mujoco.readthedocs.io/en/stable/computation/index.html) for more tips. \ No newline at end of file diff --git a/extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py b/extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py deleted file mode 100644 index 89e30956..00000000 --- a/extensions/rcs_tacto/src/rcs_tacto/tacto_wrapper.py +++ /dev/null @@ -1,102 +0,0 @@ -import logging -import os -from importlib.resources import files -from typing import Any - -import cv2 -import gymnasium as gym -import tacto -from omegaconf import OmegaConf - -logger = logging.getLogger(__name__) - - -class TactoSimWrapper(gym.Wrapper): - """Wrapper to use Tacto with RCS Sim.""" - - def __init__( - self, - env: gym.Env, - tacto_sites: list[str], - tacto_geoms: list[str], - tacto_meshes: dict[str, str] | None = None, - tacto_config: str | None = None, - tacto_bg: str | None = None, - enable_depth: bool = False, - tacto_fps: int = 60, - visualize: bool = False, - ): - """ - Initialize Tacto sensor with the given configuration. - Args: - env (gym.Env): The environment to wrap. - simulation (sim.Sim): The simulation instance. - tacto_sites (list[str]): List of sites to mount Tacto cameras. - tacto_geoms (list[str]): List of mjOBJ_GEOM names to add. - tacto_meshes (dict[str, str] | None): Dictionary mapping geom names to mesh names. - Needed when geom names are not the same as the mesh name in the XML. - tacto_config (str)=None: Absolute path to the Tacto configuration folder containing "digit.yaml". - If None, package default is used. - tacto_bg (str)=None: Absolute path to the background image for Tacto, ending with ".jpg". - If None, package default is used. - enable_depth (bool)=False: Whether to enable depth rendering. - tacto_fps (int)=60: Frames per second for Tacto rendering. - visualize (bool)=False: Whether to visualize Tacto rendering in a separate window. - """ - super().__init__(env) - self.env = env - if tacto_config is None: - tacto_config = os.path.dirname(str(files("tacto") / "cfg" / "digit.yaml")) - logger.warning(f"No tacto_config provided, using default from package: {tacto_config}/digit.yaml") - if tacto_bg is None: - tacto_bg = str(files("tacto") / "assets" / "bg_digit_240_320.jpg") - logger.warning(f"No tacto_bg provided, using default from package: {tacto_bg}") - config_path = os.path.join(tacto_config, "digit.yaml") - t_config = OmegaConf.load(config_path) - self.tacto_sensor = tacto.Sensor(**t_config.tacto, background=cv2.imread(tacto_bg)) - self.tacto_fps = tacto_fps - self.tacto_last_render = -1 - self.tacto_sites = tacto_sites - self.tacto_geoms = tacto_geoms - self.tacto_meshes = tacto_meshes if tacto_meshes is not None else {} - self.enable_depth = enable_depth - self.model = self.env.get_wrapper_attr("sim").model - self.data = self.env.get_wrapper_attr("sim").data - self.initialized = False - self.visualize = visualize - - def reset( - self, seed: int | None = None, options: dict[str, Any] | None = None - ) -> tuple[dict[str, Any], dict[str, Any]]: - obs, info = super().reset(seed=seed, options=options) - if not self.initialized: - # Set up Tacto sensor with the simulation - for site in self.tacto_sites: - self.tacto_sensor.add_camera_mujoco(site, self.model, self.data) - for geom in self.tacto_geoms: - if geom in self.tacto_meshes: - self.tacto_sensor.add_geom_mujoco(geom, self.model, self.data, self.tacto_meshes[geom]) - else: - self.tacto_sensor.add_geom_mujoco(geom, self.model, self.data) - self.initialized = True - self.tacto_last_render = -1 # Reset last render time - colors, depths = self.tacto_sensor.render(self.model, self.data) - for site, color, depth in zip(self.tacto_sites, colors, depths, strict=False): - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("rgb", {})["data"] = color - if self.enable_depth: - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("depth", {})["data"] = depth - return obs, info - - def step(self, action: dict[str, Any]): - obs, reward, done, truncated, info = super().step(action) - if self.tacto_last_render + (1 / self.tacto_fps) < self.data.time: - colors, depths = self.tacto_sensor.render(self.model, self.data) - self.tacto_sensor.updateGUI(colors, depths) if self.visualize else None - self.tacto_last_render = self.data.time - for site, color, depth in zip(self.tacto_sites, colors, depths, strict=False): - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("rgb", {})["data"] = color - if self.enable_depth: - obs.setdefault("frames", {}).setdefault(f"tactile_{site}", {}).setdefault("depth", {})[ - "data" - ] = depth - return obs, reward, done, truncated, info diff --git a/extensions/rcs_tacto/pyproject.toml b/extensions/rcs_taxim/pyproject.toml similarity index 79% rename from extensions/rcs_tacto/pyproject.toml rename to extensions/rcs_taxim/pyproject.toml index f69dd2cb..3d533373 100644 --- a/extensions/rcs_tacto/pyproject.toml +++ b/extensions/rcs_taxim/pyproject.toml @@ -3,18 +3,18 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "rcs_tacto" +name = "rcs_taxim" version = "0.6.3" -description = "RCS integration of tacto" +description = "RCS integration of mujoco-taxim" dependencies = [ "rcs>=0.6.3", "omegaconf", - "mujoco-tacto@git+https://github.com/utn-air/mujoco-tacto.git@main", + "mujoco-taxim@git+https://github.com/utn-air/mujoco-taxim.git@main", ] readme = "README.md" maintainers = [ - { name = "Tobias Jülg", email = "tobias.juelg@utn.de" }, { name = "Seongjin Bien", email = "seongjin.bien@utn.de" }, + { name = "Tobias Jülg", email = "tobias.juelg@utn.de" }, ] authors = [{ name = "Seongjin Bien", email = "seongjin.bien@utn.de" }] requires-python = ">=3.10" diff --git a/extensions/rcs_tacto/src/rcs_tacto/__init__.py b/extensions/rcs_taxim/src/rcs_taxim/__init__.py similarity index 100% rename from extensions/rcs_tacto/src/rcs_tacto/__init__.py rename to extensions/rcs_taxim/src/rcs_taxim/__init__.py diff --git a/extensions/rcs_tacto/src/rcs_tacto/creators.py b/extensions/rcs_taxim/src/rcs_taxim/creators.py similarity index 71% rename from extensions/rcs_tacto/src/rcs_tacto/creators.py rename to extensions/rcs_taxim/src/rcs_taxim/creators.py index e78581ee..70ee3232 100644 --- a/extensions/rcs_tacto/src/rcs_tacto/creators.py +++ b/extensions/rcs_taxim/src/rcs_taxim/creators.py @@ -11,7 +11,7 @@ from rcs.envs.creators import SimTaskEnvCreator from rcs.envs.utils import default_sim_robot_cfg from rcs.sim import SimGripperConfig -from rcs_tacto.tacto_wrapper import TactoSimWrapper +from rcs_taxim.taxim_wrapper import TaximSimWrapper import rcs @@ -19,7 +19,7 @@ logger.setLevel(logging.INFO) -class FR3TactoSimplePickUpSimEnvCreator(EnvCreator): +class FR3TaximSimplePickUpSimEnvCreator(EnvCreator): def __call__( # type: ignore self, render_mode: str = "human", @@ -36,7 +36,7 @@ def __call__( # type: ignore "left_side", "side_view", ), - tacto_kwargs: dict[str, typing.Any] | None = None, + taxim_kwargs: dict[str, typing.Any] | None = None, **kwargs, ) -> gym.Env: if resolution is None: @@ -84,7 +84,7 @@ def __call__( # type: ignore # Append the id to keep it consistent with the model gripper_cfg.add_id("0") - random_pos_args = {"joint_name": "yellow-box-joint"} + random_pos_args = {"joint_name": "box_joint"} env = SimTaskEnvCreator()( robot_cfg, @@ -97,14 +97,28 @@ def __call__( # type: ignore **kwargs, ) - # Here, we feed some default values for the tacto wrapper + ''' + env: gym.Env, + taxim_sites: list[str], + taxim_pad_geoms: list[str], + target_geom_mesh_dict: dict[str, str], + taxim_sensor_type: str = "digit", + taxim_bg_idx: int = 0, + taxim_bg_randomize: bool = False, + enable_depth: bool = False, + taxim_fps: int = 60, + visualize: bool = False, + ''' + # Here, we feed some default values for the taxim wrapper # that aligns with what we have in the fr3_digit_simple_pick_up - if tacto_kwargs is None: - tacto_kwargs = {} - tacto_kwargs["tacto_sites"] = ["left_tacto_pad_0", "right_tacto_pad_0"] - tacto_kwargs["tacto_geoms"] = ["yellow_box_geom"] - tacto_kwargs["tacto_fps"] = 60 - tacto_kwargs["enable_depth"] = True - tacto_kwargs["visualize"] = True + if taxim_kwargs is None: + taxim_kwargs = {} + taxim_kwargs["taxim_sites"] = ["left_taxim_pad_0", "right_taxim_pad_0"] + taxim_kwargs["taxim_pad_geoms"] = ["finger_1_left_0", "finger_1_right_0"] + taxim_kwargs["target_geom_mesh_dict"] = {"box_geom": "box_geom"} + taxim_kwargs["taxim_sensor_type"] = "digit" + taxim_kwargs["taxim_fps"] = 60 + taxim_kwargs["enable_depth"] = True + taxim_kwargs["visualize"] = True - return TactoSimWrapper(env, **tacto_kwargs) + return TaximSimWrapper(env, **taxim_kwargs) diff --git a/extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py b/extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py new file mode 100644 index 00000000..ea6a0657 --- /dev/null +++ b/extensions/rcs_taxim/src/rcs_taxim/taxim_wrapper.py @@ -0,0 +1,110 @@ +import logging +import os +from importlib.resources import files +from typing import Any + +import cv2 +import gymnasium as gym +from TaximSensor import TaximSensor +from omegaconf import OmegaConf + +logger = logging.getLogger(__name__) + + +class TaximSimWrapper(gym.Wrapper): + """Wrapper to use Taxim with RCS Sim.""" + + def __init__( + self, + env: gym.Env, + taxim_sites: list[str], + taxim_pad_geoms: list[str], + target_geom_mesh_dict: dict[str, str], + taxim_sensor_type: str = "digit", + taxim_bg_idx: int = 0, + taxim_bg_randomize: bool = False, + enable_depth: bool = False, + taxim_fps: int = 60, + visualize: bool = False, + ): + """ + Initialize Taxim sensor with the given configuration. + Args: + env (gym.Env): The environment to wrap. + simulation (sim.Sim): The simulation instance. + taxim_sites (list[str]): List of sites to mount Taxim cameras. + taxim_pad_geoms (list[str]): List of tactile sensor pad geoms which should act as the contact surfaces. + target_geom_mesh_dict (dict[str, str]): Dictionary mapping mjGeom names to mjMesh names. + taxim_sensor_type (str)="digit": The type of Taxim sensor to use. either 'digit' or 'gelsight_r1.5'. + taxim_bg_idx (int)=0: The index of the background image to use. + taxim_bg_randomize (bool)=False: Whether to randomize the background image for every contact. + enable_depth (bool)=False: Whether to enable depth rendering. + taxim_fps (int)=60: Frames per second for Taxim rendering. + visualize (bool)=False: Whether to visualize Taxim rendering in a separate window. + """ + super().__init__(env) + self.env = env + self.taxim_sensors = [] + + self.model = self.env.get_wrapper_attr("sim").model + self.data = self.env.get_wrapper_attr("sim").data + + self.taxim_sites = taxim_sites + self.taxim_pad_geoms = taxim_pad_geoms + self.target_geom_mesh_dict = target_geom_mesh_dict + self.taxim_sensor_type = taxim_sensor_type + self.taxim_bg_idx = taxim_bg_idx + self.taxim_bg_randomize = taxim_bg_randomize + + self.colors = [] + self.depths = [] + + self.taxim_fps = taxim_fps + self.taxim_last_render = -1 + self.enable_depth = enable_depth + + self.initialized = False + self.visualize = visualize + + def reset( + self, seed: int | None = None, options: dict[str, Any] | None = None + ) -> tuple[dict[str, Any], dict[str, Any]]: + obs, info = super().reset(seed=seed, options=options) + if not self.initialized: + # Create taxim sensors for each specified site + print(self.taxim_sites) + for i, site in enumerate(self.taxim_sites): + sensor = TaximSensor(resize=(240,320), sensor_type=self.taxim_sensor_type, preprocess_bg=False) + sensor.add_camera_mujoco(site, self.model, self.data) + sensor.change_bg(self.taxim_bg_idx) + # Add the target geoms to the sensor + for geom, mesh in self.target_geom_mesh_dict.items(): + sensor.add_geom_mujoco(geom, self.model, self.data, mesh) + + sensor.set_sensor_pad_geom(self.taxim_pad_geoms[i]) + self.taxim_sensors.append(sensor) + + self.initialized = True + + self.taxim_last_render = -1 # Reset last render time + + for i, sensor in enumerate(self.taxim_sensors): + rgb, depth, _ = sensor.render_taxim(self.model, self.data, visualize=False) + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("rgb", {})["data"] = rgb + if self.enable_depth: + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("depth", {})["data"] = depth + return obs, info + + def step(self, action: dict[str, Any]): + obs, reward, done, truncated, info = super().step(action) + if self.taxim_last_render + (1 / self.taxim_fps) > self.data.time: + return obs, reward, done, truncated, info + + for i, sensor in enumerate(self.taxim_sensors): + rgb, depth, _ = sensor.render_taxim(self.model, self.data, visualize=self.visualize) + self.taxim_last_render = self.data.time + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("rgb", {})["data"] = rgb + if self.enable_depth: + obs.setdefault("frames", {}).setdefault(f"tactile_{self.taxim_sites[i]}", {}).setdefault("depth", {})["data"] = depth + + return obs, reward, done, truncated, info diff --git a/pyproject.toml b/pyproject.toml index f9f56376..6cdcb7f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "simplejpeg", "mujoco==3.2.6", "pin==3.7.0", + "tqdm", ] readme = "README.md" maintainers = [{ name = "Tobias Jülg", email = "tobias.juelg@utn.de" }] @@ -71,6 +72,7 @@ dev = [ build_deps = [ "mujoco==3.2.6", "pin==3.7.0", + "scikit-build-core>=0.3.3", ] [tool.cibuildwheel] @@ -211,9 +213,9 @@ version_files = [ "extensions/rcs_ur5e/src/rcs_ur5e/__init__.py:__version__", "extensions/rcs_ur5e/pyproject.toml:\"rcs>=(.*)\"", - "extensions/rcs_tacto/pyproject.toml:version", - "extensions/rcs_tacto/src/rcs_tacto/__init__.py:__version__", - "extensions/rcs_tacto/pyproject.toml:\"rcs>=(.*)\"", + "extensions/rcs_taxim/pyproject.toml:version", + "extensions/rcs_taxim/src/rcs_taxim/__init__.py:__version__", + "extensions/rcs_taxim/pyproject.toml:\"rcs>=(.*)\"", "extensions/rcs_robotiq2f85/pyproject.toml:version", "extensions/rcs_robotiq2f85/src/rcs_robotiq2f85/__init__.py:__version__", diff --git a/python/rcs/__init__.py b/python/rcs/__init__.py index 8c4efd3a..d47a0076 100644 --- a/python/rcs/__init__.py +++ b/python/rcs/__init__.py @@ -102,7 +102,6 @@ def get_scene_urdf(scene_name: str) -> str | None: entry_point=FR3LabDigitGripperPickUpSimEnvCreator(), ) -# Genius TODO: Add the tacto version of the SimEnvCreator # TODO: gym.make("rcs/FR3SimEnv-v0") results in a pickling error: # TypeError: cannot pickle 'rcs._core.sim.SimRobotConfig' object # cf. https://pybind11.readthedocs.io/en/stable/advanced/classes.html#deepcopy-support diff --git a/python/rcs/envs/sim.py b/python/rcs/envs/sim.py index 589b49fd..1a1cf60f 100644 --- a/python/rcs/envs/sim.py +++ b/python/rcs/envs/sim.py @@ -410,12 +410,12 @@ class PickCubeSuccessWrapper(gym.Wrapper): - whether the arm is standing still once the task is solved. """ - def __init__(self, env, cube_joint_name="box_joint"): + def __init__(self, env, cube_joint_name="box_joint", cube_geom_name="box_geom"): super().__init__(env) self.unwrapped: RobotEnv assert isinstance(self.unwrapped.robot, sim.SimRobot), "Robot must be a sim.SimRobot instance." self.sim = env.get_wrapper_attr("sim") - self.cube_geom_name = "box_geom" + self.cube_geom_name = cube_geom_name self.home_pose = self.unwrapped.robot.get_cartesian_position() self._gripper_closing = 0 self._gripper = self.get_wrapper_attr("_gripper")