upload code

.gitignore

@@ -0,0 +1,14 @@
+deps/*
+!deps/.gitkeep
+outputs/*
+!outputs/.gitkeep
+src_shot/build/*
+**/__pycache__
+license/*
+!license/.gitkeep
+*.tar
+*.so
+.TimeRecord
+imgui.ini
+temp_data/*
+!temp_data/.gitkeep

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 RPMArt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,156 @@
+# RPMArt
+
+<a href='https://r-pmart.github.io/'>
+  <img src='https://img.shields.io/badge/Homepage-Website-green?style=flat&logo=homepage&logoColor=green' alt='Website'>
+</a>
+<a href='https://arxiv.org/abs/2403.16023'>
+  <img src='https://img.shields.io/badge/Arxiv-Paper-orange?style=flat&logo=arxiv&logoColor=orange' alt='Paper'>
+</a>
+<a href='https://iros2024-abudhabi.org'>
+  <img src='https://img.shields.io/badge/IROS-2024-blue?style=flat&logo=ieee&logoColor=blue' alt='IROS'>
+</a>
+<br/>
+
+<img src="assets/teaser.png" alt="teaser" width="500" />
+
+Official implementation for the paper [RPMArt: Towards Robust Perception and Manipulation for Articulated Objects](https://arxiv.org/abs/2403.16023), accepted by [IROS 2024](https://iros2024-abudhabi.org).
+
+For more information, please visit our [project website](https://r-pmart.github.io/).
+
+---
+
+## 🛠 Installation
+### 💻 Server-side
+1. Clone this repo.
+    ```bash
+    git clone [email protected]:R-PMArt/rpmart.git
+    cd rpmart
+    ```
+
+2. Create a [Conda](https://conda.org/) environment.
+    ```bash
+    conda create -n rpmart python=3.8
+    conda activate rpmart
+    ```
+
+3. Install [PyTorch](https://pytorch.org/).
+    ```bash
+    pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
+    ```
+
+4. Install [pytorch-gradual-warmup-lr](https://github.com/ildoonet/pytorch-gradual-warmup-lr).
+    ```bash
+    cd deps
+    git clone [email protected]:ildoonet/pytorch-gradual-warmup-lr.git
+    cd pytorch-gradual-warmup-lr
+    pip install .
+    cd ../..
+    ```
+
+5. Install [MinkowskiEngine](https://github.com/NVIDIA/MinkowskiEngine).
+    ```bash
+    conda install openblas-devel -c anaconda
+    export CUDA_HOME=/usr/local/cuda
+    pip install ninja
+    cd deps
+    git clone [email protected]:NVIDIA/MinkowskiEngine.git
+    cd MinkowskiEngine
+    pip install -U . --no-deps --install-option="--blas_include_dirs=${CONDA_PREFIX}/include" --install-option="--blas=openblas"
+    cd ../..
+    ```
+
+6. Install [CuPy](https://cupy.dev/).
+   ```bash
+   pip install cupy-cuda11x
+   ```
+
+7. Install special [SAPIEN](https://sapien.ucsd.edu/).
+    ```bash
+    pip install http://download.cs.stanford.edu/orion/where2act/where2act_sapien_wheels/sapien-0.8.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+    ```
+
+8. Install [AnyGrasp](https://github.com/graspnet/anygrasp_sdk).
+    ```bash
+    pip install cvxopt munch graspnetAPI
+    # follow AnyGrasp to use the licenses and weights and binary codes
+    ```
+
+9.  Install other dependencies.
+    ```bash
+    pip install -r requirements.txt
+    ```
+
+10. Build `shot`.
+    ```bash
+    # you may need first install pybind11 and pcl
+    cd src_shot
+    mkdir build
+    cd build
+    cmake ..
+    make
+    cd ../..
+    ```
+
+### 🦾 Robot-side
+1. Make sure `rt-linux` is enabled for Franka Emika Panda.
+    ```bash
+    uname -a
+    ```
+
+2. Install `frankx` for robot and `pyrealsense2` for camera.
+    ```bash
+    pip install frankx pyrealsense2
+    ```
+
+3. Install `paramiko` for connecting with server.
+    ```bash
+    pip install paramiko
+    ```
+
+## 🏃‍♂️ Run
+1. Train or download [RoArtNet](https://huggingface.co/dadadadawjb/RoArtNet).
+    ```bash
+    bash scripts/train.sh
+    ```
+
+2. Test RoArtNet.
+    ```bash
+    bash scripts/test.sh
+    ```
+
+3. Evaluate RPMArt.
+    ```bash
+    bash scripts/eval_roartnet.sh
+    ```
+
+4. Test RoArtNet on [RealArt-6](https://huggingface.co/datasets/dadadadawjb/RealArt-6).
+    ```bash
+    bash scripts/test_real.sh
+    ```
+
+5. Evaluate RPMArt in the real world.
+    ```bash
+    # server side
+    bash scripts/real_service.sh
+
+    # robot side
+    python real_eval.py
+    ```
+
+## 🙏 Acknowledgement
+* Our simulation environment is adapted from [VAT-Mart](https://github.com/warshallrho/VAT-Mart).
+* Our voting module is adapted from [CPPF](https://github.com/qq456cvb/CPPF) and [CPPF++](https://github.com/qq456cvb/CPPF2).
+
+## ✍ Citation
+If you find our work useful, please consider citing: 
+```bibtex
+@article{wang2024rpmart,
+  title={RPMArt: Towards Robust Perception and Manipulation for Articulated Objects},
+  author={Wang, Junbo and Liu, Wenhai and Yu, Qiaojun and You, Yang and Liu, Liu and Wang, Weiming and Lu, Cewu},
+  journal={arXiv preprint arXiv:2403.16023},
+  year={2024}
+}
+```
+
+## 📃 License
+This repository is released under the [MIT](https://mit-license.org/) license.

configs/algorithm/formal.yaml

@@ -0,0 +1,29 @@
+sampling:
+  sample_tuples_num: 100000
+  tuple_more_num: 3
+
+shot_encoder:
+  hidden_dims: [128, 128, 128, 128, 128]
+  feature_dim: 64
+  bn: False
+  ln: False   # bn and ln can only be set one
+  dropout: 0
+
+encoder:
+  hidden_dims: [128, 128, 128, 128, 128]
+  bn: False
+  ln: False   # bn and ln can only be set one
+  dropout: 0
+
+voting:
+  rot_bin_num: 36       # 5 degree
+  voting_num: 120       # 3 degree
+  angle_tol: 1.5        # 10 degree
+  # angle_tol: 0.35       # 5 degree
+  translation2pc: False
+  rotation_cluster: False
+  multi_candidate: False
+  candidate_threshold: 0.5
+  rotation_multi_neighbor: False
+  neighbor_threshold: 10
+  bmm_size: 100000

configs/data/camera_config.json

@@ -0,0 +1,15 @@
+{
+    "intrinsics": {
+        "fovx": [70, 70], 
+        "fovy": [40, 60], 
+        "height": 480, 
+        "width": 640, 
+        "near": 0.1, 
+        "far": 100.0
+    }, 
+    "extrinsics": {
+        "dist": [0.6, 1.2], 
+        "phi": [0, 60], 
+        "theta": [120, 240]
+    }
+}

configs/data/object_config.json

@@ -0,0 +1,30 @@
+{
+    "scale_min": 0.8, 
+    "scale_max": 1.1, 
+    "Microwave": {
+        "size": 0.4, 
+        "joint_num": 1
+    }, 
+    "StorageFurniture": {
+        "size": 0.375, 
+        "joint_num": 2, 
+        "joint_types": ["prismatic", "revolute"]
+    }, 
+    "Refrigerator": {
+        "size": 0.4, 
+        "joint_num": 1
+    }, 
+    "Safe": {
+        "size": 0.4, 
+        "joint_num": 1
+    }, 
+    "WashingMachine": {
+        "size": 0.15, 
+        "joint_num": 1
+    }, 
+    "Drawer": {
+        "size": 0.4, 
+        "joint_num": 3, 
+        "joint_types": ["prismatic", "prismatic", "prismatic"]
+    }
+}

configs/dataset/formal_drawer.yaml

@@ -0,0 +1,13 @@
+train_path: '/data2/junbo/sapien4/train'
+test_path: '/data2/junbo/sapien4/test'
+train_categories: ['Drawer']
+test_categories: ['Drawer']
+joint_num: 3
+resolution: 2.5e-2
+# resolution: 1e-2
+receptive_field: 10
+normalize: 'bound'
+# normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/formal_microwave.yaml

@@ -0,0 +1,13 @@
+train_path: '/data2/junbo/sapien4/train'
+test_path: '/data2/junbo/sapien4/test'
+train_categories: ['Microwave']
+test_categories: ['Microwave']
+joint_num: 1
+resolution: 2.5e-2
+# resolution: 1e-2
+receptive_field: 10
+normalize: 'bound'
+# normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/formal_refrigerator.yaml

@@ -0,0 +1,13 @@
+train_path: '/data2/junbo/sapien4/train'
+test_path: '/data2/junbo/sapien4/test'
+train_categories: ['Refrigerator']
+test_categories: ['Refrigerator']
+joint_num: 1
+resolution: 2.5e-2
+# resolution: 1e-2
+receptive_field: 10
+normalize: 'bound'
+# normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/formal_safe.yaml

@@ -0,0 +1,13 @@
+train_path: '/data2/junbo/sapien4/train'
+test_path: '/data2/junbo/sapien4/test'
+train_categories: ['Safe']
+test_categories: ['Safe']
+joint_num: 1
+resolution: 2.5e-2
+# resolution: 1e-2
+receptive_field: 10
+normalize: 'bound'
+# normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/formal_storagefurniture.yaml

@@ -0,0 +1,13 @@
+train_path: '/data2/junbo/sapien4/train'
+test_path: '/data2/junbo/sapien4/test'
+train_categories: ['StorageFurniture']
+test_categories: ['StorageFurniture']
+joint_num: 2
+resolution: 2.5e-2
+# resolution: 1e-2
+receptive_field: 10
+normalize: 'bound'
+# normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/formal_washingmachine.yaml

@@ -0,0 +1,13 @@
+train_path: '/data2/junbo/sapien4/train'
+test_path: '/data2/junbo/sapien4/test'
+train_categories: ['WashingMachine']
+test_categories: ['WashingMachine']
+joint_num: 1
+resolution: 2.5e-2
+# resolution: 1e-2
+receptive_field: 10
+normalize: 'bound'
+# normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/real_with_with.yaml

@@ -0,0 +1,11 @@
+path: '/data2/junbo/RealArt-6/with_table/microwave'
+instances: ['0_with_chaos', '1_with_chaos', '2_with_chaos', '3_with_chaos', '4_with_chaos']
+joint_num: 1
+# resolution: 2.5e-2
+resolution: 1e-2
+receptive_field: 10
+# normalize: 'bound'
+normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/real_with_without.yaml

@@ -0,0 +1,11 @@
+path: '/data2/junbo/RealArt-6/with_table/microwave'
+instances: ['0_without_chaos', '1_without_chaos', '2_without_chaos', '3_without_chaos', '4_without_chaos']
+joint_num: 1
+# resolution: 2.5e-2
+resolution: 1e-2
+receptive_field: 10
+# normalize: 'bound'
+normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False

configs/dataset/real_without_with.yaml

@@ -0,0 +1,11 @@
+path: '/data2/junbo/RealArt-6/without_table/microwave'
+instances: ['0_with_chaos', '1_with_chaos', '2_with_chaos', '3_with_chaos', '4_with_chaos']
+joint_num: 1
+# resolution: 2.5e-2
+resolution: 1e-2
+receptive_field: 10
+# normalize: 'bound'
+normalize: 'none'
+sample_points_num: 1024
+rgb: False
+denoise: False