Created example with DOPE and DIff-DOPE integration through ROS actinolib.

.gitignore

@@ -6,3 +6,4 @@ docs/_buid/
 *.png
 *.jpg
 *.jpeg
+*.mp4

diffdope/diffdope.py

@@ -6,10 +6,10 @@
 
 import collections
 import io
-import sys
 import math
 import pathlib
 import random
+import sys
 import warnings
 from dataclasses import dataclass
 from itertools import repeat
@@ -36,7 +36,7 @@
 
 # for better print debug
 print()
-if not hasattr(sys, 'ps1'):
+if not hasattr(sys, "ps1"):
     print = ic
 
 # A logger for this file
@@ -209,9 +209,14 @@ def render_texture_batch(
     depth = depth.reshape(shape_keep)[..., 2] * -1
 
     # mask   , _ = dr.interpolate(torch.ones(pos_idx.shape).cuda(), rast_out, pos_idx)
-    mask, _ = dr.interpolate(torch.ones(pos_idx.shape).cuda(), 
-        rast_out, pos_idx[0],rast_db=rast_out_db,diff_attrs="all")
-    mask       = dr.antialias(mask, rast_out, pos_clip_ja, pos_idx[0])
+    mask, _ = dr.interpolate(
+        torch.ones(pos_idx.shape).cuda(),
+        rast_out,
+        pos_idx[0],
+        rast_db=rast_out_db,
+        diff_attrs="all",
+    )
+    mask = dr.antialias(mask, rast_out, pos_clip_ja, pos_idx[0])
 
     # compute vertex color interpolation
     if vtx_color is None:
@@ -237,7 +242,7 @@ def render_texture_batch(
         color = color * torch.clamp(rast_out[..., -1:], 0, 1)  # Mask out background.
     if not return_rast_out:
         rast_out = None
-    return {"rgb": color, "depth": depth, "rast_out": rast_out, 'mask':mask}
+    return {"rgb": color, "depth": depth, "rast_out": rast_out, "mask": mask}
 
 
 ##############################################################################
@@ -973,7 +978,7 @@ def __init__(
         self.qx = None  # to load on cpu and not gpu
 
         if model_path is None:
-            self.mesh = None    
+            self.mesh = None
         else:
             self.mesh = Mesh(path_model=model_path, scale=scale)
 
@@ -1227,11 +1232,11 @@ def set_batchsize(self, batchsize):
         Args:
             batchsize (int): batchsize for the tensors
         """
-        if not self.path_img is None:
+        if not self.tensor_rgb is None:
             self.tensor_rgb.set_batchsize(batchsize)
-        if not self.path_depth is None:
+        if not self.tensor_depth is None:
             self.tensor_depth.set_batchsize(batchsize)
-        if not self.path_segmentation is None:
+        if not self.tensor_segmentation is None:
             self.tensor_segmentation.set_batchsize(batchsize)
 
     def get_resolution(self):
@@ -1241,17 +1246,17 @@ def get_resolution(self):
         Return
             (list): w,h of the image for optimization
         """
-        if not self.path_img is None:
+        if not self.tensor_rgb is None:
             return [
                 self.tensor_rgb.img_tensor.shape[-3],
                 self.tensor_rgb.img_tensor.shape[-2],
             ]
-        if not self.path_depth is None:
+        if not self.tensor_depth is None:
             return [
                 self.tensor_depth.img_tensor.shape[-2],
                 self.tensor_depth.img_tensor.shape[-1],
             ]
-        if not self.path_segmentation is None:
+        if not self.tensor_segmentation is None:
             return [
                 self.tensor_segmentation.img_tensor.shape[-3],
                 self.tensor_segmentation.img_tensor.shape[-2],
@@ -1262,11 +1267,11 @@ def cuda(self):
         Put on cuda the image tensors
         """
 
-        if not self.path_img is None:
+        if not self.tensor_rgb is None:
             self.tensor_rgb.cuda()
-        if not self.path_depth is None:
+        if not self.tensor_depth is None:
             self.tensor_depth.cuda()
-        if not self.path_segmentation is None:
+        if not self.tensor_segmentation is None:
             self.tensor_segmentation.cuda()
 
 
@@ -1424,7 +1429,7 @@ def render_img(
 
             else:
                 crop = find_crop(self.optimization_results[index][render_selection][0])
-        
+
         if batch_index is None:
             # make a grid
             if self.cfg.render_images.crop_around_mask:
@@ -1656,7 +1661,6 @@ def run_optimization(self):
         if self.scene.tensor_segmentation is not None:
             self.gt_tensors["segmentation"] = self.scene.tensor_segmentation.img_tensor
 
-
         pbar = tqdm(range(self.cfg.hyperparameters.nb_iterations + 1))
 
         for iteration_now in pbar:
@@ -1728,4 +1732,3 @@ def cuda(self):
         self.object3d.cuda()
         self.scene.cuda()
         self.camera.cuda()
-        pass

docs/.gitignore

@@ -0,0 +1 @@
+_build/

docs/conf.py

@@ -6,8 +6,6 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-import os
-import sys
 import os
 
 # Ensure Sphinx uses the correct Python interpreter
@@ -43,20 +41,26 @@
 html_theme = "sphinx_rtd_theme"
 # html_static_path = ["_static"]
 
+
 # In your conf.py file
 def autodoc_skip_member(app, what, name, obj, skip, options):
     # Ref: https://stackoverflow.com/a/21449475/
-    exclusions = ('make_grid',  # special-members
-                  '__doc__', '__module__', '__dict__',  # undoc-members
-                  )
+    exclusions = (
+        "make_grid",  # special-members
+        "__doc__",
+        "__module__",
+        "__dict__",  # undoc-members
+    )
     exclude = name in exclusions
     # return True if (skip or exclude) else None  # Can interfere with subsequent skip functions.
     return True if exclude else None
 
+
 def setup(app):
     app.connect("autodoc-skip-member", autodoc_skip_member)
 
+
 import sys
 
 print("Python Version:")
-print(sys.version)
+print(sys.version)

docs/demos.rst

@@ -0,0 +1,99 @@
+Demos
+================
+
+Multi-object using ROS and DOPE with RGB and Depth
+--------------------------------------------------
+
+The demo uses RGB and Depth from a RealSense camera, poses published by DOPE,
+and segments the image using
+`segment-anything <https://github.com/facebookresearch/segment-anything>`_.
+The demo comes as a catkin package under ``examples/``.
+
+Say you cloned the diff-dope repo under your home directory (i.e., ``~/diff-dope``),
+and you have a catkin workspace under your home directory too (i.e., ``~/catkin_ws``),
+you can create a symlink of the package there:
+
+.. code::
+
+    cd ~/catkin_ws/src
+    ln -s ~/diff-dope/examples/diffdope_ros .
+
+You can ``catkin_make`` under ``~/catkin_ws`` now to build the package.
+You can, of course, move the package there instead of creating a symlink.
+
+The demo uses a configuration under
+``diffdope_ros/config/multiobject_with_dope.yaml`` which uses DOPE for initial
+pose estimation, and dictates topics for RGB and Depth (assuming a RealSense
+sensor). It also uses a camera info topic to retrieve camera intrinsic
+parameters as well as the image dimensions.
+
+You can also download a ROS bag from `here <https://leeds365-my.sharepoint.com/:u:/g/personal/scsrp_leeds_ac_uk/Ec-TbyOr1QVIt6NQQP7E4pABkEUmaEGByVjLHugY7Als_A?e=JES96n>`_
+to play with this demo, without the need of a real sensor.
+
+If you wish to use the ROS bag, you can play it back in a loop like so:
+
+.. code::
+
+    rosbag play -l ~/path/to/simple.bag
+
+
+You need to install
+`segment-anything <https://github.com/facebookresearch/segment-anything>`_
+and download the weights of the model.
+
+.. code::
+
+    pip install git+https://github.com/facebookresearch/segment-anything.git
+
+Then head to the
+`model checkpoints section <https://github.com/facebookresearch/segment-anything?tab=readme-ov-file#model-checkpoints>`_,
+and download any of the model checkpoints. Place it somewhere on your
+computer and update the ``segment_anything_checkpoint_path`` of the config file
+to let it know where to find it. If you want to test the segmentation
+functionality independently, you can run the following Python script:
+``diffdope_ros/scripts/segmentator.py``.
+
+From there you can run the demo like so:
+
+.. code::
+
+    roslaunch diffdope_ros server.py # To start the actionlib server
+
+    # Refine pose for individual object
+    roslaunch diffdope_ros refine.launch object_name:=bbq_sauce
+    roslaunch diffdope_ros refine.launch object_name:=alphabet_soup
+    roslaunch diffdope_ros refine.launch object_name:=mustard
+
+    # or don't pass object_name to refine the pose of all the objects in the config
+    roslaunch diffdope_ros refine.launch
+
+The above names are derived from the config file, ``config/multiobject_with_dope.yaml``.
+The launch files pass this file to the server and refine scripts.
+
+Parameters
+************************
+
+Please inspect the ``config/multiobject_with_dope.yaml`` file to see certain 
+parameters. For example, by default this demo will produce videos of the 
+optimisation, however you can turn this off through the config file
+to speed things up. You can also adjust certain optimisation parameters from
+the config file.
+
+Dealing with DOPE and model coordinate frames
+************************
+
+Please note the following important details when you try to use a new object
+and pose from DOPE:
+
+* DOPE pose output may not match the coordinate frame used in the 3D model of
+  the object you wish to use. In this case, you need to apply a static
+  transformation to bring the DOPE pose output to match the one used in your 3D
+  model. DOPE provides a way in the config file (``model_transforms``) to define such transformation
+  per object. For more details on this subject, please read `this <https://github.com/NVlabs/Deep_Object_Pose/issues/346>`_.
+* The scaling of the object is important. We suggest that you scale your 3D object
+  in Blender to bring it closer to the scale of the examples. For example,
+  the HOPE objects as downloaded from the official repository, we had to scale them
+  by a factor of 10. Although a parameter to scale the 3D object in the config
+  is available, we had difficulties to get it to work properly and found better
+  luck by manually scaling the 3D object in Blender. You can import a reference
+  object (like the BBQ Sauce model we provide) in Blender to see the scale.

docs/index.rst

@@ -11,6 +11,7 @@ Welcome to Diff-DOPE's documentation!
    :caption: Contents:
 
    modules
+   demos
 
 Indices and tables
 ==================