From f65db8a78550860191b9fde1df98986be2d9d48a Mon Sep 17 00:00:00 2001 From: Jeff Smith Date: Thu, 7 Mar 2024 10:34:04 -0800 Subject: [PATCH] Begin reorganization of code. --- .../train2/inference => common}/cuboid.py | 0 .../inference => common}/cuboid_pnp_solver.py | 0 {train => common}/debug.py | 0 {train/inference => common}/detector.py | 0 {train => common}/models.py | 0 {train => common}/utils.py | 0 .../blenderproc_data_gen/README.md | 0 .../generate_training_data.py | 0 .../run_blenderproc_datagen.py | 0 .../nvisii_data_gen/.gitignore | 0 .../nvisii_data_gen/debug_json_ros_node.py | 0 .../doc/videos/cylinder_nosym.mp4 | Bin .../doc/videos/cylinder_sym.mp4 | Bin .../nvisii_data_gen/doc/videos/hex_screw.mp4 | Bin .../dome_hdri_haven/download.md | 0 .../download_google_scanned_objects.py | 0 .../nvisii_data_gen/generate_dataset.py | 0 .../models/Ketchup/google_16k/texture_map.png | Bin .../Ketchup/google_16k/texture_map_flat.png | Bin .../models/Ketchup/google_16k/textured.mtl | 0 .../models/Ketchup/google_16k/textured.obj | 0 .../Ketchup/google_16k/textured.obj.bin | Bin .../Ketchup/google_16k/textured.obj.json | 0 .../Ketchup/google_16k/textured_simple.mtl | 0 .../Ketchup/google_16k/textured_simple.obj | 0 .../cylinder/google_16k/model_info.json | 0 .../cylinder/google_16k/texture_map_flat.png | Bin .../cylinder/google_16k/textured.mtl | 0 .../cylinder/google_16k/textured.obj | 0 .../hex_screw/google_16k/model_info.json | 0 .../hex_screw/google_16k/texture_map_flat.png | Bin .../hex_screw/google_16k/textured.mtl | 0 .../hex_screw/google_16k/textured.obj | 0 .../output/output_example/00000.depth.exr | Bin .../output/output_example/00000.json | 0 .../output/output_example/00000.png | Bin .../output/output_example/00000.seg.exr | Bin .../nvisii_data_gen/readme.md | 0 .../nvisii_data_gen/requirements.txt | 0 .../nvisii_data_gen/single_video_pybullet.py | 0 .../nvisii_data_gen/utils.py | 0 {scripts/metrics => evaluate}/.gitignore | 0 {scripts/metrics => evaluate}/add_compute.py | 0 .../metrics => evaluate}/download_content.sh | 0 {scripts/metrics => evaluate}/kpd_compute.py | 0 {scripts/metrics => evaluate}/make_graphs.py | 0 {scripts/metrics => evaluate}/overlay.png | Bin {scripts/metrics => evaluate}/readme.md | 0 {scripts/metrics => evaluate}/render_json.py | 0 .../metrics => evaluate}/results/output.png | Bin {train/inference => inference}/README.md | 4 +- {train/inference => inference}/inference.py | 8 +- requirements.txt | 80 +- CMakeLists.txt => ros1/CMakeLists.txt | 0 {docker => ros1/docker}/Dockerfile.noetic | 0 {docker => ros1/docker}/init_workspace.sh | 0 {docker => ros1/docker}/readme.md | 0 {docker => ros1/docker}/run_dope_docker.sh | 0 {launch => ros1/launch}/camera.launch | 0 {launch => ros1/launch}/dope.launch | 0 {nodes => ros1/nodes}/camera | 0 {nodes => ros1/nodes}/dope | 0 package.xml => ros1/package.xml | 0 setup.py => ros1/setup.py | 0 scripts/train2/inference/cuboid_pnp_solver.py | 143 ---- scripts/train2/inference/detector.py | 766 ------------------ train/inference/config/camera_info.yaml | 16 - .../inference/config/camera_info_960x540.yaml | 20 - train/inference/config/config_pose.yaml | 48 -- train/inference/cuboid.py | 126 --- train/requirements.txt | 74 -- .../config_inference/camera_info.yaml | 0 .../config_inference/config_pose.yaml | 0 {scripts/train2 => train2}/inference.py | 0 {scripts/train2 => train2}/models.py | 0 {scripts/train2 => train2}/readme.md | 0 {scripts/train2 => train2}/requirements.txt | 0 {scripts/train2 => train2}/train.py | 0 {scripts/train2 => train2}/utils_dope.py | 0 79 files changed, 76 insertions(+), 1209 deletions(-) rename {scripts/train2/inference => common}/cuboid.py (100%) rename {train/inference => common}/cuboid_pnp_solver.py (100%) rename {train => common}/debug.py (100%) rename {train/inference => common}/detector.py (100%) rename {train => common}/models.py (100%) rename {train => common}/utils.py (100%) rename {scripts => data_generation}/blenderproc_data_gen/README.md (100%) rename {scripts => data_generation}/blenderproc_data_gen/generate_training_data.py (100%) rename {scripts => data_generation}/blenderproc_data_gen/run_blenderproc_datagen.py (100%) rename {scripts => data_generation}/nvisii_data_gen/.gitignore (100%) rename {scripts => data_generation}/nvisii_data_gen/debug_json_ros_node.py (100%) rename {scripts => data_generation}/nvisii_data_gen/doc/videos/cylinder_nosym.mp4 (100%) rename {scripts => data_generation}/nvisii_data_gen/doc/videos/cylinder_sym.mp4 (100%) rename {scripts => data_generation}/nvisii_data_gen/doc/videos/hex_screw.mp4 (100%) rename {scripts => data_generation}/nvisii_data_gen/dome_hdri_haven/download.md (100%) rename {scripts => data_generation}/nvisii_data_gen/download_google_scanned_objects.py (100%) rename {scripts => data_generation}/nvisii_data_gen/generate_dataset.py (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/texture_map.png (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/texture_map_flat.png (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/textured.mtl (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/textured.obj (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.bin (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.json (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.mtl (100%) rename {scripts => data_generation}/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.obj (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/model_info.json (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/texture_map_flat.png (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.mtl (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.obj (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/model_info.json (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/texture_map_flat.png (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.mtl (100%) rename {scripts => data_generation}/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.obj (100%) rename {scripts => data_generation}/nvisii_data_gen/output/output_example/00000.depth.exr (100%) rename {scripts => data_generation}/nvisii_data_gen/output/output_example/00000.json (100%) rename {scripts => data_generation}/nvisii_data_gen/output/output_example/00000.png (100%) rename {scripts => data_generation}/nvisii_data_gen/output/output_example/00000.seg.exr (100%) rename {scripts => data_generation}/nvisii_data_gen/readme.md (100%) rename {scripts => data_generation}/nvisii_data_gen/requirements.txt (100%) rename {scripts => data_generation}/nvisii_data_gen/single_video_pybullet.py (100%) rename {scripts => data_generation}/nvisii_data_gen/utils.py (100%) rename {scripts/metrics => evaluate}/.gitignore (100%) rename {scripts/metrics => evaluate}/add_compute.py (100%) rename {scripts/metrics => evaluate}/download_content.sh (100%) rename {scripts/metrics => evaluate}/kpd_compute.py (100%) rename {scripts/metrics => evaluate}/make_graphs.py (100%) rename {scripts/metrics => evaluate}/overlay.png (100%) rename {scripts/metrics => evaluate}/readme.md (100%) rename {scripts/metrics => evaluate}/render_json.py (100%) rename {scripts/metrics => evaluate}/results/output.png (100%) rename {train/inference => inference}/README.md (91%) rename {train/inference => inference}/inference.py (98%) mode change 100644 => 100755 requirements.txt rename CMakeLists.txt => ros1/CMakeLists.txt (100%) rename {docker => ros1/docker}/Dockerfile.noetic (100%) rename {docker => ros1/docker}/init_workspace.sh (100%) rename {docker => ros1/docker}/readme.md (100%) rename {docker => ros1/docker}/run_dope_docker.sh (100%) rename {launch => ros1/launch}/camera.launch (100%) rename {launch => ros1/launch}/dope.launch (100%) rename {nodes => ros1/nodes}/camera (100%) rename {nodes => ros1/nodes}/dope (100%) rename package.xml => ros1/package.xml (100%) rename setup.py => ros1/setup.py (100%) delete mode 100755 scripts/train2/inference/cuboid_pnp_solver.py delete mode 100755 scripts/train2/inference/detector.py delete mode 100644 train/inference/config/camera_info.yaml delete mode 100644 train/inference/config/camera_info_960x540.yaml delete mode 100644 train/inference/config/config_pose.yaml delete mode 100755 train/inference/cuboid.py delete mode 100755 train/requirements.txt rename {scripts/train2 => train2}/config_inference/camera_info.yaml (100%) rename {scripts/train2 => train2}/config_inference/config_pose.yaml (100%) rename {scripts/train2 => train2}/inference.py (100%) rename {scripts/train2 => train2}/models.py (100%) rename {scripts/train2 => train2}/readme.md (100%) rename {scripts/train2 => train2}/requirements.txt (100%) rename {scripts/train2 => train2}/train.py (100%) rename {scripts/train2 => train2}/utils_dope.py (100%) diff --git a/scripts/train2/inference/cuboid.py b/common/cuboid.py similarity index 100% rename from scripts/train2/inference/cuboid.py rename to common/cuboid.py diff --git a/train/inference/cuboid_pnp_solver.py b/common/cuboid_pnp_solver.py similarity index 100% rename from train/inference/cuboid_pnp_solver.py rename to common/cuboid_pnp_solver.py diff --git a/train/debug.py b/common/debug.py similarity index 100% rename from train/debug.py rename to common/debug.py diff --git a/train/inference/detector.py b/common/detector.py similarity index 100% rename from train/inference/detector.py rename to common/detector.py diff --git a/train/models.py b/common/models.py similarity index 100% rename from train/models.py rename to common/models.py diff --git a/train/utils.py b/common/utils.py similarity index 100% rename from train/utils.py rename to common/utils.py diff --git a/scripts/blenderproc_data_gen/README.md b/data_generation/blenderproc_data_gen/README.md similarity index 100% rename from scripts/blenderproc_data_gen/README.md rename to data_generation/blenderproc_data_gen/README.md diff --git a/scripts/blenderproc_data_gen/generate_training_data.py b/data_generation/blenderproc_data_gen/generate_training_data.py similarity index 100% rename from scripts/blenderproc_data_gen/generate_training_data.py rename to data_generation/blenderproc_data_gen/generate_training_data.py diff --git a/scripts/blenderproc_data_gen/run_blenderproc_datagen.py b/data_generation/blenderproc_data_gen/run_blenderproc_datagen.py similarity index 100% rename from scripts/blenderproc_data_gen/run_blenderproc_datagen.py rename to data_generation/blenderproc_data_gen/run_blenderproc_datagen.py diff --git a/scripts/nvisii_data_gen/.gitignore b/data_generation/nvisii_data_gen/.gitignore similarity index 100% rename from scripts/nvisii_data_gen/.gitignore rename to data_generation/nvisii_data_gen/.gitignore diff --git a/scripts/nvisii_data_gen/debug_json_ros_node.py b/data_generation/nvisii_data_gen/debug_json_ros_node.py similarity index 100% rename from scripts/nvisii_data_gen/debug_json_ros_node.py rename to data_generation/nvisii_data_gen/debug_json_ros_node.py diff --git a/scripts/nvisii_data_gen/doc/videos/cylinder_nosym.mp4 b/data_generation/nvisii_data_gen/doc/videos/cylinder_nosym.mp4 similarity index 100% rename from scripts/nvisii_data_gen/doc/videos/cylinder_nosym.mp4 rename to data_generation/nvisii_data_gen/doc/videos/cylinder_nosym.mp4 diff --git a/scripts/nvisii_data_gen/doc/videos/cylinder_sym.mp4 b/data_generation/nvisii_data_gen/doc/videos/cylinder_sym.mp4 similarity index 100% rename from scripts/nvisii_data_gen/doc/videos/cylinder_sym.mp4 rename to data_generation/nvisii_data_gen/doc/videos/cylinder_sym.mp4 diff --git a/scripts/nvisii_data_gen/doc/videos/hex_screw.mp4 b/data_generation/nvisii_data_gen/doc/videos/hex_screw.mp4 similarity index 100% rename from scripts/nvisii_data_gen/doc/videos/hex_screw.mp4 rename to data_generation/nvisii_data_gen/doc/videos/hex_screw.mp4 diff --git a/scripts/nvisii_data_gen/dome_hdri_haven/download.md b/data_generation/nvisii_data_gen/dome_hdri_haven/download.md similarity index 100% rename from scripts/nvisii_data_gen/dome_hdri_haven/download.md rename to data_generation/nvisii_data_gen/dome_hdri_haven/download.md diff --git a/scripts/nvisii_data_gen/download_google_scanned_objects.py b/data_generation/nvisii_data_gen/download_google_scanned_objects.py similarity index 100% rename from scripts/nvisii_data_gen/download_google_scanned_objects.py rename to data_generation/nvisii_data_gen/download_google_scanned_objects.py diff --git a/scripts/nvisii_data_gen/generate_dataset.py b/data_generation/nvisii_data_gen/generate_dataset.py similarity index 100% rename from scripts/nvisii_data_gen/generate_dataset.py rename to data_generation/nvisii_data_gen/generate_dataset.py diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/texture_map.png b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/texture_map.png similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/texture_map.png rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/texture_map.png diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/texture_map_flat.png b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/texture_map_flat.png similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/texture_map_flat.png rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/texture_map_flat.png diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.mtl b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.mtl similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.mtl rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.mtl diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.obj b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.obj similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.obj rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.obj diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.bin b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.bin similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.bin rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.bin diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.json b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.json similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.json rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured.obj.json diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.mtl b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.mtl similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.mtl rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.mtl diff --git a/scripts/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.obj b/data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.obj similarity index 100% rename from scripts/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.obj rename to data_generation/nvisii_data_gen/models/Ketchup/google_16k/textured_simple.obj diff --git a/scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/model_info.json b/data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/model_info.json similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/model_info.json rename to data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/model_info.json diff --git a/scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/texture_map_flat.png b/data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/texture_map_flat.png similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/texture_map_flat.png rename to data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/texture_map_flat.png diff --git a/scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.mtl b/data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.mtl similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.mtl rename to data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.mtl diff --git a/scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.obj b/data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.obj similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.obj rename to data_generation/nvisii_data_gen/models_with_symmetries/cylinder/google_16k/textured.obj diff --git a/scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/model_info.json b/data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/model_info.json similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/model_info.json rename to data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/model_info.json diff --git a/scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/texture_map_flat.png b/data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/texture_map_flat.png similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/texture_map_flat.png rename to data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/texture_map_flat.png diff --git a/scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.mtl b/data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.mtl similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.mtl rename to data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.mtl diff --git a/scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.obj b/data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.obj similarity index 100% rename from scripts/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.obj rename to data_generation/nvisii_data_gen/models_with_symmetries/hex_screw/google_16k/textured.obj diff --git a/scripts/nvisii_data_gen/output/output_example/00000.depth.exr b/data_generation/nvisii_data_gen/output/output_example/00000.depth.exr similarity index 100% rename from scripts/nvisii_data_gen/output/output_example/00000.depth.exr rename to data_generation/nvisii_data_gen/output/output_example/00000.depth.exr diff --git a/scripts/nvisii_data_gen/output/output_example/00000.json b/data_generation/nvisii_data_gen/output/output_example/00000.json similarity index 100% rename from scripts/nvisii_data_gen/output/output_example/00000.json rename to data_generation/nvisii_data_gen/output/output_example/00000.json diff --git a/scripts/nvisii_data_gen/output/output_example/00000.png b/data_generation/nvisii_data_gen/output/output_example/00000.png similarity index 100% rename from scripts/nvisii_data_gen/output/output_example/00000.png rename to data_generation/nvisii_data_gen/output/output_example/00000.png diff --git a/scripts/nvisii_data_gen/output/output_example/00000.seg.exr b/data_generation/nvisii_data_gen/output/output_example/00000.seg.exr similarity index 100% rename from scripts/nvisii_data_gen/output/output_example/00000.seg.exr rename to data_generation/nvisii_data_gen/output/output_example/00000.seg.exr diff --git a/scripts/nvisii_data_gen/readme.md b/data_generation/nvisii_data_gen/readme.md similarity index 100% rename from scripts/nvisii_data_gen/readme.md rename to data_generation/nvisii_data_gen/readme.md diff --git a/scripts/nvisii_data_gen/requirements.txt b/data_generation/nvisii_data_gen/requirements.txt similarity index 100% rename from scripts/nvisii_data_gen/requirements.txt rename to data_generation/nvisii_data_gen/requirements.txt diff --git a/scripts/nvisii_data_gen/single_video_pybullet.py b/data_generation/nvisii_data_gen/single_video_pybullet.py similarity index 100% rename from scripts/nvisii_data_gen/single_video_pybullet.py rename to data_generation/nvisii_data_gen/single_video_pybullet.py diff --git a/scripts/nvisii_data_gen/utils.py b/data_generation/nvisii_data_gen/utils.py similarity index 100% rename from scripts/nvisii_data_gen/utils.py rename to data_generation/nvisii_data_gen/utils.py diff --git a/scripts/metrics/.gitignore b/evaluate/.gitignore similarity index 100% rename from scripts/metrics/.gitignore rename to evaluate/.gitignore diff --git a/scripts/metrics/add_compute.py b/evaluate/add_compute.py similarity index 100% rename from scripts/metrics/add_compute.py rename to evaluate/add_compute.py diff --git a/scripts/metrics/download_content.sh b/evaluate/download_content.sh similarity index 100% rename from scripts/metrics/download_content.sh rename to evaluate/download_content.sh diff --git a/scripts/metrics/kpd_compute.py b/evaluate/kpd_compute.py similarity index 100% rename from scripts/metrics/kpd_compute.py rename to evaluate/kpd_compute.py diff --git a/scripts/metrics/make_graphs.py b/evaluate/make_graphs.py similarity index 100% rename from scripts/metrics/make_graphs.py rename to evaluate/make_graphs.py diff --git a/scripts/metrics/overlay.png b/evaluate/overlay.png similarity index 100% rename from scripts/metrics/overlay.png rename to evaluate/overlay.png diff --git a/scripts/metrics/readme.md b/evaluate/readme.md similarity index 100% rename from scripts/metrics/readme.md rename to evaluate/readme.md diff --git a/scripts/metrics/render_json.py b/evaluate/render_json.py similarity index 100% rename from scripts/metrics/render_json.py rename to evaluate/render_json.py diff --git a/scripts/metrics/results/output.png b/evaluate/results/output.png similarity index 100% rename from scripts/metrics/results/output.png rename to evaluate/results/output.png diff --git a/train/inference/README.md b/inference/README.md similarity index 91% rename from train/inference/README.md rename to inference/README.md index 4a60d0fa..d64e68ae 100644 --- a/train/inference/README.md +++ b/inference/README.md @@ -1,7 +1,7 @@ # Deep Object Pose Estimation (DOPE) - Inference -This repo contains a simplified version of the **Inference** script for DOPE. -The original repo for DOPE [can be found here](https://github.com/NVlabs/Deep_Object_Pose). +This directory contains a simple example of inference for DOPE. + ## Setup diff --git a/train/inference/inference.py b/inference/inference.py similarity index 98% rename from train/inference/inference.py rename to inference/inference.py index 77f6e614..75270792 100644 --- a/train/inference/inference.py +++ b/inference/inference.py @@ -1,18 +1,12 @@ #!/usr/bin/env python - -""" -This file starts a ROS node to run DOPE, -listening to an image topic and publishing poses. -""" - import cv2 import numpy as np from PIL import Image import sys -sys.path.append("inference") +sys.path.append("../common/") from cuboid import Cuboid3d from cuboid_pnp_solver import CuboidPNPSolver diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index 984c65ba..ae509f7e --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,74 @@ -pyrr==0.10.3 -torch==1.6.0 -torchvision==0.7.0 -numpy==1.17.4 -scipy==1.5.2 -opencv_python==4.4.0.44 -Pillow==8.1.1 +torch==2.1.2 +torchvision +tensorboardX +boto3 +albumentations +botocore +certifi +charset-normalizer configparser==5.0.0 +idna +imageio +jmespath +joblib +networkx +numpy +opencv-python-headless==4.6.0.66 +packaging +Pillow +protobuf==3.20.1 +pyparsing +pyrr==0.10.3 +python-dateutil +PyWavelets +PyYAML +qudida +requests +s3transfer +scikit-image +scikit-learn +scipy +simplejson +six +threadpoolctl +tifffile +typing_extensions +urllib3 + + +## If running into dependency issues, install the version-specific requirements below in a virtual environment +# albumentations==1.2.1 +# boto3==1.24.58 +# botocore==1.27.58 +# certifi==2022.6.15 +# charset-normalizer==2.1.1 +# idna==3.3 +# imageio==2.21.1 +# jmespath==1.0.1 +# joblib==1.1.0 +# networkx==2.8.6 +# numpy==1.23.2 +# opencv-python-headless==4.6.0.66 +# packaging==21.3 +# Pillow==9.2.0 +# protobuf==3.20.1 +# pyparsing==3.0.9 +# pyrr==0.10.3 +# python-dateutil==2.8.2 +# PyWavelets==1.3.0 +# PyYAML==6.0 +# qudida==0.0.4 +# requests==2.28.1 +# s3transfer==0.6.0 +# scikit-image==0.19.3 +# scikit-learn==1.1.2 +# scipy==1.9.0 +# simplejson==3.17.6 +# six==1.16.0 +# tensorboardX==2.5.1 +# threadpoolctl==3.1.0 +# tifffile==2022.8.12 +# torch==1.12.1 +# torchvision==0.13.1 +# typing_extensions==4.3.0 +# urllib3==1.26.12 diff --git a/CMakeLists.txt b/ros1/CMakeLists.txt similarity index 100% rename from CMakeLists.txt rename to ros1/CMakeLists.txt diff --git a/docker/Dockerfile.noetic b/ros1/docker/Dockerfile.noetic similarity index 100% rename from docker/Dockerfile.noetic rename to ros1/docker/Dockerfile.noetic diff --git a/docker/init_workspace.sh b/ros1/docker/init_workspace.sh similarity index 100% rename from docker/init_workspace.sh rename to ros1/docker/init_workspace.sh diff --git a/docker/readme.md b/ros1/docker/readme.md similarity index 100% rename from docker/readme.md rename to ros1/docker/readme.md diff --git a/docker/run_dope_docker.sh b/ros1/docker/run_dope_docker.sh similarity index 100% rename from docker/run_dope_docker.sh rename to ros1/docker/run_dope_docker.sh diff --git a/launch/camera.launch b/ros1/launch/camera.launch similarity index 100% rename from launch/camera.launch rename to ros1/launch/camera.launch diff --git a/launch/dope.launch b/ros1/launch/dope.launch similarity index 100% rename from launch/dope.launch rename to ros1/launch/dope.launch diff --git a/nodes/camera b/ros1/nodes/camera similarity index 100% rename from nodes/camera rename to ros1/nodes/camera diff --git a/nodes/dope b/ros1/nodes/dope similarity index 100% rename from nodes/dope rename to ros1/nodes/dope diff --git a/package.xml b/ros1/package.xml similarity index 100% rename from package.xml rename to ros1/package.xml diff --git a/setup.py b/ros1/setup.py similarity index 100% rename from setup.py rename to ros1/setup.py diff --git a/scripts/train2/inference/cuboid_pnp_solver.py b/scripts/train2/inference/cuboid_pnp_solver.py deleted file mode 100755 index b19fa779..00000000 --- a/scripts/train2/inference/cuboid_pnp_solver.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2018 NVIDIA Corporation. All rights reserved. -# This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. -# https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode - -import cv2 -import numpy as np -from cuboid import CuboidVertexType -from pyrr import Quaternion - - -class CuboidPNPSolver(object): - """ - This class is used to find the 6-DoF pose of a cuboid given its projected vertices. - - Runs perspective-n-point (PNP) algorithm. - """ - - # Class variables - cv2version = cv2.__version__.split('.') - cv2majorversion = int(cv2version[0]) - - def __init__(self, object_name="", camera_intrinsic_matrix = None, cuboid3d = None, - dist_coeffs = np.zeros((4, 1))): - self.object_name = object_name - if (not camera_intrinsic_matrix is None): - self._camera_intrinsic_matrix = camera_intrinsic_matrix - else: - self._camera_intrinsic_matrix = np.array([ - [0, 0, 0], - [0, 0, 0], - [0, 0, 0] - ]) - self._cuboid3d = cuboid3d - - # print (cuboid3d.get_vertices()) - # raise() - self._dist_coeffs = dist_coeffs - - def set_camera_intrinsic_matrix(self, new_intrinsic_matrix): - '''Sets the camera intrinsic matrix''' - self._camera_intrinsic_matrix = new_intrinsic_matrix - - def set_dist_coeffs(self, dist_coeffs): - '''Sets the camera intrinsic matrix''' - self._dist_coeffs = dist_coeffs - - def solve_pnp(self, cuboid2d_points, pnp_algorithm = None): - """ - Detects the rotation and traslation - of a cuboid object from its vertexes' - 2D location in the image - """ - - # Fallback to default PNP algorithm base on OpenCV version - if pnp_algorithm is None: - if CuboidPNPSolver.cv2majorversion == 2: - pnp_algorithm = cv2.CV_ITERATIVE - elif CuboidPNPSolver.cv2majorversion == 3: - pnp_algorithm = cv2.SOLVEPNP_ITERATIVE - # Alternative algorithms: - # pnp_algorithm = SOLVE_PNP_P3P - # pnp_algorithm = cv2.SOLVEPNP_EPNP - if pnp_algorithm is None: - # pnp_algorithm = 1 - pnp_algorithm = cv2.SOLVEPNP_EPNP - - - location = None - quaternion = None - projected_points = cuboid2d_points - - cuboid3d_points = np.array(self._cuboid3d.get_vertices()) - obj_2d_points = [] - obj_3d_points = [] - - for i in range(CuboidVertexType.TotalVertexCount): - check_point_2d = cuboid2d_points[i] - # Ignore invalid points - if (check_point_2d is None): - continue - obj_2d_points.append(check_point_2d) - obj_3d_points.append(cuboid3d_points[i]) - - obj_2d_points = np.array(obj_2d_points, dtype=float) - obj_3d_points = np.array(obj_3d_points, dtype=float) - - valid_point_count = len(obj_2d_points) - - # Can only do PNP if we have more than 3 valid points - is_points_valid = valid_point_count >= 4 - - if is_points_valid: - - ret, rvec, tvec = cv2.solvePnP( - obj_3d_points, - obj_2d_points, - self._camera_intrinsic_matrix, - self._dist_coeffs, - flags=pnp_algorithm - ) - - if ret: - location = list(x[0] for x in tvec) - quaternion = self.convert_rvec_to_quaternion(rvec) - - projected_points, _ = cv2.projectPoints(cuboid3d_points, rvec, tvec, self._camera_intrinsic_matrix, self._dist_coeffs) - projected_points = np.squeeze(projected_points) - - # If the location.Z is negative or object is behind the camera then flip both location and rotation - x, y, z = location - if z < 0: - # Get the opposite location - location = [-x, -y, -z] - - # Change the rotation by 180 degree - rotate_angle = np.pi - rotate_quaternion = Quaternion.from_axis_rotation(location, rotate_angle) - quaternion = rotate_quaternion.cross(quaternion) - - return location, quaternion, projected_points - - def convert_rvec_to_quaternion(self, rvec): - '''Convert rvec (which is log quaternion) to quaternion''' - theta = np.sqrt(rvec[0] * rvec[0] + rvec[1] * rvec[1] + rvec[2] * rvec[2]) # in radians - raxis = [rvec[0] / theta, rvec[1] / theta, rvec[2] / theta] - - # pyrr's Quaternion (order is XYZW), https://pyrr.readthedocs.io/en/latest/oo_api_quaternion.html - return Quaternion.from_axis_rotation(raxis, theta) - - # Alternatively: pyquaternion - # return Quaternion(axis=raxis, radians=theta) # uses OpenCV's Quaternion (order is WXYZ) - - def project_points(self, rvec, tvec): - '''Project points from model onto image using rotation, translation''' - output_points, tmp = cv2.projectPoints( - self.__object_vertex_coordinates, - rvec, - tvec, - self.__camera_intrinsic_matrix, - self.__dist_coeffs) - - output_points = np.squeeze(output_points) - return output_points \ No newline at end of file diff --git a/scripts/train2/inference/detector.py b/scripts/train2/inference/detector.py deleted file mode 100755 index b4f47fa3..00000000 --- a/scripts/train2/inference/detector.py +++ /dev/null @@ -1,766 +0,0 @@ -# Copyright (c) 2018 NVIDIA Corporation. All rights reserved. -# This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. -# https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode - -''' -Contains the following classes: - - ModelData - High level information encapsulation - - ObjectDetector - Greedy algorithm to build cuboids from belief maps -''' - -import time -import json -import os, shutil -import sys -import traceback -from os import path -import threading -from threading import Thread - -import numpy as np -import cv2 - -import torch -import torch.nn as nn -import torchvision.transforms as transforms -from torch.autograd import Variable -import torchvision.models as models - -from scipy import ndimage -import scipy -import scipy.ndimage as ndimage -import scipy.ndimage.filters as filters -from scipy.ndimage.filters import gaussian_filter -from scipy import optimize - -import sys -sys.path.append("../") -from models import * - -# Import the definition of the neural network model and cuboids -from cuboid_pnp_solver import * - -#global transform for image input -transform = transforms.Compose([ - # transforms.Scale(IMAGE_SIZE), - # transforms.CenterCrop((imagesize,imagesize)), - transforms.ToTensor(), - # transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), - transforms.Normalize( - (0.485, 0.456, 0.406), - (0.229, 0.224, 0.225) - ) - ]) - - -#================================ Models ================================ - - -class DopeNetwork(nn.Module): - def __init__( - self, - numBeliefMap=9, - numAffinity=16, - stop_at_stage=6 # number of stages to process (if less than total number of stages) - ): - super(DopeNetwork, self).__init__() - - self.stop_at_stage = stop_at_stage - - vgg_full = models.vgg19(pretrained=False).features - self.vgg = nn.Sequential() - for i_layer in range(24): - self.vgg.add_module(str(i_layer), vgg_full[i_layer]) - - # Add some layers - i_layer = 23 - self.vgg.add_module(str(i_layer), nn.Conv2d(512, 256, kernel_size=3, stride=1, padding=1)) - self.vgg.add_module(str(i_layer+1), nn.ReLU(inplace=True)) - self.vgg.add_module(str(i_layer+2), nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1)) - self.vgg.add_module(str(i_layer+3), nn.ReLU(inplace=True)) - - # print('---Belief------------------------------------------------') - # _2 are the belief map stages - self.m1_2 = DopeNetwork.create_stage(128, numBeliefMap, True) - self.m2_2 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numBeliefMap, False) - self.m3_2 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numBeliefMap, False) - self.m4_2 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numBeliefMap, False) - self.m5_2 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numBeliefMap, False) - self.m6_2 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numBeliefMap, False) - - # print('---Affinity----------------------------------------------') - # _1 are the affinity map stages - self.m1_1 = DopeNetwork.create_stage(128, numAffinity, True) - self.m2_1 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numAffinity, False) - self.m3_1 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numAffinity, False) - self.m4_1 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numAffinity, False) - self.m5_1 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numAffinity, False) - self.m6_1 = DopeNetwork.create_stage(128 + numBeliefMap + numAffinity, - numAffinity, False) - - - def forward(self, x): - '''Runs inference on the neural network''' - - out1 = self.vgg(x) - - out1_2 = self.m1_2(out1) - out1_1 = self.m1_1(out1) - - if self.stop_at_stage == 1: - return [out1_2],\ - [out1_1] - - out2 = torch.cat([out1_2, out1_1, out1], 1) - out2_2 = self.m2_2(out2) - out2_1 = self.m2_1(out2) - - if self.stop_at_stage == 2: - return [out1_2, out2_2],\ - [out1_1, out2_1] - - out3 = torch.cat([out2_2, out2_1, out1], 1) - out3_2 = self.m3_2(out3) - out3_1 = self.m3_1(out3) - - if self.stop_at_stage == 3: - return [out1_2, out2_2, out3_2],\ - [out1_1, out2_1, out3_1] - - out4 = torch.cat([out3_2, out3_1, out1], 1) - out4_2 = self.m4_2(out4) - out4_1 = self.m4_1(out4) - - if self.stop_at_stage == 4: - return [out1_2, out2_2, out3_2, out4_2],\ - [out1_1, out2_1, out3_1, out4_1] - - out5 = torch.cat([out4_2, out4_1, out1], 1) - out5_2 = self.m5_2(out5) - out5_1 = self.m5_1(out5) - - if self.stop_at_stage == 5: - return [out1_2, out2_2, out3_2, out4_2, out5_2],\ - [out1_1, out2_1, out3_1, out4_1, out5_1] - - out6 = torch.cat([out5_2, out5_1, out1], 1) - out6_2 = self.m6_2(out6) - out6_1 = self.m6_1(out6) - - return [out1_2, out2_2, out3_2, out4_2, out5_2, out6_2],\ - [out1_1, out2_1, out3_1, out4_1, out5_1, out6_1] - - @staticmethod - def create_stage(in_channels, out_channels, first=False): - '''Create the neural network layers for a single stage.''' - - model = nn.Sequential() - mid_channels = 128 - if first: - padding = 1 - kernel = 3 - count = 6 - final_channels = 512 - else: - padding = 3 - kernel = 7 - count = 10 - final_channels = mid_channels - - # First convolution - model.add_module("0", - nn.Conv2d( - in_channels, - mid_channels, - kernel_size=kernel, - stride=1, - padding=padding) - ) - - # Middle convolutions - i = 1 - while i < count - 1: - model.add_module(str(i), nn.ReLU(inplace=True)) - i += 1 - model.add_module(str(i), - nn.Conv2d( - mid_channels, - mid_channels, - kernel_size=kernel, - stride=1, - padding=padding)) - i += 1 - - # Penultimate convolution - model.add_module(str(i), nn.ReLU(inplace=True)) - i += 1 - model.add_module(str(i), nn.Conv2d(mid_channels, final_channels, kernel_size=1, stride=1)) - i += 1 - - # Last convolution - model.add_module(str(i), nn.ReLU(inplace=True)) - i += 1 - model.add_module(str(i), nn.Conv2d(final_channels, out_channels, kernel_size=1, stride=1)) - i += 1 - - return model - - - -class ModelData(object): - '''This class contains methods for loading the neural network''' - - def __init__(self, name="", net_path="", gpu_id=0,architecture='dope'): - self.name = name - self.net_path = net_path # Path to trained network model - self.net = None # Trained network - self.gpu_id = gpu_id - self.architecture = architecture - - def get_net(self): - '''Returns network''' - if not self.net: - self.load_net_model() - return self.net - - def load_net_model(self): - '''Loads network model from disk''' - if not self.net and path.exists(self.net_path): - self.net = self.load_net_model_path(self.net_path) - if not path.exists(self.net_path): - print("ERROR: Unable to find model weights: '{}'".format( - self.net_path)) - exit(0) - - def load_net_model_path(self, path): - '''Loads network model from disk with given path''' - model_loading_start_time = time.time() - print("Loading DOPE model '{}'...".format(path)) - if self.architecture == 'dope': - net = DopeNetwork() - else: - net = ResnetSimple() - net = torch.nn.DataParallel(net, [0]).cuda() - net.load_state_dict(torch.load(path)) - net.eval() - print(' Model loaded in {} seconds.'.format( - time.time() - model_loading_start_time)) - return net - - def __str__(self): - '''Converts to string''' - return "{}: {}".format(self.name, self.net_path) - - -#================================ ObjectDetector ================================ -class ObjectDetector(object): - '''This class contains methods for object detection''' - - @staticmethod - def gaussian(height, center_x, center_y, width_x, width_y): - """Returns a gaussian function with the given parameters""" - width_x = float(width_x) - width_y = float(width_y) - return lambda x,y: height*np.exp( - -(((center_x-x)/width_x)**2+((center_y-y)/width_y)**2)/2) - - @staticmethod - def moments(data): - """Returns (height, x, y, width_x, width_y) - the gaussian parameters of a 2D distribution by calculating its - moments """ - total = data.sum() - X, Y = np.indices(data.shape) - x = (X*data).sum()/total - y = (Y*data).sum()/total - col = data[:, int(y)] - width_x = np.sqrt(np.abs((np.arange(col.size)-y)**2*col).sum()/col.sum()) - row = data[int(x), :] - width_y = np.sqrt(np.abs((np.arange(row.size)-x)**2*row).sum()/row.sum()) - height = data.max() - return height, x, y, width_x, width_y - - @staticmethod - def fitgaussian(data): - """Returns (height, x, y, width_x, width_y) - the gaussian parameters of a 2D distribution found by a fit""" - params = ObjectDetector.moments(data) - errorfunction = lambda p: np.ravel(ObjectDetector.gaussian(*p)(*np.indices(data.shape)) - - data) - p, success = optimize.leastsq(errorfunction, params) - return p - @staticmethod - def make_grid(tensor, nrow=8, padding=2, - normalize=False, range_=None, scale_each=False, pad_value=0): - """Make a grid of images. - Args: - tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W) - or a list of images all of the same size. - nrow (int, optional): Number of images displayed in each row of the grid. - The Final grid size is (B / nrow, nrow). Default is 8. - padding (int, optional): amount of padding. Default is 2. - normalize (bool, optional): If True, shift the image to the range (0, 1), - by subtracting the minimum and dividing by the maximum pixel value. - range (tuple, optional): tuple (min, max) where min and max are numbers, - then these numbers are used to normalize the image. By default, min and max - are computed from the tensor. - scale_each (bool, optional): If True, scale each image in the batch of - images separately rather than the (min, max) over all images. - pad_value (float, optional): Value for the padded pixels. - Example: - See this notebook `here `_ - """ - import math - - if not (torch.is_tensor(tensor) or - (isinstance(tensor, list) and all(torch.is_tensor(t) for t in tensor))): - raise TypeError('tensor or list of tensors expected, got {}'.format(type(tensor))) - - # if list of tensors, convert to a 4D mini-batch Tensor - if isinstance(tensor, list): - tensor = torch.stack(tensor, dim=0) - - if tensor.dim() == 2: # single image H x W - tensor = tensor.view(1, tensor.size(0), tensor.size(1)) - if tensor.dim() == 3: # single image - if tensor.size(0) == 1: # if single-channel, convert to 3-channel - tensor = torch.cat((tensor, tensor, tensor), 0) - tensor = tensor.view(1, tensor.size(0), tensor.size(1), tensor.size(2)) - - if tensor.dim() == 4 and tensor.size(1) == 1: # single-channel images - tensor = torch.cat((tensor, tensor, tensor), 1) - - if normalize is True: - tensor = tensor.clone() # avoid modifying tensor in-place - if range_ is not None: - assert isinstance(range_, tuple), \ - "range has to be a tuple (min, max) if specified. min and max are numbers" - - def norm_ip(img, min, max): - img.clamp_(min=min, max=max) - img.add_(-min).div_(max - min + 1e-5) - - def norm_range(t, range_): - if range_ is not None: - norm_ip(t, range_[0], range_[1]) - else: - norm_ip(t, float(t.min()), float(t.max())) - - if scale_each is True: - for t in tensor: # loop over mini-batch dimension - norm_range(t, range) - else: - norm_range(tensor, range) - - if tensor.size(0) == 1: - return tensor.squeeze() - - # make the mini-batch of images into a grid - nmaps = tensor.size(0) - xmaps = min(nrow, nmaps) - ymaps = int(math.ceil(float(nmaps) / xmaps)) - height, width = int(tensor.size(2) + padding), int(tensor.size(3) + padding) - grid = tensor.new(3, height * ymaps + padding, width * xmaps + padding).fill_(pad_value) - k = 0 - for y in range(ymaps): - for x in range(xmaps): - if k >= nmaps: - break - grid.narrow(1, y * height + padding, height - padding)\ - .narrow(2, x * width + padding, width - padding)\ - .copy_(tensor[k]) - k = k + 1 - return grid - - @staticmethod - def get_image_grid(tensor, filename, nrow=3, padding=2,mean=None, std=None): - """ - Saves a given Tensor into an image file. - If given a mini-batch tensor, will save the tensor as a grid of images. - """ - from PIL import Image - - # tensor = tensor.cpu() - grid = ObjectDetector.make_grid(tensor, nrow=nrow, padding=10,pad_value=1) - if not mean is None: - # ndarr = grid.mul(std).add(mean).mul(255).byte().transpose(0,2).transpose(0,1).numpy() - ndarr = grid.mul(std).add(mean).mul(255).byte().transpose(0,2).transpose(0,1).numpy() - else: - ndarr = grid.mul(0.5).add(0.5).mul(255).byte().transpose(0,2).transpose(0,1).numpy() - im = Image.fromarray(ndarr) - # im.save(filename) - return im - - @staticmethod - def detect_object_in_image(net_model, pnp_solver, in_img, config, - make_belief_debug_img=False, norm_belief=True, overlay_image=True): - """ - Detect objects in a image using a specific trained network model - Returns the poses of the objects and the belief maps - """ - if in_img is None: - return [] - - # Run network inference - image_tensor = transform(in_img) - image_torch = Variable(image_tensor).cuda().unsqueeze(0) - out, seg = net_model(image_torch) - vertex2 = out[-1][0] - aff = seg[-1][0] - - # Find objects from network output - detected_objects = ObjectDetector.find_object_poses(vertex2, aff, pnp_solver, config) - - if not make_belief_debug_img: - return detected_objects, None - else: - # Run the belief maps debug display on the belief maps - tensor = vertex2 - belief_imgs = [] - if overlay_image: - upsampling = nn.UpsamplingNearest2d(size=in_img.shape[:2]) - in_img = (torch.tensor(in_img).float() / 255.0) - in_img *= 0.5 - - for j in range(tensor.size()[0]): - belief = tensor[j].clone() - if norm_belief: - belief -= float(torch.min(belief).item()) - belief /= float(torch.max(belief).item()) - - belief = torch.clamp(belief, 0, 1).cpu() - if overlay_image: - belief = upsampling(belief.unsqueeze(0).unsqueeze(0)).squeeze().squeeze().data - belief = torch.cat([ - belief.unsqueeze(0) + in_img[:, :, 0], - belief.unsqueeze(0) + in_img[:, :, 1], - belief.unsqueeze(0) + in_img[:, :, 2] - ]).unsqueeze(0) - belief = torch.clamp(belief, 0, 1) - else: - belief = torch.cat([ - belief.unsqueeze(0), - belief.unsqueeze(0), - belief.unsqueeze(0) - ]).unsqueeze(0) - belief_imgs.append(belief.data.squeeze().numpy()) - - # Create the image grid - belief_imgs = torch.tensor(np.array(belief_imgs)) - - im_belief = ObjectDetector.get_image_grid(belief_imgs, None, - mean=0, std=1) - - return detected_objects, im_belief - - - - @staticmethod - def find_object_poses(vertex2, aff, pnp_solver, config, run_sampling=False, num_sample=100,scale_factor=8): - '''Detect objects given network output''' - - # run_sampling = True - - # Detect objects from belief maps and affinities - objects, all_peaks = ObjectDetector.find_objects(vertex2, aff, config, - run_sampling=run_sampling, num_sample=num_sample,scale_factor=scale_factor) - detected_objects = [] - obj_name = pnp_solver.object_name - - # print(all_peaks) - - #print("find_object_poses: found {} objects ================".format(len(objects))) - for obj in objects: - # Run PNP - points = obj[1] + [(obj[0][0]*scale_factor, obj[0][1]*scale_factor)] - cuboid2d = np.copy(points) - location, quaternion, projected_points = pnp_solver.solve_pnp(points) - - # run multiple sample - if run_sampling: - lx,ly,lz = [],[],[] - qx,qy,qz,qw = [],[],[],[] - - for i_sample in range(num_sample): - sample = [] - for i_point in range(len(obj[-1])): - if not obj[-1][i_point][i_sample] is None: - sample.append( (obj[-1][i_point][i_sample][0]*scale_factor, - obj[-1][i_point][i_sample][1]*scale_factor)) - else: - sample.append( None) - # final_cuboids.append(sample) - pnp_sample = pnp_solver.solve_pnp(sample) - - try: - lx.append(pnp_sample[0][0]) - ly.append(pnp_sample[0][1]) - lz.append(pnp_sample[0][2]) - - qx.append(pnp_sample[1][0]) - qy.append(pnp_sample[1][1]) - qz.append(pnp_sample[1][2]) - qw.append(pnp_sample[1][3]) - except: - pass - # TODO - # RUN quaternion as well for the std and avg. - - try: - print ("----") - print ("location:") - print (location[0],location[1],location[2]) - print (np.mean(lx),np.mean(ly),np.mean(lz)) - print (np.std(lx),np.std(ly),np.std(lz)) - print ("quaternion:") - print (quaternion[0],quaternion[1],quaternion[2],quaternion[3]) - print (np.mean(qx),np.mean(qy),np.mean(qz),np.mean(qw)) - print (np.std(qx),np.std(qy),np.std(qz),np.std(qw)) - - - except: - pass - if not location is None: - detected_objects.append({ - 'name': obj_name, - 'location': location, - 'quaternion': quaternion, - 'cuboid2d': cuboid2d, - 'projected_points': projected_points, - 'confidence': obj[-1], - 'raw_points': points - }) - - #print("find_object_poses: points = ", type(points), points) - #print("find_object_poses: locn = ", location, "quat =", quaternion) - #print("find_object_poses: projected_points = ", type(projected_points), projected_points) - - return detected_objects - - @staticmethod - def find_objects(vertex2, aff, config, numvertex=8, run_sampling=False, num_sample=100,scale_factor=8): - '''Detects objects given network belief maps and affinities, using heuristic method''' - - all_peaks = [] - all_samples = [] - - peak_counter = 0 - for j in range(vertex2.size()[0]): - belief = vertex2[j].clone() - map_ori = belief.cpu().data.numpy() - - map = gaussian_filter(belief.cpu().data.numpy(), sigma=config.sigma) - p = 1 - map_left = np.zeros(map.shape) - map_left[p:,:] = map[:-p,:] - map_right = np.zeros(map.shape) - map_right[:-p,:] = map[p:,:] - map_up = np.zeros(map.shape) - map_up[:,p:] = map[:,:-p] - map_down = np.zeros(map.shape) - map_down[:,:-p] = map[:,p:] - - peaks_binary = np.logical_and.reduce( - ( - map >= map_left, - map >= map_right, - map >= map_up, - map >= map_down, - map > config.thresh_map) - ) - peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) - - # Computing the weigthed average for localizing the peaks - peaks = list(peaks) - win = 11 - ran = win//2 - peaks_avg = [] - point_sample_list = [] - for p_value in range(len(peaks)): - p = peaks[p_value] - weights = np.zeros((win,win)) - i_values = np.zeros((win,win)) - j_values = np.zeros((win,win)) - for i in range(-ran,ran+1): - for j in range(-ran,ran+1): - if p[1]+i < 0 \ - or p[1]+i >= map_ori.shape[0] \ - or p[0]+j < 0 \ - or p[0]+j >= map_ori.shape[1]: - continue - - i_values[j+ran, i+ran] = p[1] + i - j_values[j+ran, i+ran] = p[0] + j - - weights[j+ran, i+ran] = (map_ori[p[1]+i, p[0]+j]) - # if the weights are all zeros - # then add the none continuous points - OFFSET_DUE_TO_UPSAMPLING = 0.4395 - - # Sample the points using the gaussian - if run_sampling: - data = weights - params = ObjectDetector.fitgaussian(data) - fit = ObjectDetector.gaussian(*params) - _, mu_x,mu_y,std_x,std_y = params - points_sample = np.random.multivariate_normal( - np.array([p[1] + mu_x + OFFSET_DUE_TO_UPSAMPLING, - p[0] - mu_y + OFFSET_DUE_TO_UPSAMPLING]), - # np.array([[std_x*std_x,0],[0,std_y*std_y]]), size=num_sample) - np.array([[std_x,0],[0,std_y]]), size=num_sample) - point_sample_list.append(points_sample) - - - try: - peaks_avg.append( - (np.average(j_values, weights=weights) + OFFSET_DUE_TO_UPSAMPLING, \ - np.average(i_values, weights=weights) + OFFSET_DUE_TO_UPSAMPLING)) - except: - peaks_avg.append((p[0] + OFFSET_DUE_TO_UPSAMPLING, p[1] + OFFSET_DUE_TO_UPSAMPLING)) - - - # Note: Python3 doesn't support len for zip object - peaks_len = min(len(np.nonzero(peaks_binary)[1]), len(np.nonzero(peaks_binary)[0])) - - peaks_with_score = [peaks_avg[x_] + (map_ori[peaks[x_][1],peaks[x_][0]],) for x_ in range(len(peaks))] - - id = range(peak_counter, peak_counter + peaks_len) - - peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))] - - all_peaks.append(peaks_with_score_and_id) - all_samples.append(point_sample_list) - peak_counter += peaks_len - - objects = [] - - if aff is None: - # Assume there is only one object - points = [None for i in range(numvertex)] - for i_peak, peaks in enumerate(all_peaks): - # print (peaks) - for peak in peaks: - if peak[2] > config.threshold: - points[i_peak] = (peak[0],peak[1]) - - return points - - - # Check object centroid and build the objects if the centroid is found - for nb_object in range(len(all_peaks[-1])): - if all_peaks[-1][nb_object][2] > config.thresh_points: - objects.append([ - [all_peaks[-1][nb_object][:2][0],all_peaks[-1][nb_object][:2][1]], - [None for i in range(numvertex)], - [None for i in range(numvertex)], - all_peaks[-1][nb_object][2], - [[None for j in range(num_sample)] for i in range(numvertex+1)] - ]) - - # Check if the object was added before - if run_sampling and nb_object < len(objects): - # add the samples to the object centroids - objects[nb_object][4][-1] = all_samples[-1][nb_object] - - - # Working with an output that only has belief maps - if aff is None: - if len (objects) > 0 and len(all_peaks)>0 and len(all_peaks[0])>0: - for i_points in range(8): - if len(all_peaks[i_points])>0 and all_peaks[i_points][0][2] > config.threshold: - objects[0][1][i_points] = (all_peaks[i_points][0][0], all_peaks[i_points][0][1]) - else: - # For all points found - for i_lists in range(len(all_peaks[:-1])): - lists = all_peaks[i_lists] - - # Candidate refers to point that needs to be match with a centroid object - for i_candidate, candidate in enumerate(lists): - if candidate[2] < config.thresh_points: - continue - - i_best = -1 - best_dist = 10000 - best_angle = 100 - - # Find the points that links to that centroid. - for i_obj in range(len(objects)): - center = [objects[i_obj][0][0], objects[i_obj][0][1]] - - # integer is used to look into the affinity map, - # but the float version is used to run - point_int = [int(candidate[0]), int(candidate[1])] - point = [candidate[0], candidate[1]] - - # look at the distance to the vector field. - v_aff = np.array([ - aff[i_lists*2, - point_int[1], - point_int[0]].data.item(), - aff[i_lists*2+1, - point_int[1], - point_int[0]].data.item()]) * 10 - - # normalize the vector - xvec = v_aff[0] - yvec = v_aff[1] - - norms = np.sqrt(xvec * xvec + yvec * yvec) - - xvec/=norms - yvec/=norms - - v_aff = np.concatenate([[xvec],[yvec]]) - - v_center = np.array(center) - np.array(point) - xvec = v_center[0] - yvec = v_center[1] - - norms = np.sqrt(xvec * xvec + yvec * yvec) - - xvec /= norms - yvec /= norms - - v_center = np.concatenate([[xvec],[yvec]]) - - # vector affinity - dist_angle = np.linalg.norm(v_center - v_aff) - - # distance between vertexes - dist_point = np.linalg.norm(np.array(point) - np.array(center)) - - if dist_angle < config.thresh_angle \ - and best_dist > 1000 \ - or dist_angle < config.thresh_angle \ - and best_dist > dist_point: - i_best = i_obj - best_angle = dist_angle - best_dist = dist_point - - if i_best == -1: - continue - - if objects[i_best][1][i_lists] is None \ - or best_angle < config.thresh_angle \ - and best_dist < objects[i_best][2][i_lists][1]: - # set the points - objects[i_best][1][i_lists] = ((candidate[0])*scale_factor, (candidate[1])*scale_factor) - # set information about the points: angle and distance - objects[i_best][2][i_lists] = (best_angle, best_dist) - # add the sample points - if run_sampling: - # print ("---") - # print(len(all_samples[i_lists])) - # print (len(all_peaks[i_lists])) - # print(i_obj) - objects[i_best][4][i_lists] = all_samples[i_lists][i_candidate] - return objects, all_peaks diff --git a/train/inference/config/camera_info.yaml b/train/inference/config/camera_info.yaml deleted file mode 100644 index c160b2e0..00000000 --- a/train/inference/config/camera_info.yaml +++ /dev/null @@ -1,16 +0,0 @@ -image_width: 910 -image_height: 512 -camera_name: dope_webcam_0 -distortion_model: plumb_bob -distortion_coefficients: - rows: 1 - cols: 5 - data: [0, 0, 0, 0, 0] -rectification_matrix: - rows: 3 - cols: 3 - data: [1, 0, 0, 0, 1, 0, 0, 0, 1] -projection_matrix: - rows: 3 - cols: 4 - data: [768.16058349609375, 0, 256, 0, 0, 768.16058349609375, 256, 0, 0, 0, 1, 0] diff --git a/train/inference/config/camera_info_960x540.yaml b/train/inference/config/camera_info_960x540.yaml deleted file mode 100644 index 11f043b6..00000000 --- a/train/inference/config/camera_info_960x540.yaml +++ /dev/null @@ -1,20 +0,0 @@ -image_width: 960 -image_height: 540 -camera_name: dope_webcam_0 -camera_matrix: - rows: 3 - cols: 3 - data: [768.16058349609375, 0, 480, 0, 0, 768.16058349609375, 270, 0, 0, 1] -distortion_model: plumb_bob -distortion_coefficients: - rows: 1 - cols: 5 - data: [0, 0, 0, 0, 0] -rectification_matrix: - rows: 3 - cols: 3 - data: [1, 0, 0, 0, 1, 0, 0, 0, 1] -projection_matrix: - rows: 3 - cols: 4 - data: [768.16058349609375, 0, 480, 0, 0, 768.16058349609375, 270, 0, 0, 0, 1, 0] diff --git a/train/inference/config/config_pose.yaml b/train/inference/config/config_pose.yaml deleted file mode 100644 index 5fbe30dd..00000000 --- a/train/inference/config/config_pose.yaml +++ /dev/null @@ -1,48 +0,0 @@ -input_is_rectified: True # Whether the input image is rectified (strongly suggested!) -downscale_height: 512 # if the input image is larger than this, scale it down to this pixel height - -# Cuboid dimension in cm x,y,z -dimensions: { - "003_cracker_box": [16.403600692749023,21.343700408935547,7.179999828338623], - "009_gelatin_box": [8.918299674987793, 7.311500072479248, 2.9983000755310059], - "010_potted_meat_can": [10.164673805236816,8.3542995452880859,5.7600898742675781], - "006_mustard_bottle": [9.6024150848388672,19.130100250244141,5.824894905090332], - "005_tomato_soup_can": [6.7659378051757813,10.185500144958496,6.771425724029541], - "004_sugar_box": [9.267730712890625,17.625339508056641,4.5134143829345703], - "021_bleach_cleanser": [10.267730712890625,26.625339508056641,7.5134143829345703], - "071_nine_hole_peg_test": [12.6,3.9,12.6], - '008_pudding_box':[49.47199821472168, 29.923000335693359, 83.498001098632812], - "Ketchup" : [ 14.860799789428711, 4.3368000984191895, 6.4513998031616211 ], -} - -class_ids: { - "003_cracker_box": 1, - "009_gelatin_box": 2, - "010_potted_meat_can": 3, - "006_mustard_bottle": 4, - "005_tomato_soup_can": 5, - "004_sugar_box": 6, - "021_bleach_cleanser": 7, - "071_nine_hole_peg_test": 8, - '008_pudding_box': 10, - 'Ketchup': 11, -} - -draw_colors: { - "003_cracker_box": [13, 255, 128], # green - "009_gelatin_box": [255, 255, 255], # while - "010_potted_meat_can": [0, 104, 255], # blue - "006_mustard_bottle": [217,12, 232], # magenta - "005_tomato_soup_can": [255, 101, 0], # orange - "004_sugar_box": [232, 222, 12], # yellow - "021_bleach_cleanser": [232, 222, 12], # yellow - "071_nine_hole_peg_test": [232, 222, 12], # yellow - "008_pudding_box": [255,0,0], - "Ketchup": [0x76,0xb9,0x00], # NVIDIA green -} - -# Config params for DOPE -thresh_angle: 0.5 -thresh_map: 0.01 -sigma: 3 -thresh_points: 0.0 diff --git a/train/inference/cuboid.py b/train/inference/cuboid.py deleted file mode 100755 index 469df1ff..00000000 --- a/train/inference/cuboid.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) 2018 NVIDIA Corporation. All rights reserved. -# This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. -# https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode - -from enum import IntEnum, unique -import numpy as np -import cv2 - -# Related to the object's local coordinate system -# @unique -class CuboidVertexType(IntEnum): - FrontTopRight = 0 - FrontTopLeft = 1 - FrontBottomLeft = 2 - FrontBottomRight = 3 - RearTopRight = 4 - RearTopLeft = 5 - RearBottomLeft = 6 - RearBottomRight = 7 - Center = 8 - TotalCornerVertexCount = 8 # Corner vertexes doesn't include the center point - TotalVertexCount = 9 - - -# List of the vertex indexes in each line edges of the cuboid -CuboidLineIndexes = [ - # Front face - [CuboidVertexType.FrontTopLeft, CuboidVertexType.FrontTopRight], - [CuboidVertexType.FrontTopRight, CuboidVertexType.FrontBottomRight], - [CuboidVertexType.FrontBottomRight, CuboidVertexType.FrontBottomLeft], - [CuboidVertexType.FrontBottomLeft, CuboidVertexType.FrontTopLeft], - # Back face - [CuboidVertexType.RearTopLeft, CuboidVertexType.RearTopRight], - [CuboidVertexType.RearTopRight, CuboidVertexType.RearBottomRight], - [CuboidVertexType.RearBottomRight, CuboidVertexType.RearBottomLeft], - [CuboidVertexType.RearBottomLeft, CuboidVertexType.RearTopLeft], - # Left face - [CuboidVertexType.FrontBottomLeft, CuboidVertexType.RearBottomLeft], - [CuboidVertexType.FrontTopLeft, CuboidVertexType.RearTopLeft], - # Right face - [CuboidVertexType.FrontBottomRight, CuboidVertexType.RearBottomRight], - [CuboidVertexType.FrontTopRight, CuboidVertexType.RearTopRight], -] - - -# ========================= Cuboid3d ========================= -class Cuboid3d: - """This class contains a 3D cuboid.""" - - # Create a box with a certain size - def __init__( - self, - size3d=[1.0, 1.0, 1.0], - center_location=[0, 0, 0], - coord_system=None, - parent_object=None, - ): - - # NOTE: This local coordinate system is similar - # to the intrinsic transform matrix of a 3d object - self.center_location = center_location - self.coord_system = coord_system - self.size3d = size3d - self._vertices = [0, 0, 0] * CuboidVertexType.TotalVertexCount - - self.generate_vertexes() - - def get_vertex(self, vertex_type): - """Returns the location of a vertex. - - Args: - vertex_type: enum of type CuboidVertexType - - Returns: - Numpy array(3) - Location of the vertex type in the cuboid - """ - return self._vertices[vertex_type] - - def get_vertices(self): - return self._vertices - - def generate_vertexes(self): - width, height, depth = self.size3d - - # By default just use the normal OpenCV coordinate system - if self.coord_system is None: - cx, cy, cz = self.center_location - # X axis point to the right - right = cx + width / 2.0 - left = cx - width / 2.0 - # Y axis point downward - top = cy - height / 2.0 - bottom = cy + height / 2.0 - # Z axis point forward - front = cz + depth / 2.0 - rear = cz - depth / 2.0 - - # List of 8 vertices of the box - self._vertices = [ - [right, top, front], # Front Top Right - [left, top, front], # Front Top Left - [left, bottom, front], # Front Bottom Left - [right, bottom, front], # Front Bottom Right - [right, top, rear], # Rear Top Right - [left, top, rear], # Rear Top Left - [left, bottom, rear], # Rear Bottom Left - [right, bottom, rear], # Rear Bottom Right - self.center_location, # Center - ] - else: - sx, sy, sz = self.size3d - forward = np.array(self.coord_system.forward, dtype=float) * sy * 0.5 - up = np.array(self.coord_system.up, dtype=float) * sz * 0.5 - right = np.array(self.coord_system.right, dtype=float) * sx * 0.5 - center = np.array(self.center_location, dtype=float) - self._vertices = [ - center + forward + up + right, # Front Top Right - center + forward + up - right, # Front Top Left - center + forward - up - right, # Front Bottom Left - center + forward - up + right, # Front Bottom Right - center - forward + up + right, # Rear Top Right - center - forward + up - right, # Rear Top Left - center - forward - up - right, # Rear Bottom Left - center - forward - up + right, # Rear Bottom Right - self.center_location, # Center - ] diff --git a/train/requirements.txt b/train/requirements.txt deleted file mode 100755 index 26c0df7a..00000000 --- a/train/requirements.txt +++ /dev/null @@ -1,74 +0,0 @@ -torch -torchvision -tensorboardX -boto3 -albumentations - -botocore -certifi -charset-normalizer -idna -imageio -jmespath -joblib -networkx -numpy -opencv-python-headless==4.6.0.66 -packaging -Pillow -protobuf==3.20.1 -pyparsing -pyrr -python-dateutil -PyWavelets -PyYAML -qudida -requests -s3transfer -scikit-image -scikit-learn -scipy -simplejson -six -threadpoolctl -tifffile -typing_extensions -urllib3 - - -## If running into dependency issues, install the version-specific requirements below in a virtual environment -# albumentations==1.2.1 -# boto3==1.24.58 -# botocore==1.27.58 -# certifi==2022.6.15 -# charset-normalizer==2.1.1 -# idna==3.3 -# imageio==2.21.1 -# jmespath==1.0.1 -# joblib==1.1.0 -# networkx==2.8.6 -# numpy==1.23.2 -# opencv-python-headless==4.6.0.66 -# packaging==21.3 -# Pillow==9.2.0 -# protobuf==3.20.1 -# pyparsing==3.0.9 -# pyrr==0.10.3 -# python-dateutil==2.8.2 -# PyWavelets==1.3.0 -# PyYAML==6.0 -# qudida==0.0.4 -# requests==2.28.1 -# s3transfer==0.6.0 -# scikit-image==0.19.3 -# scikit-learn==1.1.2 -# scipy==1.9.0 -# simplejson==3.17.6 -# six==1.16.0 -# tensorboardX==2.5.1 -# threadpoolctl==3.1.0 -# tifffile==2022.8.12 -# torch==1.12.1 -# torchvision==0.13.1 -# typing_extensions==4.3.0 -# urllib3==1.26.12 diff --git a/scripts/train2/config_inference/camera_info.yaml b/train2/config_inference/camera_info.yaml similarity index 100% rename from scripts/train2/config_inference/camera_info.yaml rename to train2/config_inference/camera_info.yaml diff --git a/scripts/train2/config_inference/config_pose.yaml b/train2/config_inference/config_pose.yaml similarity index 100% rename from scripts/train2/config_inference/config_pose.yaml rename to train2/config_inference/config_pose.yaml diff --git a/scripts/train2/inference.py b/train2/inference.py similarity index 100% rename from scripts/train2/inference.py rename to train2/inference.py diff --git a/scripts/train2/models.py b/train2/models.py similarity index 100% rename from scripts/train2/models.py rename to train2/models.py diff --git a/scripts/train2/readme.md b/train2/readme.md similarity index 100% rename from scripts/train2/readme.md rename to train2/readme.md diff --git a/scripts/train2/requirements.txt b/train2/requirements.txt similarity index 100% rename from scripts/train2/requirements.txt rename to train2/requirements.txt diff --git a/scripts/train2/train.py b/train2/train.py similarity index 100% rename from scripts/train2/train.py rename to train2/train.py diff --git a/scripts/train2/utils_dope.py b/train2/utils_dope.py similarity index 100% rename from scripts/train2/utils_dope.py rename to train2/utils_dope.py