Skip to content

Commit

Permalink
Add OpenVLA metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@xuanlinli17
xuanlinli17 committed Aug 18, 2024
1 parent 52727f9 commit 37d8fa8
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 0 deletions.
82 changes: 82 additions & 0 deletions simpler_env/utils/metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,41 @@
import numpy as np

REAL_PERF = { # Real robot eval performance --> extract via: REAL_PERF[task][policy]
"google_robot_pick_coke_can_horizontal": {
"rt-2-x": 0.920,
"rt-1-converged": 0.960,
"rt-1-15pct": 1.000,
"rt-1-x": 0.880,
"rt-1-begin": 0.200,
"octo-base": 0.440,
"openvla-7b": 0.640,
},
"google_robot_pick_coke_can_vertical": {
"rt-2-x": 0.800,
"rt-1-converged": 0.880,
"rt-1-15pct": 0.960,
"rt-1-x": 0.560,
"rt-1-begin": 0.000,
"octo-base": 0.200,
"openvla-7b": 0.280,
},
"google_robot_pick_coke_can_standing": {
"rt-2-x": 1.000,
"rt-1-converged": 0.720,
"rt-1-15pct": 0.800,
"rt-1-x": 0.840,
"rt-1-begin": 0.200,
"octo-base": 0.240,
"openvla-7b": 0.360,
},
"google_robot_pick_coke_can": {
"rt-2-x": 0.907,
"rt-1-converged": 0.853,
"rt-1-15pct": 0.920,
"rt-1-x": 0.760,
"rt-1-begin": 0.133,
"octo-base": 0.293,
"openvla-7b": 0.427,
},
"google_robot_move_near": {
"rt-2-x": 0.733,
Expand All @@ -20,6 +48,7 @@
"rt-1-x": 0.450,
"rt-1-begin": 0.017,
"octo-base": 0.350,
"openvla-7b": 0.667,
},
"google_robot_open_drawer": {
"rt-2-x": 0.333,
Expand All @@ -28,6 +57,7 @@
"rt-1-x": 0.519,
"rt-1-begin": 0.000,
"octo-base": 0.148,
"openvla-7b": 0.111,
},
"google_robot_close_drawer": {
"rt-2-x": 0.630,
Expand All @@ -36,6 +66,16 @@
"rt-1-x": 0.741,
"rt-1-begin": 0.000,
"octo-base": 0.519,
"openvla-7b": 0.148,
},
"google_robot_drawer": {
"rt-2-x": 0.481,
"rt-1-converged": 0.870,
"rt-1-15pct": 0.796,
"rt-1-x": 0.630,
"rt-1-begin": 0.000,
"octo-base": 0.333,
"openvla-7b": 0.130,
},
"google_robot_place_apple_in_closed_top_drawer": {
"rt-2-x": 0.074,
Expand All @@ -44,6 +84,7 @@
"rt-1-x": 0.407,
"rt-1-begin": 0.000,
"octo-base": 0.000,
"openvla-7b": 0.000,
},
"widowx_spoon_on_towel": {
"rt-1-x": 0.000,
Expand All @@ -69,13 +110,41 @@


SIMPLER_PERF = { # SIMPLER simulated eval performance --> extract via: SIMPLER_PERF[task][policy]
"google_robot_pick_coke_can_horizontal": {
"rt-2-x": 0.740,
"rt-1-converged": 0.960,
"rt-1-15pct": 0.860,
"rt-1-x": 0.820,
"rt-1-begin": 0.050,
"octo-base": 0.210,
"openvla-7b": 0.310,
},
"google_robot_pick_coke_can_vertical": {
"rt-2-x": 0.740,
"rt-1-converged": 0.900,
"rt-1-15pct": 0.790,
"rt-1-x": 0.330,
"rt-1-begin": 0.000,
"octo-base": 0.210,
"openvla-7b": 0.030,
},
"google_robot_pick_coke_can_standing": {
"rt-2-x": 0.880,
"rt-1-converged": 0.710,
"rt-1-15pct": 0.480,
"rt-1-x": 0.550,
"rt-1-begin": 0.030,
"octo-base": 0.090,
"openvla-7b": 0.190,
},
"google_robot_pick_coke_can": {
"rt-2-x": 0.787,
"rt-1-converged": 0.857,
"rt-1-15pct": 0.710,
"rt-1-x": 0.567,
"rt-1-begin": 0.027,
"octo-base": 0.170,
"openvla-7b": 0.177,
},
"google_robot_move_near": {
"rt-2-x": 0.779,
Expand All @@ -84,6 +153,7 @@
"rt-1-x": 0.317,
"rt-1-begin": 0.050,
"octo-base": 0.042,
"openvla-7b": 0.492,
},
"google_robot_open_drawer": {
"rt-2-x": 0.157,
Expand All @@ -92,6 +162,7 @@
"rt-1-x": 0.296,
"rt-1-begin": 0.000,
"octo-base": 0.009,
"openvla-7b": 0.250,
},
"google_robot_close_drawer": {
"rt-2-x": 0.343,
Expand All @@ -100,6 +171,16 @@
"rt-1-x": 0.891,
"rt-1-begin": 0.278,
"octo-base": 0.444,
"openvla-7b": 0.574,
},
"google_robot_drawer": {
"rt-2-x": 0.250,
"rt-1-converged": 0.730,
"rt-1-15pct": 0.565,
"rt-1-x": 0.597,
"rt-1-begin": 0.139,
"octo-base": 0.227,
"openvla-7b": 0.412,
},
"google_robot_place_apple_in_closed_top_drawer": {
"rt-2-x": 0.037,
Expand All @@ -108,6 +189,7 @@
"rt-1-x": 0.213,
"rt-1-begin": 0.000,
"octo-base": 0.000,
"openvla-7b": 0.000,
},
"widowx_spoon_on_towel": {
"rt-1-x": 0.000,
Expand Down
15 changes: 15 additions & 0 deletions tools/calc_metrics_evaluation_videos.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def calc_pick_coke_can_stats(root_result_dir):
"rt-1-x": 0.88,
"rt-1-begin": 0.20,
"octo-base": 0.44,
"openvla-7b": 0.64,
},
"vertical": {
"rt-2-x": 0.80,
Expand All @@ -35,6 +36,7 @@ def calc_pick_coke_can_stats(root_result_dir):
"rt-1-x": 0.56,
"rt-1-begin": 0.00,
"octo-base": 0.20,
"openvla-7b": 0.28,
},
"standing": {
"rt-2-x": 1.00,
Expand All @@ -43,6 +45,7 @@ def calc_pick_coke_can_stats(root_result_dir):
"rt-1-x": 0.84,
"rt-1-begin": 0.20,
"octo-base": 0.24,
"openvla-7b": 0.36,
},
}

Expand Down Expand Up @@ -282,6 +285,7 @@ def calc_move_near_stats(root_result_dir):
"rt-1-x": 0.45,
"rt-1-begin": 0.017,
"octo-base": 0.35,
"openvla-7b": 0.667,
}

ckpt_alias_keys = list(move_near_real_success.keys())
Expand Down Expand Up @@ -413,6 +417,7 @@ def calc_drawer_stats(root_result_dir):
"rt-1-x": 0.519,
"rt-1-begin": 0.000,
"octo-base": 0.148,
"openvla-7b": 0.111,
},
"close": {
"rt-2-x": 0.630,
Expand All @@ -421,6 +426,7 @@ def calc_drawer_stats(root_result_dir):
"rt-1-x": 0.741,
"rt-1-begin": 0.000,
"octo-base": 0.519,
"openvla-7b": 0.148,
},
}

Expand Down Expand Up @@ -642,6 +648,7 @@ def calc_long_horizon_apple_in_drawer_stats(root_result_dir):
"rt-1-x": 0.407,
"rt-1-begin": 0.000,
"octo-base": 0.000,
"openvla-7b": 0.000,
},
}

Expand Down Expand Up @@ -855,39 +862,46 @@ def calc_bridge_put_on_env_stats(root_result_dir):
"rt-1-x": 0.042,
"octo-base": 0.500,
"octo-small": 0.542,
"openvla-7b": 0.10,
},
"put_carrot_on_plate": {
"rt-1-x": 0.167,
"octo-base": 0.500,
"octo-small": 0.208,
"openvla-7b": 0.10,
},
"stack_green_block_on_yellow_block": {
"rt-1-x": 0.000,
"octo-base": 0.292,
"octo-small": 0.583,
"openvla-7b": 0.10,
},
"put_eggplant_in_basket": {
"rt-1-x": 0.000,
"octo-base": 0.400,
"octo-small": 0.600,
"openvla-7b": 0.10,
},
}
real_success_dict = {
"put_spoon_on_tablecloth": {
"rt-1-x": 0.000,
"octo-base": 0.333,
"octo-small": 0.417,
"openvla-7b": 0.10,
},
"put_carrot_on_plate": {"rt-1-x": 0.00, "octo-base": 0.25, "octo-small": 0.083},
"stack_green_block_on_yellow_block": {
"rt-1-x": 0.000,
"octo-base": 0.000,
"octo-small": 0.125,
"openvla-7b": 0.10,
},
"put_eggplant_in_basket": {
"rt-1-x": 0.000,
"octo-base": 0.250,
"octo-small": 0.400,
"openvla-7b": 0.10,
},
}

Expand Down Expand Up @@ -1023,6 +1037,7 @@ def calc_bridge_put_on_env_stats(root_result_dir):
"octo-base": "octo-base",
"octo-small": "octo-small",
"octo-server": "octo-server",
"openvla-7b": "openvla-7b",
}

parser = argparse.ArgumentParser()
Expand Down

0 comments on commit 37d8fa8

Please sign in to comment.