From 301f3947d78b36ad360946453d2edc0021d0af15 Mon Sep 17 00:00:00 2001
From: Amal Nanavati <amaln@uw.edu>
Date: Mon, 18 Dec 2023 18:23:41 -0800
Subject: [PATCH 1/4] Remove dependency on web video server

---
 .../DummyRealSense.py                         |  4 ++
 .../FaceDetection.py                          | 21 +++++---
 .../SegmentFromPoint.py                       | 14 +++--
 .../feeding_web_app_dummy_nodes_launch.xml    |  5 +-
 feedingwebapp/src/App.jsx                     | 19 +++----
 feedingwebapp/src/Pages/Constants.js          |  4 +-
 feedingwebapp/src/Pages/Header/Header.jsx     | 11 +---
 .../src/Pages/Header/LiveVideoModal.jsx       |  7 ++-
 feedingwebapp/src/Pages/Home/Home.jsx         | 12 ++---
 .../Pages/Home/MealStates/BiteSelection.jsx   | 21 ++------
 .../Pages/Home/MealStates/DetectingFace.jsx   | 18 +------
 .../Pages/Home/MealStates/PlateLocator.jsx    | 18 +------
 feedingwebapp/src/Pages/Home/VideoFeed.jsx    | 53 ++++++++++++++++---
 feedingwebapp/src/buttons/MaskButton.jsx      | 16 ++----
 14 files changed, 103 insertions(+), 120 deletions(-)

diff --git a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py
index 858e4bfa..d0efdaeb 100755
--- a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py
+++ b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py
@@ -85,6 +85,9 @@ def __init__(self):
         self.camera_info_publisher = self.create_publisher(
             CameraInfo, "~/camera_info", 1
         )
+        self.aligned_depth_camera_info_publisher = self.create_publisher(
+            CameraInfo, "~/aligned_depth/camera_info", 1
+        )
         if self.video is not None:
             self.num_frames = 0
         self.bridge = CvBridge()
@@ -174,6 +177,7 @@ def publish_frames(self):
             self.compressed_image_publisher.publish(compressed_frame_msg)
             self.aligned_depth_publisher.publish(depth_frame_msg)
             self.camera_info_publisher.publish(self.camera_info_msg)
+            self.aligned_depth_camera_info_publisher.publish(self.camera_info_msg)
 
             rate.sleep()
 
diff --git a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py
index da9ead4b..cd5da709 100755
--- a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py
+++ b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py
@@ -7,7 +7,7 @@
 import numpy as np
 import rclpy
 from rclpy.node import Node
-from sensor_msgs.msg import Image
+from sensor_msgs.msg import CompressedImage
 from threading import Lock
 
 
@@ -65,7 +65,10 @@ def __init__(
 
         # Subscribe to the camera feed
         self.subscription = self.create_subscription(
-            Image, "camera/color/image_raw", self.camera_callback, 1
+            CompressedImage,
+            "camera/color/image_raw/compressed",
+            self.camera_callback,
+            1,
         )
         self.subscription  # prevent unused variable warning
 
@@ -73,7 +76,9 @@ def __init__(
         self.publisher_results = self.create_publisher(
             FaceDetection, "face_detection", 1
         )
-        self.publisher_image = self.create_publisher(Image, "face_detection_img", 1)
+        self.publisher_image = self.create_publisher(
+            CompressedImage, "face_detection_img/compressed", 1
+        )
 
     def toggle_face_detection_callback(self, request, response):
         """
@@ -150,15 +155,16 @@ def camera_callback(self, msg):
             face_detection_msg.is_face_detected = is_face_detected
             if is_face_detected:
                 # Add a dummy face marker to the sensor_msgs/Image
-                cv_image = self.bridge.imgmsg_to_cv2(msg, "bgr8")
+                cv_image = self.bridge.compressed_imgmsg_to_cv2(msg, "bgr8")
+                height, width, _ = cv_image.shape
                 cv2.circle(
                     cv_image,
-                    (msg.width // 2, msg.height // 2),
-                    msg.height // 25,
+                    (width // 2, height // 2),
+                    height // 25,
                     (0, 0, 255),
                     -1,
                 )
-                annotated_msg = self.bridge.cv2_to_imgmsg(cv_image, "bgr8")
+                annotated_msg = self.bridge.cv2_to_compressed_imgmsg(cv_image, "jpeg")
                 annotated_img = annotated_msg
                 # Publish the detected mouth center. The below is a hardcoded
                 # rough position of the mouth from the side staging location,
@@ -185,7 +191,6 @@ def camera_callback(self, msg):
                 annotated_img = msg
             face_detection_msg.is_mouth_open = open_mouth_detected
             self.publisher_results.publish(face_detection_msg)
-            self.get_logger().info("Published face detection")
             self.publisher_image.publish(annotated_img)
 
 
diff --git a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py
index 0bec0a45..c330d7ef 100644
--- a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py
+++ b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py
@@ -9,7 +9,7 @@
 from rclpy.action import ActionServer, CancelResponse, GoalResponse
 from rclpy.executors import MultiThreadedExecutor
 from rclpy.node import Node
-from sensor_msgs.msg import CompressedImage, Image, RegionOfInterest
+from sensor_msgs.msg import CompressedImage, RegionOfInterest
 from shapely.geometry import MultiPoint
 import threading
 import time
@@ -34,8 +34,8 @@ def __init__(self, sleep_time=2.0, send_feedback_hz=10):
 
         # Subscribe to the image topic, to store the latest image
         self.image_subscriber = self.create_subscription(
-            Image,
-            "/camera/color/image_raw",
+            CompressedImage,
+            "/camera/color/image_raw/compressed",
             self.image_callback,
             1,
         )
@@ -123,8 +123,8 @@ def segment_image(self, seed_point, result, segmentation_success):
         with self.latest_img_msg_lock:
             latest_img_msg = self.latest_img_msg
         result.header = latest_img_msg.header
-        width = latest_img_msg.width
-        height = latest_img_msg.height
+        img = self.bridge.compressed_imgmsg_to_cv2(latest_img_msg, "bgr8")
+        width, height, _ = img.shape
 
         # Sleep (dummy segmentation)
         time.sleep(self.sleep_time)
@@ -175,6 +175,10 @@ def segment_image(self, seed_point, result, segmentation_success):
                 format="jpeg",
                 data=cv2.imencode(".jpg", mask_img)[1].tostring(),
             )
+            mask_msg.rgb_image = CompressedImage(
+                format="jpeg",
+                data=cv2.imencode(".jpg", img[y_min:y_max, x_min:x_max])[1].tostring(),
+            )
             mask_msg.item_id = "dummy_food_id_%d" % (i)
             mask_msg.confidence = np.random.random()
             result.detected_items.append(mask_msg)
diff --git a/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml b/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml
index 467a9a47..ec32a668 100644
--- a/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml
+++ b/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml
@@ -10,8 +10,6 @@
   <group if="$(var run_web_bridge)">
     <!-- The ROSBridge Node -->
     <include file="$(find-pkg-share rosbridge_server)/launch/rosbridge_websocket_launch.xml"/>
-    <!-- The ROS web_video_server -->
-    <node pkg="web_video_server" exec="web_video_server" name="web_video_server"/>
   </group>
 
   <!-- The dummy nodes -->
@@ -23,8 +21,9 @@
       <remap from="~/compressed_image" to="/camera/color/image_raw/compressed"/>
       <remap from="~/aligned_depth" to="/camera/aligned_depth_to_color/image_raw"/>
       <remap from="~/camera_info" to="/camera/color/camera_info"/>
+      <remap from="~/aligned_depth/camera_info" to="/camera/aligned_depth_to_color/camera_info"/>
       <param name="fps" value="30"/>
-      <param name="rgb_path" value="$(find-pkg-share feeding_web_app_ros2_test)/../data/above_plate_1_rgb.jpg"/>
+      <param name="rgb_path" value="$(find-pkg-share feeding_web_app_ros2_test)/../data/2022_11_01_ada_picks_up_carrots_camera_compressed_ft_tf.mp4"/>
       <param name="depth_path" value="$(find-pkg-share feeding_web_app_ros2_test)/../data/above_plate_1_depth.png"/>
     </node>
   </group>
diff --git a/feedingwebapp/src/App.jsx b/feedingwebapp/src/App.jsx
index 7c0290eb..779b7a63 100644
--- a/feedingwebapp/src/App.jsx
+++ b/feedingwebapp/src/App.jsx
@@ -22,23 +22,22 @@ import BiteSelectionPointMask from './Pages/Home/BiteSelectionUIStates/BiteSelec
  * @param {APP_PAGE} appPage - The current app page. Must be one of the
  *        states specified in APP_PAGE.
  * @param {string} rosbridgeURL - The URL of the rosbridge server.
- * @param {string} webVideoServerURL - The URL of the web_video_server.
  * @param {bool} debug - Whether to run it in debug mode or not.
  */
-function getComponentByAppPage(appPage, rosbridgeURL, webVideoServerURL, debug) {
+function getComponentByAppPage(appPage, rosbridgeURL, debug) {
   switch (appPage) {
     case APP_PAGE.Home:
       // Must wrap a component in ROS tags for it to be able to connect to ROS
       return (
         <RosConnection url={rosbridgeURL} autoConnect>
-          <Header webVideoServerURL={webVideoServerURL} />
-          <Home debug={debug} webVideoServerURL={webVideoServerURL} />
+          <Header />
+          <Home debug={debug} />
         </RosConnection>
       )
     case APP_PAGE.Settings:
       return (
         <RosConnection url={rosbridgeURL} autoConnect>
-          <Header webVideoServerURL={webVideoServerURL} />
+          <Header />
           <Settings />
         </RosConnection>
       )
@@ -57,8 +56,6 @@ function App() {
 
   // Get the rosbridge URL
   const rosbridgeURL = 'ws://'.concat(window.location.hostname, ':', process.env.REACT_APP_ROSBRIDGE_PORT)
-  // Get the web_video_server URL
-  const webVideoServerURL = 'http://'.concat(window.location.hostname, ':', process.env.REACT_APP_WEB_VIDEO_SERVER_PORT)
 
   // Get the debug flag
   const debug = process.env.REACT_APP_DEBUG === 'true'
@@ -68,7 +65,7 @@ function App() {
     <>
       <Router>
         <Routes>
-          <Route exact path='/' element={getComponentByAppPage(appPage, rosbridgeURL, webVideoServerURL, debug)} />
+          <Route exact path='/' element={getComponentByAppPage(appPage, rosbridgeURL, debug)} />
           <Route
             exact
             path='/test_ros'
@@ -83,7 +80,7 @@ function App() {
             path='/test_bite_selection_ui/button_overlay_selection'
             element={
               <RosConnection url={rosbridgeURL} autoConnect>
-                <Header webVideoServerURL={webVideoServerURL} />
+                <Header />
                 <BiteSelectionButtonOverlay debug={debug} />
               </RosConnection>
             }
@@ -93,7 +90,7 @@ function App() {
             path='/test_bite_selection_ui/point_mask_selection'
             element={
               <RosConnection url={rosbridgeURL} autoConnect>
-                <Header webVideoServerURL={webVideoServerURL} />
+                <Header />
                 <BiteSelectionPointMask debug={debug} />
               </RosConnection>
             }
@@ -103,7 +100,7 @@ function App() {
             path='/test_bite_selection_ui/food_name_selection'
             element={
               <RosConnection url={rosbridgeURL} autoConnect>
-                <Header webVideoServerURL={webVideoServerURL} />
+                <Header />
                 <BiteSelectionName debug={debug} />
               </RosConnection>
             }
diff --git a/feedingwebapp/src/Pages/Constants.js b/feedingwebapp/src/Pages/Constants.js
index ebe256fe..dd6ede91 100644
--- a/feedingwebapp/src/Pages/Constants.js
+++ b/feedingwebapp/src/Pages/Constants.js
@@ -48,10 +48,10 @@ NON_MOVING_STATES.add(MEAL_STATE.U_PostMeal)
 export { NON_MOVING_STATES }
 
 // The names of the ROS topic(s)
-export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw'
+export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw/compressed'
 export const FACE_DETECTION_TOPIC = '/face_detection'
 export const FACE_DETECTION_TOPIC_MSG = 'ada_feeding_msgs/FaceDetection'
-export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img'
+export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img/compressed'
 
 // States from which, if they fail, it is NOT okay for the user to retry the
 // same action.
diff --git a/feedingwebapp/src/Pages/Header/Header.jsx b/feedingwebapp/src/Pages/Header/Header.jsx
index 63f47f19..5e32d6d8 100644
--- a/feedingwebapp/src/Pages/Header/Header.jsx
+++ b/feedingwebapp/src/Pages/Header/Header.jsx
@@ -4,9 +4,6 @@ import React, { useCallback, useEffect, useState } from 'react'
 import Navbar from 'react-bootstrap/Navbar'
 import Nav from 'react-bootstrap/Nav'
 import { useMediaQuery } from 'react-responsive'
-// PropTypes is used to validate that the used props are in fact passed to this
-// Component
-import PropTypes from 'prop-types'
 // Toast generates a temporary pop-up with a timeout.
 import { ToastContainer /* , toast */ } from 'react-toastify'
 import 'react-toastify/dist/ReactToastify.css'
@@ -23,7 +20,7 @@ import LiveVideoModal from './LiveVideoModal'
  * clicking "Video", and the ToastContainer popup that specifies when the user
  * cannot click Settings.
  */
-const Header = (props) => {
+const Header = () => {
   // Create a local state variable to toggle on/off the video
   // TODO: Since this local state variable is in the header, the LiveVideoModal
   // continues showing even if the state changes. Is this desirable? Perhaps
@@ -166,13 +163,9 @@ const Header = (props) => {
        * The LiveVideoModal toggles on and off with the Video button and shows the
        * robot's live camera feed.
        */}
-      <LiveVideoModal webVideoServerURL={props.webVideoServerURL} show={videoShow} onHide={() => setVideoShow(false)} />
+      <LiveVideoModal show={videoShow} onHide={() => setVideoShow(false)} />
     </>
   )
 }
-Header.propTypes = {
-  // The URL of the ROS web video server
-  webVideoServerURL: PropTypes.string.isRequired
-}
 
 export default Header
diff --git a/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx b/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx
index a46c95b7..23f92c97 100644
--- a/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx
+++ b/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx
@@ -49,7 +49,8 @@ function LiveVideoModal(props) {
       <Modal.Body ref={modalBodyRef} style={{ overflow: 'hidden' }}>
         <center>
           <VideoFeed
-            webVideoServerURL={props.webVideoServerURL}
+            topic='/local/camera/color/image_raw/compressed'
+            updateRateHz={10}
             parent={modalBodyRef}
             marginTop={margin}
             marginBottom={margin}
@@ -65,9 +66,7 @@ LiveVideoModal.propTypes = {
   // Whether or not the modal is visible
   show: PropTypes.bool.isRequired,
   // Callback function for when the modal is hidden
-  onHide: PropTypes.func.isRequired,
-  // The URL of the ROS web video server
-  webVideoServerURL: PropTypes.string.isRequired
+  onHide: PropTypes.func.isRequired
 }
 
 export default LiveVideoModal
diff --git a/feedingwebapp/src/Pages/Home/Home.jsx b/feedingwebapp/src/Pages/Home/Home.jsx
index da43c4ad..f2cf8839 100644
--- a/feedingwebapp/src/Pages/Home/Home.jsx
+++ b/feedingwebapp/src/Pages/Home/Home.jsx
@@ -22,7 +22,6 @@ import { TIME_TO_RESET_MS } from '../Constants'
  *
  * @param {boolean} debug - whether to run it in debug mode (e.g., if you aren't
  *        simulatenously running the robot) or not
- * @param {string} webVideoServerURL - the URL of the web video server
  * @returns {JSX.Element}
  */
 function Home(props) {
@@ -94,10 +93,10 @@ function Home(props) {
         )
       }
       case MEAL_STATE.U_BiteSelection: {
-        return <BiteSelection debug={props.debug} webVideoServerURL={props.webVideoServerURL} />
+        return <BiteSelection debug={props.debug} />
       }
       case MEAL_STATE.U_PlateLocator: {
-        return <PlateLocator debug={props.debug} webVideoServerURL={props.webVideoServerURL} />
+        return <PlateLocator debug={props.debug} />
       }
       case MEAL_STATE.R_BiteAcquisition: {
         /**
@@ -153,7 +152,7 @@ function Home(props) {
         )
       }
       case MEAL_STATE.R_DetectingFace: {
-        return <DetectingFace debug={props.debug} webVideoServerURL={props.webVideoServerURL} />
+        return <DetectingFace debug={props.debug} />
       }
       case MEAL_STATE.R_MovingToMouth: {
         /**
@@ -254,7 +253,6 @@ function Home(props) {
   }, [
     mealState,
     props.debug,
-    props.webVideoServerURL,
     biteAcquisitionActionInput,
     moveAbovePlateActionInput,
     moveToMouthActionInput,
@@ -278,9 +276,7 @@ Home.propTypes = {
    * Whether to run it in debug mode (e.g., if you aren't simulatenously running
    * the robot) or not
    */
-  debug: PropTypes.bool,
-  // The URL of the web video server
-  webVideoServerURL: PropTypes.string.isRequired
+  debug: PropTypes.bool
 }
 
 export default Home
diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
index 0fa9b1ab..91659c9f 100644
--- a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
+++ b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
@@ -13,10 +13,7 @@ import { useROS, createROSActionClient, callROSAction, destroyActionClient } fro
 import { useWindowSize, convertRemToPixels } from '../../../helpers'
 import MaskButton from '../../../buttons/MaskButton'
 import {
-  REALSENSE_WIDTH,
-  REALSENSE_HEIGHT,
   ROS_ACTIONS_NAMES,
-  CAMERA_FEED_TOPIC,
   ROS_ACTION_STATUS_CANCEL_GOAL,
   ROS_ACTION_STATUS_EXECUTE,
   ROS_ACTION_STATUS_SUCCEED,
@@ -328,20 +325,13 @@ const BiteSelection = (props) => {
         let maskScaleFactor = Math.min(widthScaleFactor, heightScaleFactor)
         // maskScaleFactor  = Math.min(maskScaleFactor, 1.0)
 
-        // Get the URL of the image based on the scale factor
-        let imgSize = {
-          width: Math.round(REALSENSE_WIDTH * maskScaleFactor),
-          height: Math.round(REALSENSE_HEIGHT * maskScaleFactor)
-        }
-        let imgSrc = `${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgSize.width}&height=${imgSize.height}&type=ros_compressed`
         return (
           <View style={{ flexDirection: 'row', justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
             {actionResult.detected_items.map((detected_item, i) => (
               <View key={i} style={{ flex: 1, justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
                 <MaskButton
-                  imgSrc={imgSrc}
+                  imgSrc={'data:image/jpeg;base64,' + detected_item.rgb_image.data}
                   buttonSize={buttonSize}
-                  imgSize={imgSize}
                   maskSrc={'data:image/jpeg;base64,' + detected_item.mask.data}
                   invertMask={true}
                   maskScaleFactor={maskScaleFactor}
@@ -355,7 +345,7 @@ const BiteSelection = (props) => {
         )
       }
     }
-  }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, props.webVideoServerURL, margin])
+  }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, margin])
 
   /** Get the button for continue without acquiring bite
    *
@@ -480,12 +470,12 @@ const BiteSelection = (props) => {
               style={{
                 flex: 9,
                 alignItems: 'center',
+                justifyContent: 'center',
                 width: '100%',
                 height: '100%'
               }}
             >
               <VideoFeed
-                webVideoServerURL={props.webVideoServerURL}
                 parent={videoParentRef}
                 marginTop={margin}
                 marginBottom={margin}
@@ -576,7 +566,6 @@ const BiteSelection = (props) => {
     actionStatusText,
     renderMaskButtons,
     skipAcquisisitionButton,
-    props.webVideoServerURL,
     videoParentRef,
     imageClicked,
     props.debug,
@@ -591,9 +580,7 @@ BiteSelection.propTypes = {
    * Whether to run it in debug mode (e.g., if you aren't simulatenously running
    * the robot) or not
    */
-  debug: PropTypes.bool.isRequired,
-  // The URL of the web video server
-  webVideoServerURL: PropTypes.string.isRequired
+  debug: PropTypes.bool.isRequired
 }
 
 export default BiteSelection
diff --git a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx
index 8dc12e6e..869f1a33 100644
--- a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx
+++ b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx
@@ -3,9 +3,6 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import Button from 'react-bootstrap/Button'
 import { useMediaQuery } from 'react-responsive'
 import { View } from 'react-native'
-// PropTypes is used to validate that the used props are in fact passed to this
-// Component
-import PropTypes from 'prop-types'
 
 // Local Imports
 import { useROS, createROSService, createROSServiceRequest, subscribeToROSTopic, unsubscribeFromROSTopic } from '../../../ros/ros_helpers'
@@ -26,7 +23,7 @@ import VideoFeed from '../VideoFeed'
  * configuration. It displays the output of face detection, and automatically
  * moves on to `R_MovingToMouth` when a face is detected.
  */
-const DetectingFace = (props) => {
+const DetectingFace = () => {
   // Keep track of whether a mouth has been detected or not
   const [mouthDetected, setMouthDetected] = useState(false)
   // Get the relevant global variables
@@ -203,14 +200,12 @@ const DetectingFace = (props) => {
               }}
             >
               <VideoFeed
-                webVideoServerURL={props.webVideoServerURL}
                 parent={videoParentRef}
                 marginTop={margin}
                 marginBottom={margin}
                 marginLeft={margin}
                 marginRight={margin}
                 topic={FACE_DETECTION_IMG_TOPIC}
-                type='mjpeg&quality=20'
               />
             </View>
           </View>
@@ -238,7 +233,6 @@ const DetectingFace = (props) => {
     mouthDetected,
     moveToMouthCallback,
     moveToMouthImage,
-    props.webVideoServerURL,
     textFontSize,
     buttonHeight,
     buttonWidth,
@@ -251,14 +245,4 @@ const DetectingFace = (props) => {
   // Render the component
   return fullPageView()
 }
-DetectingFace.propTypes = {
-  /**
-   * Whether to run it in debug mode (e.g., if you aren't simulatenously running
-   * the robot) or not
-   */
-  debug: PropTypes.bool.isRequired,
-  // The URL of the web video server
-  webVideoServerURL: PropTypes.string.isRequired
-}
-
 export default DetectingFace
diff --git a/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx b/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx
index bcb2dd93..ce2611ab 100644
--- a/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx
+++ b/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx
@@ -3,9 +3,6 @@ import React, { useCallback, useMemo, useRef } from 'react'
 import Button from 'react-bootstrap/Button'
 import { useMediaQuery } from 'react-responsive'
 import { View } from 'react-native'
-// PropTypes is used to validate that the used props are in fact passed to this
-// Component
-import PropTypes from 'prop-types'
 
 // Local Imports
 import '../Home.css'
@@ -20,7 +17,7 @@ import VideoFeed from '../VideoFeed'
  * the user to teleoperate the robot with Cartesian Control until the plate
  * is satisfactorily in view.
  */
-const PlateLocator = (props) => {
+const PlateLocator = () => {
   // Get the relevant global variables
   const setMealState = useGlobalState((state) => state.setMealState)
   // Get robot motion flag for plate locator
@@ -179,14 +176,7 @@ const PlateLocator = (props) => {
   return (
     <View style={{ flex: 'auto', flexDirection: dimension, justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
       <View ref={videoParentRef} style={{ flex: 5, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
-        <VideoFeed
-          webVideoServerURL={props.webVideoServerURL}
-          parent={videoParentRef}
-          marginTop={margin}
-          marginBottom={margin}
-          marginLeft={margin}
-          marginRight={margin}
-        />
+        <VideoFeed parent={videoParentRef} marginTop={margin} marginBottom={margin} marginLeft={margin} marginRight={margin} />
       </View>
       <View style={{ flex: 5, alignItems: 'center', justifyContent: 'center' }}>
         {directionalArrows()}
@@ -195,9 +185,5 @@ const PlateLocator = (props) => {
     </View>
   )
 }
-PlateLocator.propTypes = {
-  // The URL of the web video server
-  webVideoServerURL: PropTypes.string.isRequired
-}
 
 export default PlateLocator
diff --git a/feedingwebapp/src/Pages/Home/VideoFeed.jsx b/feedingwebapp/src/Pages/Home/VideoFeed.jsx
index 71299504..c8c75e64 100644
--- a/feedingwebapp/src/Pages/Home/VideoFeed.jsx
+++ b/feedingwebapp/src/Pages/Home/VideoFeed.jsx
@@ -1,11 +1,12 @@
 // React Imports
-import React, { useCallback, useEffect, useState } from 'react'
+import React, { useCallback, useEffect, useRef, useState } from 'react'
 // PropTypes is used to validate that the used props are in fact passed to this Component
 import PropTypes from 'prop-types'
 
 // Local Imports
 import { CAMERA_FEED_TOPIC, REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../Constants'
 import { useWindowSize } from '../../helpers'
+import { useROS, subscribeToROSTopic, unsubscribeFromROSTopic } from '../../ros/ros_helpers'
 
 /**
  * Takes in an imageWidth and imageHeight, and returns a width and height that
@@ -76,6 +77,46 @@ const VideoFeed = (props) => {
   const [imgWidth, setImgWidth] = useState(0)
   const [imgHeight, setImgHeight] = useState(0)
   const [scaleFactor, setScaleFactor] = useState(0.0)
+  // Store the latest image to render
+  const [latestRenderedImg, setLatestRenderedImg] = useState(null)
+
+  /**
+   * Connect to ROS, if not already connected. Put this in useRef to avoid
+   * re-connecting upon re-renders.
+   */
+  const ros = useRef(useROS().ros)
+  // Store the latest image timestamp in a ref to avoid re-generating cameraCallback
+  const latestImageTimestamp = useRef(null)
+
+  /**
+   * Subscribe to the image topic.
+   */
+  const cameraCallback = useCallback(
+    (message) => {
+      // console.log('Got camera message', message)
+      if (!latestImageTimestamp.current || props.updateRateHz <= 0) {
+        setLatestRenderedImg(message)
+      } else {
+        let currTime = message.header.stamp.sec + message.header.stamp.nanosec * 1e-9
+        if (currTime - latestImageTimestamp.current >= 1.0 / props.updateRateHz) {
+          setLatestRenderedImg(message)
+          latestImageTimestamp.current = currTime
+        }
+      }
+    },
+    [latestImageTimestamp, setLatestRenderedImg, props.updateRateHz]
+  )
+  useEffect(() => {
+    let topic = subscribeToROSTopic(ros.current, props.topic, 'sensor_msgs/CompressedImage', cameraCallback)
+    /**
+     * In practice, because the values passed in in the second argument of
+     * useEffect will not change on re-renders, this return statement will
+     * only be called when the component unmounts.
+     */
+    return () => {
+      unsubscribeFromROSTopic(topic, cameraCallback)
+    }
+  }, [cameraCallback, props.topic])
 
   // Callback to resize the image based on the parent width and height
   const resizeImage = useCallback(() => {
@@ -145,7 +186,7 @@ const VideoFeed = (props) => {
   // Render the component
   return (
     <img
-      src={`${props.webVideoServerURL}/stream?topic=${props.topic}&width=${imgWidth}&height=${imgHeight}&type=${props.type}`}
+      src={`data:image/jpeg;base64,${latestRenderedImg ? latestRenderedImg.data : ''}`}
       alt='Live video feed from the robot'
       style={{
         width: imgWidth,
@@ -161,8 +202,6 @@ const VideoFeed = (props) => {
 VideoFeed.propTypes = {
   // The ref to the parent DOM element. Null if the component is not yet mounted
   parent: PropTypes.object.isRequired,
-  // The URL of the video feed
-  webVideoServerURL: PropTypes.string.isRequired,
   // The margins around the video feed
   marginTop: PropTypes.number.isRequired,
   marginBottom: PropTypes.number.isRequired,
@@ -170,8 +209,8 @@ VideoFeed.propTypes = {
   marginRight: PropTypes.number.isRequired,
   // The topic of the video feed
   topic: PropTypes.string.isRequired,
-  // The type of the video feed
-  type: PropTypes.string.isRequired,
+  // The rate at which to update the video feed, in Hz
+  updateRateHz: PropTypes.number.isRequired,
   /**
    * An optional callback function for when the user clicks on the video feed.
    * This function should take in two parameters, `x` and `y`, which are the
@@ -182,7 +221,7 @@ VideoFeed.propTypes = {
 }
 VideoFeed.defaultProps = {
   topic: CAMERA_FEED_TOPIC,
-  type: 'ros_compressed'
+  updateRateHz: 10
 }
 
 export default VideoFeed
diff --git a/feedingwebapp/src/buttons/MaskButton.jsx b/feedingwebapp/src/buttons/MaskButton.jsx
index 862427b7..ae500199 100644
--- a/feedingwebapp/src/buttons/MaskButton.jsx
+++ b/feedingwebapp/src/buttons/MaskButton.jsx
@@ -31,7 +31,6 @@ function MaskButton(props) {
   // Get the properties
   let buttonSize = props.buttonSize
   let imgSrc = props.imgSrc
-  let imgSize = props.imgSize
   let maskSrc = props.maskSrc
   let invertMask = props.invertMask
   let maskScaleFactor = props.maskScaleFactor
@@ -67,15 +66,11 @@ function MaskButton(props) {
         >
           <img
             src={imgSrc}
-            alt='Live video feed from the robot'
+            alt='Robot camera view'
             style={{
-              width: imgSize.width,
-              height: imgSize.height,
+              width: maskBoundingBox.width * maskScaleFactor,
+              height: maskBoundingBox.height * maskScaleFactor,
               display: 'block',
-              margin: '-'.concat(
-                maskBoundingBox.y_offset * maskScaleFactor,
-                'px 0 0 -'.concat(maskBoundingBox.x_offset * maskScaleFactor, 'px')
-              ),
               pointerEvents: 'none'
             }}
           />
@@ -106,11 +101,6 @@ MaskButton.propTypes = {
   }).isRequired,
   // The image source URL
   imgSrc: PropTypes.string.isRequired,
-  // The image size
-  imgSize: PropTypes.shape({
-    width: PropTypes.number.isRequired,
-    height: PropTypes.number.isRequired
-  }).isRequired,
   // The mask source URL
   maskSrc: PropTypes.string.isRequired,
   // Whether or not to invert the mask

From dfd916f8ad41d0f8cd28179dbe3c804359249eee Mon Sep 17 00:00:00 2001
From: Amal Nanavati <amaln@cs.washington.edu>
Date: Wed, 20 Dec 2023 00:11:40 +0000
Subject: [PATCH 2/4] Unsubscribe before unload

---
 feedingwebapp/src/Pages/Home/VideoFeed.jsx | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/feedingwebapp/src/Pages/Home/VideoFeed.jsx b/feedingwebapp/src/Pages/Home/VideoFeed.jsx
index c8c75e64..266da69a 100644
--- a/feedingwebapp/src/Pages/Home/VideoFeed.jsx
+++ b/feedingwebapp/src/Pages/Home/VideoFeed.jsx
@@ -107,14 +107,21 @@ const VideoFeed = (props) => {
     [latestImageTimestamp, setLatestRenderedImg, props.updateRateHz]
   )
   useEffect(() => {
+    console.log('subscribing to img topic')
     let topic = subscribeToROSTopic(ros.current, props.topic, 'sensor_msgs/CompressedImage', cameraCallback)
+    const cleanup = () => {
+      console.log('unsubscribing from img topic')
+      unsubscribeFromROSTopic(topic, cameraCallback)
+    }
+    window.addEventListener('beforeunload', cleanup)
     /**
      * In practice, because the values passed in in the second argument of
      * useEffect will not change on re-renders, this return statement will
      * only be called when the component unmounts.
      */
     return () => {
-      unsubscribeFromROSTopic(topic, cameraCallback)
+      window.removeEventListener('beforeunload', cleanup)
+      cleanup()
     }
   }, [cameraCallback, props.topic])
 

From d0addbb2cc4d746a3d9bad9e6488fa754364a555 Mon Sep 17 00:00:00 2001
From: Amal Nanavati <amaln@uw.edu>
Date: Thu, 21 Dec 2023 10:37:18 -0800
Subject: [PATCH 3/4] Display mask middle and resize masks

---
 .../Pages/Home/MealStates/BiteSelection.jsx   | 41 +++++++++++--------
 feedingwebapp/src/buttons/MaskButton.jsx      | 36 +++++++++++++++-
 2 files changed, 59 insertions(+), 18 deletions(-)

diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
index 91659c9f..d877f647 100644
--- a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
+++ b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
@@ -52,6 +52,8 @@ const BiteSelection = (props) => {
   let textFontSize = isPortrait ? '2.5vh' : '2vw'
   // Indicator of how to arrange screen elements based on orientation
   let dimension = isPortrait ? 'column' : 'row'
+  // Whether to scale all the masks equally or not
+  const scaleMasksEqually = false
 
   /**
    * Create a local state variable to store the detected masks, the
@@ -284,17 +286,6 @@ const BiteSelection = (props) => {
     if (actionStatus.actionStatus === ROS_ACTION_STATUS_SUCCEED) {
       // If we have a result and there are detected items
       if (actionResult && actionResult.detected_items && actionResult.detected_items.length > 0) {
-        // Get the size of the largest mask
-        let [maxWidth, maxHeight] = [0, 0]
-        for (let detected_item of actionResult.detected_items) {
-          if (detected_item.roi.width > maxWidth) {
-            maxWidth = detected_item.roi.width
-          }
-          if (detected_item.roi.height > maxHeight) {
-            maxHeight = detected_item.roi.height
-          }
-        }
-
         // Get the allotted space per mask
         let parentWidth, parentHeight
         if (maskButtonParentRef.current) {
@@ -320,10 +311,26 @@ const BiteSelection = (props) => {
          * Determine how much to scale the masks so that the largest mask fits
          * into the alloted space.
          */
-        let widthScaleFactor = allottedSpaceWidth / maxWidth
-        let heightScaleFactor = allottedSpaceHeight / maxHeight
-        let maskScaleFactor = Math.min(widthScaleFactor, heightScaleFactor)
-        // maskScaleFactor  = Math.min(maskScaleFactor, 1.0)
+        // Get the size of the largest mask
+        let [maxWidth, maxHeight] = [0, 0]
+        if (scaleMasksEqually) {
+          for (let detected_item of actionResult.detected_items) {
+            if (detected_item.roi.width > maxWidth) {
+              maxWidth = detected_item.roi.width
+            }
+            if (detected_item.roi.height > maxHeight) {
+              maxHeight = detected_item.roi.height
+            }
+          }
+        }
+        // Create a list to contain the scale factors for each mask
+        let maskScaleFactors = []
+        for (let detected_item of actionResult.detected_items) {
+          let widthScaleFactor = allottedSpaceWidth / (scaleMasksEqually ? maxWidth : detected_item.roi.width)
+          let heightScaleFactor = allottedSpaceHeight / (scaleMasksEqually ? maxHeight : detected_item.roi.height)
+          let maskScaleFactor = Math.min(widthScaleFactor, heightScaleFactor)
+          maskScaleFactors.push(maskScaleFactor)
+        }
 
         return (
           <View style={{ flexDirection: 'row', justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
@@ -334,7 +341,7 @@ const BiteSelection = (props) => {
                   buttonSize={buttonSize}
                   maskSrc={'data:image/jpeg;base64,' + detected_item.mask.data}
                   invertMask={true}
-                  maskScaleFactor={maskScaleFactor}
+                  maskScaleFactor={maskScaleFactors[i]}
                   maskBoundingBox={detected_item.roi}
                   onClick={foodItemClicked}
                   value={i.toString()}
@@ -345,7 +352,7 @@ const BiteSelection = (props) => {
         )
       }
     }
-  }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, margin])
+  }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, margin, scaleMasksEqually])
 
   /** Get the button for continue without acquiring bite
    *
diff --git a/feedingwebapp/src/buttons/MaskButton.jsx b/feedingwebapp/src/buttons/MaskButton.jsx
index ae500199..de8c1a4e 100644
--- a/feedingwebapp/src/buttons/MaskButton.jsx
+++ b/feedingwebapp/src/buttons/MaskButton.jsx
@@ -1,5 +1,5 @@
 // React imports
-import React from 'react'
+import React, { useEffect, useRef } from 'react'
 import Button from 'react-bootstrap/Button'
 // PropTypes is used to validate that the used props are in fact passed to this Component
 import PropTypes from 'prop-types'
@@ -28,6 +28,9 @@ import PropTypes from 'prop-types'
  *
  */
 function MaskButton(props) {
+  // Get a reference for the canvas
+  const canvasRef = useRef(null)
+
   // Get the properties
   let buttonSize = props.buttonSize
   let imgSrc = props.imgSrc
@@ -38,6 +41,26 @@ function MaskButton(props) {
   let onClick = props.onClick
   let value = props.value
 
+  // Draw a red filled circle on the middle of the canvas
+  useEffect(() => {
+    // Get the canvas
+    const canvas = canvasRef.current
+    // Get the context
+    const ctx = canvas.getContext('2d')
+    // Clear the canvas
+    ctx.clearRect(0, 0, canvas.width, canvas.height)
+    // Draw a red filled circle
+    let radius = 5
+    ctx.beginPath()
+    ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI)
+    ctx.fillStyle = 'red'
+    ctx.fill()
+    // Cleanup function is to clear the canvas
+    return () => {
+      ctx.clearRect(0, 0, canvas.width, canvas.height)
+    }
+  }, [])
+
   return (
     <Button
       style={{ backgroundColor: invertMask ? 'white' : 'black', borderColor: invertMask ? 'black' : 'white', borderWidth: 'medium' }}
@@ -88,6 +111,17 @@ function MaskButton(props) {
               pointerEvents: 'none'
             }}
           />
+          <canvas
+            ref={canvasRef}
+            width={maskBoundingBox.width * maskScaleFactor}
+            height={maskBoundingBox.height * maskScaleFactor}
+            style={{
+              position: 'absolute',
+              top: '0px',
+              display: 'block',
+              pointerEvents: 'none'
+            }}
+          />
         </div>
       </div>
     </Button>

From 82d415548696814db52c50fa2ffde13863010481 Mon Sep 17 00:00:00 2001
From: Amal Nanavati <amaln@uw.edu>
Date: Thu, 21 Dec 2023 11:07:39 -0800
Subject: [PATCH 4/4] Re-render canvas on resize

---
 feedingwebapp/src/buttons/MaskButton.jsx | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/feedingwebapp/src/buttons/MaskButton.jsx b/feedingwebapp/src/buttons/MaskButton.jsx
index de8c1a4e..77cd91bb 100644
--- a/feedingwebapp/src/buttons/MaskButton.jsx
+++ b/feedingwebapp/src/buttons/MaskButton.jsx
@@ -1,8 +1,9 @@
 // React imports
-import React, { useEffect, useRef } from 'react'
+import React, { useCallback, useEffect, useRef } from 'react'
 import Button from 'react-bootstrap/Button'
 // PropTypes is used to validate that the used props are in fact passed to this Component
 import PropTypes from 'prop-types'
+import { useWindowSize } from '../helpers'
 
 /**
  * A component that renders an image and a mask on top of it, all within a
@@ -42,7 +43,7 @@ function MaskButton(props) {
   let value = props.value
 
   // Draw a red filled circle on the middle of the canvas
-  useEffect(() => {
+  const drawCircle = useCallback(() => {
     // Get the canvas
     const canvas = canvasRef.current
     // Get the context
@@ -55,12 +56,20 @@ function MaskButton(props) {
     ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI)
     ctx.fillStyle = 'red'
     ctx.fill()
-    // Cleanup function is to clear the canvas
-    return () => {
-      ctx.clearRect(0, 0, canvas.width, canvas.height)
-    }
   }, [])
 
+  // Draw a red filled circle on the middle of the canvas
+  useEffect(() => {
+    // Dummy log necessary to have useEffect re-run when the props change
+    console.log('props.maskBoundingBox', props.maskBoundingBox)
+
+    // Draw the circle
+    drawCircle()
+  }, [drawCircle, props.maskBoundingBox])
+
+  // Redraw the circle when the window size changes
+  useWindowSize(drawCircle)
+
   return (
     <Button
       style={{ backgroundColor: invertMask ? 'white' : 'black', borderColor: invertMask ? 'black' : 'white', borderWidth: 'medium' }}