From 818e5031621eb5ee9ef758a3ffb8a88bc175f58f Mon Sep 17 00:00:00 2001 From: Amal Nanavati Date: Wed, 28 Jun 2023 18:58:53 -0700 Subject: [PATCH] fixed video scaling responsiveness --- .../src/Pages/Header/LiveVideoModal.jsx | 65 +- .../ImageWithButtonName.jsx | 8 +- .../ImageWithButtonOverlay.jsx | 13 +- .../ImageWithPointMask.jsx | 8 +- .../Pages/Home/MealStates/BiteSelection.jsx | 586 ++++++++++-------- .../Pages/Home/MealStates/PlateLocator.jsx | 51 +- feedingwebapp/src/Pages/Home/VideoFeed.jsx | 173 ++++++ feedingwebapp/src/helpers.js | 99 +-- 8 files changed, 553 insertions(+), 450 deletions(-) create mode 100644 feedingwebapp/src/Pages/Home/VideoFeed.jsx diff --git a/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx b/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx index bf45ba45..33cccfa9 100644 --- a/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx +++ b/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx @@ -1,5 +1,5 @@ // React imports -import React, { useEffect, useRef, useState } from 'react' +import React, { useRef } from 'react' import { useMediaQuery } from 'react-responsive' // The Modal is a screen that appears on top of the main app, and can be toggled // on and off. @@ -9,8 +9,8 @@ import Modal from 'react-bootstrap/Modal' import PropTypes from 'prop-types' // Local imports -import { REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../Constants' -import { useWindowSize, convertRemToPixels, scaleWidthHeightToWindow, showVideo } from '../../helpers' +import { convertRemToPixels } from '../../helpers' +import VideoFeed from '../Home/VideoFeed' /** * The LiveVideoModal displays to the user the live video feed from the robot. @@ -18,52 +18,13 @@ import { useWindowSize, convertRemToPixels, scaleWidthHeightToWindow, showVideo * TODO: Consider what will happen if the connection to ROS isn't working. */ function LiveVideoModal(props) { - const ref = useRef(null) - // Use the default CSS properties of Modals to determine the margin around - // the image. This is necessary so the image is scaled to fit the window. - // - // NOTE: This must change if the CSS properties of the Modal change. - // - // marginTop: bs-modal-header-padding, h4 font size & line height, bs-modal-header-padding, bs-modal-padding - const marginTop = convertRemToPixels(1 + 1.5 * 1.5 + 1 + 1) - const marginBottom = convertRemToPixels(1) - const marginLeft = convertRemToPixels(1) - const marginRight = convertRemToPixels(1) + // Variables to render the VideoFeed + const modalBodyRef = useRef(null) + const margin = convertRemToPixels(1) - // Get current window size - let windowSize = useWindowSize() - // Define variables for width and height of image - const [imgWidth, setImgWidth] = useState(windowSize.width) - const [imgHeight, setImgHeight] = useState(windowSize.height) // Flag to check if the current orientation is portrait const isPortrait = useMediaQuery({ query: '(orientation: portrait)' }) - /** Factors to modify video size in landscape to fit space - * - * TODO: Adjust it accordingly when flexbox with directional arrows implemented - */ - let landscapeSizeFactor = 0.9 - - // Get final image size according to screen orientation - let finalImgWidth = isPortrait ? imgWidth : landscapeSizeFactor * imgWidth - let finalImgHeight = isPortrait ? imgHeight : landscapeSizeFactor * imgHeight - - // Update the image size when the screen changes size. - useEffect(() => { - // 640 x 480 is the standard dimension of images outputed by the RealSense - let { width: widthUpdate, height: heightUpdate } = scaleWidthHeightToWindow( - windowSize, - REALSENSE_WIDTH, - REALSENSE_HEIGHT, - marginTop, - marginBottom, - marginLeft, - marginRight - ) - setImgWidth(widthUpdate) - setImgHeight(heightUpdate) - }, [windowSize, marginTop, marginBottom, marginLeft, marginRight]) - return ( @@ -82,8 +42,17 @@ function LiveVideoModal(props) { Live Video - -
{showVideo(props.webVideoServerURL, finalImgWidth, finalImgHeight, null)}
+ +
+ +
) diff --git a/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonName.jsx b/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonName.jsx index 604d9a95..5d29ae3a 100644 --- a/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonName.jsx +++ b/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonName.jsx @@ -3,7 +3,6 @@ import React from 'react' import PropTypes from 'prop-types' import Button from 'react-bootstrap/Button' import Row from 'react-bootstrap/Row' -import { scaleWidthHeightToWindow } from '../../../helpers' import '../Button.css' @@ -20,8 +19,11 @@ import '../Button.css' * image */ const ImageWithButtonName = (props) => { - const width = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).width - const height = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).height + // NOTE: The width and height here may be broken, due to changes in and + // then deprecation of the `scaleWidthHeightToWindow` function. Changes were + // made (resulting in the below code) but not tested. + const width = props.imgWidth + const height = props.imgWidth return ( <> diff --git a/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonOverlay.jsx b/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonOverlay.jsx index 9fd11f37..3d173f11 100644 --- a/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonOverlay.jsx +++ b/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithButtonOverlay.jsx @@ -1,8 +1,8 @@ // React Imports import React from 'react' import Button from 'react-bootstrap/Button' -import { scaleWidthHeightToWindow } from '../../../helpers' import PropTypes from 'prop-types' +import { REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../../Constants' import '../Button.css' @@ -23,9 +23,14 @@ import '../Button.css' */ const ImageWithButtonOverlay = (props) => { - const width = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).width - const height = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).height - const scaleFactor = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).scaleFactor + // NOTE: The width and height here may be broken, due to changes in and + // then deprecation of the `scaleWidthHeightToWindow` function. Changes were + // made (resulting in the below code) but not tested. + const width = props.imgWidth + const height = props.imgHeight + let scaleFactorWidth = width / REALSENSE_WIDTH + let scaleFactorHeight = height / REALSENSE_HEIGHT + let scaleFactor = (scaleFactorWidth + scaleFactorHeight) / 2 return ( <> diff --git a/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithPointMask.jsx b/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithPointMask.jsx index ea9c9da5..4845e5c8 100644 --- a/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithPointMask.jsx +++ b/feedingwebapp/src/Pages/Home/BiteSelectionUIStates/ImageWithPointMask.jsx @@ -2,7 +2,6 @@ import React, { useState } from 'react' import Button from 'react-bootstrap/Button' import Row from 'react-bootstrap/Row' -import { scaleWidthHeightToWindow } from '../../../helpers' import PropTypes from 'prop-types' import '../Button.css' @@ -18,8 +17,11 @@ import '../Button.css' * the realsense camera */ const ImageWithPointMask = (props) => { - const width = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).width - const height = scaleWidthHeightToWindow(props.imgWidth, props.imgHeight, 0, 0, 0, 0).height + // NOTE: The width and height here may be broken, due to changes in and + // then deprecation of the `scaleWidthHeightToWindow` function. Changes were + // made (resulting in the below code) but not tested. + const width = props.imgWidth + const height = props.imgHeight const [foodMasksToDisplay, setFoodMasksToDisplay] = useState([]) const imageClicked = (event) => { diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx index 6309cc39..dfefaad8 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx @@ -10,7 +10,7 @@ import PropTypes from 'prop-types' // Local Imports import '../Home.css' import { useROS, createROSActionClient, callROSAction, destroyActionClient } from '../../../ros/ros_helpers' -import { useWindowSize, convertRemToPixels, scaleWidthHeightToWindow, showVideo } from '../../../helpers' +import { useWindowSize, convertRemToPixels } from '../../../helpers' import MaskButton from '../../../buttons/MaskButton' import { REALSENSE_WIDTH, @@ -26,6 +26,7 @@ import { MOVING_STATE_ICON_DICT } from '../../Constants' import { useGlobalState, MEAL_STATE } from '../../GlobalState' +import VideoFeed from '../VideoFeed' /** * The BiteSelection component appears after the robot has moved above the plate, @@ -40,23 +41,18 @@ const BiteSelection = (props) => { const setDesiredFoodItem = useGlobalState((state) => state.setDesiredFoodItem) // Get icon image for move to mouth let moveToMouthImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToMouth] - // Factors to modify video and mask size in landscape and portrait to fit space - let landscapeSizeFactor = 0.43 - let portraitSizeFactor = 0.95 - // Factor to modify mask button size in landscape and portrait with regards to window size - let maskButtonSizeLandscapeFactor = 0.09 - let maskButtonSizePortraitFactor = 0.1 + + // Reference to the DOM element of the parent of the video feed + const videoParentRef = useRef(null) + // Margin for the video feed and between the mask buttons + const margin = convertRemToPixels(1) + // Flag to check if the current orientation is portrait const isPortrait = useMediaQuery({ query: '(orientation: portrait)' }) // Text font size let textFontSize = isPortrait ? '2.5vh' : '2vw' // Indicator of how to arrange screen elements based on orientation let dimension = isPortrait ? 'column' : 'row' - // Image and margin for phantom view to replace action status elements - let phantomImage = '/robot_state_imgs/phantom_view_image.svg' - let phantomMargin = '1px' - - const [showActionText, setShowActionText] = useState(false) /** * Create a local state variable to store the detected masks, the @@ -106,38 +102,8 @@ const BiteSelection = (props) => { setMealState(MEAL_STATE.R_StowingArm) }, [setMealState]) - // Margin for scaling video - const margin = convertRemToPixels(1) // Get current window size let windowSize = useWindowSize() - // Define variables for width, height and scale factor of video - const [imgWidth, setImgWidth] = useState(windowSize.width) - const [imgHeight, setImgHeight] = useState(windowSize.height) - const [scaleFactor, setScaleFactor] = useState(0) - - // Update the image size when the screen changes size. - useEffect(() => { - // Get the size of the robot's live video stream. - let { - width: widthUpdate, - height: heightUpdate, - scaleFactor - } = scaleWidthHeightToWindow(windowSize, REALSENSE_WIDTH, REALSENSE_HEIGHT, margin, margin, margin, margin) - setImgWidth(widthUpdate) - setImgHeight(heightUpdate) - setScaleFactor(scaleFactor) - }, [windowSize, margin]) - - /** - * The imgWidth/imgHeight outputted by scaleWidthHeightToWindow is designed - * to fill up nearly the whole window width (95%) and half of window height (50%). - * However, in landscape mode, we want the video to take up only a portion of the - * window width (50%) and nearly whole window height (95%). - * - * TODO: modify these to be percentage of view flexbox instead to window size - */ - let finalImgWidth = imgWidth * (isPortrait ? portraitSizeFactor : landscapeSizeFactor) - let finalImgHeight = imgHeight * (isPortrait ? portraitSizeFactor : landscapeSizeFactor) /** * Callback function for when the user wants to move to mouth position. @@ -226,16 +192,7 @@ const BiteSelection = (props) => { * make that clear to the user. */ const imageClicked = useCallback( - (event) => { - setShowActionText(true) - // Get the position of the click relative to the raw RealSense image. - let rect = event.target.getBoundingClientRect() - let x = event.clientX - rect.left // x position within the image. - let y = event.clientY - rect.top // y position within the image. - let x_raw = Math.round(x / scaleFactor) // x position within the raw image. - let y_raw = Math.round(y / scaleFactor) // y position within the raw image. - console.log('Left? : ' + x_raw + ' ; Top? : ' + y_raw + '.') - + (x_raw, y_raw) => { // Call the food segmentation ROS action callROSAction( segmentFromPointAction.current, @@ -249,7 +206,7 @@ const BiteSelection = (props) => { ) setNumImageClicks(numImageClicks + 1) }, - [segmentFromPointAction, scaleFactor, numImageClicks, feedbackCallback, responseCallback] + [segmentFromPointAction, numImageClicks, feedbackCallback, responseCallback] ) /** @@ -262,175 +219,44 @@ const BiteSelection = (props) => { } }, [segmentFromPointAction]) - /** Get the button for continue without acquiring bite - * - * @returns {JSX.Element} the skip acquisition button - */ - const withoutAcquireButton = useCallback(() => { - return ( - <> - {/* Ask the user whether they want to skip acquisition and move above plate */} -
Skip acquisition.
- {/* Icon to move to mouth */} - - - ) - }, [moveToMouth, moveToMouthImage, textFontSize]) - /** Get the continue button when debug mode is enabled * * @returns {JSX.Element} the continue debug button */ - const debugOptions = useCallback(() => { + const debugButton = useCallback(() => { // If the user is in debug mode, give them the option to skip - props.debug ? ( + return ( - ) : ( - <> ) - }, [setMealState, props.debug, textFontSize]) + }, [setMealState, textFontSize]) /** * Render the appropriate text and/or buttons based on the action status. * - * @returns {JSX.Element} the action status text to render - * - * TODO: After view flex boxes have been configured for Bite Selection page, maybe - * remove the phantom view elements if no longer needed. Otherwise, putting them - * in a helper function will help to avoid copy-pasted code. + * @returns {string} the action status text to render */ const actionStatusText = useCallback(() => { switch (actionStatus.actionStatus) { case ROS_ACTION_STATUS_EXECUTE: if (actionStatus.feedback) { let elapsed_time = actionStatus.feedback.elapsed_time.sec + actionStatus.feedback.elapsed_time.nanosec / 10 ** 9 - return ( - <> -
- Detecting food... ({Math.round(elapsed_time * 100) / 100} sec) -
- phantom_button_img - {withoutAcquireButton()} - - ) + return 'Detecting food... ' + (Math.round(elapsed_time * 100) / 100).toString() + ' sec' } else { - return ( - <> -
Detecting food...
-

 

-

 

-

 

- - ) + return 'Detecting food...' } case ROS_ACTION_STATUS_SUCCEED: if (actionResult && actionResult.detected_items && actionResult.detected_items.length > 0) { - // Get the parameters to display the mask as buttons - let [maxWidth, maxHeight] = [0, 0] - for (let detected_item of actionResult.detected_items) { - if (detected_item.roi.width > maxWidth) { - maxWidth = detected_item.roi.width - } - if (detected_item.roi.height > maxHeight) { - maxHeight = detected_item.roi.height - } - } - // Define mask button size - let buttonSize = { - width: isPortrait ? maskButtonSizePortraitFactor * windowSize.height : maskButtonSizeLandscapeFactor * windowSize.width, - height: isPortrait ? maskButtonSizePortraitFactor * windowSize.height : maskButtonSizeLandscapeFactor * windowSize.width - } - // Compute mask scale factor from sizes of the mask button and the mask - let maskScaleFactor = Math.min(buttonSize.width / maxWidth, buttonSize.height / maxHeight) - // Choose suitable mask factor based on screen orientation - let finalMaskScaleFactor = maskScaleFactor * (isPortrait ? portraitSizeFactor : landscapeSizeFactor) - // Define image size for mask buttons - let imgSize = { - width: Math.round(REALSENSE_WIDTH * finalMaskScaleFactor), - height: Math.round(REALSENSE_HEIGHT * finalMaskScaleFactor) - } - // Define a variable for robot's live video stream. - let imgSrc = `${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgSize.width}&height=${imgSize.height}&quality=20` - return ( - <> - {showActionText === true ? ( - <> -
Select a food, or retry image click.
- - {actionResult.detected_items.map((detected_item, i) => ( - - - - ))} - - - ) : ( - phantom_button_img - )} - {withoutAcquireButton()} - {debugOptions()} - - ) + return 'Select a food, or retry image click.' } else { - return

Food detection succeeded

+ return 'Food detection succeeded' } case ROS_ACTION_STATUS_ABORT: /** @@ -439,67 +265,182 @@ const BiteSelection = (props) => { * error cases might arise, and change the UI accordingly to instruct * users on how to troubleshoot/fix it. */ - return

Error in food detection

+ return 'Error in food detection' case ROS_ACTION_STATUS_CANCELED: - return

Food detection canceled

+ return 'Food detection canceled' default: + return '' + } + }, [actionStatus, actionResult]) + + const maskButtonParentRef = useRef(null) + /** + * Renders the mask buttons + * + * @returns {JSX.Element} the mask buttons + */ + const renderMaskButtons = useCallback(() => { + // If the action succeeded + if (actionStatus.actionStatus === ROS_ACTION_STATUS_SUCCEED) { + // If we have a result and there are detected items + if (actionResult && actionResult.detected_items && actionResult.detected_items.length > 0) { + // Get the size of the largest mask + let [maxWidth, maxHeight] = [0, 0] + for (let detected_item of actionResult.detected_items) { + if (detected_item.roi.width > maxWidth) { + maxWidth = detected_item.roi.width + } + if (detected_item.roi.height > maxHeight) { + maxHeight = detected_item.roi.height + } + } + + // Get the allotted space per mask + let parentWidth, parentHeight + if (maskButtonParentRef.current) { + console.log('Get actual parent size') + parentWidth = maskButtonParentRef.current.clientWidth + parentHeight = maskButtonParentRef.current.clientHeight + } else { + /** + * The below are initial guesses for the parent size based on our + * allocation of views. As soon as the component is mounted, we will + * use the actual parent size. + */ + console.log('Guess parent size') + parentWidth = isPortrait ? windowSize.width : windowSize.width / 2.0 + parentHeight = isPortrait ? windowSize.height / 4.0 : windowSize.height / 3.0 + } + let allottedSpaceWidth = parentWidth / actionResult.detected_items.length - margin * 2 + let allottedSpaceHeight = parentHeight - margin * 2 + let buttonSize = { + width: allottedSpaceWidth, + height: allottedSpaceHeight + } + + /** + * Determine how much to scale the masks so that the largest mask fits + * into the alloted space. + */ + let widthScaleFactor = allottedSpaceWidth / maxWidth + let heightScaleFactor = allottedSpaceHeight / maxHeight + let maskScaleFactor = Math.min(widthScaleFactor, heightScaleFactor) + // maskScaleFactor = Math.min(maskScaleFactor, 1.0) + + // Get the URL of the image based on the scale factor + let imgSize = { + width: Math.round(REALSENSE_WIDTH * maskScaleFactor), + height: Math.round(REALSENSE_HEIGHT * maskScaleFactor) + } + let imgSrc = `${props.webVideoServerURL}/stream?topic=${CAMERA_FEED_TOPIC}&width=${imgSize.width}&height=${imgSize.height}&quality=20` return ( - <> + + {actionResult.detected_items.map((detected_item, i) => ( + + + + ))} + + ) + } + } + }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, props.webVideoServerURL, margin]) + + /** Get the button for continue without acquiring bite + * + * @returns {JSX.Element} the skip acquisition button + */ + const skipAcquisisitionButton = useCallback(() => { + return ( + <> + {/* Ask the user whether they want to skip acquisition and move above plate */} + +
Skip acquisition
+ + {/* Icon to move to mouth */} + + + + + ) + }, [moveToMouth, moveToMouthImage, textFontSize]) /** Get the full page view * * @returns {JSX.Element} the the full page view */ - const fullPageView = useCallback( - (flexSizeOuter) => { - let flexSizeInner = isPortrait ? null : 1 - return ( - <> - {/** - * In addition to selecting their desired food item, the user has two - * other options on this page: - * - If their desired food item is not visible on the plate, they can - * decide to teleoperate the robot until it is visible. - * - Instead of selecting their next bite, the user can indicate that - * they are done eating. - * - * TODO: issue#65 will remove these two buttons, so final implementation - * will have flex box without these in Bite Selection page - */} -
+ const fullPageView = useCallback(() => { + return ( + <> + {/** + * In addition to selecting their desired food item, the user has two + * other options on this page: + * - If their desired food item is not visible on the plate, they can + * decide to teleoperate the robot until it is visible. + * - Instead of selecting their next bite, the user can indicate that + * they are done eating. + */} + + + + -
- + + + {/** + * Below the buttons, one half of the screen will present the video feed. + * The other half will present the action status text, the food buttons + * if the action has succeeded, and a button to proceed without acquiring + * a bite. + */} +
Click on image to select food.
- {showVideo(props.webVideoServerURL, finalImgWidth, finalImgHeight, imageClicked)} + + + + + +
{actionStatusText()}
+ + - {/* Display the action status and/or results */} - {actionStatusText()} + + {renderMaskButtons()} + + + {skipAcquisisitionButton()} + + {props.debug ? ( + + {debugButton()} + + ) : ( + <> + )} - - ) - }, - [ - dimension, - actionStatusText, - imageClicked, - doneEatingClicked, - locatePlateClicked, - finalImgHeight, - finalImgWidth, - textFontSize, - props.webVideoServerURL, - isPortrait - ] - ) + + + ) + }, [ + locatePlateClicked, + doneEatingClicked, + dimension, + margin, + textFontSize, + actionStatusText, + renderMaskButtons, + skipAcquisisitionButton, + props.webVideoServerURL, + videoParentRef, + imageClicked, + props.debug, + debugButton + ]) // Render the component return <>{fullPageView()} diff --git a/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx b/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx index 13b27534..51a13970 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx @@ -1,5 +1,5 @@ // React Imports -import React, { useCallback, useEffect, useState } from 'react' +import React, { useCallback, useRef } from 'react' import Button from 'react-bootstrap/Button' import { useMediaQuery } from 'react-responsive' import { View } from 'react-native' @@ -10,9 +10,9 @@ import PropTypes from 'prop-types' // Local Imports import '../Home.css' import { useGlobalState, MEAL_STATE } from '../../GlobalState' -import { REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../../Constants' -import { useWindowSize, convertRemToPixels, scaleWidthHeightToWindow, showVideo } from '../../../helpers' +import { convertRemToPixels } from '../../../helpers' import { Col, Row, Container } from 'react-bootstrap' +import VideoFeed from '../VideoFeed' /** * The PlateLocator component appears if the user decides to adjust the position @@ -25,14 +25,16 @@ const PlateLocator = (props) => { const setMealState = useGlobalState((state) => state.setMealState) // Get robot motion flag for plate locator const setTeleopIsMoving = useGlobalState((state) => state.setTeleopIsMoving) + // Flag to check if the current orientation is portrait const isPortrait = useMediaQuery({ query: '(orientation: portrait)' }) // Indicator of how to arrange screen elements based on orientation let dimension = isPortrait ? 'column' : 'row' - // Define margin for video + + // Variables to render the VideoFeed + const videoParentRef = useRef(null) const margin = convertRemToPixels(1) - // Get current window size - let windowSize = useWindowSize() + // text font size let textFontSize = isPortrait ? '3vh' : '3vw' // done button width @@ -41,30 +43,6 @@ const PlateLocator = (props) => { let buttonHeight = isPortrait ? '6vh' : '6vw' // arrow button width let arrowButtonWidth = isPortrait ? '6vh' : '6vw' - // Factor to modify video size in landscape which has less space than portrait - let landscapeSizeFactor = 0.5 - // Define variables for width and height of video - const [imgWidth, setWidth] = useState(windowSize.width) - const [imgHeight, setHeight] = useState(windowSize.height) - - // Update the image size when the screen changes size. - useEffect(() => { - // Get the size of the robot's live video stream. - let { width: imgWidthUpdate, height: imgHeightUpdate } = scaleWidthHeightToWindow( - windowSize, - REALSENSE_WIDTH, - REALSENSE_HEIGHT, - margin, - margin, - margin, - margin - ) - setWidth(imgWidthUpdate) - setHeight(imgHeightUpdate) - }, [windowSize, margin]) - - let finalImgWidth = isPortrait ? imgWidth : landscapeSizeFactor * imgWidth - let finalImgHeight = isPortrait ? imgHeight : landscapeSizeFactor * imgHeight /** * Callback function for when the user presses one of the buttons to teleop @@ -195,9 +173,16 @@ const PlateLocator = (props) => { // Render the component return ( - - - {showVideo(props.webVideoServerURL, finalImgWidth, finalImgHeight, null)} + + + {directionalArrows()} diff --git a/feedingwebapp/src/Pages/Home/VideoFeed.jsx b/feedingwebapp/src/Pages/Home/VideoFeed.jsx new file mode 100644 index 00000000..3ba495f3 --- /dev/null +++ b/feedingwebapp/src/Pages/Home/VideoFeed.jsx @@ -0,0 +1,173 @@ +// React Imports +import React, { useCallback, useEffect, useState } from 'react' +// PropTypes is used to validate that the used props are in fact passed to this Component +import PropTypes from 'prop-types' + +// Local Imports +import { CAMERA_FEED_TOPIC, REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../Constants' +import { useWindowSize } from '../../helpers' + +/** + * Takes in an imageWidth and imageHeight, and returns a width and height that + * maintains the same aspect ratio but fits within the window. + * + * @param {number} parentWidth the width of the parent DOM element in pixels + * @param {number} parentHeight the height of the parent DOM element in pixels + * @param {number} imageWidth the original image's width in pixels + * @param {number} imageHeight the original image's height in pixels + * @param {number} marginTop the desired top margin between window and image, in pixels + * @param {number} marginBottom the desired bottom margin between window and image, in pixels + * @param {number} marginLeft the desired left margin between window and image, in pixels + * @param {number} marginRight the desired right margin between window and image, in pixels + * + * @returns {object} the width and height of the image that fits within the window and has the requested margins + */ +function scaleWidthHeightToWindow( + parentWidth, + parentHeight, + imageWidth, + imageHeight, + marginTop = 0, + marginBottom = 0, + marginLeft = 0, + marginRight = 0 +) { + // Calculate the aspect ratio of the image + let imageAspectRatio = imageWidth / imageHeight + // Get the aspect ratio of the available subset of the window + let availableWidth = parentWidth - marginLeft - marginRight + let availableHeight = parentHeight - marginTop - marginBottom + let availableAspectRatio = availableWidth / availableHeight + + // Calculate the width and height of the image that fits within the window + let returnWidth, returnHeight + if (availableAspectRatio > imageAspectRatio) { + returnHeight = Math.round(availableHeight) + returnWidth = Math.round(imageAspectRatio * returnHeight) + } else { + returnWidth = Math.round(availableWidth) + returnHeight = Math.round(returnWidth / imageAspectRatio) + } + + // Calculate the scale factor + let scaleFactorWidth = returnWidth / imageWidth + let scaleFactorHeight = returnHeight / imageHeight + let scaleFactor = (scaleFactorWidth + scaleFactorHeight) / 2 + + return { + width: returnWidth, + height: returnHeight, + scaleFactor: scaleFactor + } +} + +/** + * The VideoFeed component takes in a reference to the parent DOM element, and + * displays a video feed that takes up the maximum size within the parent DOM, + * while maintaining the aspect ratio of the video feed and respecting specified + * margins. + * + * Note that for this to work, the parent DOM element must have a specified + * height and width that does not change with its contents. One way to achieve + * this is to ensure the `width` and `height` props of the parent are set. + */ +const VideoFeed = (props) => { + // Local state variables to keep track of the width and height of the video feed + const [imgWidth, setImgWidth] = useState(0) + const [imgHeight, setImgHeight] = useState(0) + const [scaleFactor, setScaleFactor] = useState(0.0) + + // Callback to resize the image based on the parent width and height + const resizeImage = useCallback(() => { + if (!props.parent.current) { + return + } + // Get the width and height of the parent DOM element + let parentWidth = props.parent.current.clientWidth + let parentHeight = props.parent.current.clientHeight + + // Calculate the width and height of the video feed + let { + width: childWidth, + height: childHeight, + scaleFactor: childScaleFactor + } = scaleWidthHeightToWindow( + parentWidth, + parentHeight, + REALSENSE_WIDTH, + REALSENSE_HEIGHT, + props.marginTop, + props.marginBottom, + props.marginLeft, + props.marginRight + ) + + // Set the width and height of the video feed + setImgWidth(childWidth) + setImgHeight(childHeight) + setScaleFactor(childScaleFactor) + }, [props.parent, props.marginTop, props.marginBottom, props.marginLeft, props.marginRight]) + + // When the component is first mounted, resize the image + useEffect(() => { + resizeImage() + }, [resizeImage]) + + // When the window is resized, resize the image + useWindowSize(resizeImage) + + // The callback for when the image is clicked. + const imageClicked = useCallback( + (event) => { + // Get the position of the click relative to the raw RealSense image. + let rect = event.target.getBoundingClientRect() + let x = event.clientX - rect.left // x position within the image. + let y = event.clientY - rect.top // y position within the image. + let x_raw = Math.round(x / scaleFactor) // x position within the raw image. + let y_raw = Math.round(y / scaleFactor) // y position within the raw image. + console.log('Button click on unscaled image: (' + x_raw + ', ' + y_raw + ')') + + // Call the callback function if it exists + if (props.pointClicked) { + props.pointClicked(x_raw, y_raw) + } + }, + [props, scaleFactor] + ) + + // Render the component + return ( + Live video feed from the robot + ) +} +VideoFeed.propTypes = { + // The ref to the parent DOM element. Null if the component is not yet mounted + parent: PropTypes.object.isRequired, + // The URL of the video feed + webVideoServerURL: PropTypes.string.isRequired, + // The margins around the video feed + marginTop: PropTypes.number.isRequired, + marginBottom: PropTypes.number.isRequired, + marginLeft: PropTypes.number.isRequired, + marginRight: PropTypes.number.isRequired, + /** + * An optional callback function for when the user clicks on the video feed. + * This function should take in two parameters, `x` and `y`, which are the + * coordinates of the click in the **unscaled** image (e.g., the image of + * size REALSENSE_WIDTH x REALSENSE_HEIGHT). + */ + pointClicked: PropTypes.func +} + +export default VideoFeed diff --git a/feedingwebapp/src/helpers.js b/feedingwebapp/src/helpers.js index 982adacf..494de8d1 100644 --- a/feedingwebapp/src/helpers.js +++ b/feedingwebapp/src/helpers.js @@ -1,103 +1,28 @@ -import React, { useLayoutEffect, useState } from 'react' -import { CAMERA_FEED_TOPIC } from './Pages/Constants' +import { useLayoutEffect, useState } from 'react' -// Updates and returns the window size whenever the screen is re-sized. -export function useWindowSize() { +/** + * Returns the window size, which gets updated every time the window is resized. + * + * @param {func} resizeCallback an optional function to call when the window is + * resized. Note that this function cannot use any react hooks. + */ +export function useWindowSize(resizeCallback = null) { const [windowSize, setWindowSize] = useState({ width: 0, height: 0 }) useLayoutEffect(() => { // set current window size function updateWindowSize() { setWindowSize({ width: window.innerWidth, height: window.innerHeight }) + if (resizeCallback) { + resizeCallback() + } } window.addEventListener('resize', updateWindowSize) updateWindowSize() return () => window.removeEventListener('resize', updateWindowSize) - }, []) + }, [resizeCallback]) return windowSize } -/** - * Get the robot's live video stream. - * - * @param {string} webVideoServerURL The URL of the web video server - * @param {number} currentWidth the adjusted width in pixels - * @param {number} currentHeight the adjusted height in pixels - * @param {func} onclick the function trigerred in video onclick - }} - * - * @returns {JSX.Element} the robot's live video stream - */ -export function showVideo(webVideoServerURL, currentWidth, currentHeight, onclick) { - let imgWidth = Math.round(currentWidth) - let imgHeight = Math.round(currentHeight) - return ( - Live video feed from the robot - ) -} - -/** - * Takes in an imageWidth and imageHeight, and returns a width and height that - * maintains the same aspect ratio but fits within the window. - * - * @param {number} windowSize the inner width and height of the window - * @param {number} imageWidth the original image's width in pixels - * @param {number} imageHeight the original image's height in pixels - * @param {number} marginTop the desired top margin between window and image, in pixels - * @param {number} marginBottom the desired bottom margin between window and image, in pixels - * @param {number} marginLeft the desired left margin between window and image, in pixels - * @param {number} marginRight the desired right margin between window and image, in pixels - * - * @returns {object} the width and height of the image that fits within the window and has the requested margins - */ -export function scaleWidthHeightToWindow( - windowSize, - imageWidth, - imageHeight, - marginTop = 0, - marginBottom = 0, - marginLeft = 0, - marginRight = 0 -) { - // Calculate the aspect ratio of the image - let imageAspectRatio = imageWidth / imageHeight - // Get the aspect ratio of the available subset of the window - let availableWidth = windowSize.width - marginLeft - marginRight - let availableHeight = windowSize.height - marginTop - marginBottom - let availableAspectRatio = availableWidth / availableHeight - - // Calculate the width and height of the image that fits within the window - let returnWidth, returnHeight - if (availableAspectRatio > imageAspectRatio) { - returnHeight = availableHeight - returnWidth = imageAspectRatio * returnHeight - } else { - returnWidth = availableWidth - returnHeight = returnWidth / imageAspectRatio - } - - // Calculate the scale factor - let scaleFactorWidth = returnWidth / imageWidth - let scaleFactorHeight = returnHeight / imageHeight - let scaleFactor = (scaleFactorWidth + scaleFactorHeight) / 2 - - return { - width: returnWidth, - height: returnHeight, - scaleFactor: scaleFactor - } -} - /** * Takes in a number of REM units and returns the equivalent number of pixels. * See https://blog.hubspot.com/website/css-rem form more info re. REM units.