Changes from CLV Study

README.md

@@ -197,14 +197,60 @@ The 'predictive display' mode will overlay a trajectory over the video stream th
     <img src="documentation/assets/tutorial/predictive-display.gif">
 </p>
 
-## Gripper Camera
+## Gripper Camera View
 
 There are two quick actions for the gripper camera view: (1) `center wrist` and (2) `stow wrist`. Center wrist will turn the wrist out and align it with the arm. Stow wrist will rotate the wrist to the stow position.
 
 <p align="center">
     <img src="documentation/assets/tutorial/quick-actions.gif">
 </p>
 
+### Expanded Gripper View
+
+Users can toggle between the default gripper view and an expanded gripper view. The expanded gripper view cam be useful e.g., if the robot is holding a large object that is obscuring much of the gripper camera's view.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/expanded_gripper_view.gif">
+</p>
+
+### Gripper Depth Overlay
+
+Users can toggle on/off a depth overlay, which highlights all points that are within the two fingers of the gripper. This can be useful to gauge when you have moved far enough to grasp an object.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/gripper_depth_overlay.gif">
+</p>
+
+## Head Realsense Camera View
+
+The head Realsense camera view needs to be added through the customization menu. It has the same head pan/tilt buttons, "Quick Look" buttons, and "Follow Gripper" button as the Wide-Angle Camera.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/head_realsense_camera.gif">
+</p>
+
+### Head Realsense Depth Overlay
+
+The head Realsense camera also has a depth overlay, which highlights all points that are close enough to the robot to be graspable.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/head_realsense_depth_overlay.gif">
+</p>
+
+### Click-to-Pregrasp
+
+The head Realsense camera allows users to select a point in the camera view, and have the robot automatically move to align with that point. This can be done with the gripper horizontal...
+
+<p align="center">
+    <img src="documentation/assets/tutorial/head_realsense_click_to_pregrasp_horizontal.gif">
+</p>
+
+...or with the gripper vertical.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/head_realsense_click_to_pregrasp_vertical.gif">
+</p>
+
 ## Button Pads
 
 Each button pad controls a different set of joints on the robot. When you click a button the robot will move and the button will highlight blue while the robot is moving. The button will turn red when the respective joint is at its limit.
@@ -232,6 +278,54 @@ The action modes can be selected in the dropdown in the top-left corner of the i
 - **Press-Release**: Stretch will move while you are pressing and holding the button and will stop when you release.
 - **Click-Click**: Stretch will start moving when you click and will stop when you click again. You can also stop Stretch by moving the cursor outside the button you clicked.
 
+## Movement Recorder
+
+There is a movement recorder that can be used to save and playback robot arm motions. One way of using it is to record a goal state for the robot arm to move to. To do so, start recording, keep the arm stationary for a few seconds, and then stop recording.
+
+Recording a "Tuck Arm" goal:
+
+<p align="center">
+    <img src="documentation/assets/tutorial/movement_recorder_tuck_arm_record.gif">
+</p>
+
+Moving to the "Tuck Arm" goal:
+
+<p align="center">
+    <img src="documentation/assets/tutorial/movement_recorder_tuck_arm_playback.gif">
+</p>
+
+The movement recorder can also be used to record entire movements. In this case, start recording, move the arm, and then stop recording when the motion is done.
+
+## Audio Streaming
+
+To better facilitate beyond line-of-sight operation, the web interface enables bi-directional audio interaction.
+
+### Robot-to-Operator
+
+Audio from the robot can be streamed to the operator directly through the robot's microphone, by (un)muting audio in the web interface.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/robot_to_operator_audio_streaming.gif">
+</p>
+
+### Operator-to-Robot Text-to-Speech
+
+The operator can send text to be spoken on the robot. The operator can also save commonly-used text, and can stop an ongoing utterance. When used in conjunction with robot-to-operator audio streaming, the operator can also hear when the robot has finished speaking.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/text_to_speech.gif">
+</p>
+
+Text-to-speech can also be performed with a command-line interface, by running `ros2 run stretch_web_teleop text_to_speech_ui.py`. This is particulalry useful in situations where a separate operator is controlling the robot and a separate operator is controlling speech interactions.
+
+<p align="center">
+    <img src="documentation/assets/tutorial/text_to_speech_cli.gif">
+</p>
+
+## Using a Tablet as the End Effector
+
+Stay tuned for instructions on using the web interface with a tablet as an end effector!
+
 # Contributing
 
 - This repository uses pre-commit hooks to enforce consistent formatting and style.

configure_audio.sh

@@ -10,31 +10,65 @@
 # only one external mic plugged in; that one will **always** be prioritized over the
 # robot's built-in microphone. The only way to use the robot's built-in microphone
 # is to disconnect all external microphones.
+
+
+# Configure command line arguments
 SPEAKER_NAME="alsa_output.pci-0000_00_1f.3.analog-stereo"
-if getopts ":s:" opt && [[ $opt == "s" ]]; then
-    SPEAKER_NAME="$OPTARG"
-fi
 MIC_NAME=""
-if getopts ":m:" opt && [[ $opt == "m" ]]; then
-    MIC_NAME="$OPTARG"
-fi
+while getopts ":s:m:" opt
+do
+    case $opt in
+        s ) SPEAKER_NAME="$OPTARG" ;;
+        m ) MIC_NAME="$OPTARG" ;;
+    esac
+done
+
 
 echo "Configuring audio...Note that this script may not work if X-11 forwarding is enabled."
 
-echo "Setting speaker to $SPEAKER_NAME"
-pactl set-default-sink $SPEAKER_NAME
+# Set the default speaker
+sinks=$(pactl list short sinks)
+if [[ $sinks == *"$SPEAKER_NAME"* ]]; then
+    echo "    Setting speaker to $SPEAKER_NAME"
+    pactl set-default-sink $SPEAKER_NAME
+else
+    echo "    Speaker name is invalid"
+    echo "    To get valid speaker names, run:"
+    echo "        pactl list short sinks"
+fi
 
+# Set the default microphone
 if [ ${#MIC_NAME} -gt 0 ]; then
-    echo "Setting microphone to $MIC_NAME"
-    pactl set-default-source $MIC_NAME
+    sources=$(pactl list short sources)
+    if [[ $sources == *"$MIC_NAME"* ]]; then
+        echo "    Setting microphone to $MIC_NAME"
+        pactl set-default-source $MIC_NAME
+    else
+        echo "    Microphone name is invalid"
+        echo "    To get valid microphone names, run:"
+        echo "        pactl list short sources"
+    fi
 fi
 
-echo "Unmuting the speaker and setting its volume to 100%"
-amixer set 'Master' unmute -q
-amixer sset 'Master' 100% -q
+# Unmute and set volume to 100% for relevant audio devices
+amixer_output=$(amixer scontrols)
+amixer_devices=('Master' 'PCM' 'Capture' 'Mic' 'Digital')
+for device in "${amixer_devices[@]}"
+do
+    if [[ $amixer_output == *"'$device'"* ]]; then
+        echo "    Unmuting '$device'"
+        device_output=$(amixer get $device)
+        if [[ $device_output == *"[off]"* ]]; then
+            if [[ $device_output == *"Capture"* ]]; then
+                amixer set $device cap -q
+            else
+                amixer set $device unmute -q
+            fi
+        fi
 
-echo "Unmuting the microphone and setting its gain to 100%"
-amixer set 'Capture' cap -q
-amixer sset 'Capture' 100% -q
+        echo "    Setting '$device' to 100%"
+        amixer sset $device 100% -q
+    fi
+done
 
 echo "Done configuring audio!"

launch_interface.sh

@@ -23,6 +23,7 @@ fi
 
 stretch_free_robot_process.py;
 ./stop_interface.sh
+./configure_audio.sh -m alsa_input.usb-K66_K66_20190805V001-00.analog-stereo
 sudo udevadm control --reload-rules && sudo udevadm trigger
 source /opt/ros/humble/setup.bash
 source ~/ament_ws/install/setup.bash

nodes/text_to_speech_ui.py

@@ -116,7 +116,7 @@ def publish_message(self, message: str):
         # Create the message
         msg = TextToSpeech(
             text=message,
-            is_slow=False,
+            is_slow=True,
             override_behavior=(
                 TextToSpeech.OVERRIDE_BEHAVIOR_INTERRUPT
                 if len(message) == 0

requirements.txt

@@ -1,9 +1,5 @@
 gTTS
 loguru
-# numpy 1.23.2 is not required for the web teleop interface, but is required
-# for stretch_body. If we diden't include it here, pin would update
-# to the latest version of numpy, breaking stretch_body.
-numpy==1.23.2
 pin
 PyAudio==0.2.14
 pydub

src/pages/operator/css/basic_components.css

@@ -33,7 +33,7 @@
 }
 
 .popup-modal.large {
-    height: 50%;
+    height: 60%;
 }
 
 .mobile {

src/pages/operator/tsx/basic_components/RadioGroup.tsx

@@ -2,6 +2,7 @@ import React from "react";
 import { className } from "shared/util";
 import "operator/css/RadioGroup.css";
 import { isMobile } from "react-device-detect";
+import DeleteIcon from "@mui/icons-material/DeleteOutline";
 
 export const RadioButton = (props: {
     label: string;

src/pages/operator/tsx/layout_components/CameraView.tsx

@@ -49,6 +49,7 @@ import "operator/css/CameraView.css";
 import AddIcon from "@mui/icons-material/Add";
 import CancelIcon from "@mui/icons-material/Cancel";
 import PlayCircleFilledIcon from "@mui/icons-material/PlayCircleFilled";
+import PlayCircle from "@mui/icons-material/PlayCircle";
 
 /**
  * Displays a video stream with an optional button pad overlay
@@ -958,7 +959,6 @@ const UnderRealsenseButtons = (props: {
     }
 
     // Only show ShowTablet buttons if the robot has a tablet attached.
-    // TODO: Un-comment the tool conditional!
     let showTabletButtons = <></>;
     if (props.stretchTool === StretchTool.TABLET) {
         showTabletButtons = (
@@ -987,7 +987,7 @@ const UnderRealsenseButtons = (props: {
                             }}
                         >
                             <span>Cancel</span>
-                            <span className="material-icons">cancel</span>
+                            <CancelIcon />
                         </button>
                     ) : (
                         <button
@@ -1005,7 +1005,7 @@ const UnderRealsenseButtons = (props: {
                             }}
                         >
                             <span>Show Tablet</span>
-                            <span className="material-icons">play_circle</span>
+                            <PlayCircle />
                         </button>
                     ))}
             </React.Fragment>

src/pages/operator/tsx/layout_components/Map.tsx

@@ -26,6 +26,9 @@ import { PopupModal } from "../basic_components/PopupModal";
 import { Tooltip } from "operator/tsx/static_components/Tooltip";
 import { isMobile } from "react-device-detect";
 import { RadioFunctions, RadioGroup } from "../basic_components/RadioGroup";
+import PlayCircleIcon from "@mui/icons-material/PlayCircle";
+import CancelIcon from "@mui/icons-material/Cancel";
+import SaveIcon from "@mui/icons-material/Save";
 
 export enum MapFunction {
     GetMap,
@@ -353,7 +356,7 @@ const UnderMapButtons = (props: {
                         }}
                     >
                         <span>Start</span>
-                        <span className="material-icons">play_circle</span>
+                        <PlayCircleIcon/>
                     </button>
                 )}
                 {play && (
@@ -365,7 +368,7 @@ const UnderMapButtons = (props: {
                         }}
                     >
                         <span>Cancel</span>
-                        <span className="material-icons">cancel</span>
+                        <CancelIcon/>
                     </button>
                 )}
                 <button
@@ -380,7 +383,7 @@ const UnderMapButtons = (props: {
                     }}
                 >
                     <span hidden={props.hideLabels}>Save new destination</span>
-                    <span className="material-icons">save</span>
+                    <SaveIcon/>
                 </button>
             </div>
             <RadioGroup functs={radioFuncts} />
@@ -406,7 +409,7 @@ const UnderMapButtons = (props: {
                     }}
                 >
                     <span hidden={props.hideLabels}>Save new destination</span>
-                    <span className="material-icons">save</span>
+                    <SaveIcon/>
                 </div>
                 {!play && (
                     <div
@@ -433,7 +436,7 @@ const UnderMapButtons = (props: {
                         }}
                     >
                         <span>Play</span>
-                        <span className="material-icons">play_circle</span>
+                        <PlayCircleIcon/>
                     </div>
                 )}
                 {play && (
@@ -445,7 +448,7 @@ const UnderMapButtons = (props: {
                         }}
                     >
                         <span>Cancel</span>
-                        <span className="material-icons">cancel</span>
+                        <CancelIcon/>
                     </div>
                 )}
                 <CheckToggleButton

src/pages/operator/tsx/layout_components/MovementRecorder.tsx

@@ -100,6 +100,7 @@ export const MovementRecorder = (props: {
     const JointSelectionModal = (props: {
         setShow: (show: boolean) => void;
         show: boolean;
+        setIsRecording: (isRecording: boolean) => void;
     }) => {
         const [head, setHead] = React.useState<boolean>(false);
         const [arm, setArm] = React.useState<boolean>(false);
@@ -133,7 +134,11 @@ export const MovementRecorder = (props: {
                 setShow={props.setShow}
                 show={props.show}
                 onAccept={handleAccept}
-                onCancel={() => functions.StopRecording()}
+                onCancel={() => {
+                    props.setIsRecording(false);
+                    props.setShow(false);
+                    functions.StopRecording();
+                }}
                 id="save-recording-modal"
                 acceptButtonText="Save"
                 size={isMobile ? "small" : "large"}
@@ -361,6 +366,7 @@ export const MovementRecorder = (props: {
             <JointSelectionModal
                 setShow={setShowJointSelectionModal}
                 show={showJointSelectionModal}
+                setIsRecording={setIsRecording}
             />
             <SaveRecordingModal
                 setShow={setShowSaveRecordingModal}
@@ -404,6 +410,7 @@ export const MovementRecorder = (props: {
             <JointSelectionModal
                 setShow={setShowJointSelectionModal}
                 show={showJointSelectionModal}
+                setIsRecording={setIsRecording}
             />
             <SaveRecordingModal
                 setShow={setShowSaveRecordingModal}

src/pages/operator/tsx/static_components/Sidebar.tsx

@@ -499,9 +499,9 @@ const SidebarComponentProvider = (props: SidebarComponentProviderProps) => {
             case ComponentType.CameraView:
                 (definition as ParentComponentDefinition).children = [];
                 break;
-            case ComponentType.Map:
-                (definition as MapDefinition).storageHandler = storageHandler;
-                break;
+            // case ComponentType.Map:
+            //     (definition as MapDefinition).storageHandler = storageHandler;
+            //     break;
         }
 
         props.onSelect(definition);

src/shared/util.tsx

@@ -252,6 +252,7 @@ export const STOW_WRIST_GRIPPER: RobotPose = {
     joint_wrist_roll: 0.0,
     joint_wrist_pitch: -0.497,
     joint_wrist_yaw: 3.19579,
+    joint_gripper_finger_left: 0.0,
 };
 
 export const STOW_WRIST_TABLET: RobotPose = {