FG-2933: Add microphone capture (opus) to MCAP recording demo

foxglove · Dec 15, 2024 · ad01ad1 · ad01ad1
1 parent 36f7053
commit ad01ad1
Show file tree

Hide file tree

Showing 6 changed files with 314 additions and 15 deletions.
diff --git a/website/package.json b/website/package.json
@@ -27,6 +27,8 @@
     "@mcap/core": "workspace:*",
     "@mdx-js/react": "1.6.22",
     "@tsconfig/docusaurus": "1.0.7",
+    "@types/dom-mediacapture-transform": "0.1.10",
+    "@types/dom-webcodecs": "0.1.13",
     "@types/promise-queue": "2.2.0",
     "buffer": "6.0.3",
     "classnames": "2.3.2",

diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css b/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
@@ -106,7 +106,7 @@
   margin: 0;
 }
 
-.videoContainer {
+.mediaContainer {
   background-color: #6f3be80f;
   border: 1px solid #e0e0e0;
   font-size: 0.8rem;
@@ -120,11 +120,11 @@
   overflow: hidden;
 }
 
-[data-theme="dark"] .videoContainer {
+[data-theme="dark"] .mediaContainer {
   background-color: transparent;
 }
 
-.videoContainer video {
+.mediaContainer video {
   width: 100%;
   height: 100%;
   position: absolute;
@@ -133,7 +133,7 @@
   z-index: 0;
 }
 
-.videoContainer .videoErrorContainer {
+.mediaContainer .mediaErrorContainer {
   width: 100%;
   height: 100%;
   position: absolute;
@@ -143,16 +143,16 @@
   z-index: 1;
 }
 
-[data-theme="dark"] .videoContainer .videoErrorContainer {
+[data-theme="dark"] .mediaContainer .mediaErrorContainer {
   background-color: #17151ec4;
 }
 
-.videoPlaceholderText {
+.mediaPlaceholderText {
   font-weight: 600;
   cursor: pointer;
 }
 
-.videoLoadingIndicator {
+.mediaLoadingIndicator {
   position: absolute;
   top: 50%;
   left: 50%;

diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
@@ -15,6 +15,12 @@ import {
   Recorder,
   toProtobufTime,
 } from "./Recorder";
+import {
+  recordAudioStream,
+  startAudioStream,
+  CompressedAudioData,
+  startAudioCapture,
+} from "./audioCapture";
 import {
   CompressedVideoFrame,
   startVideoCapture,
@@ -37,6 +43,7 @@ type State = {
   addPoseMessage: (msg: DeviceOrientationEvent) => void;
   addJpegFrame: (blob: Blob) => void;
   addVideoFrame: (frame: CompressedVideoFrame) => void;
+  addAudioData: (data: CompressedAudioData) => void;
   closeAndRestart: () => Promise<Blob>;
 };
 
@@ -70,6 +77,9 @@ const useStore = create<State>((set) => {
     addVideoFrame(frame: CompressedVideoFrame) {
       void recorder.addVideoFrame(frame);
     },
+    addAudioData(data: CompressedAudioData) {
+      void recorder.addAudioData(data);
+    },
     async closeAndRestart() {
       return await recorder.closeAndRestart();
     },
@@ -128,19 +138,27 @@ export function McapRecordingDemo(): JSX.Element {
 
   const videoRef = useRef<HTMLVideoElement | undefined>();
   const videoContainerRef = useRef<HTMLDivElement>(null);
+  const audioProgressRef = useRef<HTMLProgressElement>(null);
   const [recordJpeg, setRecordJpeg] = useState(false);
   const [recordH264, setRecordH264] = useState(false);
   const [recordH265, setRecordH265] = useState(false);
   const [recordVP9, setRecordVP9] = useState(false);
   const [recordAV1, setRecordAV1] = useState(false);
+  const [recordOpus, setRecordOpus] = useState(false);
   const [recordMouse, setRecordMouse] = useState(true);
   const [recordOrientation, setRecordOrientation] = useState(true);
   const [videoStarted, setVideoStarted] = useState(false);
   const [videoError, setVideoError] = useState<Error | undefined>();
+  const [audioError, setAudioError] = useState<Error | undefined>();
   const [showDownloadInfo, setShowDownloadInfo] = useState(false);
 
-  const { addJpegFrame, addVideoFrame, addMouseEventMessage, addPoseMessage } =
-    state;
+  const {
+    addJpegFrame,
+    addVideoFrame,
+    addMouseEventMessage,
+    addPoseMessage,
+    addAudioData,
+  } = state;
 
   const { data: h264Support } = useAsync(supportsH264Encoding);
   const { data: h265Support } = useAsync(supportsH265Encoding);
@@ -154,7 +172,8 @@ export function McapRecordingDemo(): JSX.Element {
     (recordVP9 && !videoError) ||
     (recordH265 && !videoError) ||
     (recordH264 && !videoError) ||
-    (recordJpeg && !videoError);
+    (recordJpeg && !videoError) ||
+    (recordOpus && !audioError);
 
   // Automatically pause recording after 30 seconds to avoid unbounded growth
   useEffect(() => {
@@ -274,11 +293,62 @@ export function McapRecordingDemo(): JSX.Element {
     recordH264,
     recordH265,
     recordAV1,
+    recordVP9,
     recording,
     videoStarted,
     recordJpeg,
   ]);
 
+  const [audioStream, setAudioStream] = useState<MediaStream | undefined>(
+    undefined,
+  );
+
+  const enableMicrophone = recordOpus;
+  useEffect(() => {
+    const progress = audioProgressRef.current;
+    if (!progress || !enableMicrophone) {
+      return;
+    }
+
+    const cleanup = startAudioStream({
+      progress,
+      onAudioStream: (stream) => {
+        setAudioStream(stream);
+      },
+      onError: (err) => {
+        setAudioError(err);
+        console.error(err);
+      },
+    });
+
+    return () => {
+      cleanup();
+      setAudioStream(undefined);
+      setAudioError(undefined);
+    };
+  }, [enableMicrophone, recordOpus]);
+
+  useEffect(() => {
+    if (!enableMicrophone || !recording || !audioStream) {
+      return;
+    }
+
+    const cleanup = startAudioCapture({
+      enableOpus: recordOpus,
+      stream: audioStream,
+      onAudioData: (data) => {
+        addAudioData(data);
+      },
+      onError: (error) => {
+        setAudioError(error);
+      },
+    });
+
+    return () => {
+      cleanup();
+    };
+  }, [addAudioData, enableMicrophone, recordOpus, audioStream, recording]);
+
   const onRecordClick = useCallback(
     (event: React.MouseEvent) => {
       event.preventDefault();
@@ -417,6 +487,16 @@ export function McapRecordingDemo(): JSX.Element {
             />
             Camera (JPEG)
           </label>
+          <label>
+            <input
+              type="checkbox"
+              checked={recordOpus}
+              onChange={(event) => {
+                setRecordOpus(event.target.checked);
+              }}
+            />
+            Microphone (Opus)
+          </label>
           {!hasMouse && (
             <label>
               <input
@@ -545,22 +625,24 @@ export function McapRecordingDemo(): JSX.Element {
               </div>
             </div>
           </div>
+        </div>
 
+        <div className={styles.recordingControls}>
           <div className={styles.recordingControlsColumn}>
-            <div className={styles.videoContainer} ref={videoContainerRef}>
+            <div className={styles.mediaContainer} ref={videoContainerRef}>
               {videoError ? (
-                <div className={cx(styles.error, styles.videoErrorContainer)}>
+                <div className={cx(styles.error, styles.mediaErrorContainer)}>
                   {videoError.toString()}
                 </div>
-              ) : recordH264 || recordJpeg ? (
+              ) : enableCamera ? (
                 <>
                   {!videoStarted && (
-                    <progress className={styles.videoLoadingIndicator} />
+                    <progress className={styles.mediaLoadingIndicator} />
                   )}
                 </>
               ) : (
                 <span
-                  className={styles.videoPlaceholderText}
+                  className={styles.mediaPlaceholderText}
                   onClick={() => {
                     if (av1Support?.supported === true) {
                       setRecordAV1(true);
@@ -578,6 +660,25 @@ export function McapRecordingDemo(): JSX.Element {
               )}
             </div>
           </div>
+
+          <div className={styles.recordingControlsColumn}>
+            <div className={styles.mediaContainer}>
+              {audioError ? (
+                <div className={cx(styles.error, styles.mediaErrorContainer)}>
+                  {audioError.toString()}
+                </div>
+              ) : enableMicrophone ? (
+                <progress
+                  className={styles.mediaLoadingIndicator}
+                  ref={audioProgressRef}
+                />
+              ) : (
+                <span className={styles.mediaPlaceholderText}>
+                  Enable “Microphone” to record audio
+                </span>
+              )}
+            </div>
+          </div>
         </div>
       </div>
     </section>