From ad01ad1e2554aa8e11748f38584b1e37a84e0061 Mon Sep 17 00:00:00 2001
From: fgwt202412 <191263616+fgwt202412@users.noreply.github.com>
Date: Sun, 15 Dec 2024 14:36:02 -0500
Subject: [PATCH] FG-2933: Add microphone capture (opus) to MCAP recording demo

---
 website/package.json                          |   2 +
 .../McapRecordingDemo.module.css              |  14 +-
 .../McapRecordingDemo/McapRecordingDemo.tsx   | 117 +++++++++++-
 .../McapRecordingDemo/audioCapture.ts         | 177 ++++++++++++++++++
 website/tsconfig.json                         |   1 +
 yarn.lock                                     |  18 ++
 6 files changed, 314 insertions(+), 15 deletions(-)

diff --git a/website/package.json b/website/package.json
index 750a5925c1..d51efca219 100644
--- a/website/package.json
+++ b/website/package.json
@@ -27,6 +27,8 @@
     "@mcap/core": "workspace:*",
     "@mdx-js/react": "1.6.22",
     "@tsconfig/docusaurus": "1.0.7",
+    "@types/dom-mediacapture-transform": "0.1.10",
+    "@types/dom-webcodecs": "0.1.13",
     "@types/promise-queue": "2.2.0",
     "buffer": "6.0.3",
     "classnames": "2.3.2",
diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css b/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
index c6d5836325..2bc019bf0f 100644
--- a/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
+++ b/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
@@ -106,7 +106,7 @@
   margin: 0;
 }
 
-.videoContainer {
+.mediaContainer {
   background-color: #6f3be80f;
   border: 1px solid #e0e0e0;
   font-size: 0.8rem;
@@ -120,11 +120,11 @@
   overflow: hidden;
 }
 
-[data-theme="dark"] .videoContainer {
+[data-theme="dark"] .mediaContainer {
   background-color: transparent;
 }
 
-.videoContainer video {
+.mediaContainer video {
   width: 100%;
   height: 100%;
   position: absolute;
@@ -133,7 +133,7 @@
   z-index: 0;
 }
 
-.videoContainer .videoErrorContainer {
+.mediaContainer .mediaErrorContainer {
   width: 100%;
   height: 100%;
   position: absolute;
@@ -143,16 +143,16 @@
   z-index: 1;
 }
 
-[data-theme="dark"] .videoContainer .videoErrorContainer {
+[data-theme="dark"] .mediaContainer .mediaErrorContainer {
   background-color: #17151ec4;
 }
 
-.videoPlaceholderText {
+.mediaPlaceholderText {
   font-weight: 600;
   cursor: pointer;
 }
 
-.videoLoadingIndicator {
+.mediaLoadingIndicator {
   position: absolute;
   top: 50%;
   left: 50%;
diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
index dc65a05cf1..89bbd73748 100644
--- a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
+++ b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
@@ -15,6 +15,12 @@ import {
   Recorder,
   toProtobufTime,
 } from "./Recorder";
+import {
+  recordAudioStream,
+  startAudioStream,
+  CompressedAudioData,
+  startAudioCapture,
+} from "./audioCapture";
 import {
   CompressedVideoFrame,
   startVideoCapture,
@@ -37,6 +43,7 @@ type State = {
   addPoseMessage: (msg: DeviceOrientationEvent) => void;
   addJpegFrame: (blob: Blob) => void;
   addVideoFrame: (frame: CompressedVideoFrame) => void;
+  addAudioData: (data: CompressedAudioData) => void;
   closeAndRestart: () => Promise<Blob>;
 };
 
@@ -70,6 +77,9 @@ const useStore = create<State>((set) => {
     addVideoFrame(frame: CompressedVideoFrame) {
       void recorder.addVideoFrame(frame);
     },
+    addAudioData(data: CompressedAudioData) {
+      void recorder.addAudioData(data);
+    },
     async closeAndRestart() {
       return await recorder.closeAndRestart();
     },
@@ -128,19 +138,27 @@ export function McapRecordingDemo(): JSX.Element {
 
   const videoRef = useRef<HTMLVideoElement | undefined>();
   const videoContainerRef = useRef<HTMLDivElement>(null);
+  const audioProgressRef = useRef<HTMLProgressElement>(null);
   const [recordJpeg, setRecordJpeg] = useState(false);
   const [recordH264, setRecordH264] = useState(false);
   const [recordH265, setRecordH265] = useState(false);
   const [recordVP9, setRecordVP9] = useState(false);
   const [recordAV1, setRecordAV1] = useState(false);
+  const [recordOpus, setRecordOpus] = useState(false);
   const [recordMouse, setRecordMouse] = useState(true);
   const [recordOrientation, setRecordOrientation] = useState(true);
   const [videoStarted, setVideoStarted] = useState(false);
   const [videoError, setVideoError] = useState<Error | undefined>();
+  const [audioError, setAudioError] = useState<Error | undefined>();
   const [showDownloadInfo, setShowDownloadInfo] = useState(false);
 
-  const { addJpegFrame, addVideoFrame, addMouseEventMessage, addPoseMessage } =
-    state;
+  const {
+    addJpegFrame,
+    addVideoFrame,
+    addMouseEventMessage,
+    addPoseMessage,
+    addAudioData,
+  } = state;
 
   const { data: h264Support } = useAsync(supportsH264Encoding);
   const { data: h265Support } = useAsync(supportsH265Encoding);
@@ -154,7 +172,8 @@ export function McapRecordingDemo(): JSX.Element {
     (recordVP9 && !videoError) ||
     (recordH265 && !videoError) ||
     (recordH264 && !videoError) ||
-    (recordJpeg && !videoError);
+    (recordJpeg && !videoError) ||
+    (recordOpus && !audioError);
 
   // Automatically pause recording after 30 seconds to avoid unbounded growth
   useEffect(() => {
@@ -274,11 +293,62 @@ export function McapRecordingDemo(): JSX.Element {
     recordH264,
     recordH265,
     recordAV1,
+    recordVP9,
     recording,
     videoStarted,
     recordJpeg,
   ]);
 
+  const [audioStream, setAudioStream] = useState<MediaStream | undefined>(
+    undefined,
+  );
+
+  const enableMicrophone = recordOpus;
+  useEffect(() => {
+    const progress = audioProgressRef.current;
+    if (!progress || !enableMicrophone) {
+      return;
+    }
+
+    const cleanup = startAudioStream({
+      progress,
+      onAudioStream: (stream) => {
+        setAudioStream(stream);
+      },
+      onError: (err) => {
+        setAudioError(err);
+        console.error(err);
+      },
+    });
+
+    return () => {
+      cleanup();
+      setAudioStream(undefined);
+      setAudioError(undefined);
+    };
+  }, [enableMicrophone, recordOpus]);
+
+  useEffect(() => {
+    if (!enableMicrophone || !recording || !audioStream) {
+      return;
+    }
+
+    const cleanup = startAudioCapture({
+      enableOpus: recordOpus,
+      stream: audioStream,
+      onAudioData: (data) => {
+        addAudioData(data);
+      },
+      onError: (error) => {
+        setAudioError(error);
+      },
+    });
+
+    return () => {
+      cleanup();
+    };
+  }, [addAudioData, enableMicrophone, recordOpus, audioStream, recording]);
+
   const onRecordClick = useCallback(
     (event: React.MouseEvent) => {
       event.preventDefault();
@@ -417,6 +487,16 @@ export function McapRecordingDemo(): JSX.Element {
             />
             Camera (JPEG)
           </label>
+          <label>
+            <input
+              type="checkbox"
+              checked={recordOpus}
+              onChange={(event) => {
+                setRecordOpus(event.target.checked);
+              }}
+            />
+            Microphone (Opus)
+          </label>
           {!hasMouse && (
             <label>
               <input
@@ -545,22 +625,24 @@ export function McapRecordingDemo(): JSX.Element {
               </div>
             </div>
           </div>
+        </div>
 
+        <div className={styles.recordingControls}>
           <div className={styles.recordingControlsColumn}>
-            <div className={styles.videoContainer} ref={videoContainerRef}>
+            <div className={styles.mediaContainer} ref={videoContainerRef}>
               {videoError ? (
-                <div className={cx(styles.error, styles.videoErrorContainer)}>
+                <div className={cx(styles.error, styles.mediaErrorContainer)}>
                   {videoError.toString()}
                 </div>
-              ) : recordH264 || recordJpeg ? (
+              ) : enableCamera ? (
                 <>
                   {!videoStarted && (
-                    <progress className={styles.videoLoadingIndicator} />
+                    <progress className={styles.mediaLoadingIndicator} />
                   )}
                 </>
               ) : (
                 <span
-                  className={styles.videoPlaceholderText}
+                  className={styles.mediaPlaceholderText}
                   onClick={() => {
                     if (av1Support?.supported === true) {
                       setRecordAV1(true);
@@ -578,6 +660,25 @@ export function McapRecordingDemo(): JSX.Element {
               )}
             </div>
           </div>
+
+          <div className={styles.recordingControlsColumn}>
+            <div className={styles.mediaContainer}>
+              {audioError ? (
+                <div className={cx(styles.error, styles.mediaErrorContainer)}>
+                  {audioError.toString()}
+                </div>
+              ) : enableMicrophone ? (
+                <progress
+                  className={styles.mediaLoadingIndicator}
+                  ref={audioProgressRef}
+                />
+              ) : (
+                <span className={styles.mediaPlaceholderText}>
+                  Enable “Microphone” to record audio
+                </span>
+              )}
+            </div>
+          </div>
         </div>
       </div>
     </section>
diff --git a/website/src/components/McapRecordingDemo/audioCapture.ts b/website/src/components/McapRecordingDemo/audioCapture.ts
index e53ca5164c..ee573e3e62 100644
--- a/website/src/components/McapRecordingDemo/audioCapture.ts
+++ b/website/src/components/McapRecordingDemo/audioCapture.ts
@@ -1,3 +1,89 @@
+type AudioStreamParams = {
+  /** Progress element to display the volume level */
+  progress: HTMLProgressElement;
+  /** Called when the audio stream is available */
+  onAudioStream: (stream: MediaStream) => void;
+  /** Called when an error is encountered */
+  onError: (error: Error) => void;
+};
+
+/**
+ * Prompts the user for microphone permission and displays audio volume in the provided <progress> element
+ * @returns A function to stop the stream and clean up resources
+ */
+export function startAudioStream({
+  progress,
+  onAudioStream,
+  onError,
+}: AudioStreamParams): () => void {
+  let canceled = false;
+  let stream: MediaStream | undefined;
+  let animationID = 0;
+
+  // Although TypeScript does not believe mediaDevices is ever undefined, it may be in practice
+  // (e.g. in Safari)
+  if (typeof navigator.mediaDevices !== "object") {
+    onError(new Error("navigator.mediaDevices is not defined"));
+  } else {
+    const update = (analyzer: AnalyserNode) => {
+      if (canceled) {
+        return;
+      }
+
+      animationID = requestAnimationFrame(() => {
+        // Update the progress bar to show the audio level
+        const fbcArray = new Uint8Array(analyzer.frequencyBinCount);
+        analyzer.getByteFrequencyData(fbcArray);
+        const level =
+          fbcArray.reduce((accum, val) => accum + val, 0) / fbcArray.length;
+        progress.value = level / 100;
+
+        update(analyzer);
+      });
+    };
+
+    const context = new AudioContext();
+    void context.resume();
+    navigator.mediaDevices
+      .getUserMedia({ audio: true })
+      .then((mediaStream) => {
+        if (canceled) {
+          return;
+        }
+        stream = mediaStream;
+        onAudioStream(stream);
+
+        // For displaying volume level
+        const source = context.createMediaStreamSource(stream);
+        const analyzer = context.createAnalyser();
+        source.connect(analyzer);
+
+        update(analyzer);
+      })
+      .catch((err) => {
+        if (canceled) {
+          return;
+        }
+
+        onError(
+          new Error(
+            `${(
+              err as Error
+            ).toString()}. Ensure microphone permissions are enabled.`,
+          ),
+        );
+      });
+  }
+
+  return () => {
+    canceled = true;
+    cancelAnimationFrame(animationID);
+    for (const track of stream?.getTracks() ?? []) {
+      track.stop();
+    }
+  };
+}
+
 type CompressedAudioFormat = "opus";
 type CompressedAudioType = "key" | "delta";
 export type CompressedAudioData = {
@@ -11,3 +97,94 @@ export type CompressedAudioData = {
   /** Call this function to release the buffer so it can be reused for new frames */
   release: () => void;
 };
+
+interface AudioCaptureParams {
+  enableOpus: boolean;
+  /** MediaStream from startAudioStream */
+  stream: MediaStream;
+  /** Called when an audio frame has been encoded */
+  onAudioData: (data: CompressedAudioData) => void;
+  onError: (error: Error) => void;
+}
+
+export function startAudioCapture({
+  enableOpus,
+  stream,
+  onAudioData,
+  onError,
+}: AudioCaptureParams): () => void {
+  const framePool: ArrayBuffer[] = [];
+
+  const track = stream.getAudioTracks()[0];
+  if (!track) {
+    onError(new Error("Invariant: expected audio track"));
+    return () => {
+      // no op
+    };
+  }
+
+  const settings = track.getSettings();
+
+  const trackProcessor = new MediaStreamTrackProcessor({
+    // TODO: Don't assert
+    track: stream.getAudioTracks()[0]!,
+  });
+
+  const encoder = new AudioEncoder({
+    output: (chunk) => {
+      console.log(chunk);
+      let buffer = framePool.pop();
+      if (!buffer || buffer.byteLength < chunk.byteLength) {
+        buffer = new ArrayBuffer(chunk.byteLength);
+      }
+      chunk.copyTo(buffer);
+      onAudioData({
+        format: "opus",
+        type: chunk.type as CompressedAudioType,
+        timestamp: chunk.timestamp,
+        data: new Uint8Array(buffer, 0, chunk.byteLength),
+        sampleRate: settings.sampleRate ?? 0,
+        numberOfChannels: settings.channelCount ?? 0,
+        release() {
+          // TODO: Don't assert
+          framePool.push(buffer!);
+        },
+      });
+    },
+    error: (error) => {
+      onError(error);
+    },
+  });
+  encoder.configure({
+    codec: "opus",
+    sampleRate: settings.sampleRate ?? 0,
+    numberOfChannels: settings.channelCount ?? 0,
+  });
+
+  const reader = trackProcessor.readable.getReader();
+  let canceled = false;
+
+  const readAndEncode = () => {
+    reader
+      .read()
+      .then((result) => {
+        if (result.done || canceled) {
+          return;
+        }
+
+        encoder.encode(result.value);
+
+        readAndEncode();
+      })
+      .catch((error) => {
+        onError(error as Error);
+      });
+  };
+
+  readAndEncode();
+
+  return () => {
+    canceled = true;
+    encoder.close();
+  };
+}
diff --git a/website/tsconfig.json b/website/tsconfig.json
index 6fc529e4db..5d1563c12b 100644
--- a/website/tsconfig.json
+++ b/website/tsconfig.json
@@ -10,6 +10,7 @@
       "@mcap/core": ["../typescript/core/src"]
     },
     "moduleResolution": "node",
+    "types": ["dom-webcodecs", "dom-mediacapture-transform"],
 
     // Settings from @foxglove/tsconfig/base
     "newLine": "lf",
diff --git a/yarn.lock b/yarn.lock
index c252e3da11..334ce381f7 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4163,6 +4163,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/dom-mediacapture-transform@npm:0.1.10":
+  version: 0.1.10
+  resolution: "@types/dom-mediacapture-transform@npm:0.1.10"
+  dependencies:
+    "@types/dom-webcodecs": "npm:*"
+  checksum: 10c0/cbad14e0b78bdd36dad7780d06526ab27cdc14c90035826dec0d2a1e71d5b77cc3f58feb0d9e0c08c9f6ebdb4d5a60cc507c7126115e0829d5987acc49e4dd30
+  languageName: node
+  linkType: hard
+
+"@types/dom-webcodecs@npm:*, @types/dom-webcodecs@npm:0.1.13":
+  version: 0.1.13
+  resolution: "@types/dom-webcodecs@npm:0.1.13"
+  checksum: 10c0/16683fd0330fdfbb867affec1c18fd493d88433a4ec66257579adc5d07b07e9492268b4401e9017b7388b13750722365b0392b375e39c84e74f564d5b4f634a4
+  languageName: node
+  linkType: hard
+
 "@types/eslint-scope@npm:^3.7.3":
   version: 3.7.4
   resolution: "@types/eslint-scope@npm:3.7.4"
@@ -15625,6 +15641,8 @@ __metadata:
     "@mcap/core": "workspace:*"
     "@mdx-js/react": "npm:1.6.22"
     "@tsconfig/docusaurus": "npm:1.0.7"
+    "@types/dom-mediacapture-transform": "npm:0.1.10"
+    "@types/dom-webcodecs": "npm:0.1.13"
     "@types/promise-queue": "npm:2.2.0"
     buffer: "npm:6.0.3"
     classnames: "npm:2.3.2"