fix: Fix changing participants during audio conversation (#325)

* fix: Fix changing participants during audio conversation * chore: @inworld/web-core 2.9.2-beta.5 * fix: Stop recorder during audio session recreation * chore: @inworld/web-core 2.9.2-beta.6 --------- Co-authored-by: CI-inworld <[email protected]>
inworld-ai · Jan 13, 2025 · 7ed1279 · 7ed1279
1 parent 977110b
commit 7ed1279
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 23 deletions.
diff --git a/packages/web-core/CHANGELOG.md b/packages/web-core/CHANGELOG.md
@@ -3,6 +3,7 @@
 ### Fixed
 
 - Fix changing of scene capabilities
+- Fix changing participants during audio conversation
 
 ## [2.9.1] - 2024-12-03
 

diff --git a/packages/web-core/__tests__/services/conversation.service.spec.ts b/packages/web-core/__tests__/services/conversation.service.spec.ts
@@ -263,26 +263,22 @@ describe('update participants', () => {
   });
 
   test('should reopen audio session', async () => {
-    jest.spyOn(ConnectionService.prototype, 'send').mockImplementation(() =>
-      Promise.resolve({
-        packetId: {
-          conversationId: conversationId,
-        },
-      }),
-    );
+    const send = jest
+      .spyOn(ConnectionService.prototype, 'send')
+      .mockImplementation(() =>
+        Promise.resolve({
+          packetId: {
+            conversationId: conversationId,
+          },
+        }),
+      );
     jest
       .spyOn(ConversationService.prototype, 'getConversationId')
       .mockImplementation(() => conversationId);
 
     const getAudioSessionAction = jest
       .spyOn(ConnectionService.prototype, 'getAudioSessionAction')
       .mockImplementation(() => AudioSessionState.START);
-    const sendAudioSessionEnd = jest
-      .spyOn(ConversationService.prototype, 'sendAudioSessionEnd')
-      .mockImplementation(jest.fn());
-    const sendAudioSessionStart = jest
-      .spyOn(ConversationService.prototype, 'sendAudioSessionStart')
-      .mockImplementation(jest.fn());
 
     const service = new ConversationService(connection, {
       participants: [characters[0].resourceName],
@@ -310,8 +306,7 @@ describe('update participants', () => {
     ]);
 
     expect(getAudioSessionAction).toHaveBeenCalledTimes(1);
-    expect(sendAudioSessionEnd).toHaveBeenCalledTimes(1);
-    expect(sendAudioSessionStart).toHaveBeenCalledTimes(1);
+    expect(send).toHaveBeenCalledTimes(3);
   });
 
   test('should add characters to scene automatically', async () => {
@@ -366,7 +361,7 @@ describe('update participants', () => {
       }),
     ]);
 
-    const update = send.mock.calls[0][0]();
+    const update = send.mock.calls[1][0]();
     expect(addCharacters).toHaveBeenCalledTimes(1);
     expect(update.packetId?.conversationId).toEqual(conversationId);
     expect(update.control).toEqual({

diff --git a/packages/web-core/package.json b/packages/web-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@inworld/web-core",
-  "version": "2.9.2-beta.4",
+  "version": "2.9.2-beta.6",
   "license": "SEE LICENSE IN LICENSE.md",
   "main": "build/src/index.js",
   "types": "build/src/index.d.ts",

diff --git a/packages/web-core/src/index.ts b/packages/web-core/src/index.ts
@@ -35,6 +35,7 @@ import {
   HistoryItem,
   HistoryItemActor,
   HistoryItemBase,
+  HistoryItemLogEvent,
   HistoryItemNarratedAction,
   HistoryItemSceneChange,
   HistoryItemTaskEvent,
@@ -120,6 +121,7 @@ export {
   HistoryItem,
   HistoryItemActor,
   HistoryItemBase,
+  HistoryItemLogEvent,
   HistoryItemNarratedAction,
   HistoryItemSceneChange,
   HistoryItemTaskEvent,

diff --git a/packages/web-core/src/services/conversation.service.ts b/packages/web-core/src/services/conversation.service.ts
@@ -35,6 +35,8 @@ export class ConversationService<
   private conversationId: string;
   private participants: string[];
   private addCharacters?: (names: string[]) => Promise<void>;
+  private startRecording?: () => Promise<void>;
+  private stopRecording?: () => void;
   private packetQueue: PacketQueueItem<InworldPacketT>[] = [];
   private ttsPlaybackAction = TtsPlaybackAction.UNKNOWN;
 
@@ -44,17 +46,23 @@ export class ConversationService<
       participants,
       conversationId,
       addCharacters,
+      startRecording,
+      stopRecording,
     }: {
       participants: string[];
       conversationId?: string;
       addCharacters: (names: string[]) => Promise<void>;
+      startRecording?: () => Promise<void>;
+      stopRecording?: () => void;
     },
   ) {
     this.connection = connection;
     this.conversationId = conversationId ?? v4();
     this.participants = participants;
 
     this.addCharacters = addCharacters;
+    this.startRecording = startRecording;
+    this.stopRecording = stopRecording;
   }
 
   getConversationId() {
@@ -110,7 +118,11 @@ export class ConversationService<
     // If audio session is started, we need to end it before updating participants
     if (this.connection.getAudioSessionAction() === AudioSessionState.START) {
       needToReacreateAudioSession = true;
-      await this.sendAudioSessionEnd();
+      this.stopRecording?.();
+      this.beforeAudioSessionEnd();
+      await this.connection.send(() =>
+        this.connection.getEventFactory().audioSessionEnd({ conversationId }),
+      );
     }
 
     // Load characters if they are not loaded
@@ -143,7 +155,11 @@ export class ConversationService<
 
     // If audio session was started before, we need to restart it
     if (needToReacreateAudioSession) {
-      await this.sendAudioSessionStart();
+      this.beforeAudioSessionStart();
+      await this.connection.send(() =>
+        this.connection.getEventFactory().audioSessionStart({ conversationId }),
+      );
+      this.startRecording?.();
     }
 
     await this.resolveInterval(
@@ -204,8 +220,7 @@ export class ConversationService<
       throw Error('Audio session is already started');
     }
 
-    this.connection.setAudioSessionAction(AudioSessionState.START);
-    this.connection.setCurrentAudioConversation(this);
+    this.beforeAudioSessionStart();
 
     return this.ensureConversation(() =>
       this.connection.getEventFactory().audioSessionStart({
@@ -226,8 +241,7 @@ export class ConversationService<
     }
 
     return this.ensureConversation(() => {
-      this.connection.setAudioSessionAction(AudioSessionState.END);
-      this.connection.setCurrentAudioConversation(undefined);
+      this.beforeAudioSessionEnd();
 
       return this.connection
         .getEventFactory()
@@ -353,6 +367,16 @@ export class ConversationService<
     return sent;
   }
 
+  private beforeAudioSessionStart() {
+    this.connection.setAudioSessionAction(AudioSessionState.START);
+    this.connection.setCurrentAudioConversation(this);
+  }
+
+  private beforeAudioSessionEnd() {
+    this.connection.setAudioSessionAction(AudioSessionState.END);
+    this.connection.setCurrentAudioConversation(undefined);
+  }
+
   setTtsPlaybackAction(action: TtsPlaybackAction) {
     this.ttsPlaybackAction = action;
     this.connection.history.setAudioEnabled(

diff --git a/packages/web-core/src/services/inworld_connection.service.ts b/packages/web-core/src/services/inworld_connection.service.ts
@@ -135,6 +135,8 @@ export class InworldConnectionService<
         {
           participants: [character.resourceName, ConversationParticipant.USER],
           addCharacters: this.addCharacters.bind(this),
+          startRecording: this.recorder.start.bind(this.recorder),
+          stopRecording: this.recorder.stop.bind(this.recorder),
         },
       );
 
@@ -376,6 +378,8 @@ export class InworldConnectionService<
     const service = new ConversationService(this.connection, {
       participants,
       addCharacters: this.addCharacters.bind(this),
+      startRecording: this.recorder.start.bind(this.recorder),
+      stopRecording: this.recorder.stop.bind(this.recorder),
     });
 
     this.connection.conversations.set(service.getConversationId(), {
@@ -413,6 +417,8 @@ export class InworldConnectionService<
         {
           participants: [character.resourceName, ConversationParticipant.USER],
           addCharacters: this.addCharacters.bind(this),
+          startRecording: this.recorder.start.bind(this.recorder),
+          stopRecording: this.recorder.stop.bind(this.recorder),
         },
       );