Issue with streaming data not making it to the Deepgram API #370

BeamoINT · 2023-10-07T03:18:41Z

BeamoINT
Oct 7, 2023

I have been having some issues when attempting to use the Deepgram Nova Streaming API and I have narrowed it down to the fact that the audio data is not making it from the web socket to the Deepgram Streaming API and I just have not been able to get it working either with the web API or with the SDK. Here is the code.

wss.on("connection", function connection(ws) {
    console.log("New Connection Initiated");

    let deepgramLive = null;

    ws.on("message", function incoming(message) {
        const msg = JSON.parse(message);
        switch (msg.event) {
            case "connected":
                console.log(`A new call has connected.`);
                break;
            case "start":
                console.log(`Starting Media Stream ${msg.streamSid}`);

                deepgramLive = deepgram.transcription.live({
                    smart_format: true,
                    model: 'nova',
                    language: 'en-US'
                });

                console.log("Deepgram live transcription initialized.");

                deepgramLive.addListener('transcriptReceived', (transcription) => {
                    if (transcription.channel && transcription.channel.alternatives && transcription.channel.alternatives.length > 0) {
                        const transcriptText = transcription.channel.alternatives[0].transcript;
                        console.log("Transcription: ", transcriptText);
                        process.stdout.write("\r" + transcriptText);

                        conversationHistory.push({
                            role: 'user',
                            content: transcriptText + '\n'
                        });

                        console.log("Transcription added to conversation history.");

                        wss.clients.forEach((client) => {
                            if (client.readyState === WebSocket.OPEN) {
                                console.log("Sending transcription to WebSocket client.");

                                client.send(
                                    JSON.stringify({
                                        event: "transcription",
                                        text: transcriptText,
                                    })
                                );
                            }
                        });

                        if (!isAPIInProgress) {
                            console.log("Calling OpenAI...");
                            callOpenAI();
                        }
                    }
                });

                break;
            case "media":
                if (!isAPIInProgress && deepgramLive.getReadyState() === 1) {
                    deepgramLive.send(msg.media.payload);
                }
                break;
            case "stop":
                console.log(`Call Has Ended`);
                console.log("Finishing the Deepgram live transcription.");
                deepgramLive.finish();
                break;
        }
    });
});