✨ feat: improve WebRTC connection state handling and enhance WebSocket connection logic

api/package.json

@@ -105,12 +105,14 @@
     "passport-local": "^1.0.0",
     "rate-limit-redis": "^4.2.0",
     "sharp": "^0.33.5",
+    "socket.io": "^4.8.1",
     "tiktoken": "^1.0.15",
     "traverse": "^0.6.7",
     "ua-parser-js": "^1.0.36",
     "winston": "^3.11.0",
     "winston-daily-rotate-file": "^4.7.1",
     "youtube-transcript": "^1.2.1",
+    "wrtc": "^0.4.7",
     "zod": "^3.22.4"
   },
   "devDependencies": {

api/server/index.js

@@ -4,6 +4,7 @@ require('module-alias')({ base: path.resolve(__dirname, '..') });
 const cors = require('cors');
 const axios = require('axios');
 const express = require('express');
+const { createServer } = require('http');
 const compression = require('compression');
 const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
@@ -14,6 +15,8 @@ const { connectDb, indexSync } = require('~/lib/db');
 const { isEnabled } = require('~/server/utils');
 const { ldapLogin } = require('~/strategies');
 const { logger } = require('~/config');
+const { AudioSocketModule } = require('./services/Files/Audio/AudioSocketModule');
+const { SocketIOService } = require('./services/WebSocket/WebSocketServer');
 const validateImageRequest = require('./middleware/validateImageRequest');
 const errorController = require('./controllers/ErrorController');
 const configureSocialLogins = require('./socialLogins');
@@ -28,6 +31,9 @@ const port = Number(PORT) || 3080;
 const host = HOST || 'localhost';
 const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default */
 
+let socketIOService;
+let audioModule;
+
 const startServer = async () => {
   if (typeof Bun !== 'undefined') {
     axios.defaults.headers.common['Accept-Encoding'] = 'gzip';
@@ -37,7 +43,21 @@ const startServer = async () => {
   await indexSync();
 
   const app = express();
+  const server = createServer(app);
+
   app.disable('x-powered-by');
+  app.use(
+    cors({
+      origin: true,
+      credentials: true,
+    }),
+  );
+
+  socketIOService = new SocketIOService(server);
+  audioModule = new AudioSocketModule(socketIOService);
+
+  logger.info('WebSocket server and Audio module initialized');
+
   await AppService(app);
 
   const indexPath = path.join(app.locals.paths.dist, 'index.html');
@@ -110,6 +130,7 @@ const startServer = async () => {
   app.use('/api/agents', routes.agents);
   app.use('/api/banner', routes.banner);
   app.use('/api/bedrock', routes.bedrock);
+  app.use('/api/websocket', routes.websocket);
 
   app.use('/api/tags', routes.tags);
 
@@ -127,7 +148,7 @@ const startServer = async () => {
     res.send(updatedIndexHtml);
   });
 
-  app.listen(port, host, () => {
+  server.listen(port, host, () => {
     if (host == '0.0.0.0') {
       logger.info(
         `Server listening on all interfaces at port ${port}. Use http://localhost:${port} to access it`,
@@ -135,11 +156,26 @@ const startServer = async () => {
     } else {
       logger.info(`Server listening at http://${host == '0.0.0.0' ? 'localhost' : host}:${port}`);
     }
+
+    logger.info(`Socket.IO endpoint: http://${host}:${port}`);
   });
 };
 
 startServer();
 
+process.on('SIGINT', () => {
+  logger.info('Shutting down server...');
+  if (audioModule) {
+    audioModule.cleanup();
+    logger.info('Audio module cleaned up');
+  }
+  if (socketIOService) {
+    socketIOService.shutdown();
+    logger.info('WebSocket server shut down');
+  }
+  process.exit(0);
+});
+
 let messageCount = 0;
 process.on('uncaughtException', (err) => {
   if (!err.message.includes('fetch failed')) {

api/server/routes/files/speech/customConfigSpeech.js

@@ -3,7 +3,7 @@ const router = express.Router();
 
 const { getCustomConfigSpeech } = require('~/server/services/Files/Audio');
 
-router.get('/get', async (req, res) => {
+router.get('/', async (req, res) => {
   await getCustomConfigSpeech(req, res);
 });
 

api/server/routes/files/speech/index.js

@@ -4,6 +4,7 @@ const { createTTSLimiters, createSTTLimiters } = require('~/server/middleware');
 const stt = require('./stt');
 const tts = require('./tts');
 const customConfigSpeech = require('./customConfigSpeech');
+const realtime = require('./realtime');
 
 const router = express.Router();
 
@@ -14,4 +15,6 @@ router.use('/tts', ttsIpLimiter, ttsUserLimiter, tts);
 
 router.use('/config', customConfigSpeech);
 
+router.use('/realtime', realtime);
+
 module.exports = router;

api/server/routes/files/speech/realtime.js

@@ -0,0 +1,10 @@
+const express = require('express');
+const router = express.Router();
+
+const { getRealtimeConfig } = require('~/server/services/Files/Audio');
+
+router.get('/', async (req, res) => {
+  await getRealtimeConfig(req, res);
+});
+
+module.exports = router;

api/server/routes/index.js

@@ -2,6 +2,7 @@ const assistants = require('./assistants');
 const categories = require('./categories');
 const tokenizer = require('./tokenizer');
 const endpoints = require('./endpoints');
+const websocket = require('./websocket');
 const staticRoute = require('./static');
 const messages = require('./messages');
 const presets = require('./presets');
@@ -15,6 +16,7 @@ const models = require('./models');
 const convos = require('./convos');
 const config = require('./config');
 const agents = require('./agents');
+const banner = require('./banner');
 const roles = require('./roles');
 const oauth = require('./oauth');
 const files = require('./files');
@@ -25,7 +27,6 @@ const edit = require('./edit');
 const keys = require('./keys');
 const user = require('./user');
 const ask = require('./ask');
-const banner = require('./banner');
 
 module.exports = {
   ask,
@@ -39,6 +40,7 @@ module.exports = {
   files,
   share,
   agents,
+  banner,
   bedrock,
   convos,
   search,
@@ -50,10 +52,10 @@ module.exports = {
   presets,
   balance,
   messages,
+  websocket,
   endpoints,
   tokenizer,
   assistants,
   categories,
   staticRoute,
-  banner,
 };

api/server/routes/websocket.js

@@ -0,0 +1,19 @@
+const express = require('express');
+const optionalJwtAuth = require('~/server/middleware/optionalJwtAuth');
+const router = express.Router();
+
+router.get('/', optionalJwtAuth, async (req, res) => {
+  const isProduction = process.env.NODE_ENV === 'production';
+
+  const protocol = isProduction && req.secure ? 'https' : 'http';
+
+  const serverDomain = process.env.SERVER_DOMAIN
+    ? process.env.SERVER_DOMAIN.replace(/^https?:\/\//, '')
+    : req.headers.host;
+
+  const socketIoUrl = `${protocol}://${serverDomain}`;
+
+  res.json({ url: socketIoUrl });
+});
+
+module.exports = router;

api/server/services/Files/Audio/AudioSocketModule.js

@@ -0,0 +1,40 @@
+const { AudioHandler } = require('./WebRTCHandler');
+const { logger } = require('~/config');
+
+class AudioSocketModule {
+  constructor(socketIOService) {
+    this.socketIOService = socketIOService;
+    this.audioHandler = new AudioHandler();
+
+    this.moduleId = 'audio-handler';
+    this.registerHandlers();
+  }
+
+  registerHandlers() {
+    this.socketIOService.registerModule(this.moduleId, {
+      connection: (socket) => this.handleConnection(socket),
+      disconnect: (socket) => this.handleDisconnect(socket),
+    });
+  }
+
+  handleConnection(socket) {
+    // Register WebRTC-specific event handlers for this socket
+    this.audioHandler.registerSocketHandlers(socket, this.config);
+
+    logger.debug(`Audio handler registered for client: ${socket.id}`);
+  }
+
+  handleDisconnect(socket) {
+    // Cleanup audio resources for disconnected client
+    this.audioHandler.cleanup(socket.id);
+    logger.debug(`Audio handler cleaned up for client: ${socket.id}`);
+  }
+
+  // Used for app shutdown
+  cleanup() {
+    this.audioHandler.cleanupAll();
+    this.socketIOService.unregisterModule(this.moduleId);
+  }
+}
+
+module.exports = { AudioSocketModule };

api/server/services/Files/Audio/WebRTCHandler.js

@@ -0,0 +1,178 @@
+const { RTCPeerConnection, RTCIceCandidate, MediaStream } = require('wrtc');
+const { logger } = require('~/config');
+
+class WebRTCConnection {
+  constructor(socket, config) {
+    this.socket = socket;
+    this.config = config;
+    this.peerConnection = null;
+    this.audioTransceiver = null;
+    this.pendingCandidates = [];
+    this.state = 'idle';
+  }
+
+  async handleOffer(offer) {
+    try {
+      if (!this.peerConnection) {
+        this.peerConnection = new RTCPeerConnection(this.config.rtcConfig);
+        this.setupPeerConnectionListeners();
+      }
+
+      await this.peerConnection.setRemoteDescription(offer);
+
+      const mediaStream = new MediaStream();
+
+      this.audioTransceiver = this.peerConnection.addTransceiver('audio', {
+        direction: 'sendrecv',
+        streams: [mediaStream],
+      });
+
+      const answer = await this.peerConnection.createAnswer();
+      await this.peerConnection.setLocalDescription(answer);
+      this.socket.emit('webrtc-answer', answer);
+    } catch (error) {
+      logger.error(`Error handling offer: ${error}`);
+      this.socket.emit('webrtc-error', {
+        message: error.message,
+        code: 'OFFER_ERROR',
+      });
+    }
+  }
+
+  setupPeerConnectionListeners() {
+    if (!this.peerConnection) {
+      return;
+    }
+
+    this.peerConnection.ontrack = ({ track }) => {
+      logger.info(`Received ${track.kind} track from client`);
+
+      if (track.kind === 'audio') {
+        this.handleIncomingAudio(track);
+      }
+
+      track.onended = () => {
+        logger.info(`${track.kind} track ended`);
+      };
+    };
+
+    this.peerConnection.onicecandidate = ({ candidate }) => {
+      if (candidate) {
+        this.socket.emit('icecandidate', candidate);
+      }
+    };
+
+    this.peerConnection.onconnectionstatechange = () => {
+      if (!this.peerConnection) {
+        return;
+      }
+
+      const state = this.peerConnection.connectionState;
+      logger.info(`Connection state changed to ${state}`);
+
+      if (state === 'failed' || state === 'closed' || state === 'disconnected') {
+        this.cleanup();
+      }
+    };
+  }
+
+  handleIncomingAudio(track) {
+    if (this.peerConnection) {
+      const stream = new MediaStream([track]);
+      this.peerConnection.addTrack(track, stream);
+    }
+  }
+
+  async addIceCandidate(candidate) {
+    try {
+      if (this.peerConnection?.remoteDescription) {
+        if (candidate && candidate.candidate) {
+          await this.peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
+        } else {
+          logger.warn('Invalid ICE candidate');
+        }
+      } else {
+        this.pendingCandidates.push(candidate);
+      }
+    } catch (error) {
+      logger.error(`Error adding ICE candidate: ${error}`);
+    }
+  }
+
+  cleanup() {
+    if (this.peerConnection) {
+      try {
+        this.peerConnection.close();
+      } catch (error) {
+        logger.error(`Error closing peer connection: ${error}`);
+      }
+      this.peerConnection = null;
+    }
+
+    this.audioTransceiver = null;
+    this.pendingCandidates = [];
+    this.state = 'idle';
+  }
+}
+
+class AudioHandler {
+  constructor() {
+    this.connections = new Map();
+    this.defaultRTCConfig = {
+      iceServers: [
+        {
+          urls: ['stun:stun.l.google.com:19302', 'stun:stun1.l.google.com:19302'],
+        },
+      ],
+      iceCandidatePoolSize: 10,
+      bundlePolicy: 'max-bundle',
+      rtcpMuxPolicy: 'require',
+    };
+  }
+
+  registerSocketHandlers(socket) {
+    const rtcConfig = {
+      rtcConfig: this.defaultRTCConfig,
+    };
+
+    const rtcConnection = new WebRTCConnection(socket, rtcConfig);
+    this.connections.set(socket.id, rtcConnection);
+
+    socket.on('webrtc-offer', (offer) => {
+      logger.debug(`Received WebRTC offer from ${socket.id}`);
+      rtcConnection.handleOffer(offer);
+    });
+
+    socket.on('icecandidate', (candidate) => {
+      rtcConnection.addIceCandidate(candidate);
+    });
+
+    socket.on('vad-status', (speaking) => {
+      logger.debug(`VAD status from ${socket.id}: ${JSON.stringify(speaking)}`);
+    });
+
+    socket.on('disconnect', () => {
+      rtcConnection.cleanup();
+      this.connections.delete(socket.id);
+    });
+
+    return rtcConnection;
+  }
+
+  cleanup(socketId) {
+    const connection = this.connections.get(socketId);
+    if (connection) {
+      connection.cleanup();
+      this.connections.delete(socketId);
+    }
+  }
+
+  cleanupAll() {
+    for (const connection of this.connections.values()) {
+      connection.cleanup();
+    }
+    this.connections.clear();
+  }
+}
+
+module.exports = { AudioHandler, WebRTCConnection };

api/server/services/Files/Audio/getRealtimeConfig.js

@@ -0,0 +1,102 @@
+const { extractEnvVariable, RealtimeVoiceProviders } = require('librechat-data-provider');
+const { getCustomConfig } = require('~/server/services/Config');
+const { logger } = require('~/config');
+
+class RealtimeService {
+  constructor(customConfig) {
+    this.customConfig = customConfig;
+    this.providerStrategies = {
+      [RealtimeVoiceProviders.OPENAI]: this.openaiProvider.bind(this),
+    };
+  }
+
+  static async getInstance() {
+    const customConfig = await getCustomConfig();
+    if (!customConfig) {
+      throw new Error('Custom config not found');
+    }
+    return new RealtimeService(customConfig);
+  }
+
+  async getProviderSchema() {
+    const realtimeSchema = this.customConfig.speech.realtime;
+    if (!realtimeSchema) {
+      throw new Error('No Realtime schema is set in config');
+    }
+
+    const providers = Object.entries(realtimeSchema).filter(
+      ([, value]) => Object.keys(value).length > 0,
+    );
+
+    if (providers.length !== 1) {
+      throw new Error(providers.length > 1 ? 'Multiple providers set' : 'No provider set');
+    }
+
+    return providers[0];
+  }
+
+  async openaiProvider(schema, voice) {
+    const defaultRealtimeUrl = 'https://api.openai.com/v1/realtime';
+    const allowedVoices = ['alloy', 'ash', 'ballad', 'coral', 'echo', 'sage', 'shimmer', 'verse'];
+
+    if (!voice) {
+      throw new Error('Voice not specified');
+    }
+
+    if (!allowedVoices.includes(voice)) {
+      throw new Error(`Invalid voice: ${voice}`);
+    }
+
+    const apiKey = extractEnvVariable(schema.apiKey);
+    if (!apiKey) {
+      throw new Error('OpenAI API key not configured');
+    }
+
+    const response = await fetch('https://api.openai.com/v1/realtime/sessions', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model: 'gpt-4o-realtime-preview-2024-12-17',
+        modalities: ['audio', 'text'],
+        voice: voice,
+      }),
+    });
+
+    const token = response.json();
+
+    return {
+      provider: RealtimeVoiceProviders.OPENAI,
+      token: token,
+      url: schema.url || defaultRealtimeUrl,
+    };
+  }
+
+  async getRealtimeConfig(req, res) {
+    try {
+      const [provider, schema] = await this.getProviderSchema();
+      const strategy = this.providerStrategies[provider];
+
+      if (!strategy) {
+        throw new Error(`Unsupported provider: ${provider}`);
+      }
+
+      const voice = req.query.voice;
+
+      const config = strategy(schema, voice);
+      res.json(config);
+    } catch (error) {
+      logger.error('[RealtimeService] Config generation failed:', error);
+      res.status(500).json({ error: error.message });
+    }
+  }
+}
+
+async function getRealtimeConfig(req, res) {
+  const service = await RealtimeService.getInstance();
+  await service.getRealtimeConfig(req, res);
+}
+
+module.exports = getRealtimeConfig;

api/server/services/Files/Audio/index.js

@@ -1,4 +1,5 @@
 const getCustomConfigSpeech = require('./getCustomConfigSpeech');
+const getRealtimeConfig = require('./getRealtimeConfig');
 const TTSService = require('./TTSService');
 const STTService = require('./STTService');
 const getVoices = require('./getVoices');
@@ -6,6 +7,7 @@ const getVoices = require('./getVoices');
 module.exports = {
   getVoices,
   getCustomConfigSpeech,
+  getRealtimeConfig,
   ...STTService,
   ...TTSService,
 };

api/server/services/WebSocket/WebSocketServer.js

@@ -0,0 +1,90 @@
+const { Server } = require('socket.io');
+const { logger } = require('~/config');
+
+class SocketIOService {
+  constructor(httpServer) {
+    this.io = new Server(httpServer, {
+      path: '/socket.io',
+      cors: {
+        origin: '*',
+        methods: ['GET', 'POST'],
+      },
+    });
+
+    this.connections = new Map();
+    this.eventHandlers = new Map();
+    this.setupSocketHandlers();
+  }
+
+  setupSocketHandlers() {
+    this.io.on('connection', (socket) => {
+      this.log(`Client connected: ${socket.id}`);
+      this.connections.set(socket.id, socket);
+
+      // Emit connection event for modules to handle
+      this.emitEvent('connection', socket);
+
+      socket.on('disconnect', () => {
+        this.log(`Client disconnected: ${socket.id}`);
+        this.emitEvent('disconnect', socket);
+        this.connections.delete(socket.id);
+      });
+    });
+  }
+
+  // Register a module to handle specific events
+  registerModule(moduleId, eventHandlers) {
+    for (const [eventName, handler] of Object.entries(eventHandlers)) {
+      if (!this.eventHandlers.has(eventName)) {
+        this.eventHandlers.set(eventName, new Map());
+      }
+
+      this.eventHandlers.get(eventName).set(moduleId, handler);
+
+      // If this is a socket event, register it on all existing connections
+      if (eventName !== 'connection' && eventName !== 'disconnect') {
+        for (const socket of this.connections.values()) {
+          socket.on(eventName, (...args) => {
+            handler(socket, ...args);
+          });
+        }
+      }
+    }
+  }
+
+  // Unregister a module
+  unregisterModule(moduleId) {
+    for (const handlers of this.eventHandlers.values()) {
+      handlers.delete(moduleId);
+    }
+  }
+
+  // Emit an event to all registered handlers
+  emitEvent(eventName, ...args) {
+    const handlers = this.eventHandlers.get(eventName);
+    if (handlers) {
+      for (const handler of handlers.values()) {
+        handler(...args);
+      }
+    }
+  }
+
+  log(message, level = 'info') {
+    const timestamp = new Date().toISOString();
+
+    try {
+      logger.debug(`[WebSocket] ${message}`, level);
+    } catch (error) {
+      console.log(`[WebSocket ${timestamp}] [${level.toUpperCase()}] ${message}`);
+      console.error(`[WebSocket ${timestamp}] [ERROR] Error while logging: ${error.message}`);
+    }
+  }
+
+  shutdown() {
+    this.connections.clear();
+    this.eventHandlers.clear();
+    this.io.close();
+  }
+}
+
+module.exports = { SocketIOService };

client/package.json

@@ -53,6 +53,7 @@
     "@radix-ui/react-tabs": "^1.0.3",
     "@radix-ui/react-toast": "^1.1.5",
     "@react-spring/web": "^9.7.5",
+    "@ricky0123/vad-react": "^0.0.28",
     "@tanstack/react-query": "^4.28.0",
     "@tanstack/react-table": "^8.11.7",
     "class-variance-authority": "^0.6.0",
@@ -101,6 +102,7 @@
     "remark-gfm": "^4.0.0",
     "remark-math": "^6.0.0",
     "remark-supersub": "^1.0.0",
+    "socket.io-client": "^4.8.1",
     "sse.js": "^2.5.0",
     "tailwind-merge": "^1.9.1",
     "tailwindcss-animate": "^1.0.5",

client/src/common/types.ts

@@ -56,6 +56,32 @@ export type BadgeItem = {
   isAvailable: boolean;
 };
 
+export interface RTCMessage {
+  type:
+    | 'audio-chunk'
+    | 'audio-received'
+    | 'transcription'
+    | 'llm-response'
+    | 'tts-chunk'
+    | 'call-ended'
+    | 'webrtc-answer'
+    | 'icecandidate';
+  payload?: RTCSessionDescriptionInit | RTCIceCandidateInit;
+}
+
+export type MessagePayload =
+  | RTCSessionDescriptionInit
+  | RTCIceCandidateInit
+  | { speaking: boolean };
+
+export enum CallState {
+  IDLE = 'idle',
+  CONNECTING = 'connecting',
+  ACTIVE = 'active',
+  ERROR = 'error',
+  ENDED = 'ended',
+}
+
 export type AssistantListItem = {
   id: string;
   name: string;

client/src/components/Chat/Input/Call.tsx

@@ -0,0 +1,301 @@
+import React, { useEffect, useRef } from 'react';
+import { useRecoilState } from 'recoil';
+import {
+  Phone,
+  PhoneOff,
+  AlertCircle,
+  Mic,
+  MicOff,
+  Volume2,
+  VolumeX,
+  Activity,
+  ChevronDown,
+  ChevronUp,
+  Wifi,
+} from 'lucide-react';
+import { OGDialog, OGDialogContent, Button } from '~/components';
+import { useWebSocket, useCall } from '~/hooks';
+import { CallState } from '~/common';
+import store from '~/store';
+
+export const Call: React.FC = () => {
+  const { isConnected } = useWebSocket();
+  const {
+    callState,
+    error,
+    startCall,
+    hangUp,
+    isConnecting,
+    localStream,
+    remoteStream,
+    connectionQuality,
+    connectionMetrics,
+    isMuted,
+    toggleMute,
+  } = useCall();
+
+  const [open, setOpen] = useRecoilState(store.callDialogOpen(0));
+  const [eventLog, setEventLog] = React.useState<string[]>([]);
+  const [isAudioEnabled, setIsAudioEnabled] = React.useState(true);
+  const [showMetrics, setShowMetrics] = React.useState(false);
+
+  const remoteAudioRef = useRef<HTMLAudioElement>(null);
+
+  const logEvent = (message: string) => {
+    console.log(message);
+    setEventLog((prev) => [...prev, `${new Date().toISOString()}: ${message}`]);
+  };
+
+  useEffect(() => {
+    if (remoteAudioRef.current && remoteStream) {
+      remoteAudioRef.current.srcObject = remoteStream;
+
+      remoteAudioRef.current.play().catch((err) => console.error('Error playing audio:', err));
+    }
+  }, [remoteStream]);
+
+  useEffect(() => {
+    if (localStream) {
+      localStream.getAudioTracks().forEach((track) => {
+        track.enabled = !isMuted;
+      });
+    }
+  }, [localStream, isMuted]);
+
+  useEffect(() => {
+    if (isConnected) {
+      logEvent('Connected to server.');
+    } else {
+      logEvent('Disconnected from server.');
+    }
+  }, [isConnected]);
+
+  useEffect(() => {
+    if (error) {
+      logEvent(`Error: ${error.message} (${error.code})`);
+    }
+  }, [error]);
+
+  useEffect(() => {
+    logEvent(`Call state changed to: ${callState}`);
+  }, [callState]);
+
+  const handleStartCall = () => {
+    logEvent('Attempting to start call...');
+    startCall();
+  };
+
+  const handleHangUp = () => {
+    logEvent('Attempting to hang up call...');
+    hangUp();
+  };
+
+  const handleToggleMute = () => {
+    toggleMute();
+    logEvent(`Microphone ${isMuted ? 'unmuted' : 'muted'}`);
+  };
+
+  const toggleAudio = () => {
+    setIsAudioEnabled((prev) => !prev);
+    if (remoteAudioRef.current) {
+      remoteAudioRef.current.muted = !isAudioEnabled;
+    }
+    logEvent(`Speaker ${isAudioEnabled ? 'disabled' : 'enabled'}`);
+  };
+
+  const isActive = callState === CallState.ACTIVE;
+  const isError = callState === CallState.ERROR;
+
+  const getQualityColor = (quality: string) => {
+    switch (quality) {
+      case 'excellent':
+        return 'bg-emerald-100 text-emerald-700';
+      case 'good':
+        return 'bg-green-100 text-green-700';
+      case 'fair':
+        return 'bg-yellow-100 text-yellow-700';
+      case 'poor':
+        return 'bg-orange-100 text-orange-700';
+      case 'bad':
+        return 'bg-red-100 text-red-700';
+      default:
+        return 'bg-gray-100 text-gray-700';
+    }
+  };
+
+  const getQualityIcon = (quality: string) => {
+    switch (quality) {
+      case 'excellent':
+      case 'good':
+        return <Wifi size={16} />;
+      case 'fair':
+      case 'poor':
+        return <Wifi size={16} className="opacity-75" />;
+      case 'bad':
+        return <Wifi size={16} className="opacity-50" />;
+      default:
+        return <Activity size={16} />;
+    }
+  };
+
+  // TESTS
+
+  useEffect(() => {
+    if (remoteAudioRef.current && remoteStream) {
+      console.log('Setting up remote audio:', {
+        tracks: remoteStream.getTracks().length,
+        active: remoteStream.active,
+      });
+
+      remoteAudioRef.current.srcObject = remoteStream;
+      remoteAudioRef.current.muted = false;
+      remoteAudioRef.current.volume = 1.0;
+
+      const playPromise = remoteAudioRef.current.play();
+      if (playPromise) {
+        playPromise.catch((err) => {
+          console.error('Error playing audio:', err);
+          // Retry play on user interaction
+          document.addEventListener(
+            'click',
+            () => {
+              remoteAudioRef.current?.play();
+            },
+            { once: true },
+          );
+        });
+      }
+    }
+  }, [remoteStream]);
+
+  return (
+    <OGDialog open={open} onOpenChange={setOpen}>
+      <OGDialogContent className="w-[28rem] p-8">
+        <div className="flex flex-col items-center gap-6">
+          {/* Connection Status */}
+          <div className="flex w-full items-center justify-between">
+            <div
+              className={`flex items-center gap-2 rounded-full px-4 py-2 ${
+                isConnected ? 'bg-green-100 text-green-700' : 'bg-red-100 text-red-700'
+              }`}
+            >
+              <div
+                className={`h-2 w-2 rounded-full ${isConnected ? 'bg-green-500' : 'bg-red-500'}`}
+              />
+              <span className="text-sm font-medium">
+                {isConnected ? 'Connected' : 'Disconnected'}
+              </span>
+            </div>
+
+            {isActive && (
+              <div
+                className={`flex items-center gap-2 rounded-full px-4 py-2 ${getQualityColor(connectionQuality)}`}
+                onClick={() => setShowMetrics(!showMetrics)}
+                style={{ cursor: 'pointer' }}
+                title="Click to show detailed metrics"
+              >
+                {getQualityIcon(connectionQuality)}
+                <span className="text-sm font-medium capitalize">{connectionQuality} Quality</span>
+                {showMetrics ? <ChevronUp size={14} /> : <ChevronDown size={14} />}
+              </div>
+            )}
+          </div>
+
+          {/* Quality Metrics Panel */}
+          {isActive && showMetrics && (
+            <div className="w-full rounded-md bg-surface-secondary p-3 text-sm shadow-inner">
+              <h4 className="mb-2 font-medium">Connection Metrics</h4>
+              <ul className="space-y-1 text-text-secondary">
+                <li className="flex justify-between">
+                  <span>Round Trip Time:</span>
+                  <span className="font-mono">{(connectionMetrics.rtt * 1000).toFixed(1)} ms</span>
+                </li>
+                <li className="flex justify-between">
+                  <span>Packet Loss:</span>
+                  <span className="font-mono">{connectionMetrics.packetsLost?.toFixed(2)}%</span>
+                </li>
+                <li className="flex justify-between">
+                  <span>Jitter:</span>
+                  <span className="font-mono">
+                    {((connectionMetrics.jitter ?? 0) * 1000).toFixed(1)} ms
+                  </span>
+                </li>
+              </ul>
+            </div>
+          )}
+
+          {/* Error Display */}
+          {error && (
+            <div className="flex w-full items-center gap-2 rounded-md bg-red-100 p-3 text-red-700">
+              <AlertCircle size={16} />
+              <span className="text-sm">{error.message}</span>
+            </div>
+          )}
+
+          {/* Call Controls */}
+          <div className="flex items-center gap-4">
+            {isActive && (
+              <>
+                <Button
+                  onClick={handleToggleMute}
+                  className={`rounded-full p-3 ${
+                    isMuted ? 'bg-red-100 text-red-700' : 'bg-gray-100 text-gray-700'
+                  }`}
+                  title={isMuted ? 'Unmute microphone' : 'Mute microphone'}
+                >
+                  {isMuted ? <MicOff size={20} /> : <Mic size={20} />}
+                </Button>
+
+                <Button
+                  onClick={toggleAudio}
+                  className={`rounded-full p-3 ${
+                    !isAudioEnabled ? 'bg-red-100 text-red-700' : 'bg-gray-100 text-gray-700'
+                  }`}
+                  title={isAudioEnabled ? 'Disable speaker' : 'Enable speaker'}
+                >
+                  {isAudioEnabled ? <Volume2 size={20} /> : <VolumeX size={20} />}
+                </Button>
+              </>
+            )}
+
+            {isActive ? (
+              <Button
+                onClick={handleHangUp}
+                className="flex items-center gap-2 rounded-full bg-red-500 px-6 py-3 text-white hover:bg-red-600"
+              >
+                <PhoneOff size={20} />
+                <span>End Call</span>
+              </Button>
+            ) : (
+              <Button
+                onClick={handleStartCall}
+                disabled={!isConnected || isError || isConnecting}
+                className="flex items-center gap-2 rounded-full bg-green-500 px-6 py-3 text-white hover:bg-green-600 disabled:opacity-50"
+              >
+                <Phone size={20} />
+                <span>{isConnecting ? 'Connecting...' : 'Start Call'}</span>
+              </Button>
+            )}
+          </div>
+
+          {/* Event Log */}
+          <h3 className="mb-2 text-lg font-medium">Event Log</h3>
+          <div className="h-64 overflow-y-auto rounded-md bg-surface-secondary p-2 shadow-inner">
+            <ul className="space-y-1 text-xs text-text-secondary">
+              {eventLog.map((log, index) => (
+                <li key={index} className="font-mono">
+                  {log}
+                </li>
+              ))}
+            </ul>
+          </div>
+
+          {/* Hidden Audio Element */}
+          <audio ref={remoteAudioRef} autoPlay>
+            <track kind="captions" />
+          </audio>
+        </div>
+      </OGDialogContent>
+    </OGDialog>
+  );
+};

client/src/components/Chat/Input/CallButton.tsx

@@ -0,0 +1,40 @@
+import React, { forwardRef } from 'react';
+import { TooltipAnchor } from '~/components/ui';
+import { SendIcon } from '~/components/svg';
+import { useLocalize } from '~/hooks';
+import { cn } from '~/utils';
+
+const Button = React.memo(
+  forwardRef((props: { disabled: boolean }) => {
+    const localize = useLocalize();
+    return (
+      <TooltipAnchor
+        description={localize('com_nav_call_mode')}
+        render={
+          <button
+            aria-label={localize('com_nav_send_message')}
+            id="call-button"
+            disabled={props.disabled}
+            className={cn(
+              'rounded-full bg-text-primary p-2 text-text-primary outline-offset-4 transition-all duration-200 disabled:cursor-not-allowed disabled:text-text-secondary disabled:opacity-10',
+            )}
+            data-testid="call-button"
+            type="submit"
+          >
+            <span className="" data-state="closed">
+              <SendIcon size={24} />
+            </span>
+          </button>
+        }
+      ></TooltipAnchor>
+    );
+  }),
+);
+
+const CallButton = React.memo(
+  forwardRef((props: { disabled: boolean }) => {
+    return <Button disabled={props.disabled} />;
+  }),
+);
+
+export default CallButton;

client/src/components/Chat/Input/ChatForm.tsx

@@ -31,6 +31,7 @@ import SendButton from './SendButton';
 import { BadgeRow } from './BadgeRow';
 import EditBadges from './EditBadges';
 import Mention from './Mention';
+import { Call } from './Call';
 import store from '~/store';
 
 const ChatForm = memo(({ index = 0 }: { index?: number }) => {
@@ -189,140 +190,145 @@ const ChatForm = memo(({ index = 0 }: { index?: number }) => {
   );
 
   return (
-    <form
-      onSubmit={methods.handleSubmit(submitMessage)}
-      className={cn(
-        'mx-auto flex flex-row gap-3 sm:px-2',
-        maximizeChatSpace ? 'w-full max-w-full' : 'md:max-w-3xl xl:max-w-4xl',
-        centerFormOnLanding &&
-          (!conversation?.conversationId || conversation?.conversationId === Constants.NEW_CONVO) &&
-          !isSubmitting
-          ? 'transition-all duration-200 sm:mb-28'
-          : 'sm:mb-10',
-      )}
-    >
-      <div className="relative flex h-full flex-1 items-stretch md:flex-col">
-        <div className={cn('flex w-full items-center', isRTL && 'flex-row-reverse')}>
-          {showPlusPopover && !isAssistantsEndpoint(endpoint) && (
-            <Mention
-              setShowMentionPopover={setShowPlusPopover}
-              newConversation={generateConversation}
-              textAreaRef={textAreaRef}
-              commandChar="+"
-              placeholder="com_ui_add_model_preset"
-              includeAssistants={false}
-            />
-          )}
-          {showMentionPopover && (
-            <Mention
-              setShowMentionPopover={setShowMentionPopover}
-              newConversation={newConversation}
-              textAreaRef={textAreaRef}
-            />
-          )}
-          <PromptsCommand index={index} textAreaRef={textAreaRef} submitPrompt={submitPrompt} />
-          <div
-            onClick={handleContainerClick}
-            className={cn(
-              'relative flex w-full flex-grow flex-col overflow-hidden rounded-t-3xl border pb-4 text-text-primary transition-all duration-200 sm:rounded-3xl sm:pb-0',
-              isTextAreaFocused ? 'shadow-lg' : 'shadow-md',
-              isTemporary
-                ? 'border-violet-800/60 bg-violet-950/10'
-                : 'border-border-light bg-surface-chat',
+    <>
+      <form
+        onSubmit={methods.handleSubmit(submitMessage)}
+        className={cn(
+          'mx-auto flex flex-row gap-3 sm:px-2',
+          maximizeChatSpace ? 'w-full max-w-full' : 'md:max-w-3xl xl:max-w-4xl',
+          centerFormOnLanding &&
+            (!conversation?.conversationId ||
+              conversation?.conversationId === Constants.NEW_CONVO) &&
+            !isSubmitting
+            ? 'transition-all duration-200 sm:mb-28'
+            : 'sm:mb-10',
+        )}
+      >
+        <div className="relative flex h-full flex-1 items-stretch md:flex-col">
+          <div className={cn('flex w-full items-center', isRTL && 'flex-row-reverse')}>
+            {showPlusPopover && !isAssistantsEndpoint(endpoint) && (
+              <Mention
+                setShowMentionPopover={setShowPlusPopover}
+                newConversation={generateConversation}
+                textAreaRef={textAreaRef}
+                commandChar="+"
+                placeholder="com_ui_add_model_preset"
+                includeAssistants={false}
+              />
             )}
-          >
-            <TextareaHeader addedConvo={addedConvo} setAddedConvo={setAddedConvo} />
-            <EditBadges
-              isEditingChatBadges={isEditingBadges}
-              handleCancelBadges={handleCancelBadges}
-              handleSaveBadges={handleSaveBadges}
-              setBadges={setBadges}
-            />
-            <FileFormChat disableInputs={disableInputs} />
-            {endpoint && (
-              <div className={cn('flex', isRTL ? 'flex-row-reverse' : 'flex-row')}>
-                <TextareaAutosize
-                  {...registerProps}
-                  ref={(e) => {
-                    ref(e);
-                    (textAreaRef as React.MutableRefObject<HTMLTextAreaElement | null>).current = e;
-                  }}
-                  disabled={disableInputs}
-                  onPaste={handlePaste}
-                  onKeyDown={handleKeyDown}
-                  onKeyUp={handleKeyUp}
-                  onCompositionStart={handleCompositionStart}
-                  onCompositionEnd={handleCompositionEnd}
-                  id={mainTextareaId}
-                  tabIndex={0}
-                  data-testid="text-input"
-                  rows={1}
-                  onFocus={() => {
-                    handleFocusOrClick();
-                    setIsTextAreaFocused(true);
-                  }}
-                  onBlur={setIsTextAreaFocused.bind(null, false)}
-                  onClick={handleFocusOrClick}
-                  style={{ height: 44, overflowY: 'auto' }}
-                  className={cn(
-                    baseClasses,
-                    removeFocusRings,
-                    'transition-[max-height] duration-200',
-                  )}
-                />
-                <div className="flex flex-col items-start justify-start pt-1.5">
-                  <CollapseChat
-                    isCollapsed={isCollapsed}
-                    isScrollable={isMoreThanThreeRows}
-                    setIsCollapsed={setIsCollapsed}
-                  />
-                </div>
-              </div>
+            {showMentionPopover && (
+              <Mention
+                setShowMentionPopover={setShowMentionPopover}
+                newConversation={newConversation}
+                textAreaRef={textAreaRef}
+              />
             )}
+            <PromptsCommand index={index} textAreaRef={textAreaRef} submitPrompt={submitPrompt} />
             <div
+              onClick={handleContainerClick}
               className={cn(
-                'items-between flex gap-2 pb-2',
-                isRTL ? 'flex-row-reverse' : 'flex-row',
+                'relative flex w-full flex-grow flex-col overflow-hidden rounded-t-3xl border pb-4 text-text-primary transition-all duration-200 sm:rounded-3xl sm:pb-0',
+                isTextAreaFocused ? 'shadow-lg' : 'shadow-md',
+                isTemporary
+                  ? 'border-violet-800/60 bg-violet-950/10'
+                  : 'border-border-light bg-surface-chat',
               )}
             >
-              <div className={`${isRTL ? 'mr-2' : 'ml-2'}`}>
-                <AttachFileChat disableInputs={disableInputs} />
-              </div>
-              <BadgeRow
-                onChange={(newBadges) => setBadges(newBadges)}
-                isInChat={
-                  Array.isArray(conversation?.messages) && conversation.messages.length >= 1
-                }
+              <TextareaHeader addedConvo={addedConvo} setAddedConvo={setAddedConvo} />
+              <EditBadges
+                isEditingChatBadges={isEditingBadges}
+                handleCancelBadges={handleCancelBadges}
+                handleSaveBadges={handleSaveBadges}
+                setBadges={setBadges}
               />
-              <div className="mx-auto flex" />
-              {SpeechToText && (
-                <AudioRecorder
-                  methods={methods}
-                  ask={submitMessage}
-                  textAreaRef={textAreaRef}
-                  disabled={disableInputs}
-                  isSubmitting={isSubmitting}
-                />
-              )}
-              <div className={`${isRTL ? 'ml-2' : 'mr-2'}`}>
-                {(isSubmitting || isSubmittingAdded) && (showStopButton || showStopAdded) ? (
-                  <StopButton stop={handleStopGenerating} setShowStopButton={setShowStopButton} />
-                ) : (
-                  endpoint && (
-                    <SendButton
-                      ref={submitButtonRef}
-                      control={methods.control}
-                      disabled={filesLoading || isSubmitting || disableInputs}
+              <FileFormChat disableInputs={disableInputs} />
+              {endpoint && (
+                <div className={cn('flex', isRTL ? 'flex-row-reverse' : 'flex-row')}>
+                  <TextareaAutosize
+                    {...registerProps}
+                    ref={(e) => {
+                      ref(e);
+                      (textAreaRef as React.MutableRefObject<HTMLTextAreaElement | null>).current =
+                        e;
+                    }}
+                    disabled={disableInputs}
+                    onPaste={handlePaste}
+                    onKeyDown={handleKeyDown}
+                    onKeyUp={handleKeyUp}
+                    onCompositionStart={handleCompositionStart}
+                    onCompositionEnd={handleCompositionEnd}
+                    id={mainTextareaId}
+                    tabIndex={0}
+                    data-testid="text-input"
+                    rows={1}
+                    onFocus={() => {
+                      handleFocusOrClick();
+                      setIsTextAreaFocused(true);
+                    }}
+                    onBlur={setIsTextAreaFocused.bind(null, false)}
+                    onClick={handleFocusOrClick}
+                    style={{ height: 44, overflowY: 'auto' }}
+                    className={cn(
+                      baseClasses,
+                      removeFocusRings,
+                      'transition-[max-height] duration-200',
+                    )}
+                  />
+                  <div className="flex flex-col items-start justify-start pt-1.5">
+                    <CollapseChat
+                      isCollapsed={isCollapsed}
+                      isScrollable={isMoreThanThreeRows}
+                      setIsCollapsed={setIsCollapsed}
                     />
-                  )
+                  </div>
+                </div>
+              )}
+              <div
+                className={cn(
+                  'items-between flex gap-2 pb-2',
+                  isRTL ? 'flex-row-reverse' : 'flex-row',
                 )}
+              >
+                <div className={`${isRTL ? 'mr-2' : 'ml-2'}`}>
+                  <AttachFileChat disableInputs={disableInputs} />
+                </div>
+                <BadgeRow
+                  onChange={(newBadges) => setBadges(newBadges)}
+                  isInChat={
+                    Array.isArray(conversation?.messages) && conversation.messages.length >= 1
+                  }
+                />
+                <div className="mx-auto flex" />
+                {SpeechToText && (
+                  <AudioRecorder
+                    methods={methods}
+                    ask={submitMessage}
+                    textAreaRef={textAreaRef}
+                    disabled={disableInputs}
+                    isSubmitting={isSubmitting}
+                  />
+                )}
+                <div className={`${isRTL ? 'ml-2' : 'mr-2'}`}>
+                  {(isSubmitting || isSubmittingAdded) && (showStopButton || showStopAdded) ? (
+                    <StopButton stop={handleStopGenerating} setShowStopButton={setShowStopButton} />
+                  ) : (
+                    endpoint && (
+                      <SendButton
+                        ref={submitButtonRef}
+                        control={methods.control}
+                        disabled={filesLoading || isSubmitting || disableInputs}
+                      />
+                    )
+                  )}
+                </div>
               </div>
+              {TextToSpeech && automaticPlayback && <StreamAudio index={index} />}
             </div>
-            {TextToSpeech && automaticPlayback && <StreamAudio index={index} />}
           </div>
         </div>
-      </div>
-    </form>
+      </form>
+      <Call />
+    </>
   );
 });
 

client/src/components/Chat/Input/SendButton.tsx

@@ -1,49 +1,104 @@
 import React, { forwardRef } from 'react';
 import { useWatch } from 'react-hook-form';
+import { useSetRecoilState } from 'recoil';
+import type { TRealtimeEphemeralTokenResponse } from 'librechat-data-provider';
 import type { Control } from 'react-hook-form';
-import { TooltipAnchor } from '~/components/ui';
-import { SendIcon } from '~/components/svg';
+import { useRealtimeEphemeralTokenMutation } from '~/data-provider';
+import { TooltipAnchor, SendIcon, CallIcon } from '~/components';
+import { useToastContext } from '~/Providers/ToastContext';
 import { useLocalize } from '~/hooks';
+import store from '~/store';
 import { cn } from '~/utils';
 
-type SendButtonProps = {
+type ButtonProps = {
   disabled: boolean;
   control: Control<{ text: string }>;
 };
 
-const SubmitButton = React.memo(
-  forwardRef((props: { disabled: boolean }, ref: React.ForwardedRef<HTMLButtonElement>) => {
-    const localize = useLocalize();
+const ActionButton = forwardRef(
+  (
+    props: {
+      disabled: boolean;
+      icon: React.ReactNode;
+      tooltip: string;
+      testId: string;
+      onClick?: () => void;
+    },
+    ref: React.ForwardedRef<HTMLButtonElement>,
+  ) => {
     return (
       <TooltipAnchor
-        description={localize('com_nav_send_message')}
+        description={props.tooltip}
         render={
           <button
             ref={ref}
-            aria-label={localize('com_nav_send_message')}
-            id="send-button"
+            aria-label={props.tooltip}
+            id="action-button"
             disabled={props.disabled}
             className={cn(
-              'rounded-full bg-text-primary p-1.5 text-text-primary outline-offset-4 transition-all duration-200 disabled:cursor-not-allowed disabled:text-text-secondary disabled:opacity-10',
+              'rounded-full bg-text-primary p-1.5 text-text-primary outline-offset-4',
+              'transition-all duration-200',
+              'disabled:cursor-not-allowed disabled:text-text-secondary disabled:opacity-10',
             )}
-            data-testid="send-button"
+            data-testid={props.testId}
             type="submit"
+            onClick={props.onClick}
           >
             <span className="" data-state="closed">
-              <SendIcon size={24} />
+              {props.icon}
             </span>
           </button>
         }
       />
     );
-  }),
+  },
 );
 
-const SendButton = React.memo(
-  forwardRef((props: SendButtonProps, ref: React.ForwardedRef<HTMLButtonElement>) => {
-    const data = useWatch({ control: props.control });
-    return <SubmitButton ref={ref} disabled={props.disabled || !data.text} />;
-  }),
-);
+const SendButton = forwardRef((props: ButtonProps, ref: React.ForwardedRef<HTMLButtonElement>) => {
+  const localize = useLocalize();
+  const { showToast } = useToastContext();
+  const { text = '' } = useWatch({ control: props.control });
+  const setCallOpen = useSetRecoilState(store.callDialogOpen(0));
+
+  // const { mutate: startCall, isLoading: isProcessing } = useRealtimeEphemeralTokenMutation({
+  //   onSuccess: async (data: TRealtimeEphemeralTokenResponse) => {
+  //     showToast({
+  //       message: 'IT WORKS!!',
+  //       status: 'success',
+  //     });
+  //   },
+  //   onError: (error: unknown) => {
+  //     showToast({
+  //       message: localize('com_nav_audio_process_error', (error as Error).message),
+  //       status: 'error',
+  //     });
+  //   },
+  // });
+
+  const handleClick = () => {
+    if (text.trim() === '') {
+      setCallOpen(true);
+      // startCall({ voice: 'verse' });
+    }
+  };
+
+  const buttonProps =
+    text.trim() !== ''
+      ? {
+        icon: <SendIcon size={24} />,
+        tooltip: localize('com_nav_send_message'),
+        testId: 'send-button',
+      }
+      : {
+        icon: <CallIcon size={24} />,
+        tooltip: localize('com_nav_call'),
+        testId: 'call-button',
+        onClick: handleClick,
+      };
+
+  return <ActionButton ref={ref} disabled={props.disabled} {...buttonProps} />;
+});
+
+SendButton.displayName = 'SendButton';
 
 export default SendButton;

client/src/components/svg/CallIcon.tsx

@@ -0,0 +1,30 @@
+import { cn } from '~/utils';
+
+export default function CallIcon({ size = 24, className = '' }) {
+  return (
+    <svg
+      width={size}
+      height={size}
+      viewBox={'0 0 24 24'}
+      fill="none"
+      className={cn('text-white dark:text-black', className)}
+    >
+      <path
+        d="M9.5 4C8.67157 4 8 4.67157 8 5.5V18.5C8 19.3284 8.67157 20 9.5 20C10.3284 20 11 19.3284 11 18.5V5.5C11 4.67157 10.3284 4 9.5 4Z"
+        fill="currentColor"
+      ></path>
+      <path
+        d="M13 8.5C13 7.67157 13.6716 7 14.5 7C15.3284 7 16 7.67157 16 8.5V15.5C16 16.3284 15.3284 17 14.5 17C13.6716 17 13 16.3284 13 15.5V8.5Z"
+        fill="currentColor"
+      ></path>
+      <path
+        d="M4.5 9C3.67157 9 3 9.67157 3 10.5V13.5C3 14.3284 3.67157 15 4.5 15C5.32843 15 6 14.3284 6 13.5V10.5C6 9.67157 5.32843 9 4.5 9Z"
+        fill="currentColor"
+      ></path>
+      <path
+        d="M19.5 9C18.6716 9 18 9.67157 18 10.5V13.5C18 14.3284 18.6716 15 19.5 15C20.3284 15 21 14.3284 21 13.5V10.5C21 9.67157 20.3284 9 19.5 9Z"
+        fill="currentColor"
+      ></path>
+    </svg>
+  );
+}

client/src/components/svg/index.ts

@@ -56,3 +56,4 @@ export { default as SpeechIcon } from './SpeechIcon';
 export { default as SaveIcon } from './SaveIcon';
 export { default as CircleHelpIcon } from './CircleHelpIcon';
 export { default as BedrockIcon } from './BedrockIcon';
+export { default as CallIcon } from './CallIcon';

client/src/data-provider/mutations.ts

@@ -726,6 +726,21 @@ export const useTextToSpeechMutation = (
   });
 };
 
+export const useRealtimeEphemeralTokenMutation = (
+  options?: t.MutationOptions<t.TRealtimeEphemeralTokenResponse, t.TRealtimeEphemeralTokenRequest>,
+): UseMutationResult<
+  t.TRealtimeEphemeralTokenResponse,
+  unknown,
+  t.TRealtimeEphemeralTokenRequest,
+  unknown
+> => {
+  return useMutation([MutationKeys.realtimeEphemeralToken], {
+    mutationFn: (data: t.TRealtimeEphemeralTokenRequest) =>
+      dataService.getRealtimeEphemeralToken(data),
+    ...(options || {}),
+  });
+};
+
 /**
  * ASSISTANTS
  */

client/src/data-provider/queries.ts

@@ -532,3 +532,18 @@ export const useUserTermsQuery = (
     ...config,
   });
 };
+
+export const useGetWebsocketUrlQuery = (
+  config?: UseQueryOptions<t.TWebsocketUrlResponse>,
+): QueryObserverResult<t.TWebsocketUrlResponse> => {
+  return useQuery<t.TWebsocketUrlResponse>(
+    [QueryKeys.websocketUrl],
+    () => dataService.getWebsocketUrl(),
+    {
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+      refetchOnMount: false,
+      ...config,
+    },
+  );
+};

client/src/hooks/index.ts

@@ -21,10 +21,12 @@ export * from './Endpoint';
 
 export type { TranslationKeys } from './useLocalize';
 
+export { default as useCall } from './useCall';
 export { default as useToast } from './useToast';
 export { default as useTimeout } from './useTimeout';
 export { default as useNewConvo } from './useNewConvo';
 export { default as useLocalize } from './useLocalize';
+export { default as useWebSocket } from './useWebSocket';
 export { default as useMediaQuery } from './useMediaQuery';
 export { default as useChatBadges } from './useChatBadges';
 export { default as useScrollToRef } from './useScrollToRef';

client/src/hooks/useCall.ts

@@ -0,0 +1,325 @@
+import { useState, useRef, useCallback, useEffect } from 'react';
+import { WebRTCService, ConnectionState, useVADSetup } from '../services/WebRTC/WebRTCService';
+import useWebSocket, { WebSocketEvents } from './useWebSocket';
+
+interface CallError {
+  code: string;
+  message: string;
+}
+
+export enum CallState {
+  IDLE = 'idle',
+  CONNECTING = 'connecting',
+  ACTIVE = 'active',
+  ERROR = 'error',
+  ENDED = 'ended',
+}
+
+interface CallStatus {
+  callState: CallState;
+  isConnecting: boolean;
+  error: CallError | null;
+  localStream: MediaStream | null;
+  remoteStream: MediaStream | null;
+  connectionQuality: 'excellent' | 'good' | 'fair' | 'poor' | 'bad' | 'unknown';
+  connectionMetrics: {
+    rtt?: number;
+    packetsLost?: number;
+    jitter?: number;
+  };
+  isUserSpeaking: boolean;
+  remoteAISpeaking: boolean;
+}
+
+const INITIAL_STATUS: CallStatus = {
+  callState: CallState.IDLE,
+  isConnecting: false,
+  error: null,
+  localStream: null,
+  remoteStream: null,
+  connectionQuality: 'unknown',
+  connectionMetrics: {},
+  isUserSpeaking: false,
+  remoteAISpeaking: false,
+};
+
+const useCall = () => {
+  const { isConnected, sendMessage, addEventListener } = useWebSocket();
+  const [status, setStatus] = useState<CallStatus>(INITIAL_STATUS);
+  const webrtcServiceRef = useRef<WebRTCService | null>(null);
+  const statsIntervalRef = useRef<NodeJS.Timeout>();
+  const [isMuted, setIsMuted] = useState(false);
+
+  const vad = useVADSetup(webrtcServiceRef.current);
+
+  const updateStatus = useCallback((updates: Partial<CallStatus>) => {
+    setStatus((prev) => ({ ...prev, ...updates }));
+  }, []);
+
+  useEffect(() => {
+    updateStatus({ isUserSpeaking: vad.userSpeaking });
+  }, [vad.userSpeaking, updateStatus]);
+
+  const handleRemoteStream = (stream: MediaStream | null) => {
+    if (!stream) {
+      console.error('[WebRTC] Received null remote stream');
+      updateStatus({
+        error: {
+          code: 'NO_REMOTE_STREAM',
+          message: 'No remote stream received',
+        },
+      });
+      return;
+    }
+
+    const audioTracks = stream.getAudioTracks();
+    if (!audioTracks.length) {
+      console.error('[WebRTC] No audio tracks in remote stream');
+      updateStatus({
+        error: {
+          code: 'NO_AUDIO_TRACKS',
+          message: 'Remote stream contains no audio',
+        },
+      });
+      return;
+    }
+
+    updateStatus({
+      remoteStream: stream,
+      callState: CallState.ACTIVE,
+    });
+  };
+
+  const handleConnectionStateChange = useCallback(
+    (state: ConnectionState) => {
+      switch (state) {
+        case ConnectionState.CONNECTED:
+          updateStatus({
+            callState: CallState.ACTIVE,
+            isConnecting: false,
+          });
+          break;
+        case ConnectionState.CONNECTING:
+        case ConnectionState.RECONNECTING:
+          updateStatus({
+            callState: CallState.CONNECTING,
+            isConnecting: true,
+          });
+          break;
+        case ConnectionState.FAILED:
+          updateStatus({
+            callState: CallState.ERROR,
+            isConnecting: false,
+            error: {
+              code: 'CONNECTION_FAILED',
+              message: 'Connection failed. Please try again.',
+            },
+          });
+          break;
+        case ConnectionState.CLOSED:
+          updateStatus({
+            ...INITIAL_STATUS,
+            callState: CallState.ENDED,
+          });
+          break;
+      }
+    },
+    [updateStatus],
+  );
+
+  const startConnectionMonitoring = useCallback(() => {
+    if (!webrtcServiceRef.current) {
+      return;
+    }
+
+    statsIntervalRef.current = setInterval(async () => {
+      const stats = await webrtcServiceRef.current?.getStats();
+      if (!stats) {
+        return;
+      }
+
+      let totalRoundTripTime = 0;
+      let totalPacketsLost = 0;
+      let totalPackets = 0;
+      let totalJitter = 0;
+      let samplesCount = 0;
+      let samplesJitterCount = 0;
+
+      stats.forEach((report) => {
+        if (report.type === 'candidate-pair' && report.currentRoundTripTime) {
+          totalRoundTripTime += report.currentRoundTripTime;
+          samplesCount++;
+        }
+
+        if (report.type === 'inbound-rtp' && report.kind === 'audio') {
+          if (report.packetsLost !== undefined && report.packetsReceived !== undefined) {
+            totalPacketsLost += report.packetsLost;
+            totalPackets += report.packetsReceived + report.packetsLost;
+          }
+
+          if (report.jitter !== undefined) {
+            totalJitter += report.jitter;
+            samplesJitterCount++;
+          }
+        }
+      });
+
+      const averageRTT = samplesCount > 0 ? totalRoundTripTime / samplesCount : 0;
+      const packetLossRate = totalPackets > 0 ? (totalPacketsLost / totalPackets) * 100 : 0;
+      const averageJitter = samplesJitterCount > 0 ? totalJitter / samplesJitterCount : 0;
+
+      let quality: CallStatus['connectionQuality'] = 'unknown';
+
+      if (averageRTT < 0.15 && packetLossRate < 0.5 && averageJitter < 0.015) {
+        quality = 'excellent';
+      } else if (averageRTT < 0.25 && packetLossRate < 2 && averageJitter < 0.025) {
+        quality = 'good';
+      } else if (averageRTT < 0.4 && packetLossRate < 5 && averageJitter < 0.04) {
+        quality = 'fair';
+      } else if (averageRTT < 0.6 && packetLossRate < 10 && averageJitter < 0.06) {
+        quality = 'poor';
+      } else if (averageRTT >= 0.6 || packetLossRate >= 10 || averageJitter >= 0.06) {
+        quality = 'bad';
+      }
+
+      updateStatus({
+        connectionQuality: quality,
+        connectionMetrics: {
+          rtt: averageRTT,
+          packetsLost: packetLossRate,
+          jitter: averageJitter,
+        },
+      });
+    }, 2000);
+  }, [updateStatus]);
+
+  const startCall = useCallback(async () => {
+    if (!isConnected) {
+      console.log('Cannot start call - not connected to server');
+      updateStatus({
+        callState: CallState.ERROR,
+        error: {
+          code: 'NOT_CONNECTED',
+          message: 'Not connected to server',
+        },
+      });
+      return;
+    }
+
+    try {
+      console.log('Starting new call...');
+
+      if (webrtcServiceRef.current) {
+        console.log('Cleaning up existing WebRTC connection');
+        webrtcServiceRef.current.close();
+      }
+
+      updateStatus({
+        callState: CallState.CONNECTING,
+        isConnecting: true,
+        error: null,
+      });
+
+      webrtcServiceRef.current = new WebRTCService(sendMessage, {
+        debug: true,
+      });
+
+      webrtcServiceRef.current.on('connectionStateChange', handleConnectionStateChange);
+      webrtcServiceRef.current.on('remoteStream', handleRemoteStream);
+      webrtcServiceRef.current.on('vadStatusChange', (speaking: boolean) => {
+        updateStatus({ isUserSpeaking: speaking });
+      });
+
+      webrtcServiceRef.current.on('error', (error: string) => {
+        console.error('WebRTC error:', error);
+        updateStatus({
+          callState: CallState.ERROR,
+          isConnecting: false,
+          error: {
+            code: 'WEBRTC_ERROR',
+            message: error,
+          },
+        });
+      });
+
+      console.log('Initializing WebRTC connection...');
+      await webrtcServiceRef.current.initialize();
+      console.log('WebRTC initialization complete');
+
+      startConnectionMonitoring();
+    } catch (error) {
+      console.error('Failed to start call:', error);
+      updateStatus({
+        callState: CallState.ERROR,
+        isConnecting: false,
+        error: {
+          code: 'INITIALIZATION_FAILED',
+          message: error instanceof Error ? error.message : 'Failed to start call',
+        },
+      });
+    }
+  }, [
+    isConnected,
+    sendMessage,
+    handleConnectionStateChange,
+    startConnectionMonitoring,
+    updateStatus,
+  ]);
+
+  const hangUp = useCallback(() => {
+    if (webrtcServiceRef.current) {
+      webrtcServiceRef.current.close();
+      webrtcServiceRef.current = null;
+    }
+    if (statsIntervalRef.current) {
+      clearInterval(statsIntervalRef.current);
+    }
+    updateStatus({
+      ...INITIAL_STATUS,
+      callState: CallState.ENDED,
+    });
+  }, [updateStatus]);
+
+  useEffect(() => {
+    const cleanupFns = [
+      addEventListener(WebSocketEvents.WEBRTC_ANSWER, (answer: RTCSessionDescriptionInit) => {
+        webrtcServiceRef.current?.handleAnswer(answer);
+      }),
+      addEventListener(WebSocketEvents.ICE_CANDIDATE, (candidate: RTCIceCandidateInit) => {
+        webrtcServiceRef.current?.addIceCandidate(candidate);
+      }),
+    ];
+
+    return () => cleanupFns.forEach((fn) => fn());
+  }, [addEventListener, updateStatus]);
+
+  const toggleMute = useCallback(() => {
+    if (webrtcServiceRef.current) {
+      const newMutedState = !isMuted;
+      webrtcServiceRef.current.setMuted(newMutedState);
+      setIsMuted(newMutedState);
+    }
+  }, [isMuted]);
+
+  useEffect(() => {
+    if (webrtcServiceRef.current) {
+      const handleMuteChange = (muted: boolean) => setIsMuted(muted);
+      webrtcServiceRef.current.on('muteStateChange', handleMuteChange);
+      return () => {
+        webrtcServiceRef.current?.off('muteStateChange', handleMuteChange);
+      };
+    }
+  }, []);
+
+  return {
+    ...status,
+    isMuted,
+    toggleMute,
+    startCall,
+    hangUp,
+    vadLoading: vad.loading,
+    vadError: vad.errored,
+  };
+};
+
+export default useCall;

client/src/hooks/useWebSocket.ts

@@ -0,0 +1,150 @@
+import { useEffect, useRef, useState } from 'react';
+import { useGetWebsocketUrlQuery } from '~/data-provider';
+import type { MessagePayload } from '~/common';
+import { io, Socket } from 'socket.io-client';
+import { EventEmitter } from 'events';
+
+export const WebSocketEvents = {
+  CALL_STARTED: 'call-started',
+  CALL_ERROR: 'call-error',
+  WEBRTC_ANSWER: 'webrtc-answer',
+  ICE_CANDIDATE: 'icecandidate',
+} as const;
+
+type EventHandler = (...args: unknown[]) => void;
+
+class WebSocketManager extends EventEmitter {
+  private socket: Socket | null = null;
+  private reconnectAttempts = 0;
+  private readonly MAX_RECONNECT_ATTEMPTS = 5;
+  private isConnected = false;
+  private isConnecting = false;
+
+  connect(url: string) {
+    if (this.isConnecting || (this.socket && this.socket.connected)) {
+      return;
+    }
+
+    this.isConnecting = true;
+    this.socket = io(url, {
+      transports: ['websocket'],
+      reconnectionAttempts: this.MAX_RECONNECT_ATTEMPTS,
+      timeout: 10000,
+    });
+    this.setupEventHandlers();
+  }
+
+  private setupEventHandlers() {
+    if (!this.socket) {
+      this.isConnecting = false;
+      return;
+    }
+
+    this.socket.on('connect', () => {
+      this.isConnected = true;
+      this.isConnecting = false;
+      this.reconnectAttempts = 0;
+      this.emit('connectionChange', true);
+    });
+
+    this.socket.on('disconnect', (reason) => {
+      this.isConnected = false;
+      this.emit('connectionChange', false);
+    });
+
+    this.socket.on('connect_error', (error) => {
+      this.reconnectAttempts++;
+      this.emit('connectionChange', false);
+      if (this.reconnectAttempts >= this.MAX_RECONNECT_ATTEMPTS) {
+        this.isConnecting = false;
+        this.emit('error', 'Failed to connect after maximum attempts');
+        this.disconnect();
+      }
+    });
+
+    // WebRTC signals
+    this.socket.on(WebSocketEvents.CALL_STARTED, () => {
+      this.emit(WebSocketEvents.CALL_STARTED);
+    });
+
+    this.socket.on(WebSocketEvents.WEBRTC_ANSWER, (answer) => {
+      this.emit(WebSocketEvents.WEBRTC_ANSWER, answer);
+    });
+
+    this.socket.on(WebSocketEvents.ICE_CANDIDATE, (candidate) => {
+      this.emit(WebSocketEvents.ICE_CANDIDATE, candidate);
+    });
+
+    this.socket.on('error', (error) => {
+      this.emit('error', error);
+    });
+  }
+
+  disconnect() {
+    if (this.socket) {
+      this.socket.disconnect();
+      this.socket = null;
+    }
+    this.isConnected = false;
+    this.isConnecting = false;
+  }
+
+  sendMessage(type: string, payload?: MessagePayload) {
+    if (!this.socket || !this.socket.connected) {
+      return false;
+    }
+    this.socket.emit(type, payload);
+    return true;
+  }
+
+  getConnectionState() {
+    return { isConnected: this.isConnected, isConnecting: this.isConnecting };
+  }
+}
+
+export const webSocketManager = new WebSocketManager();
+
+const useWebSocket = () => {
+  const { data: wsConfig } = useGetWebsocketUrlQuery();
+  const [isConnected, setIsConnected] = useState(false);
+  const eventHandlersRef = useRef<Record<string, EventHandler>>({});
+
+  useEffect(() => {
+    if (wsConfig?.url) {
+      const state = webSocketManager.getConnectionState();
+      if (!state.isConnected && !state.isConnecting) {
+        webSocketManager.connect(wsConfig.url);
+      }
+
+      const handleConnectionChange = (connected: boolean) => setIsConnected(connected);
+      webSocketManager.on('connectionChange', handleConnectionChange);
+      webSocketManager.on('error', console.error);
+
+      return () => {
+        webSocketManager.off('connectionChange', handleConnectionChange);
+        webSocketManager.off('error', console.error);
+      };
+    }
+  }, [wsConfig, wsConfig?.url]);
+
+  const sendMessage = (message: { type: string; payload?: MessagePayload }) => {
+    return webSocketManager.sendMessage(message.type, message.payload);
+  };
+
+  const addEventListener = (event: string, handler: EventHandler) => {
+    eventHandlersRef.current[event] = handler;
+    webSocketManager.on(event, handler);
+    return () => {
+      webSocketManager.off(event, handler);
+      delete eventHandlersRef.current[event];
+    };
+  };
+
+  return {
+    isConnected,
+    sendMessage,
+    addEventListener,
+  };
+};
+
+export default useWebSocket;

client/src/locales/en/translation.json

@@ -442,6 +442,7 @@
   "com_nav_user_name_display": "Display username in messages",
   "com_nav_voice_select": "Voice",
   "com_nav_voices_fetch_error": "Could not retrieve voice options. Please check your internet connection.",
+  "com_nav_call": "Call",
   "com_show_agent_settings": "Show Agent Settings",
   "com_show_completion_settings": "Show Completion Settings",
   "com_show_examples": "Show Examples",

client/src/services/WebRTC/WebRTCService.ts

@@ -0,0 +1,413 @@
+import { useEffect } from 'react';
+import { EventEmitter } from 'events';
+import { useMicVAD } from '@ricky0123/vad-react';
+import type { MessagePayload } from '~/common';
+
+export enum ConnectionState {
+  IDLE = 'idle',
+  CONNECTING = 'connecting',
+  CONNECTED = 'connected',
+  RECONNECTING = 'reconnecting',
+  FAILED = 'failed',
+  CLOSED = 'closed',
+}
+
+export enum MediaState {
+  INACTIVE = 'inactive',
+  PENDING = 'pending',
+  ACTIVE = 'active',
+  FAILED = 'failed',
+}
+
+interface WebRTCConfig {
+  iceServers?: RTCIceServer[];
+  maxReconnectAttempts?: number;
+  connectionTimeout?: number;
+  debug?: boolean;
+}
+
+export function useVADSetup(webrtcService: WebRTCService | null) {
+  const vad = useMicVAD({
+    startOnLoad: true,
+    onSpeechStart: () => {
+      if (webrtcService && !webrtcService.isMuted()) {
+        webrtcService.handleVADStatusChange(true);
+      }
+    },
+    onSpeechEnd: () => {
+      if (webrtcService && !webrtcService.isMuted()) {
+        webrtcService.handleVADStatusChange(false);
+      }
+    },
+    onVADMisfire: () => {
+      if (webrtcService && !webrtcService.isMuted()) {
+        webrtcService.handleVADStatusChange(false);
+      }
+    },
+  });
+
+  useEffect(() => {
+    if (webrtcService) {
+      const handleMuteChange = (muted: boolean) => {
+        if (muted) {
+          vad.pause();
+        } else {
+          vad.start();
+        }
+      };
+
+      webrtcService.on('muteStateChange', handleMuteChange);
+      return () => {
+        webrtcService.off('muteStateChange', handleMuteChange);
+      };
+    }
+  }, [webrtcService, vad]);
+
+  return vad;
+}
+
+export class WebRTCService extends EventEmitter {
+  private peerConnection: RTCPeerConnection | null = null;
+  private localStream: MediaStream | null = null;
+  private remoteStream: MediaStream | null = null;
+  private reconnectAttempts = 0;
+  private connectionTimeoutId: NodeJS.Timeout | null = null;
+  private config: Required<WebRTCConfig>;
+  private connectionState: ConnectionState = ConnectionState.IDLE;
+  private mediaState: MediaState = MediaState.INACTIVE;
+
+  private isUserSpeaking = false;
+
+  private readonly DEFAULT_CONFIG: Required<WebRTCConfig> = {
+    iceServers: [
+      {
+        urls: ['stun:stun.l.google.com:19302', 'stun:stun1.l.google.com:19302'],
+      },
+    ],
+    maxReconnectAttempts: 3,
+    connectionTimeout: 30000,
+    debug: false,
+  };
+
+  constructor(
+    private readonly sendMessage: (message: { type: string; payload?: MessagePayload }) => boolean,
+    config: WebRTCConfig = {},
+  ) {
+    super();
+    this.config = { ...this.DEFAULT_CONFIG, ...config };
+    this.log('WebRTCService initialized with config:', this.config);
+  }
+
+  private log(...args: unknown[]) {
+    if (this.config.debug) {
+      console.log('[WebRTC]', ...args);
+    }
+  }
+
+  private setConnectionState(state: ConnectionState) {
+    this.connectionState = state;
+    this.emit('connectionStateChange', state);
+    this.log('Connection state changed to:', state);
+  }
+
+  private setMediaState(state: MediaState) {
+    this.mediaState = state;
+    this.emit('mediaStateChange', state);
+    this.log('Media state changed to:', state);
+  }
+
+  public handleVADStatusChange(isSpeaking: boolean) {
+    if (this.isUserSpeaking !== isSpeaking) {
+      this.isUserSpeaking = isSpeaking;
+      this.sendMessage({
+        type: 'vad-status',
+        payload: { speaking: isSpeaking },
+      });
+      this.emit('vadStatusChange', isSpeaking);
+    }
+  }
+
+  public setMuted(muted: boolean) {
+    if (this.localStream) {
+      this.localStream.getAudioTracks().forEach((track) => {
+        if (muted) {
+          track.stop();
+        } else {
+          this.refreshAudioTrack();
+        }
+      });
+
+      if (muted) {
+        this.handleVADStatusChange(false);
+      }
+
+      this.emit('muteStateChange', muted);
+    }
+  }
+
+  public isMuted(): boolean {
+    if (!this.localStream) {
+      return false;
+    }
+    const audioTrack = this.localStream.getAudioTracks()[0];
+    return audioTrack ? !audioTrack.enabled : false;
+  }
+
+  private async refreshAudioTrack() {
+    try {
+      const newStream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true,
+        },
+      });
+
+      const newTrack = newStream.getAudioTracks()[0];
+      if (this.localStream && this.peerConnection) {
+        const oldTrack = this.localStream.getAudioTracks()[0];
+        if (oldTrack) {
+          this.localStream.removeTrack(oldTrack);
+        }
+        this.localStream.addTrack(newTrack);
+
+        const senders = this.peerConnection.getSenders();
+        const audioSender = senders.find((sender) => sender.track?.kind === 'audio');
+        if (audioSender) {
+          audioSender.replaceTrack(newTrack);
+        }
+      }
+    } catch (error) {
+      this.handleError(error);
+    }
+  }
+
+  async initialize() {
+    try {
+      this.setConnectionState(ConnectionState.CONNECTING);
+      this.setMediaState(MediaState.PENDING);
+
+      this.localStream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true,
+        },
+      });
+
+      this.peerConnection = new RTCPeerConnection({
+        iceServers: this.config.iceServers,
+        iceCandidatePoolSize: 10,
+        bundlePolicy: 'max-bundle',
+        rtcpMuxPolicy: 'require',
+      });
+
+      this.setupPeerConnectionListeners();
+
+      this.localStream.getTracks().forEach((track) => {
+        if (this.localStream && this.peerConnection) {
+          this.peerConnection.addTrack(track, this.localStream);
+        }
+      });
+
+      this.startConnectionTimeout();
+      await this.createAndSendOffer();
+      this.setMediaState(MediaState.ACTIVE);
+    } catch (error) {
+      this.log('Initialization error:', error);
+      this.handleError(error);
+    }
+  }
+
+  private sendSignalingMessage(message: { type: string; payload?: MessagePayload }) {
+    const sent = this.sendMessage(message);
+    if (!sent) {
+      this.handleError(new Error('Failed to send signaling message - WebSocket not connected'));
+    }
+  }
+
+  private setupPeerConnectionListeners() {
+    if (!this.peerConnection) {
+      return;
+    }
+
+    this.peerConnection.ontrack = ({ track, streams }) => {
+      this.log('Track received:', {
+        kind: track.kind,
+        enabled: track.enabled,
+        readyState: track.readyState,
+      });
+
+      if (track.kind === 'audio') {
+        if (!this.remoteStream) {
+          this.remoteStream = new MediaStream();
+        }
+
+        this.remoteStream.addTrack(track);
+
+        if (this.peerConnection) {
+          this.peerConnection.addTrack(track, this.remoteStream);
+        }
+
+        this.log('Audio track added to remote stream', {
+          tracks: this.remoteStream.getTracks().length,
+          active: this.remoteStream.active,
+        });
+
+        this.emit('remoteStream', this.remoteStream);
+      }
+    };
+
+    this.peerConnection.onconnectionstatechange = () => {
+      if (!this.peerConnection) {
+        return;
+      }
+
+      const state = this.peerConnection.connectionState;
+      this.log('Connection state changed:', state);
+
+      switch (state) {
+        case 'connected':
+          this.clearConnectionTimeout();
+          this.setConnectionState(ConnectionState.CONNECTED);
+          break;
+        case 'disconnected':
+        case 'failed':
+          if (this.reconnectAttempts < this.config.maxReconnectAttempts) {
+            this.attemptReconnection();
+          } else {
+            this.handleError(new Error('Connection failed after max reconnection attempts'));
+          }
+          break;
+        case 'closed':
+          this.setConnectionState(ConnectionState.CLOSED);
+          break;
+      }
+    };
+  }
+
+  private async createAndSendOffer() {
+    if (!this.peerConnection) {
+      return;
+    }
+
+    try {
+      const offer = await this.peerConnection.createOffer({
+        offerToReceiveAudio: true,
+      });
+
+      await this.peerConnection.setLocalDescription(offer);
+
+      this.sendSignalingMessage({
+        type: 'webrtc-offer',
+        payload: offer,
+      });
+    } catch (error) {
+      this.handleError(error);
+    }
+  }
+
+  public async handleAnswer(answer: RTCSessionDescriptionInit) {
+    if (!this.peerConnection) {
+      return;
+    }
+
+    try {
+      await this.peerConnection.setRemoteDescription(new RTCSessionDescription(answer));
+      this.log('Remote description set successfully');
+    } catch (error) {
+      this.handleError(error);
+    }
+  }
+
+  public async addIceCandidate(candidate: RTCIceCandidateInit) {
+    if (!this.peerConnection?.remoteDescription) {
+      this.log('Delaying ICE candidate addition - no remote description');
+      return;
+    }
+
+    try {
+      await this.peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
+      this.log('ICE candidate added successfully');
+    } catch (error) {
+      this.handleError(error);
+    }
+  }
+
+  private startConnectionTimeout() {
+    this.clearConnectionTimeout();
+    this.connectionTimeoutId = setTimeout(() => {
+      if (
+        this.connectionState !== ConnectionState.CONNECTED &&
+        this.connectionState !== ConnectionState.CONNECTING
+      ) {
+        this.handleError(new Error('Connection timeout'));
+      }
+    }, this.config.connectionTimeout);
+  }
+
+  private clearConnectionTimeout() {
+    if (this.connectionTimeoutId) {
+      clearTimeout(this.connectionTimeoutId);
+      this.connectionTimeoutId = null;
+    }
+  }
+
+  private async attemptReconnection() {
+    this.reconnectAttempts++;
+    this.log(
+      `Attempting reconnection (${this.reconnectAttempts}/${this.config.maxReconnectAttempts})`,
+    );
+
+    this.setConnectionState(ConnectionState.RECONNECTING);
+    this.emit('reconnecting', this.reconnectAttempts);
+
+    try {
+      if (this.peerConnection) {
+        const offer = await this.peerConnection.createOffer({ iceRestart: true });
+        await this.peerConnection.setLocalDescription(offer);
+        this.sendSignalingMessage({
+          type: 'webrtc-offer',
+          payload: offer,
+        });
+      }
+    } catch (error) {
+      this.handleError(error);
+    }
+  }
+
+  private handleError(error: Error | unknown) {
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
+    this.log('Error:', errorMessage);
+
+    if (this.connectionState !== ConnectionState.CONNECTED) {
+      this.setConnectionState(ConnectionState.FAILED);
+      this.emit('error', errorMessage);
+    }
+
+    if (this.connectionState !== ConnectionState.CONNECTED) {
+      this.close();
+    }
+  }
+
+  public close() {
+    this.clearConnectionTimeout();
+
+    if (this.localStream) {
+      this.localStream.getTracks().forEach((track) => track.stop());
+      this.localStream = null;
+    }
+
+    if (this.peerConnection) {
+      this.peerConnection.close();
+      this.peerConnection = null;
+    }
+
+    this.setConnectionState(ConnectionState.CLOSED);
+    this.setMediaState(MediaState.INACTIVE);
+  }
+
+  public getStats(): Promise<RTCStatsReport> | null {
+    return this.peerConnection?.getStats() ?? null;
+  }
+}

client/src/store/families.ts

@@ -368,6 +368,11 @@ const updateConversationSelector = selectorFamily({
       },
 });
 
+const callDialogOpen = atomFamily<boolean, string | number | null>({
+  key: 'callDialogOpen',
+  default: false,
+});
+
 export default {
   conversationKeysAtom,
   conversationByIndex,
@@ -399,4 +404,5 @@ export default {
   useClearLatestMessages,
   showPromptsPopoverFamily,
   updateConversationSelector,
+  callDialogOpen,
 };

package-lock.json

@@ -121,11 +121,13 @@
         "passport-local": "^1.0.0",
         "rate-limit-redis": "^4.2.0",
         "sharp": "^0.33.5",
+        "socket.io": "^4.8.1",
         "tiktoken": "^1.0.15",
         "traverse": "^0.6.7",
         "ua-parser-js": "^1.0.36",
         "winston": "^3.11.0",
         "winston-daily-rotate-file": "^4.7.1",
+        "wrtc": "^0.4.7",
         "youtube-transcript": "^1.2.1",
         "zod": "^3.22.4"
       },
@@ -1685,6 +1687,7 @@
         "@radix-ui/react-tabs": "^1.0.3",
         "@radix-ui/react-toast": "^1.1.5",
         "@react-spring/web": "^9.7.5",
+        "@ricky0123/vad-react": "^0.0.28",
         "@tanstack/react-query": "^4.28.0",
         "@tanstack/react-table": "^8.11.7",
         "class-variance-authority": "^0.6.0",
@@ -1733,6 +1736,7 @@
         "remark-gfm": "^4.0.0",
         "remark-math": "^6.0.0",
         "remark-supersub": "^1.0.0",
+        "socket.io-client": "^4.8.1",
         "sse.js": "^2.5.0",
         "tailwind-merge": "^1.9.1",
         "tailwindcss-animate": "^1.0.5",
@@ -3663,27 +3667,6 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
-    "client/node_modules/caniuse-lite": {
-      "version": "1.0.30001700",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001700.tgz",
-      "integrity": "sha512-2S6XIXwaE7K7erT8dY+kLQcpa5ms63XlRkMkReXjle+kf6c5g38vyMl+Z5y8dSxOFDhcFe+nxnn261PLxBSQsQ==",
-      "dev": true,
-      "funding": [
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/browserslist"
-        },
-        {
-          "type": "tidelift",
-          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
-        },
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/ai"
-        }
-      ],
-      "license": "CC-BY-4.0"
-    },
     "client/node_modules/core-js-compat": {
       "version": "3.40.0",
       "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.40.0.tgz",
@@ -21202,6 +21185,29 @@
         "node": ">=14.0.0"
       }
     },
+    "node_modules/@ricky0123/vad-react": {
+      "version": "0.0.28",
+      "resolved": "https://registry.npmjs.org/@ricky0123/vad-react/-/vad-react-0.0.28.tgz",
+      "integrity": "sha512-V2vcxhT31/tXCxqlYLJz+JzywXijMWUhp2FN30OL/NeuSwwprArhaAoUZSdjg6Hzsfe5t2lwASoUaEmGrQ/S+Q==",
+      "license": "ISC",
+      "dependencies": {
+        "@ricky0123/vad-web": "0.0.22",
+        "onnxruntime-web": "1.14.0"
+      },
+      "peerDependencies": {
+        "react": "18",
+        "react-dom": "18"
+      }
+    },
+    "node_modules/@ricky0123/vad-web": {
+      "version": "0.0.22",
+      "resolved": "https://registry.npmjs.org/@ricky0123/vad-web/-/vad-web-0.0.22.tgz",
+      "integrity": "sha512-679R6sfwXx4jkquK+FJ9RC2W29oulWC+9ZINK6LVpuy90IBV7UaTGNN79oQXufpJTJs5z4X/22nw1DQ4+Rh8CA==",
+      "license": "ISC",
+      "dependencies": {
+        "onnxruntime-web": "1.14.0"
+      }
+    },
     "node_modules/@rollup/plugin-alias": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/@rollup/plugin-alias/-/plugin-alias-5.1.0.tgz",
@@ -23223,6 +23229,12 @@
         "node": ">=18.0.0"
       }
     },
+    "node_modules/@socket.io/component-emitter": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz",
+      "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==",
+      "license": "MIT"
+    },
     "node_modules/@stitches/core": {
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/@stitches/core/-/core-1.2.8.tgz",
@@ -23608,6 +23620,15 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/cors": {
+      "version": "2.8.17",
+      "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz",
+      "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/debug": {
       "version": "4.1.12",
       "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
@@ -23776,6 +23797,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/long": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
+      "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==",
+      "license": "MIT"
+    },
     "node_modules/@types/mdast": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz",
@@ -25237,6 +25264,15 @@
         }
       ]
     },
+    "node_modules/base64id": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz",
+      "integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==",
+      "license": "MIT",
+      "engines": {
+        "node": "^4.5.0 || >= 5.9"
+      }
+    },
     "node_modules/base64url": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/base64url/-/base64url-3.0.1.tgz",
@@ -25660,9 +25696,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001663",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001663.tgz",
-      "integrity": "sha512-o9C3X27GLKbLeTYZ6HBOLU1tsAcBZsLis28wrVzddShCS16RujjHp9GDHKZqrB3meE0YjhawvMFsGb/igqiPzA==",
+      "version": "1.0.30001706",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001706.tgz",
+      "integrity": "sha512-3ZczoTApMAZwPKYWmwVbQMFpXBDds3/0VciVoUwPUbldlYyVLmRVuRs/PcUZtHpbLRpzzDvrvnFuREsGt6lUug==",
       "funding": [
         {
           "type": "opencollective",
@@ -25676,7 +25712,8 @@
           "type": "github",
           "url": "https://github.com/sponsors/ai"
         }
-      ]
+      ],
+      "license": "CC-BY-4.0"
     },
     "node_modules/ccount": {
       "version": "2.0.1",
@@ -27287,6 +27324,124 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/engine.io": {
+      "version": "6.6.4",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.4.tgz",
+      "integrity": "sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/cors": "^2.8.12",
+        "@types/node": ">=10.0.0",
+        "accepts": "~1.3.4",
+        "base64id": "2.0.0",
+        "cookie": "~0.7.2",
+        "cors": "~2.8.5",
+        "debug": "~4.3.1",
+        "engine.io-parser": "~5.2.1",
+        "ws": "~8.17.1"
+      },
+      "engines": {
+        "node": ">=10.2.0"
+      }
+    },
+    "node_modules/engine.io-client": {
+      "version": "6.6.3",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.3.tgz",
+      "integrity": "sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==",
+      "license": "MIT",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.1",
+        "engine.io-parser": "~5.2.1",
+        "ws": "~8.17.1",
+        "xmlhttprequest-ssl": "~2.1.1"
+      }
+    },
+    "node_modules/engine.io-client/node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/engine.io-client/node_modules/ws": {
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/engine.io-parser": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz",
+      "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/engine.io/node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/engine.io/node_modules/ws": {
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/enhanced-resolve": {
       "version": "5.17.1",
       "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
@@ -28986,6 +29141,12 @@
         "node": ">=16"
       }
     },
+    "node_modules/flatbuffers": {
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz",
+      "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==",
+      "license": "SEE LICENSE IN LICENSE.txt"
+    },
     "node_modules/flatted": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz",
@@ -29650,6 +29811,12 @@
         "node": ">=14.0.0"
       }
     },
+    "node_modules/guid-typescript": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
+      "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
+      "license": "ISC"
+    },
     "node_modules/hamt_plus": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/hamt_plus/-/hamt_plus-1.0.2.tgz",
@@ -35532,6 +35699,73 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/onnx-proto": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/onnx-proto/-/onnx-proto-4.0.4.tgz",
+      "integrity": "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==",
+      "license": "MIT",
+      "dependencies": {
+        "protobufjs": "^6.8.8"
+      }
+    },
+    "node_modules/onnx-proto/node_modules/long": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+      "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/onnx-proto/node_modules/protobufjs": {
+      "version": "6.11.4",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz",
+      "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==",
+      "hasInstallScript": true,
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/long": "^4.0.1",
+        "@types/node": ">=13.7.0",
+        "long": "^4.0.0"
+      },
+      "bin": {
+        "pbjs": "bin/pbjs",
+        "pbts": "bin/pbts"
+      }
+    },
+    "node_modules/onnxruntime-common": {
+      "version": "1.14.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz",
+      "integrity": "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==",
+      "license": "MIT"
+    },
+    "node_modules/onnxruntime-web": {
+      "version": "1.14.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz",
+      "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==",
+      "license": "MIT",
+      "dependencies": {
+        "flatbuffers": "^1.12.0",
+        "guid-typescript": "^1.0.9",
+        "long": "^4.0.0",
+        "onnx-proto": "^4.0.4",
+        "onnxruntime-common": "~1.14.0",
+        "platform": "^1.3.6"
+      }
+    },
+    "node_modules/onnxruntime-web/node_modules/long": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+      "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==",
+      "license": "Apache-2.0"
+    },
     "node_modules/open": {
       "version": "10.1.0",
       "resolved": "https://registry.npmjs.org/open/-/open-10.1.0.tgz",
@@ -36236,6 +36470,12 @@
         "node": ">=8"
       }
     },
+    "node_modules/platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+      "license": "MIT"
+    },
     "node_modules/playwright": {
       "version": "1.50.1",
       "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.50.1.tgz",
@@ -40123,6 +40363,151 @@
       "integrity": "sha512-9LK+E7Hv5R9u4g4C3p+jjLstaLe11MDsL21UpYaCNmapvMkYhqCV4A/f/3gyH8QjMyh6l68q9xC85vihY9ahMQ==",
       "dev": true
     },
+    "node_modules/socket.io": {
+      "version": "4.8.1",
+      "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.1.tgz",
+      "integrity": "sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg==",
+      "license": "MIT",
+      "dependencies": {
+        "accepts": "~1.3.4",
+        "base64id": "~2.0.0",
+        "cors": "~2.8.5",
+        "debug": "~4.3.2",
+        "engine.io": "~6.6.0",
+        "socket.io-adapter": "~2.5.2",
+        "socket.io-parser": "~4.2.4"
+      },
+      "engines": {
+        "node": ">=10.2.0"
+      }
+    },
+    "node_modules/socket.io-adapter": {
+      "version": "2.5.5",
+      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.5.5.tgz",
+      "integrity": "sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "~4.3.4",
+        "ws": "~8.17.1"
+      }
+    },
+    "node_modules/socket.io-adapter/node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/socket.io-adapter/node_modules/ws": {
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/socket.io-client": {
+      "version": "4.8.1",
+      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.1.tgz",
+      "integrity": "sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.2",
+        "engine.io-client": "~6.6.1",
+        "socket.io-parser": "~4.2.4"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/socket.io-client/node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/socket.io-parser": {
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz",
+      "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==",
+      "license": "MIT",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.1"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/socket.io-parser/node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/socket.io/node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/socks": {
       "version": "2.8.3",
       "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz",
@@ -43513,6 +43898,43 @@
         "node": ">=6"
       }
     },
+    "node_modules/wrtc": {
+      "version": "0.4.7",
+      "resolved": "https://registry.npmjs.org/wrtc/-/wrtc-0.4.7.tgz",
+      "integrity": "sha512-P6Hn7VT4lfSH49HxLHcHhDq+aFf/jd9dPY7lDHeFhZ22N3858EKuwm2jmnlPzpsRGEPaoF6XwkcxY5SYnt4f/g==",
+      "bundleDependencies": [
+        "node-pre-gyp"
+      ],
+      "hasInstallScript": true,
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "node-pre-gyp": "^0.13.0"
+      },
+      "engines": {
+        "node": "^8.11.2 || >=10.0.0"
+      },
+      "optionalDependencies": {
+        "domexception": "^1.0.1"
+      }
+    },
+    "node_modules/wrtc/node_modules/domexception": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz",
+      "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==",
+      "deprecated": "Use your platform's native DOMException instead",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "webidl-conversions": "^4.0.2"
+      }
+    },
+    "node_modules/wrtc/node_modules/webidl-conversions": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "license": "BSD-2-Clause",
+      "optional": true
+    },
     "node_modules/ws": {
       "version": "8.18.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
@@ -43554,6 +43976,14 @@
       "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
       "devOptional": true
     },
+    "node_modules/xmlhttprequest-ssl": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz",
+      "integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/xtend": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

packages/data-provider/src/api-endpoints.ts

@@ -171,7 +171,9 @@ export const textToSpeechManual = () => `${textToSpeech()}/manual`;
 
 export const textToSpeechVoices = () => `${textToSpeech()}/voices`;
 
-export const getCustomConfigSpeech = () => `${speech()}/config/get`;
+export const getCustomConfigSpeech = () => `${speech()}/config`;
+
+export const getRealtimeEphemeralToken = () => `${speech()}/realtime`;
 
 export const getPromptGroup = (_id: string) => `${prompts()}/groups/${_id}`;
 
@@ -244,4 +246,6 @@ export const verifyTwoFactor = () => '/api/auth/2fa/verify';
 export const confirmTwoFactor = () => '/api/auth/2fa/confirm';
 export const disableTwoFactor = () => '/api/auth/2fa/disable';
 export const regenerateBackupCodes = () => '/api/auth/2fa/backup/regenerate';
-export const verifyTwoFactorTemp = () => '/api/auth/2fa/verify-temp';
+export const verifyTwoFactorTemp = () => '/api/auth/2fa/verify-temp';
+
+export const websocket = () => '/api/websocket';

packages/data-provider/src/config.ts

@@ -408,6 +408,18 @@ const speechTab = z
   })
   .optional();
 
+const realtime = z
+  .object({
+    openai: z
+      .object({
+        url: z.string().optional(),
+        apiKey: z.string().optional(),
+        voices: z.array(z.string()).optional(),
+      })
+      .optional(),
+  })
+  .optional();
+
 export enum RateLimitPrefix {
   FILE_UPLOAD = 'FILE_UPLOAD',
   IMPORT = 'IMPORT',
@@ -595,6 +607,7 @@ export const configSchema = z.object({
       tts: ttsSchema.optional(),
       stt: sttSchema.optional(),
       speechTab: speechTab.optional(),
+      realtime: realtime.optional(),
     })
     .optional(),
   rateLimits: rateLimitSchema.optional(),
@@ -1216,6 +1229,17 @@ export enum TTSProviders {
   LOCALAI = 'localai',
 }
 
+export enum RealtimeVoiceProviders {
+  /**
+   * Provider for OpenAI Realtime Voice API
+   */
+  OPENAI = 'openai',
+  /**
+   * Provider for Google Realtime Voice API
+   */
+  GOOGLE = 'google',
+}
+
 /** Enum for app-wide constants */
 export enum Constants {
   /** Key for the app's version. */

packages/data-provider/src/data-service.ts

@@ -576,6 +576,12 @@ export const getCustomConfigSpeech = (): Promise<t.TCustomConfigSpeechResponse>
   return request.get(endpoints.getCustomConfigSpeech());
 };
 
+export const getRealtimeEphemeralToken = (
+  data: t.TRealtimeEphemeralTokenRequest,
+): Promise<t.TRealtimeEphemeralTokenResponse> => {
+  return request.get(endpoints.getRealtimeEphemeralToken(), { params: data });
+};
+
 /* conversations */
 
 export function duplicateConversation(
@@ -803,4 +809,8 @@ export function verifyTwoFactorTemp(
   payload: t.TVerify2FATempRequest,
 ): Promise<t.TVerify2FATempResponse> {
   return request.post(endpoints.verifyTwoFactorTemp(), payload);
-}
+}
+
+export function getWebsocketUrl(): Promise<t.TWebsocketUrlResponse> {
+  return request.get(endpoints.websocket());
+}

packages/data-provider/src/keys.ts

@@ -46,6 +46,7 @@ export enum QueryKeys {
   health = 'health',
   userTerms = 'userTerms',
   banner = 'banner',
+  websocketUrl = 'websocketUrl',
 }
 
 export enum MutationKeys {
@@ -69,4 +70,5 @@ export enum MutationKeys {
   updateRole = 'updateRole',
   enableTwoFactor = 'enableTwoFactor',
   verifyTwoFactor = 'verifyTwoFactor',
+  realtimeEphemeralToken = 'realtimeEphemeralToken',
 }

packages/data-provider/src/types.ts

@@ -10,6 +10,7 @@ import type {
   TConversationTag,
   TBanner,
 } from './schemas';
+
 export type TOpenAIMessage = OpenAI.Chat.ChatCompletionMessageParam;
 
 export * from './schemas';
@@ -532,3 +533,16 @@ export type TAcceptTermsResponse = {
 };
 
 export type TBannerResponse = TBanner | null;
+
+export type TRealtimeEphemeralTokenRequest = {
+  voice: string;
+};
+
+export type TRealtimeEphemeralTokenResponse = {
+  token: string;
+  url: string;
+};
+
+export type TWebsocketUrlResponse = {
+  url: string;
+};