From d6a94c7b4d432f2ec38b17f9b28b64c28a3ebb26 Mon Sep 17 00:00:00 2001 From: lan Date: Tue, 2 Jun 2026 13:28:16 +0800 Subject: [PATCH] feat(livekit): improve native audio/video handling and stability Implement robust WebRTC polyfills and native audio session management for LiveKit on mobile platforms. This includes configuring iOS audio sessions to prevent issues during calls and implementing a delayed video activation strategy to avoid main thread deadlocks on iOS. - Add WebRTC and environment polyfills in `src/polyfills.ts` - Implement iOS-specific `AudioSession` configuration in `LiveKitService` - Add support for toggling speaker output on iOS - Update `callStore` to handle video enablement after connection settles - Fix `useUnreadCountQuery` to correctly handle async fetching --- src/hooks/useUnreadCount.ts | 5 ++- src/polyfills.ts | 37 +++++++++++++++++++- src/services/livekit/LiveKitService.ts | 41 ++++++++++++++++++++-- src/stores/call/callStore.ts | 47 +++++++++++++++++++++----- 4 files changed, 118 insertions(+), 12 deletions(-) diff --git a/src/hooks/useUnreadCount.ts b/src/hooks/useUnreadCount.ts index bb1ce0b..fb60e94 100644 --- a/src/hooks/useUnreadCount.ts +++ b/src/hooks/useUnreadCount.ts @@ -8,7 +8,10 @@ export const messageKeys = { export function useUnreadCountQuery() { return useQuery({ queryKey: messageKeys.unread, - queryFn: () => messageManager.fetchUnreadCount(), + queryFn: async () => { + await messageManager.fetchUnreadCount(); + return null; + }, staleTime: 30 * 1000, refetchInterval: 60 * 1000, }); diff --git a/src/polyfills.ts b/src/polyfills.ts index a92c5a1..aba7e5b 100644 --- a/src/polyfills.ts +++ b/src/polyfills.ts @@ -1,3 +1,38 @@ +import { Platform } from 'react-native'; + +// Register WebRTC globals for LiveKit (native only — web has built-in WebRTC) +if (Platform.OS !== 'web') { + // Use @livekit/react-native's registerGlobals which sets up: + // - WebRTC native modules (via @livekit/react-native-webrtc) + // - iOS audio session management + // - LiveKitReactNativeGlobal (so livekit-client detects RN environment) + // - RN-specific polyfills (Promise.allSettled, webstreams, crypto, etc.) + const { registerGlobals } = require('@livekit/react-native'); + registerGlobals(); +} + +if (typeof globalThis.Event === 'undefined') { + (globalThis as any).Event = class Event { + type: string; + bubbles: boolean; + cancelable: boolean; + defaultPrevented: boolean; + target: any; + currentTarget: any; + constructor(type: string, opts?: { bubbles?: boolean; cancelable?: boolean }) { + this.type = type; + this.bubbles = opts?.bubbles ?? false; + this.cancelable = opts?.cancelable ?? false; + this.defaultPrevented = false; + this.target = null; + this.currentTarget = null; + } + preventDefault() { this.defaultPrevented = true; } + stopPropagation() {} + stopImmediatePropagation() {} + }; +} + if (typeof globalThis.DOMException === 'undefined') { (globalThis as any).DOMException = class DOMException extends Error { constructor(message?: string, name?: string) { @@ -5,4 +40,4 @@ if (typeof globalThis.DOMException === 'undefined') { this.name = name || 'DOMException'; } }; -} +} \ No newline at end of file diff --git a/src/services/livekit/LiveKitService.ts b/src/services/livekit/LiveKitService.ts index d97cd24..1dfc6d7 100644 --- a/src/services/livekit/LiveKitService.ts +++ b/src/services/livekit/LiveKitService.ts @@ -1,4 +1,5 @@ import '../../polyfills'; +import { Platform } from 'react-native'; import { Room, RoomEvent, Track, ConnectionState, LocalParticipant, RemoteParticipant, RemoteTrackPublication, TrackPublication, Participant as LKParticipant } from 'livekit-client'; export interface LiveKitServiceConfig { @@ -44,11 +45,26 @@ class LiveKitServiceImpl { return this.room?.remoteParticipants ?? new Map(); } - async connect(url: string, token: string): Promise { + async connect(url: string, token: string, isVideoCall: boolean = false): Promise { if (this.room) { await this.disconnect(); } + // Configure native audio session before connecting (iOS only) + if (Platform.OS === 'ios') { + try { + const { AudioSession } = require('@livekit/react-native'); + await AudioSession.configureAudio({ + ios: { + defaultOutput: isVideoCall ? 'speaker' : 'default', + }, + }); + await AudioSession.startAudioSession(); + } catch (err) { + console.warn('[LiveKit] Audio session configuration failed:', err); + } + } + this.room = new Room({ adaptiveStream: true, dynacast: true, @@ -87,6 +103,14 @@ class LiveKitServiceImpl { await this.room.disconnect(); this.room = null; this._connectionStatus = 'disconnected'; + + if (Platform.OS === 'ios') { + try { + const { AudioSession } = require('@livekit/react-native'); + await AudioSession.stopAudioSession(); + } catch {} + } + this.emit('connectionStatusChanged', 'disconnected'); this.emit('disconnected', undefined); } @@ -98,7 +122,20 @@ class LiveKitServiceImpl { async setVideoEnabled(enabled: boolean): Promise { if (!this.room?.localParticipant) return; - await this.room.localParticipant.setCameraEnabled(enabled); + console.log('[LiveKit] setVideoEnabled called:', enabled); + try { + await this.room.localParticipant.setCameraEnabled(enabled); + console.log('[LiveKit] setVideoEnabled succeeded:', enabled); + } catch (err) { + console.error('[LiveKit] setVideoEnabled failed:', err); + throw err; + } + } + + async setSpeakerOn(speakerOn: boolean): Promise { + if (Platform.OS === 'web' || Platform.OS === 'android') return; + const AudioSession = require('@livekit/react-native').AudioSession; + await AudioSession.selectAudioOutput(speakerOn ? 'force_speaker' : 'default'); } isMuted(): boolean { diff --git a/src/stores/call/callStore.ts b/src/stores/call/callStore.ts index bf75805..ece2cf6 100644 --- a/src/stores/call/callStore.ts +++ b/src/stores/call/callStore.ts @@ -81,6 +81,7 @@ let callTimeoutTimer: ReturnType | null = null; let initCallUnsub: (() => void) | null = null; let unsubInvited: (() => void) | null = null; let liveKitUnsubs: (() => void)[] = []; +let pendingVideoEnabled: boolean | null = null; // Constants const CALL_LIFETIME_MS = 55000; @@ -100,6 +101,7 @@ function cleanupProcessedCallIds() { /** * Fetch LiveKit token from the backend and connect to the room. + * Connects audio-only first, then enables video separately. */ async function joinLiveKitRoom(callId: string, videoEnabled: boolean): Promise { const res = await api.get<{ token: string; url: string }>('/calls/token', { @@ -111,12 +113,35 @@ async function joinLiveKitRoom(callId: string, videoEnabled: boolean): Promise { + liveKitService.setVideoEnabled(enable).catch((err) => { + console.warn('[CallStore] Failed to enable video:', err); + }); + }, 1000); } /** @@ -139,10 +164,14 @@ function setupLiveKitEvents(callId: string): void { wsService.sendCallReady(callId); + // Start duration timer immediately if (durationTimer) clearInterval(durationTimer); durationTimer = setInterval(() => { callStore.setState((s) => ({ callDuration: s.callDuration + 1 })); }, 1000); + + // Schedule video enable after settling (avoids iOS deadlock) + enablePendingVideo(); }) ); @@ -414,7 +443,7 @@ export const callStore = create((set, get) => ({ processedCallIds.set(msg.call_id, Date.now()); if (currentCall?.id === msg.call_id) { - console.log('[CallStore] Call ended, duration:', msg.duration); + console.log('[CallStore] Call ended, reason:', msg.reason, 'ended_by:', msg.ended_by, 'duration:', msg.duration); get().endCall('ended'); return; } @@ -523,7 +552,7 @@ export const callStore = create((set, get) => ({ status: 'calling', duration: 0, isMuted: false, - isSpeakerOn: false, + isSpeakerOn: callType === 'video', isVideoEnabled: callType === 'video', isPeerVideoEnabled: false, isInitiator: true, @@ -579,7 +608,7 @@ export const callStore = create((set, get) => ({ status: 'connecting', duration: 0, isMuted: false, - isSpeakerOn: false, + isSpeakerOn: isVideoCall, isVideoEnabled: isVideoCall, isPeerVideoEnabled: false, isInitiator: false, @@ -654,14 +683,16 @@ export const callStore = create((set, get) => ({ })); }, - toggleSpeaker: () => { + toggleSpeaker: async () => { const { currentCall } = get(); if (!currentCall) return; + const newSpeakerOn = !currentCall.isSpeakerOn; set((s) => ({ currentCall: s.currentCall - ? { ...s.currentCall, isSpeakerOn: !s.currentCall.isSpeakerOn } + ? { ...s.currentCall, isSpeakerOn: newSpeakerOn } : null, })); + await liveKitService.setSpeakerOn(newSpeakerOn); }, toggleMinimize: () => {