feat(ws): implement WebSocket cluster mode with Redis Pub/Sub

Introduce a new WebSocket messaging architecture that supports both standalone and cluster modes. This allows for horizontal scaling of WebSocket servers by using Redis Pub/Sub to synchronize messages across multiple instances. Key changes: - Added `ws.MessagePublisher` interface to abstract message distribution. - Implemented `ws.Bus` to handle cluster-mode messaging via Redis. - Added `ws.OnlineTracker` to manage user online status across the cluster. - Refactored multiple services (Chat, Group, Push, Call, etc.) to use the new `MessagePublisher` instead of a concrete `ws.Hub`. - Added WebSocket configuration options (mode, instance ID, channel, TTL, heartbeat) to `config.yaml` and `config.go`. - Updated dependency injection with Wire to support the new publisher and Redis client. - Improved logging by replacing standard `log` with `zap` in several service components.
2026-05-06 12:39:11 +08:00
parent d481742790
commit c630cbf4d0
22 changed files with 1082 additions and 120 deletions
--- a/internal/service/call_service.go
+++ b/internal/service/call_service.go
@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"

+	redispkg "with_you/internal/pkg/redis"
+
 	"with_you/internal/config"
 	apperrors "with_you/internal/errors"
 	"with_you/internal/model"
@@ -22,8 +24,27 @@ import (
 const (
 	CallLifetimeMs = 60000  // 通话邀请有效期 60秒 (参考 Matrix)
 	CallTimeoutMs  = 120000 // 通话超时(无人接听) 120秒
+
+	redisCallKeyPrefix      = "call:"
+	redisCallByUserPrefix   = "call_by_user:"
+	redisCallAcceptPrefix   = "call:accept:"
+	redisCallTTL             = 120 * time.Second
+	redisCallAcceptLockTTL  = 10 * time.Second
 )

+// activeCallRedisData Redis 中存储的通话数据（精简字段）
+type activeCallRedisData struct {
+	ID             string `json:"id"`
+	ConversationID string `json:"conversation_id"`
+	CallerID       string `json:"caller_id"`
+	CalleeID       string `json:"callee_id"`
+	CallType       string `json:"call_type"`
+	Status         string `json:"status"`
+	MediaType      string `json:"media_type"`
+	CreatedAt      int64  `json:"created_at"`
+	StartedAt      int64  `json:"started_at"` // 0 means nil
+}
+
 // ActiveCall 内存中的活跃通话
 type ActiveCall struct {
 	ID             string
@@ -65,9 +86,10 @@ type CallService interface {

 type callService struct {
 	callRepo repository.CallRepository
-	hub      *ws.Hub
+	hub      ws.MessagePublisher
 	config   *config.Config
 	db       *gorm.DB
+	redis    *redispkg.Client

 	// 内存中的活跃通话状态
 	activeCalls       map[string]*ActiveCall     // callID -> ActiveCall
@@ -79,22 +101,24 @@ type callService struct {
 // NewCallService 创建通话服务
 func NewCallService(
 	callRepo repository.CallRepository,
-	hub *ws.Hub,
+	publisher ws.MessagePublisher,
 	cfg *config.Config,
 	db *gorm.DB,
+	redisClient *redispkg.Client,
 ) CallService {
 	svc := &callService{
 		callRepo:          callRepo,
-		hub:               hub,
+		hub:               publisher,
 		config:            cfg,
 		db:                db,
+		redis:             redisClient,
 		activeCalls:       make(map[string]*ActiveCall),
 		activeCallsByUser: make(map[string]map[string]bool),
 		activeCallsByConv: make(map[string]string),
 	}

 	// 订阅 WebSocket 断开事件
-	hub.OnDisconnect(svc.handleUserDisconnect)
+	publisher.OnDisconnect(svc.handleUserDisconnect)

 	// 自动启动超时清理
 	svc.StartCleanupTicker()
@@ -108,11 +132,159 @@ func (s *callService) generateCallID() string {
 	return strconv.FormatInt(id, 10)
 }

-// getActiveCall 从内存获取活跃通话
+// redisStoreCall 存储通话数据到 Redis
+func (s *callService) redisStoreCall(call *ActiveCall) {
+	if s.redis == nil {
+		return
+	}
+	ctx := context.Background()
+
+	var startedAt int64
+	if call.StartedAt != nil {
+		startedAt = call.StartedAt.UnixMilli()
+	}
+	data := activeCallRedisData{
+		ID:             call.ID,
+		ConversationID: call.ConversationID,
+		CallerID:       call.CallerID,
+		CalleeID:       call.CalleeID,
+		CallType:       string(call.CallType),
+		Status:         string(call.Status),
+		MediaType:      call.MediaType,
+		CreatedAt:      call.CreatedAt.UnixMilli(),
+		StartedAt:      startedAt,
+	}
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		zap.L().Error("Failed to marshal call for Redis", zap.Error(err))
+		return
+	}
+
+	callKey := redisCallKeyPrefix + call.ID
+	callerKey := redisCallByUserPrefix + call.CallerID
+	calleeKey := redisCallByUserPrefix + call.CalleeID
+
+	pipe := s.redis.Pipeline()
+	pipe.Set(ctx, callKey, jsonData, redisCallTTL)
+	pipe.Set(ctx, callerKey, call.ID, redisCallTTL)
+	pipe.Set(ctx, calleeKey, call.ID, redisCallTTL)
+	if _, err := pipe.Exec(ctx); err != nil {
+		zap.L().Error("Failed to store call in Redis", zap.String("call_id", call.ID), zap.Error(err))
+	}
+}
+
+// redisRemoveCall 从 Redis 移除通话数据
+func (s *callService) redisRemoveCall(call *ActiveCall) {
+	if s.redis == nil {
+		return
+	}
+	ctx := context.Background()
+
+	keys := []string{
+		redisCallKeyPrefix + call.ID,
+		redisCallByUserPrefix + call.CallerID,
+		redisCallByUserPrefix + call.CalleeID,
+		redisCallAcceptPrefix + call.ID,
+	}
+	if err := s.redis.Del(ctx, keys...); err != nil {
+		zap.L().Error("Failed to remove call from Redis", zap.String("call_id", call.ID), zap.Error(err))
+	}
+}
+
+// redisGetCall 从 Redis 获取通话数据
+func (s *callService) redisGetCall(callID string) *ActiveCall {
+	if s.redis == nil {
+		return nil
+	}
+	ctx := context.Background()
+
+	raw, err := s.redis.Get(ctx, redisCallKeyPrefix+callID)
+	if err != nil {
+		return nil
+	}
+
+	var data activeCallRedisData
+	if err := json.Unmarshal([]byte(raw), &data); err != nil {
+		zap.L().Error("Failed to unmarshal call from Redis", zap.Error(err))
+		return nil
+	}
+
+	call := &ActiveCall{
+		ID:             data.ID,
+		ConversationID: data.ConversationID,
+		CallerID:       data.CallerID,
+		CalleeID:       data.CalleeID,
+		CallType:       model.CallType(data.CallType),
+		Status:         model.CallStatus(data.Status),
+		MediaType:      data.MediaType,
+		ICEServers:     s.config.WebRTC.ICEServers,
+		Participants: map[string]*ActiveParticipant{
+			data.CallerID: {UserID: data.CallerID, Status: model.ParticipantStatusJoined},
+			data.CalleeID: {UserID: data.CalleeID, Status: model.ParticipantStatusInvited},
+		},
+	}
+
+	call.CreatedAt = time.UnixMilli(data.CreatedAt)
+	if data.StartedAt > 0 {
+		t := time.UnixMilli(data.StartedAt)
+		call.StartedAt = &t
+		// 如果已经开始，被叫方也已加入
+		call.Participants[data.CalleeID] = &ActiveParticipant{
+			UserID:   data.CalleeID,
+			Status:   model.ParticipantStatusJoined,
+			JoinedAt: call.StartedAt,
+		}
+	}
+
+	return call
+}
+
+// redisRefreshTTL 刷新通话相关 Redis 键的 TTL
+func (s *callService) redisRefreshTTL(call *ActiveCall) {
+	if s.redis == nil {
+		return
+	}
+	ctx := context.Background()
+
+	keys := []string{
+		redisCallKeyPrefix + call.ID,
+		redisCallByUserPrefix + call.CallerID,
+		redisCallByUserPrefix + call.CalleeID,
+	}
+	for _, key := range keys {
+		_, _ = s.redis.Expire(ctx, key, redisCallTTL)
+	}
+}
+
+// redisTryAcceptLock 尝试获取 Accept 分布式锁
+func (s *callService) redisTryAcceptLock(callID string) bool {
+	if s.redis == nil {
+		return true // 无 Redis 时允许通过（降级为本地锁）
+	}
+	ctx := context.Background()
+	ok, err := s.redis.GetClient().SetNX(ctx, redisCallAcceptPrefix+callID, "1", redisCallAcceptLockTTL).Result()
+	if err != nil {
+		zap.L().Error("Redis SETNX error, allowing accept", zap.Error(err))
+		return true
+	}
+	return ok
+}
+
+// getActiveCall 从内存获取活跃通话，L1 未命中时回退到 Redis
 func (s *callService) getActiveCall(callID string) *ActiveCall {
 	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.activeCalls[callID]
+	call := s.activeCalls[callID]
+	s.mu.RUnlock()
+	if call != nil {
+		return call
+	}
+
+	// L1 miss → Redis fallback
+	call = s.redisGetCall(callID)
+	if call != nil {
+		s.storeActiveCall(call)
+	}
+	return call
 }

 // getActiveCallByConversation 从内存获取会话的活跃通话
@@ -224,6 +396,9 @@ func (s *callService) Invite(ctx context.Context, callerID, calleeID, conversati
 	// 存储到内存
 	s.storeActiveCall(call)

+	// 写入 Redis
+	s.redisStoreCall(call)
+
 	// 检查被叫方是否在线
 	calleeOnline := s.hub.HasClients(calleeID)

@@ -272,6 +447,11 @@ func (s *callService) Accept(ctx context.Context, callID, userID string) (*Activ
 		return nil, apperrors.ErrCallAlreadyAnswered
 	}

+	// 分布式锁：防止跨实例重复 Accept
+	if !s.redisTryAcceptLock(callID) {
+		return nil, apperrors.ErrCallAlreadyAnswered
+	}
+
 	// 使用原子操作更新状态
 	s.mu.Lock()
 	if call.Status != model.CallStatusCalling {
@@ -285,6 +465,10 @@ func (s *callService) Accept(ctx context.Context, callID, userID string) (*Activ
 	call.Participants[userID].JoinedAt = &now
 	s.mu.Unlock()

+	// 更新 Redis
+	s.redisStoreCall(call)
+	s.redisRefreshTTL(call)
+
 	// 通知拨打方
 	s.hub.PublishToUserOnline(call.CallerID, "call_accepted", map[string]any{
 		"call_id":     callID,
@@ -324,6 +508,9 @@ func (s *callService) Reject(ctx context.Context, callID, userID string) error {
 	// 从内存移除
 	s.removeActiveCall(callID)

+	// 从 Redis 移除
+	s.redisRemoveCall(call)
+
 	// 保存到数据库作为历史记录
 	s.saveCallHistory(call, model.CallStatusRejected, 0)

@@ -358,6 +545,9 @@ func (s *callService) Busy(ctx context.Context, callID, userID string) error {
 	// 从内存移除
 	s.removeActiveCall(callID)

+	// 从 Redis 移除
+	s.redisRemoveCall(call)
+
 	// 保存到数据库作为历史记录
 	s.saveCallHistory(call, model.CallStatusMissed, 0)

@@ -414,6 +604,9 @@ func (s *callService) End(ctx context.Context, callID, userID string, reason str
 	// 从内存移除
 	s.removeActiveCall(callID)

+	// 从 Redis 移除
+	s.redisRemoveCall(call)
+
 	// 保存到数据库作为历史记录
 	s.saveCallHistory(call, endStatus, duration)