feat(ws): implement WebSocket cluster mode with Redis Pub/Sub

Introduce a new WebSocket messaging architecture that supports both standalone and cluster modes. This allows for horizontal scaling of WebSocket servers by using Redis Pub/Sub to synchronize messages across multiple instances. Key changes: - Added `ws.MessagePublisher` interface to abstract message distribution. - Implemented `ws.Bus` to handle cluster-mode messaging via Redis. - Added `ws.OnlineTracker` to manage user online status across the cluster. - Refactored multiple services (Chat, Group, Push, Call, etc.) to use the new `MessagePublisher` instead of a concrete `ws.Hub`. - Added WebSocket configuration options (mode, instance ID, channel, TTL, heartbeat) to `config.yaml` and `config.go`. - Updated dependency injection with Wire to support the new publisher and Redis client. - Improved logging by replacing standard `log` with `zap` in several service components.
2026-05-06 12:39:11 +08:00
parent d481742790
commit c630cbf4d0
22 changed files with 1082 additions and 120 deletions
--- a/internal/service/account_cleanup_service.go
+++ b/internal/service/account_cleanup_service.go
@@ -2,12 +2,12 @@ package service

 import (
 	"context"
-	"log"
 	"time"

 	"with_you/internal/model"
 	"with_you/internal/repository"

+	"go.uber.org/zap"
 	"gorm.io/gorm"
 )

@@ -50,13 +50,18 @@ func (s *accountCleanupServiceImpl) CleanupExpiredAccounts(ctx context.Context)
 	var deletedCount int64
 	for _, user := range users {
 		if err := s.anonymizeAndDeleteUser(ctx, user); err != nil {
-			log.Printf("[AccountCleanup] failed to cleanup user %s: %v", user.ID, err)
+			zap.L().Error("Failed to cleanup user",
+				zap.String("user_id", user.ID),
+				zap.Error(err),
+			)
 			continue
 		}
 		deletedCount++
 	}

-	log.Printf("[AccountCleanup] cleaned up %d expired accounts", deletedCount)
+	zap.L().Info("Cleaned up expired accounts",
+		zap.Int64("count", deletedCount),
+	)
 	return deletedCount, nil
 }

--- a/internal/service/call_service.go
+++ b/internal/service/call_service.go
@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"

+	redispkg "with_you/internal/pkg/redis"
+
 	"with_you/internal/config"
 	apperrors "with_you/internal/errors"
 	"with_you/internal/model"
@@ -22,8 +24,27 @@ import (
 const (
 	CallLifetimeMs = 60000  // 通话邀请有效期 60秒 (参考 Matrix)
 	CallTimeoutMs  = 120000 // 通话超时(无人接听) 120秒
+
+	redisCallKeyPrefix      = "call:"
+	redisCallByUserPrefix   = "call_by_user:"
+	redisCallAcceptPrefix   = "call:accept:"
+	redisCallTTL             = 120 * time.Second
+	redisCallAcceptLockTTL  = 10 * time.Second
 )

+// activeCallRedisData Redis 中存储的通话数据（精简字段）
+type activeCallRedisData struct {
+	ID             string `json:"id"`
+	ConversationID string `json:"conversation_id"`
+	CallerID       string `json:"caller_id"`
+	CalleeID       string `json:"callee_id"`
+	CallType       string `json:"call_type"`
+	Status         string `json:"status"`
+	MediaType      string `json:"media_type"`
+	CreatedAt      int64  `json:"created_at"`
+	StartedAt      int64  `json:"started_at"` // 0 means nil
+}
+
 // ActiveCall 内存中的活跃通话
 type ActiveCall struct {
 	ID             string
@@ -65,9 +86,10 @@ type CallService interface {

 type callService struct {
 	callRepo repository.CallRepository
-	hub      *ws.Hub
+	hub      ws.MessagePublisher
 	config   *config.Config
 	db       *gorm.DB
+	redis    *redispkg.Client

 	// 内存中的活跃通话状态
 	activeCalls       map[string]*ActiveCall     // callID -> ActiveCall
@@ -79,22 +101,24 @@ type callService struct {
 // NewCallService 创建通话服务
 func NewCallService(
 	callRepo repository.CallRepository,
-	hub *ws.Hub,
+	publisher ws.MessagePublisher,
 	cfg *config.Config,
 	db *gorm.DB,
+	redisClient *redispkg.Client,
 ) CallService {
 	svc := &callService{
 		callRepo:          callRepo,
-		hub:               hub,
+		hub:               publisher,
 		config:            cfg,
 		db:                db,
+		redis:             redisClient,
 		activeCalls:       make(map[string]*ActiveCall),
 		activeCallsByUser: make(map[string]map[string]bool),
 		activeCallsByConv: make(map[string]string),
 	}

 	// 订阅 WebSocket 断开事件
-	hub.OnDisconnect(svc.handleUserDisconnect)
+	publisher.OnDisconnect(svc.handleUserDisconnect)

 	// 自动启动超时清理
 	svc.StartCleanupTicker()
@@ -108,11 +132,159 @@ func (s *callService) generateCallID() string {
 	return strconv.FormatInt(id, 10)
 }

-// getActiveCall 从内存获取活跃通话
+// redisStoreCall 存储通话数据到 Redis
+func (s *callService) redisStoreCall(call *ActiveCall) {
+	if s.redis == nil {
+		return
+	}
+	ctx := context.Background()
+
+	var startedAt int64
+	if call.StartedAt != nil {
+		startedAt = call.StartedAt.UnixMilli()
+	}
+	data := activeCallRedisData{
+		ID:             call.ID,
+		ConversationID: call.ConversationID,
+		CallerID:       call.CallerID,
+		CalleeID:       call.CalleeID,
+		CallType:       string(call.CallType),
+		Status:         string(call.Status),
+		MediaType:      call.MediaType,
+		CreatedAt:      call.CreatedAt.UnixMilli(),
+		StartedAt:      startedAt,
+	}
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		zap.L().Error("Failed to marshal call for Redis", zap.Error(err))
+		return
+	}
+
+	callKey := redisCallKeyPrefix + call.ID
+	callerKey := redisCallByUserPrefix + call.CallerID
+	calleeKey := redisCallByUserPrefix + call.CalleeID
+
+	pipe := s.redis.Pipeline()
+	pipe.Set(ctx, callKey, jsonData, redisCallTTL)
+	pipe.Set(ctx, callerKey, call.ID, redisCallTTL)
+	pipe.Set(ctx, calleeKey, call.ID, redisCallTTL)
+	if _, err := pipe.Exec(ctx); err != nil {
+		zap.L().Error("Failed to store call in Redis", zap.String("call_id", call.ID), zap.Error(err))
+	}
+}
+
+// redisRemoveCall 从 Redis 移除通话数据
+func (s *callService) redisRemoveCall(call *ActiveCall) {
+	if s.redis == nil {
+		return
+	}
+	ctx := context.Background()
+
+	keys := []string{
+		redisCallKeyPrefix + call.ID,
+		redisCallByUserPrefix + call.CallerID,
+		redisCallByUserPrefix + call.CalleeID,
+		redisCallAcceptPrefix + call.ID,
+	}
+	if err := s.redis.Del(ctx, keys...); err != nil {
+		zap.L().Error("Failed to remove call from Redis", zap.String("call_id", call.ID), zap.Error(err))
+	}
+}
+
+// redisGetCall 从 Redis 获取通话数据
+func (s *callService) redisGetCall(callID string) *ActiveCall {
+	if s.redis == nil {
+		return nil
+	}
+	ctx := context.Background()
+
+	raw, err := s.redis.Get(ctx, redisCallKeyPrefix+callID)
+	if err != nil {
+		return nil
+	}
+
+	var data activeCallRedisData
+	if err := json.Unmarshal([]byte(raw), &data); err != nil {
+		zap.L().Error("Failed to unmarshal call from Redis", zap.Error(err))
+		return nil
+	}
+
+	call := &ActiveCall{
+		ID:             data.ID,
+		ConversationID: data.ConversationID,
+		CallerID:       data.CallerID,
+		CalleeID:       data.CalleeID,
+		CallType:       model.CallType(data.CallType),
+		Status:         model.CallStatus(data.Status),
+		MediaType:      data.MediaType,
+		ICEServers:     s.config.WebRTC.ICEServers,
+		Participants: map[string]*ActiveParticipant{
+			data.CallerID: {UserID: data.CallerID, Status: model.ParticipantStatusJoined},
+			data.CalleeID: {UserID: data.CalleeID, Status: model.ParticipantStatusInvited},
+		},
+	}
+
+	call.CreatedAt = time.UnixMilli(data.CreatedAt)
+	if data.StartedAt > 0 {
+		t := time.UnixMilli(data.StartedAt)
+		call.StartedAt = &t
+		// 如果已经开始，被叫方也已加入
+		call.Participants[data.CalleeID] = &ActiveParticipant{
+			UserID:   data.CalleeID,
+			Status:   model.ParticipantStatusJoined,
+			JoinedAt: call.StartedAt,
+		}
+	}
+
+	return call
+}
+
+// redisRefreshTTL 刷新通话相关 Redis 键的 TTL
+func (s *callService) redisRefreshTTL(call *ActiveCall) {
+	if s.redis == nil {
+		return
+	}
+	ctx := context.Background()
+
+	keys := []string{
+		redisCallKeyPrefix + call.ID,
+		redisCallByUserPrefix + call.CallerID,
+		redisCallByUserPrefix + call.CalleeID,
+	}
+	for _, key := range keys {
+		_, _ = s.redis.Expire(ctx, key, redisCallTTL)
+	}
+}
+
+// redisTryAcceptLock 尝试获取 Accept 分布式锁
+func (s *callService) redisTryAcceptLock(callID string) bool {
+	if s.redis == nil {
+		return true // 无 Redis 时允许通过（降级为本地锁）
+	}
+	ctx := context.Background()
+	ok, err := s.redis.GetClient().SetNX(ctx, redisCallAcceptPrefix+callID, "1", redisCallAcceptLockTTL).Result()
+	if err != nil {
+		zap.L().Error("Redis SETNX error, allowing accept", zap.Error(err))
+		return true
+	}
+	return ok
+}
+
+// getActiveCall 从内存获取活跃通话，L1 未命中时回退到 Redis
 func (s *callService) getActiveCall(callID string) *ActiveCall {
 	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.activeCalls[callID]
+	call := s.activeCalls[callID]
+	s.mu.RUnlock()
+	if call != nil {
+		return call
+	}
+
+	// L1 miss → Redis fallback
+	call = s.redisGetCall(callID)
+	if call != nil {
+		s.storeActiveCall(call)
+	}
+	return call
 }

 // getActiveCallByConversation 从内存获取会话的活跃通话
@@ -224,6 +396,9 @@ func (s *callService) Invite(ctx context.Context, callerID, calleeID, conversati
 	// 存储到内存
 	s.storeActiveCall(call)

+	// 写入 Redis
+	s.redisStoreCall(call)
+
 	// 检查被叫方是否在线
 	calleeOnline := s.hub.HasClients(calleeID)

@@ -272,6 +447,11 @@ func (s *callService) Accept(ctx context.Context, callID, userID string) (*Activ
 		return nil, apperrors.ErrCallAlreadyAnswered
 	}

+	// 分布式锁：防止跨实例重复 Accept
+	if !s.redisTryAcceptLock(callID) {
+		return nil, apperrors.ErrCallAlreadyAnswered
+	}
+
 	// 使用原子操作更新状态
 	s.mu.Lock()
 	if call.Status != model.CallStatusCalling {
@@ -285,6 +465,10 @@ func (s *callService) Accept(ctx context.Context, callID, userID string) (*Activ
 	call.Participants[userID].JoinedAt = &now
 	s.mu.Unlock()

+	// 更新 Redis
+	s.redisStoreCall(call)
+	s.redisRefreshTTL(call)
+
 	// 通知拨打方
 	s.hub.PublishToUserOnline(call.CallerID, "call_accepted", map[string]any{
 		"call_id":     callID,
@@ -324,6 +508,9 @@ func (s *callService) Reject(ctx context.Context, callID, userID string) error {
 	// 从内存移除
 	s.removeActiveCall(callID)

+	// 从 Redis 移除
+	s.redisRemoveCall(call)
+
 	// 保存到数据库作为历史记录
 	s.saveCallHistory(call, model.CallStatusRejected, 0)

@@ -358,6 +545,9 @@ func (s *callService) Busy(ctx context.Context, callID, userID string) error {
 	// 从内存移除
 	s.removeActiveCall(callID)

+	// 从 Redis 移除
+	s.redisRemoveCall(call)
+
 	// 保存到数据库作为历史记录
 	s.saveCallHistory(call, model.CallStatusMissed, 0)

@@ -414,6 +604,9 @@ func (s *callService) End(ctx context.Context, callID, userID string, reason str
 	// 从内存移除
 	s.removeActiveCall(callID)

+	// 从 Redis 移除
+	s.redisRemoveCall(call)
+
 	// 保存到数据库作为历史记录
 	s.saveCallHistory(call, endStatus, duration)

--- a/internal/service/chat_service.go
+++ b/internal/service/chat_service.go
@@ -67,7 +67,7 @@ type chatServiceImpl struct {
 	repo        repository.MessageRepository
 	userRepo    repository.UserRepository
 	sensitive   SensitiveService
-	wsHub       *ws.Hub
+	wsHub       ws.MessagePublisher
 	pushSvc     PushService
 	cache       cache.Cache

@@ -80,7 +80,7 @@ func NewChatService(
 	repo repository.MessageRepository,
 	userRepo repository.UserRepository,
 	sensitive SensitiveService,
-	wsHub *ws.Hub,
+	publisher ws.MessagePublisher,
 	cacheBackend cache.Cache,
 	uploadService *UploadService,
 	pushSvc PushService,
@@ -101,7 +101,7 @@ func NewChatService(
 		repo:              repo,
 		userRepo:          userRepo,
 		sensitive:         sensitive,
-		wsHub:             wsHub,
+		wsHub:             publisher,
 		pushSvc:           pushSvc,
 		cache:             cacheBackend,
 		conversationCache: conversationCache,
--- a/internal/service/group_service.go
+++ b/internal/service/group_service.go
@@ -112,13 +112,13 @@ type groupService struct {
 	messageRepo       repository.MessageRepository
 	requestRepo       repository.GroupJoinRequestRepository
 	notifyRepo        repository.SystemNotificationRepository
-	wsHub             *ws.Hub
+	wsHub             ws.MessagePublisher
 	cache             cache.Cache
 	conversationCache *cache.ConversationCache
 }

 // NewGroupService 创建群组服务
-func NewGroupService(groupRepo repository.GroupRepository, userRepo repository.UserRepository, messageRepo repository.MessageRepository, requestRepo repository.GroupJoinRequestRepository, notifyRepo repository.SystemNotificationRepository, wsHub *ws.Hub, cacheBackend cache.Cache) GroupService {
+func NewGroupService(groupRepo repository.GroupRepository, userRepo repository.UserRepository, messageRepo repository.MessageRepository, requestRepo repository.GroupJoinRequestRepository, notifyRepo repository.SystemNotificationRepository, publisher ws.MessagePublisher, cacheBackend cache.Cache) GroupService {
 	convRepoAdapter := cache.NewConversationRepositoryAdapter(messageRepo)
 	msgRepoAdapter := cache.NewMessageRepositoryAdapter(messageRepo)
 	conversationCache := cache.NewConversationCache(
@@ -134,7 +134,7 @@ func NewGroupService(groupRepo repository.GroupRepository, userRepo repository.U
 		messageRepo:       messageRepo,
 		requestRepo:       requestRepo,
 		notifyRepo:        notifyRepo,
-		wsHub:             wsHub,
+		wsHub:             publisher,
 		cache:             cacheBackend,
 		conversationCache: conversationCache,
 	}
--- a/internal/service/push_service.go
+++ b/internal/service/push_service.go
@@ -82,7 +82,7 @@ type pushServiceImpl struct {
 	pushRepo    repository.PushRecordRepository
 	deviceRepo  repository.DeviceTokenRepository
 	messageRepo repository.MessageRepository
-	wsHub       *ws.Hub
+	wsHub       ws.MessagePublisher
 	jpushClient *jpush.Client

 	pushQueue chan *pushTask
@@ -102,14 +102,14 @@ func NewPushService(
 	pushRepo repository.PushRecordRepository,
 	deviceRepo repository.DeviceTokenRepository,
 	messageRepo repository.MessageRepository,
-	wsHub *ws.Hub,
+	publisher ws.MessagePublisher,
 	jpushClient *jpush.Client,
 ) PushService {
 	return &pushServiceImpl{
 		pushRepo:    pushRepo,
 		deviceRepo:  deviceRepo,
 		messageRepo: messageRepo,
-		wsHub:       wsHub,
+		wsHub:       publisher,
 		jpushClient: jpushClient,
 		pushQueue:   make(chan *pushTask, PushQueueSize),
 		stopChan:    make(chan struct{}),
--- a/internal/service/qrcode_login_service.go
+++ b/internal/service/qrcode_login_service.go
@@ -54,7 +54,7 @@ type QRCodeSession struct {
 // QRCodeLoginService 二维码登录服务
 type QRCodeLoginService struct {
 	redis           *redis.Client
-	wsHub           *ws.Hub
+	wsHub           ws.MessagePublisher
 	jwtService      *JWTService
 	userService     UserService
 	activityService UserActivityService
@@ -62,10 +62,10 @@ type QRCodeLoginService struct {
 }

 // NewQRCodeLoginService 创建二维码登录服务
-func NewQRCodeLoginService(redis *redis.Client, wsHub *ws.Hub, jwtService *JWTService, userService UserService, activityService UserActivityService, logService *LogService) *QRCodeLoginService {
+func NewQRCodeLoginService(redis *redis.Client, publisher ws.MessagePublisher, jwtService *JWTService, userService UserService, activityService UserActivityService, logService *LogService) *QRCodeLoginService {
 	return &QRCodeLoginService{
 		redis:           redis,
-		wsHub:           wsHub,
+		wsHub:           publisher,
 		jwtService:      jwtService,
 		userService:     userService,
 		activityService: activityService,
@@ -280,9 +280,16 @@ func (s *QRCodeLoginService) Cancel(ctx context.Context, sessionID, userID strin
 	return nil
 }

-// GetWSHub 获取WS Hub
+// GetWSHub 获取底层 WS Hub（用于 Subscribe 等本地操作）
 func (s *QRCodeLoginService) GetWSHub() *ws.Hub {
-	return s.wsHub
+	switch p := s.wsHub.(type) {
+	case *ws.Bus:
+		return p.Hub()
+	case *ws.Hub:
+		return p
+	default:
+		return nil
+	}
 }

 // GetUserService 获取用户服务
--- a/internal/service/trade_service.go
+++ b/internal/service/trade_service.go
@@ -79,8 +79,6 @@ func (s *tradeService) GetByID(ctx context.Context, id string, currentUserID *st
 		}
 	}

-	_ = s.tradeRepo.IncrementViews(ctx, id)
-
 	return dto.ConvertTradeItemToResponse(item, isFavorited), nil
 }

--- a/internal/service/user_profile_audit_service.go
+++ b/internal/service/user_profile_audit_service.go
@@ -2,7 +2,6 @@ package service

 import (
 	"context"
-	"log"
 	"time"

 	"with_you/internal/model"
@@ -179,7 +178,10 @@ func (s *userProfileAuditServiceImpl) GetPendingCounts(ctx context.Context) (int
 func (s *userProfileAuditServiceImpl) reviewAvatarAsync(auditID, userID, avatarURL string) {
 	defer func() {
 		if r := recover(); r != nil {
-			log.Printf("[ERROR] Panic in avatar moderation async flow, audit=%s panic=%v", auditID, r)
+			zap.L().Error("Panic in avatar moderation async flow",
+				zap.String("audit_id", auditID),
+				zap.Any("panic", r),
+			)
 		}
 	}()

@@ -245,7 +247,10 @@ func (s *userProfileAuditServiceImpl) reviewAvatarAsync(auditID, userID, avatarU
 func (s *userProfileAuditServiceImpl) reviewCoverAsync(auditID, userID, coverURL string) {
 	defer func() {
 		if r := recover(); r != nil {
-			log.Printf("[ERROR] Panic in cover moderation async flow, audit=%s panic=%v", auditID, r)
+			zap.L().Error("Panic in cover moderation async flow",
+				zap.String("audit_id", auditID),
+				zap.Any("panic", r),
+			)
 		}
 	}()

@@ -311,7 +316,10 @@ func (s *userProfileAuditServiceImpl) reviewCoverAsync(auditID, userID, coverURL
 func (s *userProfileAuditServiceImpl) reviewBioAsync(auditID, userID, bio string) {
 	defer func() {
 		if r := recover(); r != nil {
-			log.Printf("[ERROR] Panic in bio moderation async flow, audit=%s panic=%v", auditID, r)
+			zap.L().Error("Panic in bio moderation async flow",
+				zap.String("audit_id", auditID),
+				zap.Any("panic", r),
+			)
 		}
 	}()