feat(core): optimize performance and reliability through batching and redis-backed unread counts
This commit introduces several significant architectural improvements to enhance system performance, scalability, and reliability:
- **Performance Optimization (N+1 Problem Resolution)**: Implemented batching mechanisms across `MessageHandler`, `ChatService`, `GroupService`, `MessageService`, and `UserService`. This replaces multiple individual database/cache queries with single batch operations for fetching unread counts, last messages, participants, and user information.
- **Enhanced Unread Count Management**: Migrated unread count tracking to a Redis Hash-based approach (`unread#️⃣{userID}`). This allows for atomic increments/decrements and efficient retrieval of both individual conversation unread counts and total unread counts for a user.
- **Improved Message Sequencing**: Integrated Redis-based sequence (`seq`) pre-allocation for messages to ensure strict ordering and reduce database contention during high-concurrency message creation.
- **Push Service Reliability**: Refactored the push notification system to include persistent `PushRecord` tracking. Added a recovery mechanism (`recoverPendingPushes`) to reload pending notifications from the database upon service startup, ensuring better delivery guarantees.
- **WebSocket Reliability**: Updated the WebSocket hub to move away from in-memory history replay in favor of a client-driven synchronization model (`sync_required` event and `ack` handling), reducing memory overhead and improving connection stability.
- **Cache Layer Enhancements**: Added `HIncrBy` and `IncrBySeq` to the `Cache` interface and its implementations (`RedisCache`, `LayeredCache`) to support the new unread and sequence management logic.
This commit is contained in:
@@ -27,6 +27,7 @@ type GroupRepository interface {
|
||||
UpdateMember(member *model.GroupMember) error
|
||||
RemoveMember(groupID string, userID string) error
|
||||
GetMemberCount(groupID string) (int64, error)
|
||||
GetMemberCountBatch(ctx context.Context, groupIDs []string) (map[string]int64, error)
|
||||
IsMember(groupID string, userID string) (bool, error)
|
||||
GetUserGroups(userID string, page, pageSize int) ([]model.Group, int64, error)
|
||||
|
||||
@@ -225,6 +226,34 @@ func (r *groupRepository) GetMemberCount(groupID string) (int64, error) {
|
||||
return count, err
|
||||
}
|
||||
|
||||
// GetMemberCountBatch 批量获取多个群的成员数量
|
||||
func (r *groupRepository) GetMemberCountBatch(ctx context.Context, groupIDs []string) (map[string]int64, error) {
|
||||
result := make(map[string]int64, len(groupIDs))
|
||||
if len(groupIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type row struct {
|
||||
GroupID string
|
||||
Count int64
|
||||
}
|
||||
var rows []row
|
||||
err := r.db.WithContext(ctx).
|
||||
Model(&model.GroupMember{}).
|
||||
Select("group_id, COUNT(*) as count").
|
||||
Where("group_id IN ?", groupIDs).
|
||||
Group("group_id").
|
||||
Scan(&rows).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
result[r.GroupID] = r.Count
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IsMember 检查是否是群成员
|
||||
func (r *groupRepository) IsMember(groupID string, userID string) (bool, error) {
|
||||
var count int64
|
||||
|
||||
@@ -54,6 +54,12 @@ type MessageRepository interface {
|
||||
UpdateConversationLastSeqWithTx(tx *gorm.DB, convID string, lastSeq int64, lastMsgTime time.Time) error
|
||||
GetMessagesByCursor(ctx context.Context, conversationID string, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Message], error)
|
||||
GetConversationsByCursor(ctx context.Context, userID string, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Conversation], error)
|
||||
|
||||
// 批量查询方法(解决 N+1 问题)
|
||||
GetUnreadCountBatch(ctx context.Context, userID string, convIDs []string) (map[string]int64, error)
|
||||
GetLastMessagesBatch(ctx context.Context, convIDs []string) (map[string]*model.Message, error)
|
||||
GetParticipantsBatch(ctx context.Context, convIDs []string) (map[string][]*model.ConversationParticipant, error)
|
||||
GetMyParticipantsBatch(ctx context.Context, convIDs []string, userID string) (map[string]*model.ConversationParticipant, error)
|
||||
}
|
||||
|
||||
// messageRepository 消息仓储实现
|
||||
@@ -405,17 +411,10 @@ func (r *messageRepository) GetNextSeq(conversationID string) (int64, error) {
|
||||
}
|
||||
|
||||
// CreateMessageWithSeq 创建消息并更新seq(事务操作)
|
||||
// msg.Seq 应由调用方通过 Redis INCR 预分配
|
||||
func (r *messageRepository) CreateMessageWithSeq(msg *model.Message) error {
|
||||
return r.db.Transaction(func(tx *gorm.DB) error {
|
||||
// 获取当前seq并+1
|
||||
var conv model.Conversation
|
||||
if err := tx.Select("last_seq").Where("id = ?", msg.ConversationID).First(&conv).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
msg.Seq = conv.LastSeq + 1
|
||||
|
||||
// 创建消息
|
||||
// 创建消息(seq 已由调用方预分配)
|
||||
if err := tx.Create(msg).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -430,7 +429,7 @@ func (r *messageRepository) CreateMessageWithSeq(msg *model.Message) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// 新消息到达后,自动恢复被“仅自己删除”的会话
|
||||
// 新消息到达后,自动恢复被"仅自己删除"的会话
|
||||
if err := tx.Model(&model.ConversationParticipant{}).
|
||||
Where("conversation_id = ?", msg.ConversationID).
|
||||
Update("hidden_at", nil).Error; err != nil {
|
||||
@@ -883,3 +882,101 @@ func (r *messageRepository) GetConversationsByCursor(ctx context.Context, userID
|
||||
|
||||
return cursor.NewCursorPageResult(conversations, nextCursor, prevCursor, hasMore), nil
|
||||
}
|
||||
|
||||
// GetUnreadCountBatch 批量获取用户在多个会话中的未读数
|
||||
func (r *messageRepository) GetUnreadCountBatch(ctx context.Context, userID string, convIDs []string) (map[string]int64, error) {
|
||||
result := make(map[string]int64, len(convIDs))
|
||||
if len(convIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type row struct {
|
||||
ConversationID string
|
||||
Count int64
|
||||
}
|
||||
var rows []row
|
||||
err := r.db.WithContext(ctx).
|
||||
Table("conversation_participants cp").
|
||||
Select("cp.conversation_id, COALESCE(cnt.unread, 0) as count").
|
||||
Joins("LEFT JOIN (SELECT m.conversation_id, COUNT(m.id) as unread FROM messages m WHERE m.sender_id <> ? AND m.deleted_at IS NULL AND m.conversation_id IN (?) GROUP BY m.conversation_id) cnt ON cnt.conversation_id = cp.conversation_id AND cnt.unread > cp.last_read_seq", userID, convIDs).
|
||||
Where("cp.user_id = ? AND cp.conversation_id IN (?)", userID, convIDs).
|
||||
Scan(&rows).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, id := range convIDs {
|
||||
result[id] = 0
|
||||
}
|
||||
for _, r := range rows {
|
||||
result[r.ConversationID] = r.Count
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetLastMessagesBatch 批量获取每个会话的最后一条消息
|
||||
func (r *messageRepository) GetLastMessagesBatch(ctx context.Context, convIDs []string) (map[string]*model.Message, error) {
|
||||
result := make(map[string]*model.Message, len(convIDs))
|
||||
if len(convIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var messages []*model.Message
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("id IN (?)", r.db.
|
||||
Select("MAX(id)").
|
||||
Table("messages").
|
||||
Where("conversation_id IN (?) AND deleted_at IS NULL", convIDs).
|
||||
Group("conversation_id")).
|
||||
Find(&messages).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, m := range messages {
|
||||
result[m.ConversationID] = m
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetParticipantsBatch 批量获取多个会话的参与者列表
|
||||
func (r *messageRepository) GetParticipantsBatch(ctx context.Context, convIDs []string) (map[string][]*model.ConversationParticipant, error) {
|
||||
result := make(map[string][]*model.ConversationParticipant, len(convIDs))
|
||||
if len(convIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var participants []*model.ConversationParticipant
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("conversation_id IN (?)", convIDs).
|
||||
Find(&participants).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, p := range participants {
|
||||
result[p.ConversationID] = append(result[p.ConversationID], p)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetMyParticipantsBatch 批量获取用户在多个会话中的参与者信息
|
||||
func (r *messageRepository) GetMyParticipantsBatch(ctx context.Context, convIDs []string, userID string) (map[string]*model.ConversationParticipant, error) {
|
||||
result := make(map[string]*model.ConversationParticipant, len(convIDs))
|
||||
if len(convIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var participants []*model.ConversationParticipant
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("conversation_id IN (?) AND user_id = ?", convIDs, userID).
|
||||
Find(&participants).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, p := range participants {
|
||||
result[p.ConversationID] = p
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
"with_you/internal/model"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
type UserRepository interface {
|
||||
Create(user *model.User) error
|
||||
GetByID(id string) (*model.User, error)
|
||||
GetUsersByIDs(ctx context.Context, ids []string) (map[string]*model.User, error)
|
||||
GetByUsername(username string) (*model.User, error)
|
||||
GetByEmail(email string) (*model.User, error)
|
||||
GetByPhone(phone string) (*model.User, error)
|
||||
@@ -607,3 +609,22 @@ func (r *userRepository) GetPendingDeletionUsers(beforeTime time.Time) ([]*model
|
||||
func (r *userRepository) HardDeleteUser(userID string) error {
|
||||
return r.db.Unscoped().Delete(&model.User{}, "id = ?", userID).Error
|
||||
}
|
||||
|
||||
// GetUsersByIDs 批量获取用户信息
|
||||
func (r *userRepository) GetUsersByIDs(ctx context.Context, ids []string) (map[string]*model.User, error) {
|
||||
result := make(map[string]*model.User, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var users []*model.User
|
||||
err := r.db.WithContext(ctx).Where("id IN ?", ids).Find(&users).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, u := range users {
|
||||
result[u.ID] = u
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user