feat(core): optimize performance and reliability through batching and redis-backed unread counts
This commit introduces several significant architectural improvements to enhance system performance, scalability, and reliability:
- **Performance Optimization (N+1 Problem Resolution)**: Implemented batching mechanisms across `MessageHandler`, `ChatService`, `GroupService`, `MessageService`, and `UserService`. This replaces multiple individual database/cache queries with single batch operations for fetching unread counts, last messages, participants, and user information.
- **Enhanced Unread Count Management**: Migrated unread count tracking to a Redis Hash-based approach (`unread#️⃣{userID}`). This allows for atomic increments/decrements and efficient retrieval of both individual conversation unread counts and total unread counts for a user.
- **Improved Message Sequencing**: Integrated Redis-based sequence (`seq`) pre-allocation for messages to ensure strict ordering and reduce database contention during high-concurrency message creation.
- **Push Service Reliability**: Refactored the push notification system to include persistent `PushRecord` tracking. Added a recovery mechanism (`recoverPendingPushes`) to reload pending notifications from the database upon service startup, ensuring better delivery guarantees.
- **WebSocket Reliability**: Updated the WebSocket hub to move away from in-memory history replay in favor of a client-driven synchronization model (`sync_required` event and `ack` handling), reducing memory overhead and improving connection stability.
- **Cache Layer Enhancements**: Added `HIncrBy` and `IncrBySeq` to the `Cache` interface and its implementations (`RedisCache`, `LayeredCache`) to support the new unread and sequence management logic.
This commit is contained in:
@@ -150,6 +150,11 @@ func (c *Client) HDel(ctx context.Context, key string, fields ...string) error {
|
||||
return c.rdb.HDel(ctx, key, fields...).Err()
|
||||
}
|
||||
|
||||
// HIncrBy 原子递增 Hash 字段的值
|
||||
func (c *Client) HIncrBy(ctx context.Context, key string, field string, incr int64) (int64, error) {
|
||||
return c.rdb.HIncrBy(ctx, key, field, incr).Result()
|
||||
}
|
||||
|
||||
// HExists 检查 Hash 字段是否存在
|
||||
func (c *Client) HExists(ctx context.Context, key string, field string) (bool, error) {
|
||||
return c.rdb.HExists(ctx, key, field).Result()
|
||||
|
||||
@@ -12,11 +12,12 @@ import (
|
||||
|
||||
const (
|
||||
defaultUserBufferSize = 128
|
||||
maxReplayEvents = 200
|
||||
groupPushConcurrency = 64
|
||||
groupPushQueueSize = 512
|
||||
maxTotalConnections = 100000
|
||||
maxConnectionsPerUser = 10
|
||||
ackTimeout = 30 * time.Second
|
||||
ackCheckInterval = 10 * time.Second
|
||||
)
|
||||
|
||||
// Event 表示一个WebSocket事件
|
||||
@@ -30,10 +31,11 @@ type Event struct {
|
||||
|
||||
// Client 表示一个WebSocket客户端连接
|
||||
type Client struct {
|
||||
ID uint64
|
||||
UserID string
|
||||
Send chan []byte
|
||||
Quit chan struct{}
|
||||
ID uint64
|
||||
UserID string
|
||||
Send chan []byte
|
||||
Quit chan struct{}
|
||||
PendingAcks map[string]time.Time
|
||||
}
|
||||
|
||||
// ErrConnectionLimit 连接数限制错误
|
||||
@@ -86,7 +88,6 @@ type Hub struct {
|
||||
|
||||
mu sync.RWMutex
|
||||
clients map[string]map[uint64]*Client
|
||||
history map[string][]Event
|
||||
subscribers map[string]map[uint64]*subscriber
|
||||
disconnectHandlers []DisconnectHandler
|
||||
connectHandlers []ConnectHandler
|
||||
@@ -96,7 +97,6 @@ type Hub struct {
|
||||
func NewHub() *Hub {
|
||||
return &Hub{
|
||||
clients: make(map[string]map[uint64]*Client),
|
||||
history: make(map[string][]Event),
|
||||
subscribers: make(map[string]map[uint64]*subscriber),
|
||||
disconnectHandlers: make([]DisconnectHandler, 0),
|
||||
connectHandlers: make([]ConnectHandler, 0),
|
||||
@@ -108,17 +108,17 @@ func (h *Hub) NextID() uint64 {
|
||||
return atomic.AddUint64(&h.seq, 1)
|
||||
}
|
||||
|
||||
// Register 注册客户端连接，返回回放事件和可能的错误（连接数超限）
|
||||
func (h *Hub) Register(client *Client) ([]Event, error) {
|
||||
// Register 注册客户端连接，返回可能的错误（连接数超限）
|
||||
func (h *Hub) Register(client *Client) error {
|
||||
if h.connCount.Load() >= int64(maxTotalConnections) {
|
||||
return nil, ErrConnectionLimit
|
||||
return ErrConnectionLimit
|
||||
}
|
||||
|
||||
h.mu.Lock()
|
||||
|
||||
if len(h.clients[client.UserID]) >= maxConnectionsPerUser {
|
||||
h.mu.Unlock()
|
||||
return nil, ErrConnectionLimit
|
||||
return ErrConnectionLimit
|
||||
}
|
||||
|
||||
_, existed := h.clients[client.UserID]
|
||||
@@ -130,11 +130,6 @@ func (h *Hub) Register(client *Client) ([]Event, error) {
|
||||
h.clients[client.UserID][client.ID] = client
|
||||
h.connCount.Add(1)
|
||||
|
||||
replay := make([]Event, 0)
|
||||
for _, e := range h.history[client.UserID] {
|
||||
replay = append(replay, e)
|
||||
}
|
||||
|
||||
zap.L().Debug("WebSocket client registered",
|
||||
zap.String("user_id", client.UserID),
|
||||
zap.Uint64("client_id", client.ID),
|
||||
@@ -155,7 +150,7 @@ func (h *Hub) Register(client *Client) ([]Event, error) {
|
||||
}()
|
||||
}
|
||||
|
||||
return replay, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unregister 注销客户端连接
|
||||
@@ -290,18 +285,12 @@ func (h *Hub) PublishToUsers(userIDs []string, eventType string, payload any) {
|
||||
|
||||
// publishPreMarshaled 使用预序列化的数据发送事件
|
||||
func (h *Hub) publishPreMarshaled(userID string, ev Event, data []byte) {
|
||||
h.mu.Lock()
|
||||
history := append(h.history[userID], ev)
|
||||
if len(history) > maxReplayEvents {
|
||||
history = history[len(history)-maxReplayEvents:]
|
||||
}
|
||||
h.history[userID] = history
|
||||
|
||||
h.mu.RLock()
|
||||
targets := make([]*Client, 0, len(h.clients[userID]))
|
||||
for _, c := range h.clients[userID] {
|
||||
targets = append(targets, c)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.mu.RUnlock()
|
||||
|
||||
for _, c := range targets {
|
||||
select {
|
||||
@@ -392,20 +381,12 @@ func (h *Hub) PublishToUserOnlineReliable(userID string, eventType string, paylo
|
||||
|
||||
// publish 内部发布方法
|
||||
func (h *Hub) publish(userID string, ev Event) {
|
||||
h.mu.Lock()
|
||||
// 存储历史事件（最多保留200条）
|
||||
history := append(h.history[userID], ev)
|
||||
if len(history) > maxReplayEvents {
|
||||
history = history[len(history)-maxReplayEvents:]
|
||||
}
|
||||
h.history[userID] = history
|
||||
|
||||
// 复制客户端列表避免锁竞争
|
||||
h.mu.RLock()
|
||||
targets := make([]*Client, 0, len(h.clients[userID]))
|
||||
for _, c := range h.clients[userID] {
|
||||
targets = append(targets, c)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.mu.RUnlock()
|
||||
|
||||
// 构建消息
|
||||
msg := ResponseMessage{
|
||||
@@ -532,17 +513,6 @@ func (h *Hub) PublishToUsersConcurrent(userIDs []string, eventType string, paylo
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Store history for each user for reconnection replay
|
||||
h.mu.Lock()
|
||||
for _, uid := range userIDs {
|
||||
history := append(h.history[uid], ev)
|
||||
if len(history) > maxReplayEvents {
|
||||
history = history[len(history)-maxReplayEvents:]
|
||||
}
|
||||
h.history[uid] = history
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// GetOnlineUsers 获取在线用户列表
|
||||
@@ -588,7 +558,7 @@ func (h *Hub) CloseAllConnections() {
|
||||
}
|
||||
|
||||
// Subscribe 订阅事件
|
||||
func (h *Hub) Subscribe(userID string, afterID uint64) (chan Event, func(), []Event) {
|
||||
func (h *Hub) Subscribe(userID string, afterID uint64) (chan Event, func()) {
|
||||
subID := h.NextID()
|
||||
sub := &subscriber{
|
||||
id: subID,
|
||||
@@ -601,13 +571,6 @@ func (h *Hub) Subscribe(userID string, afterID uint64) (chan Event, func(), []Ev
|
||||
h.subscribers[userID] = make(map[uint64]*subscriber)
|
||||
}
|
||||
h.subscribers[userID][subID] = sub
|
||||
|
||||
replay := make([]Event, 0)
|
||||
for _, e := range h.history[userID] {
|
||||
if e.ID > afterID {
|
||||
replay = append(replay, e)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
|
||||
cancel := func() {
|
||||
@@ -625,7 +588,59 @@ func (h *Hub) Subscribe(userID string, afterID uint64) (chan Event, func(), []Ev
|
||||
}
|
||||
}
|
||||
|
||||
return sub.ch, cancel, replay
|
||||
return sub.ch, cancel
|
||||
}
|
||||
|
||||
// AckMessage 确认消息已送达，从 pendingAcks 中移除
|
||||
func (h *Hub) AckMessage(userID, messageID string) bool {
|
||||
h.mu.RLock()
|
||||
defer h.mu.RUnlock()
|
||||
|
||||
for _, client := range h.clients[userID] {
|
||||
if _, ok := client.PendingAcks[messageID]; ok {
|
||||
delete(client.PendingAcks, messageID)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// startAckChecker 启动超时 ACK 检查器
|
||||
func (h *Hub) startAckChecker(onTimeout func(userID, messageID string)) {
|
||||
ticker := time.NewTicker(ackCheckInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
h.mu.RLock()
|
||||
// 收集超时的 ack
|
||||
type timeoutEntry struct {
|
||||
userID string
|
||||
messageID string
|
||||
}
|
||||
var timeouts []timeoutEntry
|
||||
|
||||
now := time.Now()
|
||||
for uid, userClients := range h.clients {
|
||||
for _, client := range userClients {
|
||||
for msgID, sentAt := range client.PendingAcks {
|
||||
if now.Sub(sentAt) > ackTimeout {
|
||||
timeouts = append(timeouts, timeoutEntry{uid, msgID})
|
||||
delete(client.PendingAcks, msgID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.RUnlock()
|
||||
|
||||
for _, entry := range timeouts {
|
||||
onTimeout(entry.userID, entry.messageID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StartAckChecker 启动 ACK 检查器（公开方法）
|
||||
func (h *Hub) StartAckChecker(onTimeout func(userID, messageID string)) {
|
||||
go h.startAckChecker(onTimeout)
|
||||
}
|
||||
|
||||
// EncodeData 编码事件数据
|
||||
|
||||
Reference in New Issue
Block a user