refactor: improve system stability, performance, and code structure
All checks were successful
Build Backend / build (push) Successful in 3m2s
Build Backend / build-docker (push) Successful in 2m44s

This commit introduces several architectural improvements and optimizations across the codebase:

- **Performance & Reliability**:
  - Implemented Redis pipelining in `ConversationCache.CacheMessage` to reduce network round-trips.
  - Added a circuit breaker to the JPush client to prevent cascading failures.
  - Introduced batch deletion and batch member addition capabilities in repositories.
  - Added message idempotency support using `client_msg_id` and a Redis-based cache.
  - Optimized WebSocket handling with connection limits (total and per-user) and improved error logging.

- **Code Refactoring**:
  - Refactored `Router` to use a `RouterDeps` struct, simplifying the constructor and improving maintainability.
  - Unified model ID generation logic using new `id_helper.go` (supporting UUID and Snowflake).
  - Standardized JSON serialization/deserialization in models using `json_helper.go`.
  - Refactored DTO conversion logic, specifically for `UserResponse` (using functional options) and `Report` responses.
  - Removed redundant/deprecated DTOs like `PostDetailResponse` and `TradeItemDetailResponse`.

- **Cache Improvements**:
  - Enhanced `LayeredCache` with `SetRaw` to avoid double-encoding when promoting values from Redis to local cache.
  - Added `DeleteBatch` support to the cache interface.

- **Other Changes**:
  - Cleaned up `config.go` by removing redundant default values and explicit environment variable overrides.
  - Improved WebSocket registration flow to handle connection limits gracefully.
This commit is contained in:
2026-05-04 13:07:03 +08:00
parent b2b55ea52d
commit ee78071d4d
65 changed files with 1293 additions and 975 deletions

View File

@@ -32,6 +32,8 @@ type Cache interface {
Increment(key string) int64
// IncrementBy 增加指定值
IncrementBy(key string, value int64) int64
// DeleteBatch 批量删除多个 key使用 Pipeline 减少网络往返)
DeleteBatch(keys []string)
// ==================== Hash 操作 ====================
// HSet 设置 Hash 字段

View File

@@ -6,6 +6,8 @@ import (
"fmt"
"time"
"github.com/redis/go-redis/v9"
"with_you/internal/model"
"go.uber.org/zap"
@@ -618,11 +620,11 @@ func (c *ConversationCache) GetMessagesBeforeSeq(ctx context.Context, convID str
return messages, nil
}
// CacheMessage 缓存单条消息(立即写入缓存
// 写入 Hash、Sorted Set、更新计数
// CacheMessage 缓存单条消息(使用 Redis Pipeline 减少网络往返
func (c *ConversationCache) CacheMessage(ctx context.Context, convID string, msg *model.Message) error {
hashKey := MessageHashKey(convID)
indexKey := MessageIndexKey(convID)
countKey := MessageCountKey(convID)
msgData := MessageCacheDataFromModel(msg)
data, err := json.Marshal(msgData)
@@ -630,23 +632,34 @@ func (c *ConversationCache) CacheMessage(ctx context.Context, convID string, msg
return fmt.Errorf("failed to marshal message: %w", err)
}
// HSET 消息详情
if err := c.cache.HSet(ctx, hashKey, fmt.Sprintf("%d", msg.Seq), string(data)); err != nil {
hashKey = ResolveKey(hashKey)
indexKey = ResolveKey(indexKey)
countKey = ResolveKey(countKey)
redisCache, ok := c.cache.(*RedisCache)
if ok {
rdb := redisCache.GetRedisClient().GetClient()
pipe := rdb.Pipeline()
pipe.HSet(ctx, hashKey, fmt.Sprintf("%d", msg.Seq), string(data))
pipe.ZAdd(ctx, indexKey, redis.Z{Score: float64(msg.Seq), Member: fmt.Sprintf("%d", msg.Seq)})
pipe.Expire(ctx, hashKey, c.settings.MessageDetailTTL)
pipe.Expire(ctx, indexKey, c.settings.MessageIndexTTL)
pipe.Incr(ctx, countKey)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("failed to pipeline cache message: %w", err)
}
return nil
}
if err := c.cache.HSet(ctx, MessageHashKey(convID), fmt.Sprintf("%d", msg.Seq), string(data)); err != nil {
return fmt.Errorf("failed to set hash: %w", err)
}
// ZADD 消息索引
if err := c.cache.ZAdd(ctx, indexKey, float64(msg.Seq), fmt.Sprintf("%d", msg.Seq)); err != nil {
if err := c.cache.ZAdd(ctx, MessageIndexKey(convID), float64(msg.Seq), fmt.Sprintf("%d", msg.Seq)); err != nil {
return fmt.Errorf("failed to add to sorted set: %w", err)
}
// 设置 TTL
c.cache.Expire(ctx, hashKey, c.settings.MessageDetailTTL)
c.cache.Expire(ctx, indexKey, c.settings.MessageIndexTTL)
// INCR 消息计数
c.cache.Expire(ctx, MessageHashKey(convID), c.settings.MessageDetailTTL)
c.cache.Expire(ctx, MessageIndexKey(convID), c.settings.MessageIndexTTL)
c.cache.Incr(ctx, MessageCountKey(convID))
return nil
}

View File

@@ -35,11 +35,12 @@ const (
PrefixChannelsAllList = "channels:all_list"
// 消息缓存相关
keyPrefixMsgHash = "msg_hash" // 消息详情 Hash
keyPrefixMsgIndex = "msg_index" // 消息索引 Sorted Set
keyPrefixMsgCount = "msg_count" // 消息计数
keyPrefixMsgSeq = "msg_seq" // Seq 计数器
keyPrefixMsgPage = "msg_page" // 分页缓存
keyPrefixMsgHash = "msg_hash" // 消息详情 Hash
keyPrefixMsgIndex = "msg_index" // 消息索引 Sorted Set
keyPrefixMsgCount = "msg_count" // 消息计数
keyPrefixMsgSeq = "msg_seq" // Seq 计数器
keyPrefixMsgPage = "msg_page" // 分页缓存
keyPrefixMsgIdempotent = "msg_idem" // 消息幂等键
)
// PostListKey 生成帖子列表缓存键
@@ -221,3 +222,13 @@ func MessagePageKey(convID string, page, pageSize int) string {
func InvalidateMessagePages(cache Cache, conversationID string) {
cache.DeleteByPrefix(fmt.Sprintf("%s:%s:", keyPrefixMsgPage, conversationID))
}
// MessageIdempotentKey 消息幂等键 (clientMsgID -> server MsgID)
func MessageIdempotentKey(senderID, clientMsgID string) string {
return fmt.Sprintf("%s:%s:%s", keyPrefixMsgIdempotent, senderID, clientMsgID)
}
// PushDedupKey 推送去重键,防止同一消息重复推送给同一用户
func PushDedupKey(userID, messageID string) string {
return fmt.Sprintf("push_dedup:%s:%s", userID, messageID)
}

View File

@@ -111,7 +111,11 @@ func (c *LayeredCache) Get(key string) (any, bool) {
c.mu.Unlock()
if ttl := c.getLocalTTL(); ttl > 0 {
c.local.Set(key, val, ttl)
if str, ok := val.(string); ok {
c.local.SetRaw(key, []byte(str), ttl)
} else {
c.local.Set(key, val, ttl)
}
}
return val, true
}
@@ -157,6 +161,23 @@ func (c *LayeredCache) DeleteByPrefix(prefix string) {
}
}
func (c *LayeredCache) DeleteBatch(keys []string) {
if !c.enabled || len(keys) == 0 {
return
}
c.mu.Lock()
c.stats.Invalidates += int64(len(keys))
c.mu.Unlock()
for _, k := range keys {
c.local.Delete(k)
}
if c.redis != nil {
c.redis.DeleteBatch(keys)
}
}
func (c *LayeredCache) Clear() {
c.local.Clear()
if c.redis != nil {

33
internal/cache/lru.go vendored
View File

@@ -87,6 +87,39 @@ func (l *LRU) set(key string, value any, ttl time.Duration) {
}
}
// SetRaw stores raw bytes directly without re-serializing.
// This avoids double-encoding when promoting values from Redis to local cache.
func (l *LRU) SetRaw(key string, raw []byte, ttl time.Duration) {
var expires time.Time
if ttl > 0 {
expires = time.Now().Add(ttl)
}
l.lock.Lock()
defer l.lock.Unlock()
if ent, ok := l.items[key]; ok {
l.evictList.MoveToFront(ent)
e := ent.Value.(*lruEntry)
e.value = raw
e.expires = expires
if e.generation != l.currentGeneration {
e.generation = l.currentGeneration
l.len++
}
return
}
ent := &lruEntry{key, raw, expires, l.currentGeneration}
entry := l.evictList.PushFront(ent)
l.items[key] = entry
l.len++
if l.evictList.Len() > l.size {
l.removeElement(l.evictList.Back())
}
}
func (l *LRU) Get(key string) (any, bool) {
val, err := l.getItem(key)
if err != nil {

View File

@@ -106,11 +106,20 @@ func (l *LRUStriped) Name() string {
}
func (l *LRUStriped) DeleteMulti(keys []string) error {
var err error
for _, key := range keys {
l.Delete(key)
}
return err
return nil
}
func (l *LRUStriped) DeleteBatch(keys []string) {
for _, key := range keys {
l.Delete(key)
}
}
func (l *LRUStriped) SetRaw(key string, raw []byte, ttl time.Duration) {
l.keyBucket(key).SetRaw(key, raw, ttl)
}
func (l *LRUStriped) GetMulti(keys []string, values []any) []error {

View File

@@ -82,7 +82,6 @@ func (c *RedisCache) Delete(key string) {
// DeleteByPrefix 根据前缀删除缓存
func (c *RedisCache) DeleteByPrefix(prefix string) {
prefix = normalizePrefix(prefix)
// 使用原生客户端执行SCAN命令
rdb := c.client.GetClient()
var cursor uint64
for {
@@ -112,6 +111,30 @@ func (c *RedisCache) DeleteByPrefix(prefix string) {
}
}
// DeleteBatch 批量删除多个 key使用 Redis Pipeline 减少网络往返)
func (c *RedisCache) DeleteBatch(keys []string) {
if len(keys) == 0 {
return
}
normalizedKeys := make([]string, len(keys))
for i, k := range keys {
normalizedKeys[i] = normalizeKey(k)
}
recordInvalidateMultiple(int64(len(normalizedKeys)))
// 使用 pipeline 批量删除
pipe := c.client.GetClient().Pipeline()
for _, k := range normalizedKeys {
pipe.Del(c.ctx, k)
}
if _, err := pipe.Exec(c.ctx); err != nil {
zap.L().Error("Failed to batch delete keys",
zap.Int("count", len(normalizedKeys)),
zap.Error(err),
)
}
}
// Clear 清空所有缓存
func (c *RedisCache) Clear() {
if settings.DisableFlushDB {