refactor: remove Gorse integration and implement HotRank feature
Some checks failed
Build Backend / build-docker (push) Has been cancelled
Build Backend / build (push) Has been cancelled

- Removed Gorse-related configurations, handlers, and dependencies from the codebase.
- Introduced HotRank feature with configuration options for ranking posts based on recent activity.
- Updated application structure to support HotRank processing, including new caching mechanisms and database interactions.
- Cleaned up related DTOs and repository methods to reflect the removal of Gorse and the addition of HotRank functionality.
This commit is contained in:
lafay
2026-03-24 05:18:30 +08:00
parent b41567a39a
commit 176cd20847
32 changed files with 735 additions and 1128 deletions

View File

@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"log"
"math/rand"
"strconv"
"strings"
"time"
@@ -12,7 +11,6 @@ import (
"carrot_bbs/internal/cache"
"carrot_bbs/internal/model"
"carrot_bbs/internal/pkg/cursor"
"carrot_bbs/internal/pkg/gorse"
"carrot_bbs/internal/pkg/hook"
"carrot_bbs/internal/repository"
)
@@ -22,7 +20,6 @@ const (
PostListTTL = 30 * time.Second // 帖子列表缓存30秒
PostListNullTTL = 5 * time.Second
PostListJitterRatio = 0.15
anonymousViewUserID = "_anon_view"
)
// PostService 帖子服务接口
@@ -48,12 +45,10 @@ type PostService interface {
GetUserPostsByCursor(ctx context.Context, userID string, includePending bool, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Post], error)
GetFollowingPostsByCursor(ctx context.Context, userID string, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Post], error)
GetHotPostsByCursor(ctx context.Context, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Post], error)
GetRecommendedPostsByCursor(ctx context.Context, userID string, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Post], error)
// 关注和推荐
GetFollowingPosts(ctx context.Context, userID string, page, pageSize int) ([]*model.Post, int64, error)
GetHotPosts(ctx context.Context, page, pageSize int) ([]*model.Post, int64, error)
GetRecommendedPosts(ctx context.Context, userID string, page, pageSize int) ([]*model.Post, int64, error)
// 交互功能
Like(ctx context.Context, postID, userID string) error
@@ -79,19 +74,17 @@ type postServiceImpl struct {
postRepo *repository.PostRepository
systemMessageService SystemMessageService
cache cache.Cache
gorseClient gorse.Client
postAIService *PostAIService
txManager repository.TransactionManager
logService *LogService
hookManager *hook.Manager
}
func NewPostService(postRepo *repository.PostRepository, systemMessageService SystemMessageService, gorseClient gorse.Client, postAIService *PostAIService, cacheBackend cache.Cache, txManager repository.TransactionManager, hookManager *hook.Manager) PostService {
func NewPostService(postRepo *repository.PostRepository, systemMessageService SystemMessageService, postAIService *PostAIService, cacheBackend cache.Cache, txManager repository.TransactionManager, hookManager *hook.Manager) PostService {
return &postServiceImpl{
postRepo: postRepo,
systemMessageService: systemMessageService,
cache: cacheBackend,
gorseClient: gorseClient,
postAIService: postAIService,
txManager: txManager,
logService: nil,
@@ -139,7 +132,7 @@ func (s *postServiceImpl) Create(ctx context.Context, userID, title, content str
// 失效帖子列表缓存
cache.InvalidatePostList(s.cache)
// 同步到Gorse推荐系统异步
// 异步执行审核流
go s.reviewPostAsync(post.ID, userID, title, content, images)
// 重新查询以获取关联的 User 和 Images
@@ -205,19 +198,6 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
}
s.invalidatePostCaches(postID)
if s.gorseClient.IsEnabled() {
post, getErr := s.postRepo.GetByID(postID)
if getErr != nil {
log.Printf("[WARN] Failed to load published post for gorse sync: %v", getErr)
return
}
categories := s.buildPostCategories(post)
comment := post.Title
textToEmbed := post.Title + " " + post.Content
if upsertErr := s.gorseClient.UpsertItemWithEmbedding(context.Background(), post.ID, categories, comment, textToEmbed); upsertErr != nil {
log.Printf("[WARN] Failed to upsert item to Gorse: %v", upsertErr)
}
}
}
func (s *postServiceImpl) updateModerationStatusWithRetry(postID string, status model.PostStatus, rejectReason string, reviewedBy string) error {
@@ -302,15 +282,6 @@ func (s *postServiceImpl) Delete(ctx context.Context, id string) error {
cache.InvalidatePostDetail(s.cache, id)
cache.InvalidatePostList(s.cache)
// 从Gorse中删除帖子异步
go func() {
if s.gorseClient.IsEnabled() {
if err := s.gorseClient.DeleteItem(context.Background(), id); err != nil {
log.Printf("[WARN] Failed to delete item from Gorse: %v", err)
}
}
}()
return nil
}
@@ -420,15 +391,6 @@ func (s *postServiceImpl) Like(ctx context.Context, postID, userID string) error
}()
}
// 推送点赞行为到Gorse异步
go func() {
if s.gorseClient.IsEnabled() {
if err := s.gorseClient.InsertFeedback(context.Background(), gorse.FeedbackTypeLike, userID, postID); err != nil {
log.Printf("[WARN] Failed to insert like feedback to Gorse: %v", err)
}
}
}()
return nil
}
@@ -442,15 +404,6 @@ func (s *postServiceImpl) Unlike(ctx context.Context, postID, userID string) err
// 失效帖子详情缓存
cache.InvalidatePostDetail(s.cache, postID)
// 删除Gorse中的点赞反馈异步
go func() {
if s.gorseClient.IsEnabled() {
if err := s.gorseClient.DeleteFeedback(context.Background(), gorse.FeedbackTypeLike, userID, postID); err != nil {
log.Printf("[WARN] Failed to delete like feedback from Gorse: %v", err)
}
}
}()
return nil
}
@@ -485,15 +438,6 @@ func (s *postServiceImpl) Favorite(ctx context.Context, postID, userID string) e
}()
}
// 推送收藏行为到Gorse异步
go func() {
if s.gorseClient.IsEnabled() {
if err := s.gorseClient.InsertFeedback(context.Background(), gorse.FeedbackTypeStar, userID, postID); err != nil {
log.Printf("[WARN] Failed to insert favorite feedback to Gorse: %v", err)
}
}
}()
return nil
}
@@ -507,15 +451,6 @@ func (s *postServiceImpl) Unfavorite(ctx context.Context, postID, userID string)
// 失效帖子详情缓存
cache.InvalidatePostDetail(s.cache, postID)
// 删除Gorse中的收藏反馈异步
go func() {
if s.gorseClient.IsEnabled() {
if err := s.gorseClient.DeleteFeedback(context.Background(), gorse.FeedbackTypeStar, userID, postID); err != nil {
log.Printf("[WARN] Failed to delete favorite feedback from Gorse: %v", err)
}
}
}()
return nil
}
@@ -548,28 +483,12 @@ func (s *postServiceImpl) GetPostInteractionStatusSingle(ctx context.Context, po
return s.postRepo.IsLiked(postID, userID), s.postRepo.IsFavorited(postID, userID)
}
// IncrementViews 增加帖子观看量并同步到Gorse
// IncrementViews 增加帖子观看量
func (s *postServiceImpl) IncrementViews(ctx context.Context, postID, userID string) error {
if err := s.postRepo.IncrementViews(postID); err != nil {
return err
}
// 同步浏览行为到Gorse异步
go func() {
if !s.gorseClient.IsEnabled() {
return
}
feedbackUserID := userID
if feedbackUserID == "" {
feedbackUserID = anonymousViewUserID
}
if err := s.gorseClient.InsertFeedback(context.Background(), gorse.FeedbackTypeRead, feedbackUserID, postID); err != nil {
log.Printf("[WARN] Failed to insert read feedback to Gorse: %v", err)
}
}()
return nil
}
@@ -625,43 +544,70 @@ func (s *postServiceImpl) GetFollowingPosts(ctx context.Context, userID string,
return result.Posts, result.Total, nil
}
// GetHotPosts 获取热门帖子(使用Gorse非个性化推荐
// GetHotPosts 获取热门帖子(优先分层缓存 top_ids → Redis ZSET未就绪时回源 DB 按最新排序
func (s *postServiceImpl) GetHotPosts(ctx context.Context, page, pageSize int) ([]*model.Post, int64, error) {
// 如果Gorse启用使用自定义的非个性化推荐器
if s.gorseClient.IsEnabled() {
offset := (page - 1) * pageSize
// 使用 most_liked_weekly 推荐器获取周热门
// 多取1条用于判断是否还有下一页
itemIDs, err := s.gorseClient.GetNonPersonalized(ctx, "most_liked_weekly", pageSize+1, offset, "")
if err != nil {
log.Printf("[WARN] Gorse GetNonPersonalized failed: %v, fallback to database", err)
return s.getHotPostsFromDB(ctx, page, pageSize)
}
if len(itemIDs) > 0 {
hasNext := len(itemIDs) > pageSize
if hasNext {
itemIDs = itemIDs[:pageSize]
}
posts, err := s.postRepo.GetByIDs(itemIDs)
if err != nil {
return nil, 0, err
}
// 近似 total当 hasNext 为 true 时,按分页窗口估算,避免因脏数据/缺失数据导致总页数被低估
estimatedTotal := int64(offset + len(posts))
if hasNext {
estimatedTotal = int64(offset + pageSize + 1)
}
return posts, estimatedTotal, nil
}
offset := (page - 1) * pageSize
if posts, total, ok, err := s.tryHotPostsFromTopIDsCache(offset, pageSize); ok {
return posts, total, err
} else if err != nil {
return nil, 0, err
}
if posts, total, ok, err := s.tryHotPostsFromZSet(ctx, offset, pageSize); ok {
return posts, total, err
} else if err != nil {
return nil, 0, err
}
// 降级:从数据库获取
return s.getHotPostsFromDB(ctx, page, pageSize)
}
// getHotPostsFromDB 从数据库获取热门帖子(降级路径
// tryHotPostsFromTopIDsCache 从热门榜有序 ID 列表取帖(命中 LayeredCache 本地层时可不访问 Redis
func (s *postServiceImpl) tryHotPostsFromTopIDsCache(offset, pageSize int) (posts []*model.Post, total int64, ok bool, err error) {
if s.cache == nil || pageSize <= 0 {
return nil, 0, false, nil
}
ids, hit := cache.GetTyped[[]string](s.cache, cache.HotRankTopIDsKey())
if !hit || len(ids) == 0 {
return nil, 0, false, nil
}
if offset >= len(ids) {
return []*model.Post{}, int64(len(ids)), true, nil
}
end := offset + pageSize
if end > len(ids) {
end = len(ids)
}
pageIDs := ids[offset:end]
posts, err = s.postRepo.GetByIDs(pageIDs)
if err != nil {
return nil, 0, false, err
}
return posts, int64(len(ids)), true, nil
}
// tryHotPostsFromZSet 从定时任务写入的 ZSET 按排名取帖ok=false 表示应回源
func (s *postServiceImpl) tryHotPostsFromZSet(ctx context.Context, offset, pageSize int) (posts []*model.Post, total int64, ok bool, err error) {
if s.cache == nil || pageSize <= 0 {
return nil, 0, false, nil
}
key := cache.HotRankZSetKey()
card, zerr := s.cache.ZCard(ctx, key)
if zerr != nil || card == 0 {
return nil, 0, false, nil
}
stop := int64(offset + pageSize - 1)
ids, zerr := s.cache.ZRevRange(ctx, key, int64(offset), stop)
if zerr != nil || len(ids) == 0 {
return nil, 0, false, nil
}
posts, err = s.postRepo.GetByIDs(ids)
if err != nil {
return nil, 0, false, err
}
return posts, card, true, nil
}
// getHotPostsFromDB 热门未就绪时按最新发布降级
func (s *postServiceImpl) getHotPostsFromDB(ctx context.Context, page, pageSize int) ([]*model.Post, int64, error) {
// 直接查询数据库不再使用本地缓存Gorse失败降级时使用
posts, total, err := s.postRepo.GetHotPosts(page, pageSize)
if err != nil {
return nil, 0, err
@@ -669,94 +615,6 @@ func (s *postServiceImpl) getHotPostsFromDB(ctx context.Context, page, pageSize
return posts, total, nil
}
// GetRecommendedPosts 获取推荐帖子
func (s *postServiceImpl) GetRecommendedPosts(ctx context.Context, userID string, page, pageSize int) ([]*model.Post, int64, error) {
// 如果Gorse未启用或用户未登录降级为热门帖子
if !s.gorseClient.IsEnabled() || userID == "" {
return s.GetHotPosts(ctx, page, pageSize)
}
// 计算偏移量:第一页使用随机偏移量,增加推荐多样性
var offset int
if page == 1 {
// 第一页随机偏移 0-50让每次刷新看到不同的推荐内容
offset = rand.Intn(51)
} else {
offset = (page - 1) * pageSize
}
// 从Gorse获取推荐列表
// 多取1条用于判断是否还有下一页
itemIDs, err := s.gorseClient.GetRecommend(ctx, userID, pageSize+1, offset)
if err != nil {
log.Printf("[WARN] Gorse recommendation failed: %v, fallback to hot posts", err)
return s.GetHotPosts(ctx, page, pageSize)
}
// 如果没有推荐结果,降级为热门帖子
if len(itemIDs) == 0 {
return s.GetHotPosts(ctx, page, pageSize)
}
hasNext := len(itemIDs) > pageSize
if hasNext {
itemIDs = itemIDs[:pageSize]
}
// 根据ID列表查询帖子详情
posts, err := s.postRepo.GetByIDs(itemIDs)
if err != nil {
return nil, 0, err
}
// 近似 total当 hasNext 为 true 时,按分页窗口估算,避免因脏数据/缺失数据导致总页数被低估
estimatedTotal := int64(offset + len(posts))
if hasNext {
estimatedTotal = int64(offset + pageSize + 1)
}
return posts, estimatedTotal, nil
}
// buildPostCategories 构建帖子的类别标签
func (s *postServiceImpl) buildPostCategories(post *model.Post) []string {
var categories []string
// 热度标签
if post.ViewsCount > 1000 {
categories = append(categories, "hot_high")
} else if post.ViewsCount > 100 {
categories = append(categories, "hot_medium")
}
// 点赞标签
if post.LikesCount > 100 {
categories = append(categories, "likes_100+")
} else if post.LikesCount > 50 {
categories = append(categories, "likes_50+")
} else if post.LikesCount > 10 {
categories = append(categories, "likes_10+")
}
// 评论标签
if post.CommentsCount > 50 {
categories = append(categories, "comments_50+")
} else if post.CommentsCount > 10 {
categories = append(categories, "comments_10+")
}
// 时间标签
age := time.Since(post.CreatedAt)
if age < 24*time.Hour {
categories = append(categories, "today")
} else if age < 7*24*time.Hour {
categories = append(categories, "this_week")
} else if age < 30*24*time.Hour {
categories = append(categories, "this_month")
}
return categories
}
// ========== 事务管理器示例方法 ==========
// DeletePostWithTransaction 使用事务管理器删除帖子(示例)
@@ -810,103 +668,92 @@ func (s *postServiceImpl) GetHotPostsByCursor(ctx context.Context, req *cursor.P
// 规范化请求参数
req.Normalize()
// 如果Gorse启用使用自定义的非个性化推荐器
if s.gorseClient.IsEnabled() {
// 计算偏移量
offset := 0
if req.Cursor != "" {
// 尝试解析游标作为偏移量
if parsed, err := strconv.Atoi(req.Cursor); err == nil {
offset = parsed
}
}
// 使用 most_liked_weekly 推荐器获取周热门
// 多取1条用于判断是否还有下一页
itemIDs, err := s.gorseClient.GetNonPersonalized(ctx, "most_liked_weekly", req.PageSize+1, offset, "")
if err != nil {
log.Printf("[WARN] Gorse GetNonPersonalized failed: %v, fallback to database", err)
return s.getHotPostsByCursorFromDB(ctx, req)
}
if len(itemIDs) > 0 {
hasMore := len(itemIDs) > req.PageSize
if hasMore {
itemIDs = itemIDs[:req.PageSize]
}
posts, err := s.postRepo.GetByIDs(itemIDs)
if err != nil {
return nil, err
}
// 下一页的游标是当前偏移量 + 已获取数量
nextCursor := ""
if hasMore {
nextCursor = strconv.Itoa(offset + req.PageSize)
}
return &cursor.CursorPageResult[*model.Post]{
Items: posts,
NextCursor: nextCursor,
PrevCursor: "",
HasMore: hasMore,
}, nil
}
}
// 降级:从数据库获取
return s.getHotPostsByCursorFromDB(ctx, req)
}
// getHotPostsByCursorFromDB 从数据库游标分页获取热门帖子(降级路径)
// getHotPostsByCursorFromDB 热门游标:优先 ZSET一次多取 1 条判断 hasMore否则回源 DB
func (s *postServiceImpl) getHotPostsByCursorFromDB(ctx context.Context, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Post], error) {
return s.postRepo.GetHotPostsByCursor(ctx, req)
}
// GetRecommendedPostsByCursor 游标分页获取推荐帖子
func (s *postServiceImpl) GetRecommendedPostsByCursor(ctx context.Context, userID string, req *cursor.PageRequest) (*cursor.CursorPageResult[*model.Post], error) {
// 规范化请求参数
req.Normalize()
// 如果Gorse未启用或用户未登录降级为热门帖子
if !s.gorseClient.IsEnabled() || userID == "" {
return s.GetHotPostsByCursor(ctx, req)
}
// 计算偏移量:第一页使用随机偏移量,增加推荐多样性
var offset int
if req.Cursor == "" {
// 第一页随机偏移 0-50让每次刷新看到不同的推荐内容
offset = rand.Intn(51)
} else {
// 尝试解析游标作为偏移量
if parsed, err := strconv.Atoi(req.Cursor); err == nil {
offset := 0
if req.Cursor != "" {
if parsed, err := strconv.Atoi(req.Cursor); err == nil && parsed >= 0 {
offset = parsed
}
}
// 从Gorse获取推荐列表
// 多取1条用于判断是否还有下一页
itemIDs, err := s.gorseClient.GetRecommend(ctx, userID, req.PageSize+1, offset)
if err != nil {
log.Printf("[WARN] Gorse recommendation failed: %v, fallback to hot posts", err)
return s.GetHotPostsByCursor(ctx, req)
if s.cache != nil {
if ids, hit := cache.GetTyped[[]string](s.cache, cache.HotRankTopIDsKey()); hit && len(ids) > 0 {
if offset < len(ids) {
stop := offset + req.PageSize + 1
if stop > len(ids) {
stop = len(ids)
}
slice := ids[offset:stop]
hasMore := len(slice) > req.PageSize
if hasMore {
slice = slice[:req.PageSize]
}
posts, gerr := s.postRepo.GetByIDs(slice)
if gerr != nil {
return nil, gerr
}
nextCursor := ""
if hasMore {
nextCursor = strconv.Itoa(offset + req.PageSize)
}
return &cursor.CursorPageResult[*model.Post]{
Items: posts,
NextCursor: nextCursor,
PrevCursor: "",
HasMore: hasMore,
}, nil
}
return &cursor.CursorPageResult[*model.Post]{
Items: []*model.Post{},
NextCursor: "",
PrevCursor: "",
HasMore: false,
}, nil
}
key := cache.HotRankZSetKey()
card, zerr := s.cache.ZCard(ctx, key)
if zerr == nil && card > 0 {
stop := int64(offset + req.PageSize)
ids, rerr := s.cache.ZRevRange(ctx, key, int64(offset), stop)
if rerr == nil && len(ids) > 0 {
hasMore := len(ids) > req.PageSize
if hasMore {
ids = ids[:req.PageSize]
}
posts, gerr := s.postRepo.GetByIDs(ids)
if gerr != nil {
return nil, gerr
}
nextCursor := ""
if hasMore {
nextCursor = strconv.Itoa(offset + req.PageSize)
}
return &cursor.CursorPageResult[*model.Post]{
Items: posts,
NextCursor: nextCursor,
PrevCursor: "",
HasMore: hasMore,
}, nil
}
}
}
// 如果没有推荐结果,降级为热门帖子
if len(itemIDs) == 0 {
return s.GetHotPostsByCursor(ctx, req)
}
hasMore := len(itemIDs) > req.PageSize
if hasMore {
itemIDs = itemIDs[:req.PageSize]
}
// 根据ID列表查询帖子详情
posts, err := s.postRepo.GetByIDs(itemIDs)
page := offset/req.PageSize + 1
posts, _, err := s.postRepo.GetHotPosts(page, req.PageSize+1)
if err != nil {
return nil, err
}
// 下一页的游标是当前偏移量 + 已获取数量
hasMore := len(posts) > req.PageSize
if hasMore {
posts = posts[:req.PageSize]
}
nextCursor := ""
if hasMore {
nextCursor = strconv.Itoa(offset + req.PageSize)