feat(moderation): add user profile content moderation for avatars, covers, and bios
All checks were successful
Build Backend / build (push) Successful in 4m24s
Build Backend / build-docker (push) Successful in 1m14s

This commit is contained in:
lafay
2026-04-15 11:50:47 +08:00
parent b6f2df87c4
commit 90495385cd
23 changed files with 1561 additions and 55 deletions

View File

@@ -94,7 +94,7 @@ jobs:
-e APP_OPENAI_ENABLED=true \ -e APP_OPENAI_ENABLED=true \
-e APP_OPENAI_BASE_URL=https://api.littlelan.cn/ \ -e APP_OPENAI_BASE_URL=https://api.littlelan.cn/ \
-e "APP_OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" \ -e "APP_OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" \
-e APP_OPENAI_MODERATION_MODEL=qwen3.5-plus \ -e APP_OPENAI_MODERATION_MODEL=qwen3.6-plus \
-e APP_OPENAI_MODERATION_MAX_IMAGES_PER_REQUEST=3 \ -e APP_OPENAI_MODERATION_MAX_IMAGES_PER_REQUEST=3 \
-e APP_OPENAI_REQUEST_TIMEOUT=30 \ -e APP_OPENAI_REQUEST_TIMEOUT=30 \
-e APP_OPENAI_STRICT_MODERATION=false \ -e APP_OPENAI_STRICT_MODERATION=false \

View File

@@ -54,6 +54,7 @@ func ProvideRouter(
adminReportHandler *handler.AdminReportHandler, adminReportHandler *handler.AdminReportHandler,
verificationHandler *handler.VerificationHandler, verificationHandler *handler.VerificationHandler,
adminVerificationHandler *handler.AdminVerificationHandler, adminVerificationHandler *handler.AdminVerificationHandler,
adminProfileAuditHandler *handler.AdminProfileAuditHandler,
logService *service.LogService, logService *service.LogService,
activityService service.UserActivityService, activityService service.UserActivityService,
casbinService service.CasbinService, casbinService service.CasbinService,
@@ -89,6 +90,7 @@ func ProvideRouter(
adminReportHandler, adminReportHandler,
verificationHandler, verificationHandler,
adminVerificationHandler, adminVerificationHandler,
adminProfileAuditHandler,
logService, logService,
activityService, activityService,
casbinService, casbinService,

View File

@@ -51,13 +51,17 @@ func InitializeApp() (*App, error) {
userService := wire.ProvideUserService(userRepository, systemMessageService, emailService, cache, logService) userService := wire.ProvideUserService(userRepository, systemMessageService, emailService, cache, logService)
userActivityRepository := wire.ProvideUserActivityRepository(db, cache) userActivityRepository := wire.ProvideUserActivityRepository(db, cache)
userActivityService := wire.ProvideUserActivityService(userActivityRepository) userActivityService := wire.ProvideUserActivityService(userActivityRepository)
userHandler := handler.NewUserHandler(userService, userActivityService, logService) userProfileAuditRepository := repository.NewUserProfileAuditRepository(db)
manager := wire.ProvideHookManager()
openaiClient := wire.ProvideOpenAIClient(config) openaiClient := wire.ProvideOpenAIClient(config)
postAIService := wire.ProvidePostAIService(openaiClient) postAIService := wire.ProvidePostAIService(openaiClient)
transactionManager := wire.ProvideTransactionManager(db)
manager := wire.ProvideHookManager()
commentRepository := repository.NewCommentRepository(db) commentRepository := repository.NewCommentRepository(db)
moderationHooks := wire.ProvideModerationHooks(manager, postAIService, postRepository, commentRepository, config) sensitiveWordRepository := repository.NewSensitiveWordRepository(db)
sensitiveService := wire.ProvideSensitiveService(sensitiveWordRepository, client, config)
moderationHooks := wire.ProvideModerationHooks(manager, postAIService, postRepository, commentRepository, userRepository, sensitiveService, config)
userProfileAuditService := wire.ProvideUserProfileAuditService(userProfileAuditRepository, userRepository, manager, moderationHooks)
userHandler := wire.ProvideUserHandler(userService, userActivityService, logService, userProfileAuditService)
transactionManager := wire.ProvideTransactionManager(db)
builtinHooks := wire.ProvideBuiltinHooks(manager) builtinHooks := wire.ProvideBuiltinHooks(manager)
postService := wire.ProvidePostService(postRepository, systemMessageService, postAIService, cache, transactionManager, manager, moderationHooks, builtinHooks, logService) postService := wire.ProvidePostService(postRepository, systemMessageService, postAIService, cache, transactionManager, manager, moderationHooks, builtinHooks, logService)
channelRepository := repository.NewChannelRepository(db) channelRepository := repository.NewChannelRepository(db)
@@ -132,8 +136,9 @@ func InitializeApp() (*App, error) {
verificationHandler := handler.NewVerificationHandler(verificationService) verificationHandler := handler.NewVerificationHandler(verificationService)
adminVerificationService := wire.ProvideAdminVerificationService(verificationRepository, userRepository) adminVerificationService := wire.ProvideAdminVerificationService(verificationRepository, userRepository)
adminVerificationHandler := wire.ProvideAdminVerificationHandler(adminVerificationService, userRepository) adminVerificationHandler := wire.ProvideAdminVerificationHandler(adminVerificationService, userRepository)
adminProfileAuditHandler := handler.NewAdminProfileAuditHandler(userProfileAuditService)
wsHandler := wire.ProvideWSHandler(hub, chatService, groupService, jwtService, callService, userRepository) wsHandler := wire.ProvideWSHandler(hub, chatService, groupService, jwtService, callService, userRepository)
router := ProvideRouter(userRepository, userHandler, postHandler, commentHandler, messageHandler, notificationHandler, uploadHandler, jwtService, pushHandler, systemMessageHandler, groupHandler, stickerHandler, voteHandler, channelHandler, scheduleHandler, roleHandler, adminUserHandler, adminPostHandler, adminCommentHandler, adminGroupHandler, adminDashboardHandler, adminLogHandler, qrCodeHandler, materialHandler, callHandler, reportHandler, adminReportHandler, verificationHandler, adminVerificationHandler, logService, userActivityService, casbinService, wsHandler) router := ProvideRouter(userRepository, userHandler, postHandler, commentHandler, messageHandler, notificationHandler, uploadHandler, jwtService, pushHandler, systemMessageHandler, groupHandler, stickerHandler, voteHandler, channelHandler, scheduleHandler, roleHandler, adminUserHandler, adminPostHandler, adminCommentHandler, adminGroupHandler, adminDashboardHandler, adminLogHandler, qrCodeHandler, materialHandler, callHandler, reportHandler, adminReportHandler, verificationHandler, adminVerificationHandler, adminProfileAuditHandler, logService, userActivityService, casbinService, wsHandler)
hotRankWorker := wire.ProvideHotRankWorker(config, postRepository, cache) hotRankWorker := wire.ProvideHotRankWorker(config, postRepository, cache)
app := NewApp(config, db, router, pushService, hotRankWorker, server, logger) app := NewApp(config, db, router, pushService, hotRankWorker, server, logger)
return app, nil return app, nil
@@ -172,6 +177,7 @@ func ProvideRouter(
adminReportHandler *handler.AdminReportHandler, adminReportHandler *handler.AdminReportHandler,
verificationHandler *handler.VerificationHandler, verificationHandler *handler.VerificationHandler,
adminVerificationHandler *handler.AdminVerificationHandler, adminVerificationHandler *handler.AdminVerificationHandler,
adminProfileAuditHandler *handler.AdminProfileAuditHandler,
logService *service.LogService, logService *service.LogService,
activityService service.UserActivityService, activityService service.UserActivityService,
casbinService service.CasbinService, casbinService service.CasbinService,
@@ -207,6 +213,7 @@ func ProvideRouter(
adminReportHandler, adminReportHandler,
verificationHandler, verificationHandler,
adminVerificationHandler, adminVerificationHandler,
adminProfileAuditHandler,
logService, logService,
activityService, activityService,
casbinService, casbinService,

View File

@@ -1163,6 +1163,9 @@ type DashboardStatsResponse struct {
PendingReview int64 `json:"pending_review"` // 待审核总数 PendingReview int64 `json:"pending_review"` // 待审核总数
PendingPostsCount int64 `json:"pending_posts_count"` // 待审核帖子数 PendingPostsCount int64 `json:"pending_posts_count"` // 待审核帖子数
PendingComments int64 `json:"pending_comments_count"` // 待审核评论数 PendingComments int64 `json:"pending_comments_count"` // 待审核评论数
PendingAvatars int64 `json:"pending_avatars"` // 待审核头像数
PendingCovers int64 `json:"pending_covers"` // 待审核背景图数
PendingBios int64 `json:"pending_bios"` // 待审核签名数
} }
// UserActivityTrendResponse 用户活跃度趋势响应 // UserActivityTrendResponse 用户活跃度趋势响应

View File

@@ -0,0 +1,129 @@
package handler
import (
"strconv"
"carrot_bbs/internal/model"
"carrot_bbs/internal/pkg/response"
"carrot_bbs/internal/service"
"github.com/gin-gonic/gin"
)
type AdminProfileAuditHandler struct {
profileAuditService service.UserProfileAuditService
}
func NewAdminProfileAuditHandler(profileAuditService service.UserProfileAuditService) *AdminProfileAuditHandler {
return &AdminProfileAuditHandler{
profileAuditService: profileAuditService,
}
}
func (h *AdminProfileAuditHandler) GetPendingList(c *gin.Context) {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20"))
if page <= 0 {
page = 1
}
if pageSize <= 0 || pageSize > 100 {
pageSize = 20
}
contentType := c.Query("content_type")
var ct model.UserProfileContentType
switch contentType {
case "avatar":
ct = model.UserProfileContentTypeAvatar
case "cover":
ct = model.UserProfileContentTypeCover
case "bio":
ct = model.UserProfileContentTypeBio
default:
ct = ""
}
audits, total, err := h.profileAuditService.GetPendingList(c.Request.Context(), ct, page, pageSize)
if err != nil {
response.InternalServerError(c, "failed to get pending profile audits")
return
}
items := make([]map[string]any, 0, len(audits))
for _, audit := range audits {
item := map[string]any{
"id": audit.ID,
"user_id": audit.UserID,
"content_type": audit.ContentType,
"content": audit.Content,
"status": audit.Status,
"created_at": audit.CreatedAt.Format("2006-01-02T15:04:05Z07:00"),
}
if audit.User != nil {
item["user"] = map[string]any{
"id": audit.User.ID,
"username": audit.User.Username,
"nickname": audit.User.Nickname,
"avatar": audit.User.Avatar,
}
}
items = append(items, item)
}
response.Paginated(c, items, total, page, pageSize)
}
func (h *AdminProfileAuditHandler) ModerateAudit(c *gin.Context) {
auditID := c.Param("id")
if auditID == "" {
response.BadRequest(c, "audit id is required")
return
}
var req struct {
Status string `json:"status" binding:"required"`
Reason string `json:"reason"`
}
if err := c.ShouldBindJSON(&req); err != nil {
response.BadRequest(c, err.Error())
return
}
if req.Status != string(model.UserProfileAuditStatusApproved) && req.Status != string(model.UserProfileAuditStatusRejected) {
response.BadRequest(c, "invalid status, must be 'approved' or 'rejected'")
return
}
reviewedBy := c.GetString("user_id")
if err := h.profileAuditService.ModerateAudit(c.Request.Context(), auditID, model.UserProfileAuditStatus(req.Status), req.Reason, reviewedBy); err != nil {
response.HandleError(c, err, "failed to moderate profile audit")
return
}
response.Success(c, map[string]any{"success": true})
}
func (h *AdminProfileAuditHandler) BatchModerate(c *gin.Context) {
var req struct {
IDs []string `json:"ids" binding:"required"`
Status string `json:"status" binding:"required"`
Reason string `json:"reason"`
}
if err := c.ShouldBindJSON(&req); err != nil {
response.BadRequest(c, err.Error())
return
}
if req.Status != string(model.UserProfileAuditStatusApproved) && req.Status != string(model.UserProfileAuditStatusRejected) {
response.BadRequest(c, "invalid status, must be 'approved' or 'rejected'")
return
}
reviewedBy := c.GetString("user_id")
if err := h.profileAuditService.BatchModerateAudits(c.Request.Context(), req.IDs, model.UserProfileAuditStatus(req.Status), req.Reason, reviewedBy); err != nil {
response.InternalServerError(c, "failed to batch moderate profile audits")
return
}
response.Success(c, map[string]any{"success": true})
}

View File

@@ -32,13 +32,13 @@ func normalizePagination(page, pageSize int) (int, int) {
// UserHandler 用户处理器 // UserHandler 用户处理器
type UserHandler struct { type UserHandler struct {
userService service.UserService userService service.UserService
activityService service.UserActivityService // 用户活跃统计服务 activityService service.UserActivityService
jwtService *service.JWTService profileAuditService service.UserProfileAuditService
logService *service.LogService jwtService *service.JWTService
logService *service.LogService
} }
// NewUserHandler 创建用户处理器
func NewUserHandler(userService service.UserService, activityService service.UserActivityService, logService *service.LogService) *UserHandler { func NewUserHandler(userService service.UserService, activityService service.UserActivityService, logService *service.LogService) *UserHandler {
return &UserHandler{ return &UserHandler{
userService: userService, userService: userService,
@@ -47,6 +47,10 @@ func NewUserHandler(userService service.UserService, activityService service.Use
} }
} }
func (h *UserHandler) SetProfileAuditService(profileAuditService service.UserProfileAuditService) {
h.profileAuditService = profileAuditService
}
// SetJWTService 设置JWT服务 // SetJWTService 设置JWT服务
func (h *UserHandler) SetJWTService(jwtService *service.JWTService) { func (h *UserHandler) SetJWTService(jwtService *service.JWTService) {
h.jwtService = jwtService h.jwtService = jwtService
@@ -318,6 +322,7 @@ func (h *UserHandler) UpdateUser(c *gin.Context) {
Website string `json:"website"` Website string `json:"website"`
Location string `json:"location"` Location string `json:"location"`
Avatar string `json:"avatar"` Avatar string `json:"avatar"`
CoverURL string `json:"cover_url"`
Phone *string `json:"phone"` Phone *string `json:"phone"`
Email *string `json:"email"` Email *string `json:"email"`
} }
@@ -337,18 +342,12 @@ func (h *UserHandler) UpdateUser(c *gin.Context) {
if req.Nickname != "" { if req.Nickname != "" {
user.Nickname = req.Nickname user.Nickname = req.Nickname
} }
if req.Bio != "" {
user.Bio = req.Bio
}
if req.Website != "" { if req.Website != "" {
user.Website = req.Website user.Website = req.Website
} }
if req.Location != "" { if req.Location != "" {
user.Location = req.Location user.Location = req.Location
} }
if req.Avatar != "" {
user.Avatar = req.Avatar
}
if req.Phone != nil { if req.Phone != nil {
user.Phone = req.Phone user.Phone = req.Phone
} }
@@ -359,20 +358,53 @@ func (h *UserHandler) UpdateUser(c *gin.Context) {
user.Email = req.Email user.Email = req.Email
} }
avatarNeedsAudit := req.Avatar != "" && req.Avatar != user.Avatar && h.profileAuditService != nil
coverNeedsAudit := req.CoverURL != "" && req.CoverURL != user.CoverURL && h.profileAuditService != nil
bioNeedsAudit := req.Bio != "" && req.Bio != user.Bio && h.profileAuditService != nil
if avatarNeedsAudit {
if _, err := h.profileAuditService.SubmitAvatarAudit(c.Request.Context(), userID, req.Avatar); err != nil {
avatarNeedsAudit = false
user.Avatar = req.Avatar
}
} else if req.Avatar != "" {
user.Avatar = req.Avatar
}
if coverNeedsAudit {
if _, err := h.profileAuditService.SubmitCoverAudit(c.Request.Context(), userID, req.CoverURL); err != nil {
coverNeedsAudit = false
user.CoverURL = req.CoverURL
}
} else if req.CoverURL != "" {
user.CoverURL = req.CoverURL
}
if bioNeedsAudit {
if _, err := h.profileAuditService.SubmitBioAudit(c.Request.Context(), userID, req.Bio); err != nil {
bioNeedsAudit = false
user.Bio = req.Bio
}
} else if req.Bio != "" {
user.Bio = req.Bio
}
err = h.userService.UpdateUser(c.Request.Context(), user) err = h.userService.UpdateUser(c.Request.Context(), user)
if err != nil { if err != nil {
response.InternalServerError(c, "failed to update user") response.InternalServerError(c, "failed to update user")
return return
} }
// 实时计算帖子数量
postsCount, err := h.userService.GetUserPostCount(c.Request.Context(), userID) postsCount, err := h.userService.GetUserPostCount(c.Request.Context(), userID)
if err != nil { if err != nil {
// 如果获取失败,使用数据库中的值
postsCount = int64(user.PostsCount) postsCount = int64(user.PostsCount)
} }
response.Success(c, dto.ConvertUserToDetailResponseWithPostsCount(user, int(postsCount))) resp := gin.H{"user": dto.ConvertUserToDetailResponseWithPostsCount(user, int(postsCount))}
if avatarNeedsAudit || coverNeedsAudit || bioNeedsAudit {
resp["audit_pending"] = true
}
response.Success(c, resp)
} }
// SendEmailVerifyCode 发送当前用户邮箱验证码 // SendEmailVerifyCode 发送当前用户邮箱验证码

View File

@@ -11,11 +11,15 @@ import (
type AuditTargetType string type AuditTargetType string
const ( const (
AuditTargetTypePost AuditTargetType = "post" // 帖子 AuditTargetTypePost AuditTargetType = "post" // 帖子
AuditTargetTypeComment AuditTargetType = "comment" // 评论 AuditTargetTypeComment AuditTargetType = "comment" // 评论
AuditTargetTypeMessage AuditTargetType = "message" // 私信 AuditTargetTypeMessage AuditTargetType = "message" // 私信
AuditTargetTypeUser AuditTargetType = "user" // 用户资料 AuditTargetTypeUser AuditTargetType = "user" // 用户资料
AuditTargetTypeImage AuditTargetType = "image" // 图片 AuditTargetTypeImage AuditTargetType = "image" // 图片
AuditTargetTypeAvatar AuditTargetType = "avatar" // 头像
AuditTargetTypeCover AuditTargetType = "cover" // 背景图
AuditTargetTypeBio AuditTargetType = "bio" // 个性签名
AuditTargetTypeUserProfile AuditTargetType = "user_profile" // 用户资料审核任务
) )
// AuditResult 审核结果 // AuditResult 审核结果

View File

@@ -173,6 +173,9 @@ func autoMigrate(db *gorm.DB) error {
// 身份认证相关 // 身份认证相关
&VerificationRecord{}, &VerificationRecord{},
// 用户资料审核相关
&UserProfileAudit{},
) )
if err != nil { if err != nil {
return err return err

View File

@@ -0,0 +1,51 @@
package model
import (
"time"
"github.com/google/uuid"
"gorm.io/gorm"
)
type UserProfileContentType string
const (
UserProfileContentTypeAvatar UserProfileContentType = "avatar"
UserProfileContentTypeCover UserProfileContentType = "cover"
UserProfileContentTypeBio UserProfileContentType = "bio"
)
type UserProfileAuditStatus string
const (
UserProfileAuditStatusPending UserProfileAuditStatus = "pending"
UserProfileAuditStatusApproved UserProfileAuditStatus = "approved"
UserProfileAuditStatusRejected UserProfileAuditStatus = "rejected"
)
type UserProfileAudit struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
UserID string `json:"user_id" gorm:"type:varchar(36);index;not null"`
ContentType UserProfileContentType `json:"content_type" gorm:"type:varchar(20);index;not null"`
Content string `json:"content" gorm:"type:text;not null"`
Status UserProfileAuditStatus `json:"status" gorm:"type:varchar(20);default:pending;index"`
ReviewedAt *time.Time `json:"reviewed_at" gorm:"type:timestamp"`
ReviewedBy string `json:"reviewed_by" gorm:"type:varchar(50)"`
RejectReason string `json:"reject_reason" gorm:"type:text"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
User *User `json:"user" gorm:"foreignKey:UserID"`
}
func (a *UserProfileAudit) BeforeCreate(tx *gorm.DB) error {
if a.ID == "" {
a.ID = uuid.New().String()
}
return nil
}
func (UserProfileAudit) TableName() string {
return "user_profile_audits"
}

View File

@@ -11,31 +11,37 @@ import (
type HookType string type HookType string
const ( const (
HookPostCreated HookType = "post.created" HookPostCreated HookType = "post.created"
HookPostUpdated HookType = "post.updated" HookPostUpdated HookType = "post.updated"
HookPostDeleted HookType = "post.deleted" HookPostDeleted HookType = "post.deleted"
HookPostPreModerate HookType = "post.pre_moderate" HookPostPreModerate HookType = "post.pre_moderate"
HookPostModerated HookType = "post.moderated" HookPostModerated HookType = "post.moderated"
HookCommentCreated HookType = "comment.created" HookCommentCreated HookType = "comment.created"
HookCommentDeleted HookType = "comment.deleted" HookCommentDeleted HookType = "comment.deleted"
HookCommentPreModerate HookType = "comment.pre_moderate" HookCommentPreModerate HookType = "comment.pre_moderate"
HookCommentModerated HookType = "comment.moderated" HookCommentModerated HookType = "comment.moderated"
HookUserCreated HookType = "user.created" HookUserCreated HookType = "user.created"
HookUserUpdated HookType = "user.updated" HookUserUpdated HookType = "user.updated"
HookUserDeleted HookType = "user.deleted" HookUserDeleted HookType = "user.deleted"
HookUserLogin HookType = "user.login" HookUserLogin HookType = "user.login"
HookUserLogout HookType = "user.logout" HookUserLogout HookType = "user.logout"
HookGroupCreated HookType = "group.created" HookGroupCreated HookType = "group.created"
HookGroupUpdated HookType = "group.updated" HookGroupUpdated HookType = "group.updated"
HookGroupDeleted HookType = "group.deleted" HookGroupDeleted HookType = "group.deleted"
HookGroupJoined HookType = "group.joined" HookGroupJoined HookType = "group.joined"
HookGroupLeft HookType = "group.left" HookGroupLeft HookType = "group.left"
HookMessageSent HookType = "message.sent" HookMessageSent HookType = "message.sent"
HookMessageRead HookType = "message.read" HookMessageRead HookType = "message.read"
HookNotificationSent HookType = "notification.sent" HookNotificationSent HookType = "notification.sent"
HookVoteCreated HookType = "vote.created" HookVoteCreated HookType = "vote.created"
HookVoteUpdated HookType = "vote.updated" HookVoteUpdated HookType = "vote.updated"
HookFileUploaded HookType = "file.uploaded" HookFileUploaded HookType = "file.uploaded"
HookUserAvatarPreModerate HookType = "user.avatar.pre_moderate"
HookUserAvatarModerated HookType = "user.avatar.moderated"
HookUserCoverPreModerate HookType = "user.cover.pre_moderate"
HookUserCoverModerated HookType = "user.cover.moderated"
HookUserBioPreModerate HookType = "user.bio.pre_moderate"
HookUserBioModerated HookType = "user.bio.moderated"
) )
type Priority int type Priority int

View File

@@ -18,6 +18,8 @@ type PostAIService interface {
IsEnabled() bool IsEnabled() bool
ModeratePost(ctx context.Context, title, content string, images []string) error ModeratePost(ctx context.Context, title, content string, images []string) error
ModerateComment(ctx context.Context, content string, images []string) error ModerateComment(ctx context.Context, content string, images []string) error
ModerateImage(ctx context.Context, imageURL string) error
ModerateBio(ctx context.Context, bio string) error
} }
type PostModerationHook struct { type PostModerationHook struct {
@@ -324,6 +326,7 @@ type ModerationHooks struct {
sensitiveService SensitiveService sensitiveService SensitiveService
postRepo repository.PostRepository postRepo repository.PostRepository
commentRepo repository.CommentRepository commentRepo repository.CommentRepository
userRepo repository.UserRepository
strictMode bool strictMode bool
replaceStr string replaceStr string
} }
@@ -333,6 +336,7 @@ func NewModerationHooks(
sensitiveService SensitiveService, sensitiveService SensitiveService,
postRepo repository.PostRepository, postRepo repository.PostRepository,
commentRepo repository.CommentRepository, commentRepo repository.CommentRepository,
userRepo repository.UserRepository,
strictMode bool, strictMode bool,
replaceStr string, replaceStr string,
) *ModerationHooks { ) *ModerationHooks {
@@ -341,6 +345,7 @@ func NewModerationHooks(
sensitiveService: sensitiveService, sensitiveService: sensitiveService,
postRepo: postRepo, postRepo: postRepo,
commentRepo: commentRepo, commentRepo: commentRepo,
userRepo: userRepo,
strictMode: strictMode, strictMode: strictMode,
replaceStr: replaceStr, replaceStr: replaceStr,
} }
@@ -359,6 +364,15 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode) commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager) commentHook.Register(manager)
avatarHook := NewUserAvatarModerationHook(m.postAIService, m.userRepo, m.strictMode)
avatarHook.Register(manager)
coverHook := NewUserCoverModerationHook(m.postAIService, m.userRepo, m.strictMode)
coverHook.Register(manager)
bioHook := NewUserBioModerationHook(m.postAIService, m.sensitiveService, m.userRepo, m.strictMode)
bioHook.Register(manager)
zap.L().Info("Registered moderation hooks", zap.L().Info("Registered moderation hooks",
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()), zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
zap.Bool("sensitive_enabled", m.sensitiveService != nil), zap.Bool("sensitive_enabled", m.sensitiveService != nil),
@@ -381,3 +395,27 @@ func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager,
}) })
return result return result
} }
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}

View File

@@ -101,6 +101,42 @@ type CommentModeratedHookData struct {
ReviewedBy string ReviewedBy string
} }
type UserAvatarModerateHookData struct {
UserID string
AvatarURL string
}
type UserCoverModerateHookData struct {
UserID string
CoverURL string
}
type UserBioModerateHookData struct {
UserID string
Bio string
}
type UserAvatarModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type UserCoverModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type UserBioModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type ModerationResult struct { type ModerationResult struct {
Approved bool Approved bool
NeedsReview bool NeedsReview bool
@@ -271,3 +307,33 @@ func (h *HookTriggerHelper) TriggerCommentPreModerate(ctx context.Context, userI
func (h *HookTriggerHelper) TriggerCommentModerated(ctx context.Context, userID uint, data *CommentModeratedHookData) { func (h *HookTriggerHelper) TriggerCommentModerated(ctx context.Context, userID uint, data *CommentModeratedHookData) {
h.manager.Trigger(ctx, HookCommentModerated, userID, data) h.manager.Trigger(ctx, HookCommentModerated, userID, data)
} }
func (h *HookTriggerHelper) TriggerUserAvatarPreModerate(ctx context.Context, userID uint, data *UserAvatarModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserAvatarModerated(ctx context.Context, userID uint, data *UserAvatarModeratedHookData) {
h.manager.Trigger(ctx, HookUserAvatarModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserCoverPreModerate(ctx context.Context, userID uint, data *UserCoverModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserCoverModerated(ctx context.Context, userID uint, data *UserCoverModeratedHookData) {
h.manager.Trigger(ctx, HookUserCoverModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserBioPreModerate(ctx context.Context, userID uint, data *UserBioModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserBioModerated(ctx context.Context, userID uint, data *UserBioModeratedHookData) {
h.manager.Trigger(ctx, HookUserBioModerated, userID, data)
}

View File

@@ -0,0 +1,317 @@
package hook
import (
"strings"
"time"
"carrot_bbs/internal/repository"
"go.uber.org/zap"
)
type UserAvatarModerationHook struct {
name string
postAIService PostAIService
userRepo repository.UserRepository
strictMode bool
}
func NewUserAvatarModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserAvatarModerationHook {
return &UserAvatarModerationHook{
name: "moderation.user.avatar.ai",
postAIService: postAIService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserAvatarModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserAvatarPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserAvatarModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserAvatarModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateImage(ctx.Ctx, data.AvatarURL)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User avatar flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User avatar rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI avatar moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve avatar",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}
type UserCoverModerationHook struct {
name string
postAIService PostAIService
userRepo repository.UserRepository
strictMode bool
}
func NewUserCoverModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserCoverModerationHook {
return &UserCoverModerationHook{
name: "moderation.user.cover.ai",
postAIService: postAIService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserCoverModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserCoverPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserCoverModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserCoverModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateImage(ctx.Ctx, data.CoverURL)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User cover flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User cover rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI cover moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve cover",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}
type UserBioModerationHook struct {
name string
postAIService PostAIService
sensitiveService SensitiveService
userRepo repository.UserRepository
strictMode bool
}
func NewUserBioModerationHook(postAIService PostAIService, sensitiveService SensitiveService, userRepo repository.UserRepository, strictMode bool) *UserBioModerationHook {
return &UserBioModerationHook{
name: "moderation.user.bio.ai",
postAIService: postAIService,
sensitiveService: sensitiveService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserBioModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserBioPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserBioModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserBioModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.sensitiveService != nil {
hasSensitive, words := h.sensitiveService.Check(ctx.Ctx, data.Bio)
if hasSensitive {
result.Approved = false
result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
result.ReviewedBy = "sensitive_word"
zap.L().Info("User bio rejected by sensitive word filter",
zap.String("user_id", data.UserID),
zap.Strings("words", words),
)
return nil
}
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateBio(ctx.Ctx, data.Bio)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User bio flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User bio rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI bio moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve bio",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}

View File

@@ -93,6 +93,8 @@ type Client interface {
Config() Config Config() Config
ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error) ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error)
ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error) ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error)
ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error)
ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error)
} }
type clientImpl struct { type clientImpl struct {
@@ -163,6 +165,157 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
return c.moderateContentInBatches(ctx, prompt.String(), images) return c.moderateContentInBatches(ctx, prompt.String(), images)
} }
const imageModerationSystemPrompt = `你是图片内容审核助手负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
硬性违规(命中任一项必须 result=block
- 色情低俗、裸露、性暗示内容
- 暴力血腥、恐怖画面
- 违法犯罪相关内容
- 明确的政治敏感宣传
- 恶意广告、诈骗信息
- 仇恨歧视符号或标志
可疑内容result=review
- 图片含义模糊,无法确定是否存在违规
- 轻微争议性内容但不完全违规
- AI无法准确判断的内容
放行规则result=pass
- 正常风景、人物、动物、动漫等图片
- 表情包、梗图
- 正常的艺术创作、摄影作品
输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}`
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
硬性违规(命中任一项必须 result=block
- 明确的违法犯罪、暴力威胁内容
- 色情低俗、性暗示描述
- 诈骗引流、恶意广告
- 明确煽动群体对立、仇恨
- 恶意公开他人隐私信息
可疑内容result=review
- 内容含义模糊,无法确定违规意图
- 涉及敏感话题但未明确违规
- AI无法准确判断的表达
放行规则result=pass
- 正常的自我介绍、心情表达
- 玩梗、幽默、吐槽
- 引用、歌词、诗句等
- 理性的观点表达
输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}`
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
if strings.TrimSpace(imageURL) == "" {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
userPrompt := "请审核以下用户图片"
cleanImages := normalizeImageURLs([]string{imageURL})
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
}
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
if strings.TrimSpace(bio) == "" {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
userPrompt := "个性签名内容:" + bio
return c.moderateWithCustomPrompt(ctx, bioModerationSystemPrompt, userPrompt)
}
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
}
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) { func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
cleanImages := normalizeImageURLs(images) cleanImages := normalizeImageURLs(images)
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages) optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)

View File

@@ -0,0 +1,123 @@
package repository
import (
"carrot_bbs/internal/model"
"gorm.io/gorm"
)
type UserProfileAuditRepository interface {
Create(audit *model.UserProfileAudit) error
GetByID(id string) (*model.UserProfileAudit, error)
GetByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
GetPendingByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
Update(audit *model.UserProfileAudit) error
Delete(id string) error
GetPendingList(contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error)
CountPending(contentType model.UserProfileContentType) (int64, error)
CountPendingAll() (int64, int64, int64, error)
}
type userProfileAuditRepository struct {
db *gorm.DB
}
func NewUserProfileAuditRepository(db *gorm.DB) UserProfileAuditRepository {
return &userProfileAuditRepository{db: db}
}
func (r *userProfileAuditRepository) Create(audit *model.UserProfileAudit) error {
return r.db.Create(audit).Error
}
func (r *userProfileAuditRepository) GetByID(id string) (*model.UserProfileAudit, error) {
var audit model.UserProfileAudit
err := r.db.Preload("User").First(&audit, "id = ?", id).Error
if err != nil {
return nil, err
}
return &audit, nil
}
func (r *userProfileAuditRepository) GetByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
var audit model.UserProfileAudit
err := r.db.Where("user_id = ? AND content_type = ?", userID, contentType).
Order("created_at DESC").
First(&audit).Error
if err != nil {
return nil, err
}
return &audit, nil
}
func (r *userProfileAuditRepository) GetPendingByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
var audit model.UserProfileAudit
err := r.db.Where("user_id = ? AND content_type = ? AND status = ?", userID, contentType, model.UserProfileAuditStatusPending).
Order("created_at DESC").
First(&audit).Error
if err != nil {
return nil, err
}
return &audit, nil
}
func (r *userProfileAuditRepository) Update(audit *model.UserProfileAudit) error {
return r.db.Save(audit).Error
}
func (r *userProfileAuditRepository) Delete(id string) error {
return r.db.Delete(&model.UserProfileAudit{}, "id = ?", id).Error
}
func (r *userProfileAuditRepository) GetPendingList(contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error) {
var audits []*model.UserProfileAudit
var total int64
query := r.db.Model(&model.UserProfileAudit{}).Where("status = ?", model.UserProfileAuditStatusPending)
if contentType != "" {
query = query.Where("content_type = ?", contentType)
}
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
offset := (page - 1) * pageSize
if err := query.Preload("User").Order("created_at DESC").Offset(offset).Limit(pageSize).Find(&audits).Error; err != nil {
return nil, 0, err
}
return audits, total, nil
}
func (r *userProfileAuditRepository) CountPending(contentType model.UserProfileContentType) (int64, error) {
var count int64
err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", contentType, model.UserProfileAuditStatusPending).
Count(&count).Error
return count, err
}
func (r *userProfileAuditRepository) CountPendingAll() (int64, int64, int64, error) {
var avatarCount, coverCount, bioCount int64
if err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", model.UserProfileContentTypeAvatar, model.UserProfileAuditStatusPending).
Count(&avatarCount).Error; err != nil {
return 0, 0, 0, err
}
if err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", model.UserProfileContentTypeCover, model.UserProfileAuditStatusPending).
Count(&coverCount).Error; err != nil {
return 0, 0, 0, err
}
if err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", model.UserProfileContentTypeBio, model.UserProfileAuditStatusPending).
Count(&bioCount).Error; err != nil {
return 0, 0, 0, err
}
return avatarCount, coverCount, bioCount, nil
}

View File

@@ -41,6 +41,7 @@ type Router struct {
adminReportHandler *handler.AdminReportHandler adminReportHandler *handler.AdminReportHandler
verificationHandler *handler.VerificationHandler verificationHandler *handler.VerificationHandler
adminVerificationHandler *handler.AdminVerificationHandler adminVerificationHandler *handler.AdminVerificationHandler
adminProfileAuditHandler *handler.AdminProfileAuditHandler
wsHandler *handler.WSHandler wsHandler *handler.WSHandler
logService *service.LogService logService *service.LogService
jwtService *service.JWTService jwtService *service.JWTService
@@ -78,6 +79,7 @@ func New(
adminReportHandler *handler.AdminReportHandler, adminReportHandler *handler.AdminReportHandler,
verificationHandler *handler.VerificationHandler, verificationHandler *handler.VerificationHandler,
adminVerificationHandler *handler.AdminVerificationHandler, adminVerificationHandler *handler.AdminVerificationHandler,
adminProfileAuditHandler *handler.AdminProfileAuditHandler,
logService *service.LogService, logService *service.LogService,
activityService service.UserActivityService, activityService service.UserActivityService,
casbinService service.CasbinService, casbinService service.CasbinService,
@@ -118,6 +120,7 @@ func New(
adminReportHandler: adminReportHandler, adminReportHandler: adminReportHandler,
verificationHandler: verificationHandler, verificationHandler: verificationHandler,
adminVerificationHandler: adminVerificationHandler, adminVerificationHandler: adminVerificationHandler,
adminProfileAuditHandler: adminProfileAuditHandler,
logService: logService, logService: logService,
jwtService: jwtService, jwtService: jwtService,
casbinService: casbinService, casbinService: casbinService,
@@ -619,6 +622,13 @@ func (r *Router) setupRoutes() {
admin.GET("/verifications/:id", r.adminVerificationHandler.GetVerificationDetail) admin.GET("/verifications/:id", r.adminVerificationHandler.GetVerificationDetail)
admin.PUT("/verifications/:id/review", r.adminVerificationHandler.ReviewVerification) admin.PUT("/verifications/:id/review", r.adminVerificationHandler.ReviewVerification)
} }
// 用户资料审核管理
if r.adminProfileAuditHandler != nil {
admin.GET("/profile-audits", r.adminProfileAuditHandler.GetPendingList)
admin.PUT("/profile-audits/:id/moderate", r.adminProfileAuditHandler.ModerateAudit)
admin.POST("/profile-audits/batch-moderate", r.adminProfileAuditHandler.BatchModerate)
}
} }
} }
} }

View File

@@ -164,8 +164,16 @@ func (s *adminDashboardServiceImpl) GetStats(ctx context.Context) (*dto.Dashboar
} }
stats.PendingComments = pendingComments stats.PendingComments = pendingComments
var pendingAvatars, pendingCovers, pendingBios int64
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeAvatar, model.UserProfileAuditStatusPending).Count(&pendingAvatars)
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeCover, model.UserProfileAuditStatusPending).Count(&pendingCovers)
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeBio, model.UserProfileAuditStatusPending).Count(&pendingBios)
stats.PendingAvatars = pendingAvatars
stats.PendingCovers = pendingCovers
stats.PendingBios = pendingBios
// 计算待审核总数 // 计算待审核总数
stats.PendingReview = stats.PendingPosts + stats.PendingComments stats.PendingReview = stats.PendingPosts + stats.PendingComments + stats.PendingAvatars + stats.PendingCovers + stats.PendingBios
stats.PendingPostsCount = stats.PendingPosts stats.PendingPostsCount = stats.PendingPosts
return &stats, nil return &stats, nil

View File

@@ -89,6 +89,86 @@ func (e *CommentModerationReviewError) IsReview() bool {
return true return true
} }
type ImageModerationRejectedError struct {
Reason string
}
func (e *ImageModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "image rejected by moderation"
}
return "image rejected by moderation: " + e.Reason
}
func (e *ImageModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "图片未通过审核,请更换后重试"
}
return strings.TrimSpace(e.Reason)
}
type ImageModerationReviewError struct {
Reason string
}
func (e *ImageModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "image needs manual review"
}
return "image needs manual review: " + e.Reason
}
func (e *ImageModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "图片需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *ImageModerationReviewError) IsReview() bool {
return true
}
type BioModerationRejectedError struct {
Reason string
}
func (e *BioModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "bio rejected by moderation"
}
return "bio rejected by moderation: " + e.Reason
}
func (e *BioModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "个性签名未通过审核,请修改后重试"
}
return strings.TrimSpace(e.Reason)
}
type BioModerationReviewError struct {
Reason string
}
func (e *BioModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "bio needs manual review"
}
return "bio needs manual review: " + e.Reason
}
func (e *BioModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "个性签名需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *BioModerationReviewError) IsReview() bool {
return true
}
type PostAIService struct { type PostAIService struct {
openAIClient openai.Client openAIClient openai.Client
} }
@@ -154,3 +234,55 @@ func (s *PostAIService) ModerateComment(ctx context.Context, content string, ima
return nil return nil
} }
} }
func (s *PostAIService) ModerateImage(ctx context.Context, imageURL string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateImage(ctx, imageURL)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI image moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &ImageModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &ImageModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}
func (s *PostAIService) ModerateBio(ctx context.Context, bio string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateBio(ctx, bio)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI bio moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &BioModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &BioModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}

View File

@@ -0,0 +1,395 @@
package service
import (
"context"
"log"
"time"
"carrot_bbs/internal/model"
"carrot_bbs/internal/pkg/hook"
"carrot_bbs/internal/repository"
"go.uber.org/zap"
)
type UserProfileAuditService interface {
SubmitAvatarAudit(ctx context.Context, userID, avatarURL string) (*model.UserProfileAudit, error)
SubmitCoverAudit(ctx context.Context, userID, coverURL string) (*model.UserProfileAudit, error)
SubmitBioAudit(ctx context.Context, userID, bio string) (*model.UserProfileAudit, error)
GetPendingAudit(ctx context.Context, userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
GetAuditStatus(ctx context.Context, userID string) (*ProfileAuditStatusResponse, error)
ModerateAudit(ctx context.Context, auditID string, status model.UserProfileAuditStatus, reason, reviewedBy string) error
BatchModerateAudits(ctx context.Context, ids []string, status model.UserProfileAuditStatus, reason, reviewedBy string) error
GetPendingList(ctx context.Context, contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error)
GetPendingCounts(ctx context.Context) (int64, int64, int64, error)
}
type ProfileAuditStatusResponse struct {
AvatarStatus string `json:"avatar_status"`
CoverStatus string `json:"cover_status"`
BioStatus string `json:"bio_status"`
}
type userProfileAuditServiceImpl struct {
auditRepo repository.UserProfileAuditRepository
userRepo repository.UserRepository
hookManager *hook.Manager
moderationHooks *hook.ModerationHooks
}
func NewUserProfileAuditService(
auditRepo repository.UserProfileAuditRepository,
userRepo repository.UserRepository,
hookManager *hook.Manager,
moderationHooks *hook.ModerationHooks,
) UserProfileAuditService {
return &userProfileAuditServiceImpl{
auditRepo: auditRepo,
userRepo: userRepo,
hookManager: hookManager,
moderationHooks: moderationHooks,
}
}
func (s *userProfileAuditServiceImpl) SubmitAvatarAudit(ctx context.Context, userID, avatarURL string) (*model.UserProfileAudit, error) {
audit := &model.UserProfileAudit{
UserID: userID,
ContentType: model.UserProfileContentTypeAvatar,
Content: avatarURL,
Status: model.UserProfileAuditStatusPending,
}
if err := s.auditRepo.Create(audit); err != nil {
return nil, err
}
go s.reviewAvatarAsync(audit.ID, userID, avatarURL)
return audit, nil
}
func (s *userProfileAuditServiceImpl) SubmitCoverAudit(ctx context.Context, userID, coverURL string) (*model.UserProfileAudit, error) {
audit := &model.UserProfileAudit{
UserID: userID,
ContentType: model.UserProfileContentTypeCover,
Content: coverURL,
Status: model.UserProfileAuditStatusPending,
}
if err := s.auditRepo.Create(audit); err != nil {
return nil, err
}
go s.reviewCoverAsync(audit.ID, userID, coverURL)
return audit, nil
}
func (s *userProfileAuditServiceImpl) SubmitBioAudit(ctx context.Context, userID, bio string) (*model.UserProfileAudit, error) {
audit := &model.UserProfileAudit{
UserID: userID,
ContentType: model.UserProfileContentTypeBio,
Content: bio,
Status: model.UserProfileAuditStatusPending,
}
if err := s.auditRepo.Create(audit); err != nil {
return nil, err
}
go s.reviewBioAsync(audit.ID, userID, bio)
return audit, nil
}
func (s *userProfileAuditServiceImpl) GetPendingAudit(ctx context.Context, userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
return s.auditRepo.GetPendingByUserID(userID, contentType)
}
func (s *userProfileAuditServiceImpl) GetAuditStatus(ctx context.Context, userID string) (*ProfileAuditStatusResponse, error) {
resp := &ProfileAuditStatusResponse{
AvatarStatus: string(model.UserProfileAuditStatusApproved),
CoverStatus: string(model.UserProfileAuditStatusApproved),
BioStatus: string(model.UserProfileAuditStatusApproved),
}
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeAvatar); err == nil {
resp.AvatarStatus = string(audit.Status)
}
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeCover); err == nil {
resp.CoverStatus = string(audit.Status)
}
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeBio); err == nil {
resp.BioStatus = string(audit.Status)
}
return resp, nil
}
func (s *userProfileAuditServiceImpl) ModerateAudit(ctx context.Context, auditID string, status model.UserProfileAuditStatus, reason, reviewedBy string) error {
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
return err
}
now := time.Now()
audit.Status = status
audit.ReviewedAt = &now
audit.ReviewedBy = reviewedBy
audit.RejectReason = reason
if err := s.auditRepo.Update(audit); err != nil {
return err
}
if status == model.UserProfileAuditStatusApproved {
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved audit content",
zap.String("audit_id", audit.ID),
zap.String("user_id", audit.UserID),
zap.Error(err),
)
}
}
return nil
}
func (s *userProfileAuditServiceImpl) BatchModerateAudits(ctx context.Context, ids []string, status model.UserProfileAuditStatus, reason string, reviewedBy string) error {
for _, id := range ids {
if err := s.ModerateAudit(ctx, id, status, reason, reviewedBy); err != nil {
zap.L().Warn("Failed to moderate audit in batch",
zap.String("audit_id", id),
zap.Error(err),
)
}
}
return nil
}
func (s *userProfileAuditServiceImpl) GetPendingList(ctx context.Context, contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error) {
return s.auditRepo.GetPendingList(contentType, page, pageSize)
}
func (s *userProfileAuditServiceImpl) GetPendingCounts(ctx context.Context) (int64, int64, int64, error) {
return s.auditRepo.CountPendingAll()
}
func (s *userProfileAuditServiceImpl) reviewAvatarAsync(auditID, userID, avatarURL string) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] Panic in avatar moderation async flow, audit=%s panic=%v", auditID, r)
}
}()
result := &hook.ModerationResult{}
if s.moderationHooks != nil && s.hookManager != nil {
result = s.moderationHooks.ModerateAvatar(context.Background(), s.hookManager, &hook.UserAvatarModerateHookData{
UserID: userID,
AvatarURL: avatarURL,
})
} else {
result.Approved = true
result.ReviewedBy = "system"
}
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
zap.L().Error("Failed to get audit for avatar moderation result",
zap.String("audit_id", auditID),
zap.Error(err),
)
return
}
now := time.Now()
if !result.Approved {
if result.NeedsReview {
audit.ReviewedBy = "ai"
zap.L().Info("User avatar flagged for manual review",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
)
} else {
audit.Status = model.UserProfileAuditStatusRejected
audit.RejectReason = result.RejectReason
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
zap.L().Info("User avatar rejected by moderation",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
zap.String("reason", result.RejectReason),
)
}
} else {
audit.Status = model.UserProfileAuditStatusApproved
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved avatar",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
if err := s.auditRepo.Update(audit); err != nil {
zap.L().Error("Failed to update avatar audit status",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
func (s *userProfileAuditServiceImpl) reviewCoverAsync(auditID, userID, coverURL string) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] Panic in cover moderation async flow, audit=%s panic=%v", auditID, r)
}
}()
result := &hook.ModerationResult{}
if s.moderationHooks != nil && s.hookManager != nil {
result = s.moderationHooks.ModerateCover(context.Background(), s.hookManager, &hook.UserCoverModerateHookData{
UserID: userID,
CoverURL: coverURL,
})
} else {
result.Approved = true
result.ReviewedBy = "system"
}
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
zap.L().Error("Failed to get audit for cover moderation result",
zap.String("audit_id", auditID),
zap.Error(err),
)
return
}
now := time.Now()
if !result.Approved {
if result.NeedsReview {
audit.ReviewedBy = "ai"
zap.L().Info("User cover flagged for manual review",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
)
} else {
audit.Status = model.UserProfileAuditStatusRejected
audit.RejectReason = result.RejectReason
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
zap.L().Info("User cover rejected by moderation",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
zap.String("reason", result.RejectReason),
)
}
} else {
audit.Status = model.UserProfileAuditStatusApproved
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved cover",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
if err := s.auditRepo.Update(audit); err != nil {
zap.L().Error("Failed to update cover audit status",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
func (s *userProfileAuditServiceImpl) reviewBioAsync(auditID, userID, bio string) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] Panic in bio moderation async flow, audit=%s panic=%v", auditID, r)
}
}()
result := &hook.ModerationResult{}
if s.moderationHooks != nil && s.hookManager != nil {
result = s.moderationHooks.ModerateBio(context.Background(), s.hookManager, &hook.UserBioModerateHookData{
UserID: userID,
Bio: bio,
})
} else {
result.Approved = true
result.ReviewedBy = "system"
}
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
zap.L().Error("Failed to get audit for bio moderation result",
zap.String("audit_id", auditID),
zap.Error(err),
)
return
}
now := time.Now()
if !result.Approved {
if result.NeedsReview {
audit.ReviewedBy = "ai"
zap.L().Info("User bio flagged for manual review",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
)
} else {
audit.Status = model.UserProfileAuditStatusRejected
audit.RejectReason = result.RejectReason
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
zap.L().Info("User bio rejected by moderation",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
zap.String("reason", result.RejectReason),
)
}
} else {
audit.Status = model.UserProfileAuditStatusApproved
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved bio",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
if err := s.auditRepo.Update(audit); err != nil {
zap.L().Error("Failed to update bio audit status",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
func (s *userProfileAuditServiceImpl) applyAuditContent(audit *model.UserProfileAudit) error {
user, err := s.userRepo.GetByID(audit.UserID)
if err != nil {
return err
}
switch audit.ContentType {
case model.UserProfileContentTypeAvatar:
user.Avatar = audit.Content
case model.UserProfileContentTypeCover:
user.CoverURL = audit.Content
case model.UserProfileContentTypeBio:
user.Bio = audit.Content
default:
return nil
}
return s.userRepo.Update(user)
}

View File

@@ -32,10 +32,11 @@ var HandlerSet = wire.NewSet(
handler.NewReportHandler, handler.NewReportHandler,
handler.NewAdminReportHandler, handler.NewAdminReportHandler,
handler.NewVerificationHandler, handler.NewVerificationHandler,
handler.NewAdminProfileAuditHandler,
handler.NewPostHandler,
// 需要特殊处理的 Handler // 需要特殊处理的 Handler
handler.NewUserHandler, ProvideUserHandler,
handler.NewPostHandler,
ProvideMessageHandler, ProvideMessageHandler,
ProvideSystemMessageHandler, ProvideSystemMessageHandler,
ProvideGroupHandler, ProvideGroupHandler,
@@ -44,7 +45,17 @@ var HandlerSet = wire.NewSet(
ProvideAdminVerificationHandler, ProvideAdminVerificationHandler,
) )
// ProvideMessageHandler 提供消息处理器 func ProvideUserHandler(
userService service.UserService,
activityService service.UserActivityService,
logService *service.LogService,
profileAuditService service.UserProfileAuditService,
) *handler.UserHandler {
h := handler.NewUserHandler(userService, activityService, logService)
h.SetProfileAuditService(profileAuditService)
return h
}
func ProvideMessageHandler( func ProvideMessageHandler(
chatService service.ChatService, chatService service.ChatService,
messageService *service.MessageService, messageService *service.MessageService,

View File

@@ -137,15 +137,18 @@ func ProvideModerationHooks(
postAIService *service.PostAIService, postAIService *service.PostAIService,
postRepo repository.PostRepository, postRepo repository.PostRepository,
commentRepo repository.CommentRepository, commentRepo repository.CommentRepository,
userRepo repository.UserRepository,
sensitiveService service.SensitiveService,
cfg *config.Config, cfg *config.Config,
) *hook.ModerationHooks { ) *hook.ModerationHooks {
strictMode := cfg.OpenAI.StrictModeration strictMode := cfg.OpenAI.StrictModeration
moderationHooks := hook.NewModerationHooks( moderationHooks := hook.NewModerationHooks(
postAIService, postAIService,
nil, sensitiveService,
postRepo, postRepo,
commentRepo, commentRepo,
userRepo,
strictMode, strictMode,
"***", "***",
) )

View File

@@ -40,6 +40,9 @@ var RepositorySet = wire.NewSet(
// 敏感词相关仓储 // 敏感词相关仓储
repository.NewSensitiveWordRepository, repository.NewSensitiveWordRepository,
// 用户资料审核相关仓储
repository.NewUserProfileAuditRepository,
) )
// ProvideUserActivityRepository 提供用户活跃数据仓储 // ProvideUserActivityRepository 提供用户活跃数据仓储

View File

@@ -61,6 +61,7 @@ var ServiceSet = wire.NewSet(
ProvideVerificationService, ProvideVerificationService,
ProvideAdminVerificationService, ProvideAdminVerificationService,
ProvideSensitiveService, ProvideSensitiveService,
ProvideUserProfileAuditService,
// 日志服务 // 日志服务
ProvideAsyncLogManager, ProvideAsyncLogManager,
@@ -465,3 +466,12 @@ func ProvideSensitiveService(
} }
return service.NewSensitiveService(sensitiveWordRepo, redisClient, sensitiveCfg) return service.NewSensitiveService(sensitiveWordRepo, redisClient, sensitiveCfg)
} }
func ProvideUserProfileAuditService(
auditRepo repository.UserProfileAuditRepository,
userRepo repository.UserRepository,
hookManager *hook.Manager,
moderationHooks *hook.ModerationHooks,
) service.UserProfileAuditService {
return service.NewUserProfileAuditService(auditRepo, userRepo, hookManager, moderationHooks)
}