feat(moderation): add user profile content moderation for avatars, covers, and bios
All checks were successful
Build Backend / build (push) Successful in 4m24s
Build Backend / build-docker (push) Successful in 1m14s

This commit is contained in:
lafay
2026-04-15 11:50:47 +08:00
parent b6f2df87c4
commit 90495385cd
23 changed files with 1561 additions and 55 deletions

View File

@@ -94,7 +94,7 @@ jobs:
-e APP_OPENAI_ENABLED=true \
-e APP_OPENAI_BASE_URL=https://api.littlelan.cn/ \
-e "APP_OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" \
-e APP_OPENAI_MODERATION_MODEL=qwen3.5-plus \
-e APP_OPENAI_MODERATION_MODEL=qwen3.6-plus \
-e APP_OPENAI_MODERATION_MAX_IMAGES_PER_REQUEST=3 \
-e APP_OPENAI_REQUEST_TIMEOUT=30 \
-e APP_OPENAI_STRICT_MODERATION=false \

View File

@@ -54,6 +54,7 @@ func ProvideRouter(
adminReportHandler *handler.AdminReportHandler,
verificationHandler *handler.VerificationHandler,
adminVerificationHandler *handler.AdminVerificationHandler,
adminProfileAuditHandler *handler.AdminProfileAuditHandler,
logService *service.LogService,
activityService service.UserActivityService,
casbinService service.CasbinService,
@@ -89,6 +90,7 @@ func ProvideRouter(
adminReportHandler,
verificationHandler,
adminVerificationHandler,
adminProfileAuditHandler,
logService,
activityService,
casbinService,

View File

@@ -51,13 +51,17 @@ func InitializeApp() (*App, error) {
userService := wire.ProvideUserService(userRepository, systemMessageService, emailService, cache, logService)
userActivityRepository := wire.ProvideUserActivityRepository(db, cache)
userActivityService := wire.ProvideUserActivityService(userActivityRepository)
userHandler := handler.NewUserHandler(userService, userActivityService, logService)
userProfileAuditRepository := repository.NewUserProfileAuditRepository(db)
manager := wire.ProvideHookManager()
openaiClient := wire.ProvideOpenAIClient(config)
postAIService := wire.ProvidePostAIService(openaiClient)
transactionManager := wire.ProvideTransactionManager(db)
manager := wire.ProvideHookManager()
commentRepository := repository.NewCommentRepository(db)
moderationHooks := wire.ProvideModerationHooks(manager, postAIService, postRepository, commentRepository, config)
sensitiveWordRepository := repository.NewSensitiveWordRepository(db)
sensitiveService := wire.ProvideSensitiveService(sensitiveWordRepository, client, config)
moderationHooks := wire.ProvideModerationHooks(manager, postAIService, postRepository, commentRepository, userRepository, sensitiveService, config)
userProfileAuditService := wire.ProvideUserProfileAuditService(userProfileAuditRepository, userRepository, manager, moderationHooks)
userHandler := wire.ProvideUserHandler(userService, userActivityService, logService, userProfileAuditService)
transactionManager := wire.ProvideTransactionManager(db)
builtinHooks := wire.ProvideBuiltinHooks(manager)
postService := wire.ProvidePostService(postRepository, systemMessageService, postAIService, cache, transactionManager, manager, moderationHooks, builtinHooks, logService)
channelRepository := repository.NewChannelRepository(db)
@@ -132,8 +136,9 @@ func InitializeApp() (*App, error) {
verificationHandler := handler.NewVerificationHandler(verificationService)
adminVerificationService := wire.ProvideAdminVerificationService(verificationRepository, userRepository)
adminVerificationHandler := wire.ProvideAdminVerificationHandler(adminVerificationService, userRepository)
adminProfileAuditHandler := handler.NewAdminProfileAuditHandler(userProfileAuditService)
wsHandler := wire.ProvideWSHandler(hub, chatService, groupService, jwtService, callService, userRepository)
router := ProvideRouter(userRepository, userHandler, postHandler, commentHandler, messageHandler, notificationHandler, uploadHandler, jwtService, pushHandler, systemMessageHandler, groupHandler, stickerHandler, voteHandler, channelHandler, scheduleHandler, roleHandler, adminUserHandler, adminPostHandler, adminCommentHandler, adminGroupHandler, adminDashboardHandler, adminLogHandler, qrCodeHandler, materialHandler, callHandler, reportHandler, adminReportHandler, verificationHandler, adminVerificationHandler, logService, userActivityService, casbinService, wsHandler)
router := ProvideRouter(userRepository, userHandler, postHandler, commentHandler, messageHandler, notificationHandler, uploadHandler, jwtService, pushHandler, systemMessageHandler, groupHandler, stickerHandler, voteHandler, channelHandler, scheduleHandler, roleHandler, adminUserHandler, adminPostHandler, adminCommentHandler, adminGroupHandler, adminDashboardHandler, adminLogHandler, qrCodeHandler, materialHandler, callHandler, reportHandler, adminReportHandler, verificationHandler, adminVerificationHandler, adminProfileAuditHandler, logService, userActivityService, casbinService, wsHandler)
hotRankWorker := wire.ProvideHotRankWorker(config, postRepository, cache)
app := NewApp(config, db, router, pushService, hotRankWorker, server, logger)
return app, nil
@@ -172,6 +177,7 @@ func ProvideRouter(
adminReportHandler *handler.AdminReportHandler,
verificationHandler *handler.VerificationHandler,
adminVerificationHandler *handler.AdminVerificationHandler,
adminProfileAuditHandler *handler.AdminProfileAuditHandler,
logService *service.LogService,
activityService service.UserActivityService,
casbinService service.CasbinService,
@@ -207,6 +213,7 @@ func ProvideRouter(
adminReportHandler,
verificationHandler,
adminVerificationHandler,
adminProfileAuditHandler,
logService,
activityService,
casbinService,

View File

@@ -1163,6 +1163,9 @@ type DashboardStatsResponse struct {
PendingReview int64 `json:"pending_review"` // 待审核总数
PendingPostsCount int64 `json:"pending_posts_count"` // 待审核帖子数
PendingComments int64 `json:"pending_comments_count"` // 待审核评论数
PendingAvatars int64 `json:"pending_avatars"` // 待审核头像数
PendingCovers int64 `json:"pending_covers"` // 待审核背景图数
PendingBios int64 `json:"pending_bios"` // 待审核签名数
}
// UserActivityTrendResponse 用户活跃度趋势响应

View File

@@ -0,0 +1,129 @@
package handler
import (
"strconv"
"carrot_bbs/internal/model"
"carrot_bbs/internal/pkg/response"
"carrot_bbs/internal/service"
"github.com/gin-gonic/gin"
)
type AdminProfileAuditHandler struct {
profileAuditService service.UserProfileAuditService
}
func NewAdminProfileAuditHandler(profileAuditService service.UserProfileAuditService) *AdminProfileAuditHandler {
return &AdminProfileAuditHandler{
profileAuditService: profileAuditService,
}
}
func (h *AdminProfileAuditHandler) GetPendingList(c *gin.Context) {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20"))
if page <= 0 {
page = 1
}
if pageSize <= 0 || pageSize > 100 {
pageSize = 20
}
contentType := c.Query("content_type")
var ct model.UserProfileContentType
switch contentType {
case "avatar":
ct = model.UserProfileContentTypeAvatar
case "cover":
ct = model.UserProfileContentTypeCover
case "bio":
ct = model.UserProfileContentTypeBio
default:
ct = ""
}
audits, total, err := h.profileAuditService.GetPendingList(c.Request.Context(), ct, page, pageSize)
if err != nil {
response.InternalServerError(c, "failed to get pending profile audits")
return
}
items := make([]map[string]any, 0, len(audits))
for _, audit := range audits {
item := map[string]any{
"id": audit.ID,
"user_id": audit.UserID,
"content_type": audit.ContentType,
"content": audit.Content,
"status": audit.Status,
"created_at": audit.CreatedAt.Format("2006-01-02T15:04:05Z07:00"),
}
if audit.User != nil {
item["user"] = map[string]any{
"id": audit.User.ID,
"username": audit.User.Username,
"nickname": audit.User.Nickname,
"avatar": audit.User.Avatar,
}
}
items = append(items, item)
}
response.Paginated(c, items, total, page, pageSize)
}
func (h *AdminProfileAuditHandler) ModerateAudit(c *gin.Context) {
auditID := c.Param("id")
if auditID == "" {
response.BadRequest(c, "audit id is required")
return
}
var req struct {
Status string `json:"status" binding:"required"`
Reason string `json:"reason"`
}
if err := c.ShouldBindJSON(&req); err != nil {
response.BadRequest(c, err.Error())
return
}
if req.Status != string(model.UserProfileAuditStatusApproved) && req.Status != string(model.UserProfileAuditStatusRejected) {
response.BadRequest(c, "invalid status, must be 'approved' or 'rejected'")
return
}
reviewedBy := c.GetString("user_id")
if err := h.profileAuditService.ModerateAudit(c.Request.Context(), auditID, model.UserProfileAuditStatus(req.Status), req.Reason, reviewedBy); err != nil {
response.HandleError(c, err, "failed to moderate profile audit")
return
}
response.Success(c, map[string]any{"success": true})
}
func (h *AdminProfileAuditHandler) BatchModerate(c *gin.Context) {
var req struct {
IDs []string `json:"ids" binding:"required"`
Status string `json:"status" binding:"required"`
Reason string `json:"reason"`
}
if err := c.ShouldBindJSON(&req); err != nil {
response.BadRequest(c, err.Error())
return
}
if req.Status != string(model.UserProfileAuditStatusApproved) && req.Status != string(model.UserProfileAuditStatusRejected) {
response.BadRequest(c, "invalid status, must be 'approved' or 'rejected'")
return
}
reviewedBy := c.GetString("user_id")
if err := h.profileAuditService.BatchModerateAudits(c.Request.Context(), req.IDs, model.UserProfileAuditStatus(req.Status), req.Reason, reviewedBy); err != nil {
response.InternalServerError(c, "failed to batch moderate profile audits")
return
}
response.Success(c, map[string]any{"success": true})
}

View File

@@ -32,13 +32,13 @@ func normalizePagination(page, pageSize int) (int, int) {
// UserHandler 用户处理器
type UserHandler struct {
userService service.UserService
activityService service.UserActivityService // 用户活跃统计服务
jwtService *service.JWTService
logService *service.LogService
userService service.UserService
activityService service.UserActivityService
profileAuditService service.UserProfileAuditService
jwtService *service.JWTService
logService *service.LogService
}
// NewUserHandler 创建用户处理器
func NewUserHandler(userService service.UserService, activityService service.UserActivityService, logService *service.LogService) *UserHandler {
return &UserHandler{
userService: userService,
@@ -47,6 +47,10 @@ func NewUserHandler(userService service.UserService, activityService service.Use
}
}
func (h *UserHandler) SetProfileAuditService(profileAuditService service.UserProfileAuditService) {
h.profileAuditService = profileAuditService
}
// SetJWTService 设置JWT服务
func (h *UserHandler) SetJWTService(jwtService *service.JWTService) {
h.jwtService = jwtService
@@ -318,6 +322,7 @@ func (h *UserHandler) UpdateUser(c *gin.Context) {
Website string `json:"website"`
Location string `json:"location"`
Avatar string `json:"avatar"`
CoverURL string `json:"cover_url"`
Phone *string `json:"phone"`
Email *string `json:"email"`
}
@@ -337,18 +342,12 @@ func (h *UserHandler) UpdateUser(c *gin.Context) {
if req.Nickname != "" {
user.Nickname = req.Nickname
}
if req.Bio != "" {
user.Bio = req.Bio
}
if req.Website != "" {
user.Website = req.Website
}
if req.Location != "" {
user.Location = req.Location
}
if req.Avatar != "" {
user.Avatar = req.Avatar
}
if req.Phone != nil {
user.Phone = req.Phone
}
@@ -359,20 +358,53 @@ func (h *UserHandler) UpdateUser(c *gin.Context) {
user.Email = req.Email
}
avatarNeedsAudit := req.Avatar != "" && req.Avatar != user.Avatar && h.profileAuditService != nil
coverNeedsAudit := req.CoverURL != "" && req.CoverURL != user.CoverURL && h.profileAuditService != nil
bioNeedsAudit := req.Bio != "" && req.Bio != user.Bio && h.profileAuditService != nil
if avatarNeedsAudit {
if _, err := h.profileAuditService.SubmitAvatarAudit(c.Request.Context(), userID, req.Avatar); err != nil {
avatarNeedsAudit = false
user.Avatar = req.Avatar
}
} else if req.Avatar != "" {
user.Avatar = req.Avatar
}
if coverNeedsAudit {
if _, err := h.profileAuditService.SubmitCoverAudit(c.Request.Context(), userID, req.CoverURL); err != nil {
coverNeedsAudit = false
user.CoverURL = req.CoverURL
}
} else if req.CoverURL != "" {
user.CoverURL = req.CoverURL
}
if bioNeedsAudit {
if _, err := h.profileAuditService.SubmitBioAudit(c.Request.Context(), userID, req.Bio); err != nil {
bioNeedsAudit = false
user.Bio = req.Bio
}
} else if req.Bio != "" {
user.Bio = req.Bio
}
err = h.userService.UpdateUser(c.Request.Context(), user)
if err != nil {
response.InternalServerError(c, "failed to update user")
return
}
// 实时计算帖子数量
postsCount, err := h.userService.GetUserPostCount(c.Request.Context(), userID)
if err != nil {
// 如果获取失败,使用数据库中的值
postsCount = int64(user.PostsCount)
}
response.Success(c, dto.ConvertUserToDetailResponseWithPostsCount(user, int(postsCount)))
resp := gin.H{"user": dto.ConvertUserToDetailResponseWithPostsCount(user, int(postsCount))}
if avatarNeedsAudit || coverNeedsAudit || bioNeedsAudit {
resp["audit_pending"] = true
}
response.Success(c, resp)
}
// SendEmailVerifyCode 发送当前用户邮箱验证码

View File

@@ -11,11 +11,15 @@ import (
type AuditTargetType string
const (
AuditTargetTypePost AuditTargetType = "post" // 帖子
AuditTargetTypeComment AuditTargetType = "comment" // 评论
AuditTargetTypeMessage AuditTargetType = "message" // 私信
AuditTargetTypeUser AuditTargetType = "user" // 用户资料
AuditTargetTypeImage AuditTargetType = "image" // 图片
AuditTargetTypePost AuditTargetType = "post" // 帖子
AuditTargetTypeComment AuditTargetType = "comment" // 评论
AuditTargetTypeMessage AuditTargetType = "message" // 私信
AuditTargetTypeUser AuditTargetType = "user" // 用户资料
AuditTargetTypeImage AuditTargetType = "image" // 图片
AuditTargetTypeAvatar AuditTargetType = "avatar" // 头像
AuditTargetTypeCover AuditTargetType = "cover" // 背景图
AuditTargetTypeBio AuditTargetType = "bio" // 个性签名
AuditTargetTypeUserProfile AuditTargetType = "user_profile" // 用户资料审核任务
)
// AuditResult 审核结果

View File

@@ -173,6 +173,9 @@ func autoMigrate(db *gorm.DB) error {
// 身份认证相关
&VerificationRecord{},
// 用户资料审核相关
&UserProfileAudit{},
)
if err != nil {
return err

View File

@@ -0,0 +1,51 @@
package model
import (
"time"
"github.com/google/uuid"
"gorm.io/gorm"
)
type UserProfileContentType string
const (
UserProfileContentTypeAvatar UserProfileContentType = "avatar"
UserProfileContentTypeCover UserProfileContentType = "cover"
UserProfileContentTypeBio UserProfileContentType = "bio"
)
type UserProfileAuditStatus string
const (
UserProfileAuditStatusPending UserProfileAuditStatus = "pending"
UserProfileAuditStatusApproved UserProfileAuditStatus = "approved"
UserProfileAuditStatusRejected UserProfileAuditStatus = "rejected"
)
type UserProfileAudit struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
UserID string `json:"user_id" gorm:"type:varchar(36);index;not null"`
ContentType UserProfileContentType `json:"content_type" gorm:"type:varchar(20);index;not null"`
Content string `json:"content" gorm:"type:text;not null"`
Status UserProfileAuditStatus `json:"status" gorm:"type:varchar(20);default:pending;index"`
ReviewedAt *time.Time `json:"reviewed_at" gorm:"type:timestamp"`
ReviewedBy string `json:"reviewed_by" gorm:"type:varchar(50)"`
RejectReason string `json:"reject_reason" gorm:"type:text"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
User *User `json:"user" gorm:"foreignKey:UserID"`
}
func (a *UserProfileAudit) BeforeCreate(tx *gorm.DB) error {
if a.ID == "" {
a.ID = uuid.New().String()
}
return nil
}
func (UserProfileAudit) TableName() string {
return "user_profile_audits"
}

View File

@@ -11,31 +11,37 @@ import (
type HookType string
const (
HookPostCreated HookType = "post.created"
HookPostUpdated HookType = "post.updated"
HookPostDeleted HookType = "post.deleted"
HookPostPreModerate HookType = "post.pre_moderate"
HookPostModerated HookType = "post.moderated"
HookCommentCreated HookType = "comment.created"
HookCommentDeleted HookType = "comment.deleted"
HookCommentPreModerate HookType = "comment.pre_moderate"
HookCommentModerated HookType = "comment.moderated"
HookUserCreated HookType = "user.created"
HookUserUpdated HookType = "user.updated"
HookUserDeleted HookType = "user.deleted"
HookUserLogin HookType = "user.login"
HookUserLogout HookType = "user.logout"
HookGroupCreated HookType = "group.created"
HookGroupUpdated HookType = "group.updated"
HookGroupDeleted HookType = "group.deleted"
HookGroupJoined HookType = "group.joined"
HookGroupLeft HookType = "group.left"
HookMessageSent HookType = "message.sent"
HookMessageRead HookType = "message.read"
HookNotificationSent HookType = "notification.sent"
HookVoteCreated HookType = "vote.created"
HookVoteUpdated HookType = "vote.updated"
HookFileUploaded HookType = "file.uploaded"
HookPostCreated HookType = "post.created"
HookPostUpdated HookType = "post.updated"
HookPostDeleted HookType = "post.deleted"
HookPostPreModerate HookType = "post.pre_moderate"
HookPostModerated HookType = "post.moderated"
HookCommentCreated HookType = "comment.created"
HookCommentDeleted HookType = "comment.deleted"
HookCommentPreModerate HookType = "comment.pre_moderate"
HookCommentModerated HookType = "comment.moderated"
HookUserCreated HookType = "user.created"
HookUserUpdated HookType = "user.updated"
HookUserDeleted HookType = "user.deleted"
HookUserLogin HookType = "user.login"
HookUserLogout HookType = "user.logout"
HookGroupCreated HookType = "group.created"
HookGroupUpdated HookType = "group.updated"
HookGroupDeleted HookType = "group.deleted"
HookGroupJoined HookType = "group.joined"
HookGroupLeft HookType = "group.left"
HookMessageSent HookType = "message.sent"
HookMessageRead HookType = "message.read"
HookNotificationSent HookType = "notification.sent"
HookVoteCreated HookType = "vote.created"
HookVoteUpdated HookType = "vote.updated"
HookFileUploaded HookType = "file.uploaded"
HookUserAvatarPreModerate HookType = "user.avatar.pre_moderate"
HookUserAvatarModerated HookType = "user.avatar.moderated"
HookUserCoverPreModerate HookType = "user.cover.pre_moderate"
HookUserCoverModerated HookType = "user.cover.moderated"
HookUserBioPreModerate HookType = "user.bio.pre_moderate"
HookUserBioModerated HookType = "user.bio.moderated"
)
type Priority int

View File

@@ -18,6 +18,8 @@ type PostAIService interface {
IsEnabled() bool
ModeratePost(ctx context.Context, title, content string, images []string) error
ModerateComment(ctx context.Context, content string, images []string) error
ModerateImage(ctx context.Context, imageURL string) error
ModerateBio(ctx context.Context, bio string) error
}
type PostModerationHook struct {
@@ -324,6 +326,7 @@ type ModerationHooks struct {
sensitiveService SensitiveService
postRepo repository.PostRepository
commentRepo repository.CommentRepository
userRepo repository.UserRepository
strictMode bool
replaceStr string
}
@@ -333,6 +336,7 @@ func NewModerationHooks(
sensitiveService SensitiveService,
postRepo repository.PostRepository,
commentRepo repository.CommentRepository,
userRepo repository.UserRepository,
strictMode bool,
replaceStr string,
) *ModerationHooks {
@@ -341,6 +345,7 @@ func NewModerationHooks(
sensitiveService: sensitiveService,
postRepo: postRepo,
commentRepo: commentRepo,
userRepo: userRepo,
strictMode: strictMode,
replaceStr: replaceStr,
}
@@ -359,6 +364,15 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager)
avatarHook := NewUserAvatarModerationHook(m.postAIService, m.userRepo, m.strictMode)
avatarHook.Register(manager)
coverHook := NewUserCoverModerationHook(m.postAIService, m.userRepo, m.strictMode)
coverHook.Register(manager)
bioHook := NewUserBioModerationHook(m.postAIService, m.sensitiveService, m.userRepo, m.strictMode)
bioHook.Register(manager)
zap.L().Info("Registered moderation hooks",
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
zap.Bool("sensitive_enabled", m.sensitiveService != nil),
@@ -381,3 +395,27 @@ func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager,
})
return result
}
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}

View File

@@ -101,6 +101,42 @@ type CommentModeratedHookData struct {
ReviewedBy string
}
type UserAvatarModerateHookData struct {
UserID string
AvatarURL string
}
type UserCoverModerateHookData struct {
UserID string
CoverURL string
}
type UserBioModerateHookData struct {
UserID string
Bio string
}
type UserAvatarModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type UserCoverModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type UserBioModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type ModerationResult struct {
Approved bool
NeedsReview bool
@@ -271,3 +307,33 @@ func (h *HookTriggerHelper) TriggerCommentPreModerate(ctx context.Context, userI
func (h *HookTriggerHelper) TriggerCommentModerated(ctx context.Context, userID uint, data *CommentModeratedHookData) {
h.manager.Trigger(ctx, HookCommentModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserAvatarPreModerate(ctx context.Context, userID uint, data *UserAvatarModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserAvatarModerated(ctx context.Context, userID uint, data *UserAvatarModeratedHookData) {
h.manager.Trigger(ctx, HookUserAvatarModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserCoverPreModerate(ctx context.Context, userID uint, data *UserCoverModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserCoverModerated(ctx context.Context, userID uint, data *UserCoverModeratedHookData) {
h.manager.Trigger(ctx, HookUserCoverModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserBioPreModerate(ctx context.Context, userID uint, data *UserBioModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserBioModerated(ctx context.Context, userID uint, data *UserBioModeratedHookData) {
h.manager.Trigger(ctx, HookUserBioModerated, userID, data)
}

View File

@@ -0,0 +1,317 @@
package hook
import (
"strings"
"time"
"carrot_bbs/internal/repository"
"go.uber.org/zap"
)
type UserAvatarModerationHook struct {
name string
postAIService PostAIService
userRepo repository.UserRepository
strictMode bool
}
func NewUserAvatarModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserAvatarModerationHook {
return &UserAvatarModerationHook{
name: "moderation.user.avatar.ai",
postAIService: postAIService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserAvatarModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserAvatarPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserAvatarModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserAvatarModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateImage(ctx.Ctx, data.AvatarURL)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User avatar flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User avatar rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI avatar moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve avatar",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}
type UserCoverModerationHook struct {
name string
postAIService PostAIService
userRepo repository.UserRepository
strictMode bool
}
func NewUserCoverModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserCoverModerationHook {
return &UserCoverModerationHook{
name: "moderation.user.cover.ai",
postAIService: postAIService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserCoverModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserCoverPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserCoverModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserCoverModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateImage(ctx.Ctx, data.CoverURL)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User cover flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User cover rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI cover moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve cover",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}
type UserBioModerationHook struct {
name string
postAIService PostAIService
sensitiveService SensitiveService
userRepo repository.UserRepository
strictMode bool
}
func NewUserBioModerationHook(postAIService PostAIService, sensitiveService SensitiveService, userRepo repository.UserRepository, strictMode bool) *UserBioModerationHook {
return &UserBioModerationHook{
name: "moderation.user.bio.ai",
postAIService: postAIService,
sensitiveService: sensitiveService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserBioModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserBioPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserBioModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserBioModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.sensitiveService != nil {
hasSensitive, words := h.sensitiveService.Check(ctx.Ctx, data.Bio)
if hasSensitive {
result.Approved = false
result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
result.ReviewedBy = "sensitive_word"
zap.L().Info("User bio rejected by sensitive word filter",
zap.String("user_id", data.UserID),
zap.Strings("words", words),
)
return nil
}
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateBio(ctx.Ctx, data.Bio)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User bio flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User bio rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI bio moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve bio",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}

View File

@@ -93,6 +93,8 @@ type Client interface {
Config() Config
ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error)
ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error)
ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error)
ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error)
}
type clientImpl struct {
@@ -163,6 +165,157 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
return c.moderateContentInBatches(ctx, prompt.String(), images)
}
const imageModerationSystemPrompt = `你是图片内容审核助手负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
硬性违规(命中任一项必须 result=block
- 色情低俗、裸露、性暗示内容
- 暴力血腥、恐怖画面
- 违法犯罪相关内容
- 明确的政治敏感宣传
- 恶意广告、诈骗信息
- 仇恨歧视符号或标志
可疑内容result=review
- 图片含义模糊,无法确定是否存在违规
- 轻微争议性内容但不完全违规
- AI无法准确判断的内容
放行规则result=pass
- 正常风景、人物、动物、动漫等图片
- 表情包、梗图
- 正常的艺术创作、摄影作品
输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}`
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
硬性违规(命中任一项必须 result=block
- 明确的违法犯罪、暴力威胁内容
- 色情低俗、性暗示描述
- 诈骗引流、恶意广告
- 明确煽动群体对立、仇恨
- 恶意公开他人隐私信息
可疑内容result=review
- 内容含义模糊,无法确定违规意图
- 涉及敏感话题但未明确违规
- AI无法准确判断的表达
放行规则result=pass
- 正常的自我介绍、心情表达
- 玩梗、幽默、吐槽
- 引用、歌词、诗句等
- 理性的观点表达
输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}`
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
if strings.TrimSpace(imageURL) == "" {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
userPrompt := "请审核以下用户图片"
cleanImages := normalizeImageURLs([]string{imageURL})
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
}
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
if strings.TrimSpace(bio) == "" {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
userPrompt := "个性签名内容:" + bio
return c.moderateWithCustomPrompt(ctx, bioModerationSystemPrompt, userPrompt)
}
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
}
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
cleanImages := normalizeImageURLs(images)
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)

View File

@@ -0,0 +1,123 @@
package repository
import (
"carrot_bbs/internal/model"
"gorm.io/gorm"
)
type UserProfileAuditRepository interface {
Create(audit *model.UserProfileAudit) error
GetByID(id string) (*model.UserProfileAudit, error)
GetByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
GetPendingByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
Update(audit *model.UserProfileAudit) error
Delete(id string) error
GetPendingList(contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error)
CountPending(contentType model.UserProfileContentType) (int64, error)
CountPendingAll() (int64, int64, int64, error)
}
type userProfileAuditRepository struct {
db *gorm.DB
}
func NewUserProfileAuditRepository(db *gorm.DB) UserProfileAuditRepository {
return &userProfileAuditRepository{db: db}
}
func (r *userProfileAuditRepository) Create(audit *model.UserProfileAudit) error {
return r.db.Create(audit).Error
}
func (r *userProfileAuditRepository) GetByID(id string) (*model.UserProfileAudit, error) {
var audit model.UserProfileAudit
err := r.db.Preload("User").First(&audit, "id = ?", id).Error
if err != nil {
return nil, err
}
return &audit, nil
}
func (r *userProfileAuditRepository) GetByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
var audit model.UserProfileAudit
err := r.db.Where("user_id = ? AND content_type = ?", userID, contentType).
Order("created_at DESC").
First(&audit).Error
if err != nil {
return nil, err
}
return &audit, nil
}
func (r *userProfileAuditRepository) GetPendingByUserID(userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
var audit model.UserProfileAudit
err := r.db.Where("user_id = ? AND content_type = ? AND status = ?", userID, contentType, model.UserProfileAuditStatusPending).
Order("created_at DESC").
First(&audit).Error
if err != nil {
return nil, err
}
return &audit, nil
}
func (r *userProfileAuditRepository) Update(audit *model.UserProfileAudit) error {
return r.db.Save(audit).Error
}
func (r *userProfileAuditRepository) Delete(id string) error {
return r.db.Delete(&model.UserProfileAudit{}, "id = ?", id).Error
}
func (r *userProfileAuditRepository) GetPendingList(contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error) {
var audits []*model.UserProfileAudit
var total int64
query := r.db.Model(&model.UserProfileAudit{}).Where("status = ?", model.UserProfileAuditStatusPending)
if contentType != "" {
query = query.Where("content_type = ?", contentType)
}
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
offset := (page - 1) * pageSize
if err := query.Preload("User").Order("created_at DESC").Offset(offset).Limit(pageSize).Find(&audits).Error; err != nil {
return nil, 0, err
}
return audits, total, nil
}
func (r *userProfileAuditRepository) CountPending(contentType model.UserProfileContentType) (int64, error) {
var count int64
err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", contentType, model.UserProfileAuditStatusPending).
Count(&count).Error
return count, err
}
func (r *userProfileAuditRepository) CountPendingAll() (int64, int64, int64, error) {
var avatarCount, coverCount, bioCount int64
if err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", model.UserProfileContentTypeAvatar, model.UserProfileAuditStatusPending).
Count(&avatarCount).Error; err != nil {
return 0, 0, 0, err
}
if err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", model.UserProfileContentTypeCover, model.UserProfileAuditStatusPending).
Count(&coverCount).Error; err != nil {
return 0, 0, 0, err
}
if err := r.db.Model(&model.UserProfileAudit{}).
Where("content_type = ? AND status = ?", model.UserProfileContentTypeBio, model.UserProfileAuditStatusPending).
Count(&bioCount).Error; err != nil {
return 0, 0, 0, err
}
return avatarCount, coverCount, bioCount, nil
}

View File

@@ -41,6 +41,7 @@ type Router struct {
adminReportHandler *handler.AdminReportHandler
verificationHandler *handler.VerificationHandler
adminVerificationHandler *handler.AdminVerificationHandler
adminProfileAuditHandler *handler.AdminProfileAuditHandler
wsHandler *handler.WSHandler
logService *service.LogService
jwtService *service.JWTService
@@ -78,6 +79,7 @@ func New(
adminReportHandler *handler.AdminReportHandler,
verificationHandler *handler.VerificationHandler,
adminVerificationHandler *handler.AdminVerificationHandler,
adminProfileAuditHandler *handler.AdminProfileAuditHandler,
logService *service.LogService,
activityService service.UserActivityService,
casbinService service.CasbinService,
@@ -118,6 +120,7 @@ func New(
adminReportHandler: adminReportHandler,
verificationHandler: verificationHandler,
adminVerificationHandler: adminVerificationHandler,
adminProfileAuditHandler: adminProfileAuditHandler,
logService: logService,
jwtService: jwtService,
casbinService: casbinService,
@@ -619,6 +622,13 @@ func (r *Router) setupRoutes() {
admin.GET("/verifications/:id", r.adminVerificationHandler.GetVerificationDetail)
admin.PUT("/verifications/:id/review", r.adminVerificationHandler.ReviewVerification)
}
// 用户资料审核管理
if r.adminProfileAuditHandler != nil {
admin.GET("/profile-audits", r.adminProfileAuditHandler.GetPendingList)
admin.PUT("/profile-audits/:id/moderate", r.adminProfileAuditHandler.ModerateAudit)
admin.POST("/profile-audits/batch-moderate", r.adminProfileAuditHandler.BatchModerate)
}
}
}
}

View File

@@ -164,8 +164,16 @@ func (s *adminDashboardServiceImpl) GetStats(ctx context.Context) (*dto.Dashboar
}
stats.PendingComments = pendingComments
var pendingAvatars, pendingCovers, pendingBios int64
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeAvatar, model.UserProfileAuditStatusPending).Count(&pendingAvatars)
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeCover, model.UserProfileAuditStatusPending).Count(&pendingCovers)
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeBio, model.UserProfileAuditStatusPending).Count(&pendingBios)
stats.PendingAvatars = pendingAvatars
stats.PendingCovers = pendingCovers
stats.PendingBios = pendingBios
// 计算待审核总数
stats.PendingReview = stats.PendingPosts + stats.PendingComments
stats.PendingReview = stats.PendingPosts + stats.PendingComments + stats.PendingAvatars + stats.PendingCovers + stats.PendingBios
stats.PendingPostsCount = stats.PendingPosts
return &stats, nil

View File

@@ -89,6 +89,86 @@ func (e *CommentModerationReviewError) IsReview() bool {
return true
}
type ImageModerationRejectedError struct {
Reason string
}
func (e *ImageModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "image rejected by moderation"
}
return "image rejected by moderation: " + e.Reason
}
func (e *ImageModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "图片未通过审核,请更换后重试"
}
return strings.TrimSpace(e.Reason)
}
type ImageModerationReviewError struct {
Reason string
}
func (e *ImageModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "image needs manual review"
}
return "image needs manual review: " + e.Reason
}
func (e *ImageModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "图片需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *ImageModerationReviewError) IsReview() bool {
return true
}
type BioModerationRejectedError struct {
Reason string
}
func (e *BioModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "bio rejected by moderation"
}
return "bio rejected by moderation: " + e.Reason
}
func (e *BioModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "个性签名未通过审核,请修改后重试"
}
return strings.TrimSpace(e.Reason)
}
type BioModerationReviewError struct {
Reason string
}
func (e *BioModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "bio needs manual review"
}
return "bio needs manual review: " + e.Reason
}
func (e *BioModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "个性签名需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *BioModerationReviewError) IsReview() bool {
return true
}
type PostAIService struct {
openAIClient openai.Client
}
@@ -154,3 +234,55 @@ func (s *PostAIService) ModerateComment(ctx context.Context, content string, ima
return nil
}
}
func (s *PostAIService) ModerateImage(ctx context.Context, imageURL string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateImage(ctx, imageURL)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI image moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &ImageModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &ImageModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}
func (s *PostAIService) ModerateBio(ctx context.Context, bio string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateBio(ctx, bio)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI bio moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &BioModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &BioModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}

View File

@@ -0,0 +1,395 @@
package service
import (
"context"
"log"
"time"
"carrot_bbs/internal/model"
"carrot_bbs/internal/pkg/hook"
"carrot_bbs/internal/repository"
"go.uber.org/zap"
)
type UserProfileAuditService interface {
SubmitAvatarAudit(ctx context.Context, userID, avatarURL string) (*model.UserProfileAudit, error)
SubmitCoverAudit(ctx context.Context, userID, coverURL string) (*model.UserProfileAudit, error)
SubmitBioAudit(ctx context.Context, userID, bio string) (*model.UserProfileAudit, error)
GetPendingAudit(ctx context.Context, userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
GetAuditStatus(ctx context.Context, userID string) (*ProfileAuditStatusResponse, error)
ModerateAudit(ctx context.Context, auditID string, status model.UserProfileAuditStatus, reason, reviewedBy string) error
BatchModerateAudits(ctx context.Context, ids []string, status model.UserProfileAuditStatus, reason, reviewedBy string) error
GetPendingList(ctx context.Context, contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error)
GetPendingCounts(ctx context.Context) (int64, int64, int64, error)
}
type ProfileAuditStatusResponse struct {
AvatarStatus string `json:"avatar_status"`
CoverStatus string `json:"cover_status"`
BioStatus string `json:"bio_status"`
}
type userProfileAuditServiceImpl struct {
auditRepo repository.UserProfileAuditRepository
userRepo repository.UserRepository
hookManager *hook.Manager
moderationHooks *hook.ModerationHooks
}
func NewUserProfileAuditService(
auditRepo repository.UserProfileAuditRepository,
userRepo repository.UserRepository,
hookManager *hook.Manager,
moderationHooks *hook.ModerationHooks,
) UserProfileAuditService {
return &userProfileAuditServiceImpl{
auditRepo: auditRepo,
userRepo: userRepo,
hookManager: hookManager,
moderationHooks: moderationHooks,
}
}
func (s *userProfileAuditServiceImpl) SubmitAvatarAudit(ctx context.Context, userID, avatarURL string) (*model.UserProfileAudit, error) {
audit := &model.UserProfileAudit{
UserID: userID,
ContentType: model.UserProfileContentTypeAvatar,
Content: avatarURL,
Status: model.UserProfileAuditStatusPending,
}
if err := s.auditRepo.Create(audit); err != nil {
return nil, err
}
go s.reviewAvatarAsync(audit.ID, userID, avatarURL)
return audit, nil
}
func (s *userProfileAuditServiceImpl) SubmitCoverAudit(ctx context.Context, userID, coverURL string) (*model.UserProfileAudit, error) {
audit := &model.UserProfileAudit{
UserID: userID,
ContentType: model.UserProfileContentTypeCover,
Content: coverURL,
Status: model.UserProfileAuditStatusPending,
}
if err := s.auditRepo.Create(audit); err != nil {
return nil, err
}
go s.reviewCoverAsync(audit.ID, userID, coverURL)
return audit, nil
}
func (s *userProfileAuditServiceImpl) SubmitBioAudit(ctx context.Context, userID, bio string) (*model.UserProfileAudit, error) {
audit := &model.UserProfileAudit{
UserID: userID,
ContentType: model.UserProfileContentTypeBio,
Content: bio,
Status: model.UserProfileAuditStatusPending,
}
if err := s.auditRepo.Create(audit); err != nil {
return nil, err
}
go s.reviewBioAsync(audit.ID, userID, bio)
return audit, nil
}
func (s *userProfileAuditServiceImpl) GetPendingAudit(ctx context.Context, userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
return s.auditRepo.GetPendingByUserID(userID, contentType)
}
func (s *userProfileAuditServiceImpl) GetAuditStatus(ctx context.Context, userID string) (*ProfileAuditStatusResponse, error) {
resp := &ProfileAuditStatusResponse{
AvatarStatus: string(model.UserProfileAuditStatusApproved),
CoverStatus: string(model.UserProfileAuditStatusApproved),
BioStatus: string(model.UserProfileAuditStatusApproved),
}
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeAvatar); err == nil {
resp.AvatarStatus = string(audit.Status)
}
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeCover); err == nil {
resp.CoverStatus = string(audit.Status)
}
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeBio); err == nil {
resp.BioStatus = string(audit.Status)
}
return resp, nil
}
func (s *userProfileAuditServiceImpl) ModerateAudit(ctx context.Context, auditID string, status model.UserProfileAuditStatus, reason, reviewedBy string) error {
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
return err
}
now := time.Now()
audit.Status = status
audit.ReviewedAt = &now
audit.ReviewedBy = reviewedBy
audit.RejectReason = reason
if err := s.auditRepo.Update(audit); err != nil {
return err
}
if status == model.UserProfileAuditStatusApproved {
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved audit content",
zap.String("audit_id", audit.ID),
zap.String("user_id", audit.UserID),
zap.Error(err),
)
}
}
return nil
}
func (s *userProfileAuditServiceImpl) BatchModerateAudits(ctx context.Context, ids []string, status model.UserProfileAuditStatus, reason string, reviewedBy string) error {
for _, id := range ids {
if err := s.ModerateAudit(ctx, id, status, reason, reviewedBy); err != nil {
zap.L().Warn("Failed to moderate audit in batch",
zap.String("audit_id", id),
zap.Error(err),
)
}
}
return nil
}
func (s *userProfileAuditServiceImpl) GetPendingList(ctx context.Context, contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error) {
return s.auditRepo.GetPendingList(contentType, page, pageSize)
}
func (s *userProfileAuditServiceImpl) GetPendingCounts(ctx context.Context) (int64, int64, int64, error) {
return s.auditRepo.CountPendingAll()
}
func (s *userProfileAuditServiceImpl) reviewAvatarAsync(auditID, userID, avatarURL string) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] Panic in avatar moderation async flow, audit=%s panic=%v", auditID, r)
}
}()
result := &hook.ModerationResult{}
if s.moderationHooks != nil && s.hookManager != nil {
result = s.moderationHooks.ModerateAvatar(context.Background(), s.hookManager, &hook.UserAvatarModerateHookData{
UserID: userID,
AvatarURL: avatarURL,
})
} else {
result.Approved = true
result.ReviewedBy = "system"
}
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
zap.L().Error("Failed to get audit for avatar moderation result",
zap.String("audit_id", auditID),
zap.Error(err),
)
return
}
now := time.Now()
if !result.Approved {
if result.NeedsReview {
audit.ReviewedBy = "ai"
zap.L().Info("User avatar flagged for manual review",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
)
} else {
audit.Status = model.UserProfileAuditStatusRejected
audit.RejectReason = result.RejectReason
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
zap.L().Info("User avatar rejected by moderation",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
zap.String("reason", result.RejectReason),
)
}
} else {
audit.Status = model.UserProfileAuditStatusApproved
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved avatar",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
if err := s.auditRepo.Update(audit); err != nil {
zap.L().Error("Failed to update avatar audit status",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
func (s *userProfileAuditServiceImpl) reviewCoverAsync(auditID, userID, coverURL string) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] Panic in cover moderation async flow, audit=%s panic=%v", auditID, r)
}
}()
result := &hook.ModerationResult{}
if s.moderationHooks != nil && s.hookManager != nil {
result = s.moderationHooks.ModerateCover(context.Background(), s.hookManager, &hook.UserCoverModerateHookData{
UserID: userID,
CoverURL: coverURL,
})
} else {
result.Approved = true
result.ReviewedBy = "system"
}
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
zap.L().Error("Failed to get audit for cover moderation result",
zap.String("audit_id", auditID),
zap.Error(err),
)
return
}
now := time.Now()
if !result.Approved {
if result.NeedsReview {
audit.ReviewedBy = "ai"
zap.L().Info("User cover flagged for manual review",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
)
} else {
audit.Status = model.UserProfileAuditStatusRejected
audit.RejectReason = result.RejectReason
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
zap.L().Info("User cover rejected by moderation",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
zap.String("reason", result.RejectReason),
)
}
} else {
audit.Status = model.UserProfileAuditStatusApproved
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved cover",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
if err := s.auditRepo.Update(audit); err != nil {
zap.L().Error("Failed to update cover audit status",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
func (s *userProfileAuditServiceImpl) reviewBioAsync(auditID, userID, bio string) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] Panic in bio moderation async flow, audit=%s panic=%v", auditID, r)
}
}()
result := &hook.ModerationResult{}
if s.moderationHooks != nil && s.hookManager != nil {
result = s.moderationHooks.ModerateBio(context.Background(), s.hookManager, &hook.UserBioModerateHookData{
UserID: userID,
Bio: bio,
})
} else {
result.Approved = true
result.ReviewedBy = "system"
}
audit, err := s.auditRepo.GetByID(auditID)
if err != nil {
zap.L().Error("Failed to get audit for bio moderation result",
zap.String("audit_id", auditID),
zap.Error(err),
)
return
}
now := time.Now()
if !result.Approved {
if result.NeedsReview {
audit.ReviewedBy = "ai"
zap.L().Info("User bio flagged for manual review",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
)
} else {
audit.Status = model.UserProfileAuditStatusRejected
audit.RejectReason = result.RejectReason
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
zap.L().Info("User bio rejected by moderation",
zap.String("audit_id", auditID),
zap.String("user_id", userID),
zap.String("reason", result.RejectReason),
)
}
} else {
audit.Status = model.UserProfileAuditStatusApproved
audit.ReviewedBy = result.ReviewedBy
audit.ReviewedAt = &now
if err := s.applyAuditContent(audit); err != nil {
zap.L().Error("Failed to apply approved bio",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
if err := s.auditRepo.Update(audit); err != nil {
zap.L().Error("Failed to update bio audit status",
zap.String("audit_id", auditID),
zap.Error(err),
)
}
}
func (s *userProfileAuditServiceImpl) applyAuditContent(audit *model.UserProfileAudit) error {
user, err := s.userRepo.GetByID(audit.UserID)
if err != nil {
return err
}
switch audit.ContentType {
case model.UserProfileContentTypeAvatar:
user.Avatar = audit.Content
case model.UserProfileContentTypeCover:
user.CoverURL = audit.Content
case model.UserProfileContentTypeBio:
user.Bio = audit.Content
default:
return nil
}
return s.userRepo.Update(user)
}

View File

@@ -32,10 +32,11 @@ var HandlerSet = wire.NewSet(
handler.NewReportHandler,
handler.NewAdminReportHandler,
handler.NewVerificationHandler,
handler.NewAdminProfileAuditHandler,
handler.NewPostHandler,
// 需要特殊处理的 Handler
handler.NewUserHandler,
handler.NewPostHandler,
ProvideUserHandler,
ProvideMessageHandler,
ProvideSystemMessageHandler,
ProvideGroupHandler,
@@ -44,7 +45,17 @@ var HandlerSet = wire.NewSet(
ProvideAdminVerificationHandler,
)
// ProvideMessageHandler 提供消息处理器
func ProvideUserHandler(
userService service.UserService,
activityService service.UserActivityService,
logService *service.LogService,
profileAuditService service.UserProfileAuditService,
) *handler.UserHandler {
h := handler.NewUserHandler(userService, activityService, logService)
h.SetProfileAuditService(profileAuditService)
return h
}
func ProvideMessageHandler(
chatService service.ChatService,
messageService *service.MessageService,

View File

@@ -137,15 +137,18 @@ func ProvideModerationHooks(
postAIService *service.PostAIService,
postRepo repository.PostRepository,
commentRepo repository.CommentRepository,
userRepo repository.UserRepository,
sensitiveService service.SensitiveService,
cfg *config.Config,
) *hook.ModerationHooks {
strictMode := cfg.OpenAI.StrictModeration
moderationHooks := hook.NewModerationHooks(
postAIService,
nil,
sensitiveService,
postRepo,
commentRepo,
userRepo,
strictMode,
"***",
)

View File

@@ -40,6 +40,9 @@ var RepositorySet = wire.NewSet(
// 敏感词相关仓储
repository.NewSensitiveWordRepository,
// 用户资料审核相关仓储
repository.NewUserProfileAuditRepository,
)
// ProvideUserActivityRepository 提供用户活跃数据仓储

View File

@@ -61,6 +61,7 @@ var ServiceSet = wire.NewSet(
ProvideVerificationService,
ProvideAdminVerificationService,
ProvideSensitiveService,
ProvideUserProfileAuditService,
// 日志服务
ProvideAsyncLogManager,
@@ -465,3 +466,12 @@ func ProvideSensitiveService(
}
return service.NewSensitiveService(sensitiveWordRepo, redisClient, sensitiveCfg)
}
func ProvideUserProfileAuditService(
auditRepo repository.UserProfileAuditRepository,
userRepo repository.UserRepository,
hookManager *hook.Manager,
moderationHooks *hook.ModerationHooks,
) service.UserProfileAuditService {
return service.NewUserProfileAuditService(auditRepo, userRepo, hookManager, moderationHooks)
}