feat(moderation): add user profile content moderation for avatars, covers, and bios
All checks were successful
Build Backend / build (push) Successful in 4m24s
Build Backend / build-docker (push) Successful in 1m14s

This commit is contained in:
lafay
2026-04-15 11:50:47 +08:00
parent b6f2df87c4
commit 90495385cd
23 changed files with 1561 additions and 55 deletions

View File

@@ -11,31 +11,37 @@ import (
type HookType string
const (
HookPostCreated HookType = "post.created"
HookPostUpdated HookType = "post.updated"
HookPostDeleted HookType = "post.deleted"
HookPostPreModerate HookType = "post.pre_moderate"
HookPostModerated HookType = "post.moderated"
HookCommentCreated HookType = "comment.created"
HookCommentDeleted HookType = "comment.deleted"
HookCommentPreModerate HookType = "comment.pre_moderate"
HookCommentModerated HookType = "comment.moderated"
HookUserCreated HookType = "user.created"
HookUserUpdated HookType = "user.updated"
HookUserDeleted HookType = "user.deleted"
HookUserLogin HookType = "user.login"
HookUserLogout HookType = "user.logout"
HookGroupCreated HookType = "group.created"
HookGroupUpdated HookType = "group.updated"
HookGroupDeleted HookType = "group.deleted"
HookGroupJoined HookType = "group.joined"
HookGroupLeft HookType = "group.left"
HookMessageSent HookType = "message.sent"
HookMessageRead HookType = "message.read"
HookNotificationSent HookType = "notification.sent"
HookVoteCreated HookType = "vote.created"
HookVoteUpdated HookType = "vote.updated"
HookFileUploaded HookType = "file.uploaded"
HookPostCreated HookType = "post.created"
HookPostUpdated HookType = "post.updated"
HookPostDeleted HookType = "post.deleted"
HookPostPreModerate HookType = "post.pre_moderate"
HookPostModerated HookType = "post.moderated"
HookCommentCreated HookType = "comment.created"
HookCommentDeleted HookType = "comment.deleted"
HookCommentPreModerate HookType = "comment.pre_moderate"
HookCommentModerated HookType = "comment.moderated"
HookUserCreated HookType = "user.created"
HookUserUpdated HookType = "user.updated"
HookUserDeleted HookType = "user.deleted"
HookUserLogin HookType = "user.login"
HookUserLogout HookType = "user.logout"
HookGroupCreated HookType = "group.created"
HookGroupUpdated HookType = "group.updated"
HookGroupDeleted HookType = "group.deleted"
HookGroupJoined HookType = "group.joined"
HookGroupLeft HookType = "group.left"
HookMessageSent HookType = "message.sent"
HookMessageRead HookType = "message.read"
HookNotificationSent HookType = "notification.sent"
HookVoteCreated HookType = "vote.created"
HookVoteUpdated HookType = "vote.updated"
HookFileUploaded HookType = "file.uploaded"
HookUserAvatarPreModerate HookType = "user.avatar.pre_moderate"
HookUserAvatarModerated HookType = "user.avatar.moderated"
HookUserCoverPreModerate HookType = "user.cover.pre_moderate"
HookUserCoverModerated HookType = "user.cover.moderated"
HookUserBioPreModerate HookType = "user.bio.pre_moderate"
HookUserBioModerated HookType = "user.bio.moderated"
)
type Priority int

View File

@@ -18,6 +18,8 @@ type PostAIService interface {
IsEnabled() bool
ModeratePost(ctx context.Context, title, content string, images []string) error
ModerateComment(ctx context.Context, content string, images []string) error
ModerateImage(ctx context.Context, imageURL string) error
ModerateBio(ctx context.Context, bio string) error
}
type PostModerationHook struct {
@@ -324,6 +326,7 @@ type ModerationHooks struct {
sensitiveService SensitiveService
postRepo repository.PostRepository
commentRepo repository.CommentRepository
userRepo repository.UserRepository
strictMode bool
replaceStr string
}
@@ -333,6 +336,7 @@ func NewModerationHooks(
sensitiveService SensitiveService,
postRepo repository.PostRepository,
commentRepo repository.CommentRepository,
userRepo repository.UserRepository,
strictMode bool,
replaceStr string,
) *ModerationHooks {
@@ -341,6 +345,7 @@ func NewModerationHooks(
sensitiveService: sensitiveService,
postRepo: postRepo,
commentRepo: commentRepo,
userRepo: userRepo,
strictMode: strictMode,
replaceStr: replaceStr,
}
@@ -359,6 +364,15 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager)
avatarHook := NewUserAvatarModerationHook(m.postAIService, m.userRepo, m.strictMode)
avatarHook.Register(manager)
coverHook := NewUserCoverModerationHook(m.postAIService, m.userRepo, m.strictMode)
coverHook.Register(manager)
bioHook := NewUserBioModerationHook(m.postAIService, m.sensitiveService, m.userRepo, m.strictMode)
bioHook.Register(manager)
zap.L().Info("Registered moderation hooks",
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
zap.Bool("sensitive_enabled", m.sensitiveService != nil),
@@ -381,3 +395,27 @@ func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager,
})
return result
}
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
"result": result,
})
return result
}

View File

@@ -101,6 +101,42 @@ type CommentModeratedHookData struct {
ReviewedBy string
}
type UserAvatarModerateHookData struct {
UserID string
AvatarURL string
}
type UserCoverModerateHookData struct {
UserID string
CoverURL string
}
type UserBioModerateHookData struct {
UserID string
Bio string
}
type UserAvatarModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type UserCoverModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type UserBioModeratedHookData struct {
UserID string
Approved bool
RejectReason string
ReviewedBy string
}
type ModerationResult struct {
Approved bool
NeedsReview bool
@@ -271,3 +307,33 @@ func (h *HookTriggerHelper) TriggerCommentPreModerate(ctx context.Context, userI
func (h *HookTriggerHelper) TriggerCommentModerated(ctx context.Context, userID uint, data *CommentModeratedHookData) {
h.manager.Trigger(ctx, HookCommentModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserAvatarPreModerate(ctx context.Context, userID uint, data *UserAvatarModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserAvatarModerated(ctx context.Context, userID uint, data *UserAvatarModeratedHookData) {
h.manager.Trigger(ctx, HookUserAvatarModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserCoverPreModerate(ctx context.Context, userID uint, data *UserCoverModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserCoverModerated(ctx context.Context, userID uint, data *UserCoverModeratedHookData) {
h.manager.Trigger(ctx, HookUserCoverModerated, userID, data)
}
func (h *HookTriggerHelper) TriggerUserBioPreModerate(ctx context.Context, userID uint, data *UserBioModerateHookData, result *ModerationResult) {
h.manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, userID, data, map[string]any{
"result": result,
})
}
func (h *HookTriggerHelper) TriggerUserBioModerated(ctx context.Context, userID uint, data *UserBioModeratedHookData) {
h.manager.Trigger(ctx, HookUserBioModerated, userID, data)
}

View File

@@ -0,0 +1,317 @@
package hook
import (
"strings"
"time"
"carrot_bbs/internal/repository"
"go.uber.org/zap"
)
type UserAvatarModerationHook struct {
name string
postAIService PostAIService
userRepo repository.UserRepository
strictMode bool
}
func NewUserAvatarModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserAvatarModerationHook {
return &UserAvatarModerationHook{
name: "moderation.user.avatar.ai",
postAIService: postAIService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserAvatarModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserAvatarPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserAvatarModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserAvatarModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateImage(ctx.Ctx, data.AvatarURL)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User avatar flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User avatar rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI avatar moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve avatar",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}
type UserCoverModerationHook struct {
name string
postAIService PostAIService
userRepo repository.UserRepository
strictMode bool
}
func NewUserCoverModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserCoverModerationHook {
return &UserCoverModerationHook{
name: "moderation.user.cover.ai",
postAIService: postAIService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserCoverModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserCoverPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserCoverModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserCoverModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateImage(ctx.Ctx, data.CoverURL)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User cover flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User cover rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI cover moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve cover",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}
type UserBioModerationHook struct {
name string
postAIService PostAIService
sensitiveService SensitiveService
userRepo repository.UserRepository
strictMode bool
}
func NewUserBioModerationHook(postAIService PostAIService, sensitiveService SensitiveService, userRepo repository.UserRepository, strictMode bool) *UserBioModerationHook {
return &UserBioModerationHook{
name: "moderation.user.bio.ai",
postAIService: postAIService,
sensitiveService: sensitiveService,
userRepo: userRepo,
strictMode: strictMode,
}
}
func (h *UserBioModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookUserBioPreModerate,
Priority: PriorityHigh,
Func: h.Execute,
Async: false,
Timeout: 30 * time.Second,
})
}
func (h *UserBioModerationHook) Execute(ctx *HookContext) error {
data, ok := ctx.Data.(*UserBioModerateHookData)
if !ok {
return nil
}
result, ok := ctx.Metadata["result"].(*ModerationResult)
if !ok || result == nil {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.sensitiveService != nil {
hasSensitive, words := h.sensitiveService.Check(ctx.Ctx, data.Bio)
if hasSensitive {
result.Approved = false
result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
result.ReviewedBy = "sensitive_word"
zap.L().Info("User bio rejected by sensitive word filter",
zap.String("user_id", data.UserID),
zap.Strings("words", words),
)
return nil
}
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
return nil
}
err := h.postAIService.ModerateBio(ctx.Ctx, data.Bio)
if err != nil {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User bio flagged for manual review by AI",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("User bio rejected by AI moderation",
zap.String("user_id", data.UserID),
zap.String("reason", result.RejectReason),
)
return nil
}
if h.strictMode {
result.Approved = false
result.Error = err
result.ReviewedBy = "ai"
zap.L().Error("AI bio moderation failed in strict mode, reject",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI moderation error, fallback approve bio",
zap.String("user_id", data.UserID),
zap.Error(err),
)
return nil
}
result.Approved = true
result.ReviewedBy = "ai"
return nil
}

View File

@@ -93,6 +93,8 @@ type Client interface {
Config() Config
ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error)
ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error)
ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error)
ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error)
}
type clientImpl struct {
@@ -163,6 +165,157 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
return c.moderateContentInBatches(ctx, prompt.String(), images)
}
const imageModerationSystemPrompt = `你是图片内容审核助手负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
硬性违规(命中任一项必须 result=block
- 色情低俗、裸露、性暗示内容
- 暴力血腥、恐怖画面
- 违法犯罪相关内容
- 明确的政治敏感宣传
- 恶意广告、诈骗信息
- 仇恨歧视符号或标志
可疑内容result=review
- 图片含义模糊,无法确定是否存在违规
- 轻微争议性内容但不完全违规
- AI无法准确判断的内容
放行规则result=pass
- 正常风景、人物、动物、动漫等图片
- 表情包、梗图
- 正常的艺术创作、摄影作品
输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}`
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
硬性违规(命中任一项必须 result=block
- 明确的违法犯罪、暴力威胁内容
- 色情低俗、性暗示描述
- 诈骗引流、恶意广告
- 明确煽动群体对立、仇恨
- 恶意公开他人隐私信息
可疑内容result=review
- 内容含义模糊,无法确定违规意图
- 涉及敏感话题但未明确违规
- AI无法准确判断的表达
放行规则result=pass
- 正常的自我介绍、心情表达
- 玩梗、幽默、吐槽
- 引用、歌词、诗句等
- 理性的观点表达
输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}`
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
if strings.TrimSpace(imageURL) == "" {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
userPrompt := "请审核以下用户图片"
cleanImages := normalizeImageURLs([]string{imageURL})
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
}
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
if strings.TrimSpace(bio) == "" {
return &ModerationResponse{Result: ModerationResultPass}, nil
}
userPrompt := "个性签名内容:" + bio
return c.moderateWithCustomPrompt(ctx, bioModerationSystemPrompt, userPrompt)
}
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
}
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
cleanImages := normalizeImageURLs(images)
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)