feat(moderation): add user profile content moderation for avatars, covers, and bios

2026-04-15 11:50:47 +08:00
parent b6f2df87c4
commit 90495385cd
23 changed files with 1561 additions and 55 deletions
--- a/internal/pkg/hook/hook.go
+++ b/internal/pkg/hook/hook.go
@@ -11,31 +11,37 @@ import (
 type HookType string

 const (
-	HookPostCreated        HookType = "post.created"
-	HookPostUpdated        HookType = "post.updated"
-	HookPostDeleted        HookType = "post.deleted"
-	HookPostPreModerate    HookType = "post.pre_moderate"
-	HookPostModerated      HookType = "post.moderated"
-	HookCommentCreated     HookType = "comment.created"
-	HookCommentDeleted     HookType = "comment.deleted"
-	HookCommentPreModerate HookType = "comment.pre_moderate"
-	HookCommentModerated   HookType = "comment.moderated"
-	HookUserCreated        HookType = "user.created"
-	HookUserUpdated        HookType = "user.updated"
-	HookUserDeleted        HookType = "user.deleted"
-	HookUserLogin          HookType = "user.login"
-	HookUserLogout         HookType = "user.logout"
-	HookGroupCreated       HookType = "group.created"
-	HookGroupUpdated       HookType = "group.updated"
-	HookGroupDeleted       HookType = "group.deleted"
-	HookGroupJoined        HookType = "group.joined"
-	HookGroupLeft          HookType = "group.left"
-	HookMessageSent        HookType = "message.sent"
-	HookMessageRead        HookType = "message.read"
-	HookNotificationSent   HookType = "notification.sent"
-	HookVoteCreated        HookType = "vote.created"
-	HookVoteUpdated        HookType = "vote.updated"
-	HookFileUploaded       HookType = "file.uploaded"
+	HookPostCreated           HookType = "post.created"
+	HookPostUpdated           HookType = "post.updated"
+	HookPostDeleted           HookType = "post.deleted"
+	HookPostPreModerate       HookType = "post.pre_moderate"
+	HookPostModerated         HookType = "post.moderated"
+	HookCommentCreated        HookType = "comment.created"
+	HookCommentDeleted        HookType = "comment.deleted"
+	HookCommentPreModerate    HookType = "comment.pre_moderate"
+	HookCommentModerated      HookType = "comment.moderated"
+	HookUserCreated           HookType = "user.created"
+	HookUserUpdated           HookType = "user.updated"
+	HookUserDeleted           HookType = "user.deleted"
+	HookUserLogin             HookType = "user.login"
+	HookUserLogout            HookType = "user.logout"
+	HookGroupCreated          HookType = "group.created"
+	HookGroupUpdated          HookType = "group.updated"
+	HookGroupDeleted          HookType = "group.deleted"
+	HookGroupJoined           HookType = "group.joined"
+	HookGroupLeft             HookType = "group.left"
+	HookMessageSent           HookType = "message.sent"
+	HookMessageRead           HookType = "message.read"
+	HookNotificationSent      HookType = "notification.sent"
+	HookVoteCreated           HookType = "vote.created"
+	HookVoteUpdated           HookType = "vote.updated"
+	HookFileUploaded          HookType = "file.uploaded"
+	HookUserAvatarPreModerate HookType = "user.avatar.pre_moderate"
+	HookUserAvatarModerated   HookType = "user.avatar.moderated"
+	HookUserCoverPreModerate  HookType = "user.cover.pre_moderate"
+	HookUserCoverModerated    HookType = "user.cover.moderated"
+	HookUserBioPreModerate    HookType = "user.bio.pre_moderate"
+	HookUserBioModerated      HookType = "user.bio.moderated"
 )

 type Priority int
--- a/internal/pkg/hook/moderation_hooks.go
+++ b/internal/pkg/hook/moderation_hooks.go
@@ -18,6 +18,8 @@ type PostAIService interface {
 	IsEnabled() bool
 	ModeratePost(ctx context.Context, title, content string, images []string) error
 	ModerateComment(ctx context.Context, content string, images []string) error
+	ModerateImage(ctx context.Context, imageURL string) error
+	ModerateBio(ctx context.Context, bio string) error
 }

 type PostModerationHook struct {
@@ -324,6 +326,7 @@ type ModerationHooks struct {
 	sensitiveService SensitiveService
 	postRepo         repository.PostRepository
 	commentRepo      repository.CommentRepository
+	userRepo         repository.UserRepository
 	strictMode       bool
 	replaceStr       string
 }
@@ -333,6 +336,7 @@ func NewModerationHooks(
 	sensitiveService SensitiveService,
 	postRepo repository.PostRepository,
 	commentRepo repository.CommentRepository,
+	userRepo repository.UserRepository,
 	strictMode bool,
 	replaceStr string,
 ) *ModerationHooks {
@@ -341,6 +345,7 @@ func NewModerationHooks(
 		sensitiveService: sensitiveService,
 		postRepo:         postRepo,
 		commentRepo:      commentRepo,
+		userRepo:         userRepo,
 		strictMode:       strictMode,
 		replaceStr:       replaceStr,
 	}
@@ -359,6 +364,15 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
 	commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
 	commentHook.Register(manager)

+	avatarHook := NewUserAvatarModerationHook(m.postAIService, m.userRepo, m.strictMode)
+	avatarHook.Register(manager)
+
+	coverHook := NewUserCoverModerationHook(m.postAIService, m.userRepo, m.strictMode)
+	coverHook.Register(manager)
+
+	bioHook := NewUserBioModerationHook(m.postAIService, m.sensitiveService, m.userRepo, m.strictMode)
+	bioHook.Register(manager)
+
 	zap.L().Info("Registered moderation hooks",
 		zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
 		zap.Bool("sensitive_enabled", m.sensitiveService != nil),
@@ -381,3 +395,27 @@ func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager,
 	})
 	return result
 }
+
+func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
+	result := &ModerationResult{}
+	manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
+		"result": result,
+	})
+	return result
+}
+
+func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
+	result := &ModerationResult{}
+	manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
+		"result": result,
+	})
+	return result
+}
+
+func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
+	result := &ModerationResult{}
+	manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
+		"result": result,
+	})
+	return result
+}
--- a/internal/pkg/hook/types.go
+++ b/internal/pkg/hook/types.go
@@ -101,6 +101,42 @@ type CommentModeratedHookData struct {
 	ReviewedBy   string
 }

+type UserAvatarModerateHookData struct {
+	UserID    string
+	AvatarURL string
+}
+
+type UserCoverModerateHookData struct {
+	UserID   string
+	CoverURL string
+}
+
+type UserBioModerateHookData struct {
+	UserID string
+	Bio    string
+}
+
+type UserAvatarModeratedHookData struct {
+	UserID       string
+	Approved     bool
+	RejectReason string
+	ReviewedBy   string
+}
+
+type UserCoverModeratedHookData struct {
+	UserID       string
+	Approved     bool
+	RejectReason string
+	ReviewedBy   string
+}
+
+type UserBioModeratedHookData struct {
+	UserID       string
+	Approved     bool
+	RejectReason string
+	ReviewedBy   string
+}
+
 type ModerationResult struct {
 	Approved     bool
 	NeedsReview  bool
@@ -271,3 +307,33 @@ func (h *HookTriggerHelper) TriggerCommentPreModerate(ctx context.Context, userI
 func (h *HookTriggerHelper) TriggerCommentModerated(ctx context.Context, userID uint, data *CommentModeratedHookData) {
 	h.manager.Trigger(ctx, HookCommentModerated, userID, data)
 }
+
+func (h *HookTriggerHelper) TriggerUserAvatarPreModerate(ctx context.Context, userID uint, data *UserAvatarModerateHookData, result *ModerationResult) {
+	h.manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, userID, data, map[string]any{
+		"result": result,
+	})
+}
+
+func (h *HookTriggerHelper) TriggerUserAvatarModerated(ctx context.Context, userID uint, data *UserAvatarModeratedHookData) {
+	h.manager.Trigger(ctx, HookUserAvatarModerated, userID, data)
+}
+
+func (h *HookTriggerHelper) TriggerUserCoverPreModerate(ctx context.Context, userID uint, data *UserCoverModerateHookData, result *ModerationResult) {
+	h.manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, userID, data, map[string]any{
+		"result": result,
+	})
+}
+
+func (h *HookTriggerHelper) TriggerUserCoverModerated(ctx context.Context, userID uint, data *UserCoverModeratedHookData) {
+	h.manager.Trigger(ctx, HookUserCoverModerated, userID, data)
+}
+
+func (h *HookTriggerHelper) TriggerUserBioPreModerate(ctx context.Context, userID uint, data *UserBioModerateHookData, result *ModerationResult) {
+	h.manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, userID, data, map[string]any{
+		"result": result,
+	})
+}
+
+func (h *HookTriggerHelper) TriggerUserBioModerated(ctx context.Context, userID uint, data *UserBioModeratedHookData) {
+	h.manager.Trigger(ctx, HookUserBioModerated, userID, data)
+}
--- a/internal/pkg/hook/user_profile_moderation_hooks.go
+++ b/internal/pkg/hook/user_profile_moderation_hooks.go
@@ -0,0 +1,317 @@
+package hook
+
+import (
+	"strings"
+	"time"
+
+	"carrot_bbs/internal/repository"
+
+	"go.uber.org/zap"
+)
+
+type UserAvatarModerationHook struct {
+	name          string
+	postAIService PostAIService
+	userRepo      repository.UserRepository
+	strictMode    bool
+}
+
+func NewUserAvatarModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserAvatarModerationHook {
+	return &UserAvatarModerationHook{
+		name:          "moderation.user.avatar.ai",
+		postAIService: postAIService,
+		userRepo:      userRepo,
+		strictMode:    strictMode,
+	}
+}
+
+func (h *UserAvatarModerationHook) Register(manager *Manager) {
+	manager.Register(&Hook{
+		Name:     h.name,
+		HookType: HookUserAvatarPreModerate,
+		Priority: PriorityHigh,
+		Func:     h.Execute,
+		Async:    false,
+		Timeout:  30 * time.Second,
+	})
+}
+
+func (h *UserAvatarModerationHook) Execute(ctx *HookContext) error {
+	data, ok := ctx.Data.(*UserAvatarModerateHookData)
+	if !ok {
+		return nil
+	}
+
+	result, ok := ctx.Metadata["result"].(*ModerationResult)
+	if !ok || result == nil {
+		return nil
+	}
+
+	if result.RejectReason != "" || result.NeedsReview {
+		return nil
+	}
+
+	if h.postAIService == nil || !h.postAIService.IsEnabled() {
+		result.Approved = true
+		result.ReviewedBy = "system"
+		return nil
+	}
+
+	err := h.postAIService.ModerateImage(ctx.Ctx, data.AvatarURL)
+	if err != nil {
+		if userErr, ok := err.(interface{ UserMessage() string }); ok {
+			if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
+				result.Approved = false
+				result.NeedsReview = true
+				result.RejectReason = userErr.UserMessage()
+				result.ReviewedBy = "ai"
+				zap.L().Info("User avatar flagged for manual review by AI",
+					zap.String("user_id", data.UserID),
+					zap.String("reason", result.RejectReason),
+				)
+				return nil
+			}
+			result.Approved = false
+			result.RejectReason = userErr.UserMessage()
+			result.ReviewedBy = "ai"
+			zap.L().Info("User avatar rejected by AI moderation",
+				zap.String("user_id", data.UserID),
+				zap.String("reason", result.RejectReason),
+			)
+			return nil
+		}
+
+		if h.strictMode {
+			result.Approved = false
+			result.Error = err
+			result.ReviewedBy = "ai"
+			zap.L().Error("AI avatar moderation failed in strict mode, reject",
+				zap.String("user_id", data.UserID),
+				zap.Error(err),
+			)
+			return nil
+		}
+
+		result.Approved = true
+		result.ReviewedBy = "system"
+		zap.L().Warn("AI moderation error, fallback approve avatar",
+			zap.String("user_id", data.UserID),
+			zap.Error(err),
+		)
+		return nil
+	}
+
+	result.Approved = true
+	result.ReviewedBy = "ai"
+	return nil
+}
+
+type UserCoverModerationHook struct {
+	name          string
+	postAIService PostAIService
+	userRepo      repository.UserRepository
+	strictMode    bool
+}
+
+func NewUserCoverModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserCoverModerationHook {
+	return &UserCoverModerationHook{
+		name:          "moderation.user.cover.ai",
+		postAIService: postAIService,
+		userRepo:      userRepo,
+		strictMode:    strictMode,
+	}
+}
+
+func (h *UserCoverModerationHook) Register(manager *Manager) {
+	manager.Register(&Hook{
+		Name:     h.name,
+		HookType: HookUserCoverPreModerate,
+		Priority: PriorityHigh,
+		Func:     h.Execute,
+		Async:    false,
+		Timeout:  30 * time.Second,
+	})
+}
+
+func (h *UserCoverModerationHook) Execute(ctx *HookContext) error {
+	data, ok := ctx.Data.(*UserCoverModerateHookData)
+	if !ok {
+		return nil
+	}
+
+	result, ok := ctx.Metadata["result"].(*ModerationResult)
+	if !ok || result == nil {
+		return nil
+	}
+
+	if result.RejectReason != "" || result.NeedsReview {
+		return nil
+	}
+
+	if h.postAIService == nil || !h.postAIService.IsEnabled() {
+		result.Approved = true
+		result.ReviewedBy = "system"
+		return nil
+	}
+
+	err := h.postAIService.ModerateImage(ctx.Ctx, data.CoverURL)
+	if err != nil {
+		if userErr, ok := err.(interface{ UserMessage() string }); ok {
+			if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
+				result.Approved = false
+				result.NeedsReview = true
+				result.RejectReason = userErr.UserMessage()
+				result.ReviewedBy = "ai"
+				zap.L().Info("User cover flagged for manual review by AI",
+					zap.String("user_id", data.UserID),
+					zap.String("reason", result.RejectReason),
+				)
+				return nil
+			}
+			result.Approved = false
+			result.RejectReason = userErr.UserMessage()
+			result.ReviewedBy = "ai"
+			zap.L().Info("User cover rejected by AI moderation",
+				zap.String("user_id", data.UserID),
+				zap.String("reason", result.RejectReason),
+			)
+			return nil
+		}
+
+		if h.strictMode {
+			result.Approved = false
+			result.Error = err
+			result.ReviewedBy = "ai"
+			zap.L().Error("AI cover moderation failed in strict mode, reject",
+				zap.String("user_id", data.UserID),
+				zap.Error(err),
+			)
+			return nil
+		}
+
+		result.Approved = true
+		result.ReviewedBy = "system"
+		zap.L().Warn("AI moderation error, fallback approve cover",
+			zap.String("user_id", data.UserID),
+			zap.Error(err),
+		)
+		return nil
+	}
+
+	result.Approved = true
+	result.ReviewedBy = "ai"
+	return nil
+}
+
+type UserBioModerationHook struct {
+	name             string
+	postAIService    PostAIService
+	sensitiveService SensitiveService
+	userRepo         repository.UserRepository
+	strictMode       bool
+}
+
+func NewUserBioModerationHook(postAIService PostAIService, sensitiveService SensitiveService, userRepo repository.UserRepository, strictMode bool) *UserBioModerationHook {
+	return &UserBioModerationHook{
+		name:             "moderation.user.bio.ai",
+		postAIService:    postAIService,
+		sensitiveService: sensitiveService,
+		userRepo:         userRepo,
+		strictMode:       strictMode,
+	}
+}
+
+func (h *UserBioModerationHook) Register(manager *Manager) {
+	manager.Register(&Hook{
+		Name:     h.name,
+		HookType: HookUserBioPreModerate,
+		Priority: PriorityHigh,
+		Func:     h.Execute,
+		Async:    false,
+		Timeout:  30 * time.Second,
+	})
+}
+
+func (h *UserBioModerationHook) Execute(ctx *HookContext) error {
+	data, ok := ctx.Data.(*UserBioModerateHookData)
+	if !ok {
+		return nil
+	}
+
+	result, ok := ctx.Metadata["result"].(*ModerationResult)
+	if !ok || result == nil {
+		return nil
+	}
+
+	if result.RejectReason != "" || result.NeedsReview {
+		return nil
+	}
+
+	if h.sensitiveService != nil {
+		hasSensitive, words := h.sensitiveService.Check(ctx.Ctx, data.Bio)
+		if hasSensitive {
+			result.Approved = false
+			result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
+			result.ReviewedBy = "sensitive_word"
+			zap.L().Info("User bio rejected by sensitive word filter",
+				zap.String("user_id", data.UserID),
+				zap.Strings("words", words),
+			)
+			return nil
+		}
+	}
+
+	if h.postAIService == nil || !h.postAIService.IsEnabled() {
+		result.Approved = true
+		result.ReviewedBy = "system"
+		return nil
+	}
+
+	err := h.postAIService.ModerateBio(ctx.Ctx, data.Bio)
+	if err != nil {
+		if userErr, ok := err.(interface{ UserMessage() string }); ok {
+			if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
+				result.Approved = false
+				result.NeedsReview = true
+				result.RejectReason = userErr.UserMessage()
+				result.ReviewedBy = "ai"
+				zap.L().Info("User bio flagged for manual review by AI",
+					zap.String("user_id", data.UserID),
+					zap.String("reason", result.RejectReason),
+				)
+				return nil
+			}
+			result.Approved = false
+			result.RejectReason = userErr.UserMessage()
+			result.ReviewedBy = "ai"
+			zap.L().Info("User bio rejected by AI moderation",
+				zap.String("user_id", data.UserID),
+				zap.String("reason", result.RejectReason),
+			)
+			return nil
+		}
+
+		if h.strictMode {
+			result.Approved = false
+			result.Error = err
+			result.ReviewedBy = "ai"
+			zap.L().Error("AI bio moderation failed in strict mode, reject",
+				zap.String("user_id", data.UserID),
+				zap.Error(err),
+			)
+			return nil
+		}
+
+		result.Approved = true
+		result.ReviewedBy = "system"
+		zap.L().Warn("AI moderation error, fallback approve bio",
+			zap.String("user_id", data.UserID),
+			zap.Error(err),
+		)
+		return nil
+	}
+
+	result.Approved = true
+	result.ReviewedBy = "ai"
+	return nil
+}