feat(moderation): add user profile content moderation for avatars, covers, and bios

internal/pkg/hook/user_profile_moderation_hooks.go

@@ -0,0 +1,317 @@
+package hook
+
+import (
+	"strings"
+	"time"
+
+	"carrot_bbs/internal/repository"
+
+	"go.uber.org/zap"
+)
+
+type UserAvatarModerationHook struct {
+	name          string
+	postAIService PostAIService
+	userRepo      repository.UserRepository
+	strictMode    bool
+}
+
+func NewUserAvatarModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserAvatarModerationHook {
+	return &UserAvatarModerationHook{
+		name:          "moderation.user.avatar.ai",
+		postAIService: postAIService,
+		userRepo:      userRepo,
+		strictMode:    strictMode,
+	}
+}
+
+func (h *UserAvatarModerationHook) Register(manager *Manager) {
+	manager.Register(&Hook{
+		Name:     h.name,
+		HookType: HookUserAvatarPreModerate,
+		Priority: PriorityHigh,
+		Func:     h.Execute,
+		Async:    false,
+		Timeout:  30 * time.Second,
+	})
+}
+
+func (h *UserAvatarModerationHook) Execute(ctx *HookContext) error {
+	data, ok := ctx.Data.(*UserAvatarModerateHookData)
+	if !ok {
+		return nil
+	}
+
+	result, ok := ctx.Metadata["result"].(*ModerationResult)
+	if !ok || result == nil {
+		return nil
+	}
+
+	if result.RejectReason != "" || result.NeedsReview {
+		return nil
+	}
+
+	if h.postAIService == nil || !h.postAIService.IsEnabled() {
+		result.Approved = true
+		result.ReviewedBy = "system"
+		return nil
+	}
+
+	err := h.postAIService.ModerateImage(ctx.Ctx, data.AvatarURL)
+	if err != nil {
+		if userErr, ok := err.(interface{ UserMessage() string }); ok {
+			if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
+				result.Approved = false
+				result.NeedsReview = true
+				result.RejectReason = userErr.UserMessage()
+				result.ReviewedBy = "ai"
+				zap.L().Info("User avatar flagged for manual review by AI",
+					zap.String("user_id", data.UserID),
+					zap.String("reason", result.RejectReason),
+				)
+				return nil
+			}
+			result.Approved = false
+			result.RejectReason = userErr.UserMessage()
+			result.ReviewedBy = "ai"
+			zap.L().Info("User avatar rejected by AI moderation",
+				zap.String("user_id", data.UserID),
+				zap.String("reason", result.RejectReason),
+			)
+			return nil
+		}
+
+		if h.strictMode {
+			result.Approved = false
+			result.Error = err
+			result.ReviewedBy = "ai"
+			zap.L().Error("AI avatar moderation failed in strict mode, reject",
+				zap.String("user_id", data.UserID),
+				zap.Error(err),
+			)
+			return nil
+		}
+
+		result.Approved = true
+		result.ReviewedBy = "system"
+		zap.L().Warn("AI moderation error, fallback approve avatar",
+			zap.String("user_id", data.UserID),
+			zap.Error(err),
+		)
+		return nil
+	}
+
+	result.Approved = true
+	result.ReviewedBy = "ai"
+	return nil
+}
+
+type UserCoverModerationHook struct {
+	name          string
+	postAIService PostAIService
+	userRepo      repository.UserRepository
+	strictMode    bool
+}
+
+func NewUserCoverModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserCoverModerationHook {
+	return &UserCoverModerationHook{
+		name:          "moderation.user.cover.ai",
+		postAIService: postAIService,
+		userRepo:      userRepo,
+		strictMode:    strictMode,
+	}
+}
+
+func (h *UserCoverModerationHook) Register(manager *Manager) {
+	manager.Register(&Hook{
+		Name:     h.name,
+		HookType: HookUserCoverPreModerate,
+		Priority: PriorityHigh,
+		Func:     h.Execute,
+		Async:    false,
+		Timeout:  30 * time.Second,
+	})
+}
+
+func (h *UserCoverModerationHook) Execute(ctx *HookContext) error {
+	data, ok := ctx.Data.(*UserCoverModerateHookData)
+	if !ok {
+		return nil
+	}
+
+	result, ok := ctx.Metadata["result"].(*ModerationResult)
+	if !ok || result == nil {
+		return nil
+	}
+
+	if result.RejectReason != "" || result.NeedsReview {
+		return nil
+	}
+
+	if h.postAIService == nil || !h.postAIService.IsEnabled() {
+		result.Approved = true
+		result.ReviewedBy = "system"
+		return nil
+	}
+
+	err := h.postAIService.ModerateImage(ctx.Ctx, data.CoverURL)
+	if err != nil {
+		if userErr, ok := err.(interface{ UserMessage() string }); ok {
+			if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
+				result.Approved = false
+				result.NeedsReview = true
+				result.RejectReason = userErr.UserMessage()
+				result.ReviewedBy = "ai"
+				zap.L().Info("User cover flagged for manual review by AI",
+					zap.String("user_id", data.UserID),
+					zap.String("reason", result.RejectReason),
+				)
+				return nil
+			}
+			result.Approved = false
+			result.RejectReason = userErr.UserMessage()
+			result.ReviewedBy = "ai"
+			zap.L().Info("User cover rejected by AI moderation",
+				zap.String("user_id", data.UserID),
+				zap.String("reason", result.RejectReason),
+			)
+			return nil
+		}
+
+		if h.strictMode {
+			result.Approved = false
+			result.Error = err
+			result.ReviewedBy = "ai"
+			zap.L().Error("AI cover moderation failed in strict mode, reject",
+				zap.String("user_id", data.UserID),
+				zap.Error(err),
+			)
+			return nil
+		}
+
+		result.Approved = true
+		result.ReviewedBy = "system"
+		zap.L().Warn("AI moderation error, fallback approve cover",
+			zap.String("user_id", data.UserID),
+			zap.Error(err),
+		)
+		return nil
+	}
+
+	result.Approved = true
+	result.ReviewedBy = "ai"
+	return nil
+}
+
+type UserBioModerationHook struct {
+	name             string
+	postAIService    PostAIService
+	sensitiveService SensitiveService
+	userRepo         repository.UserRepository
+	strictMode       bool
+}
+
+func NewUserBioModerationHook(postAIService PostAIService, sensitiveService SensitiveService, userRepo repository.UserRepository, strictMode bool) *UserBioModerationHook {
+	return &UserBioModerationHook{
+		name:             "moderation.user.bio.ai",
+		postAIService:    postAIService,
+		sensitiveService: sensitiveService,
+		userRepo:         userRepo,
+		strictMode:       strictMode,
+	}
+}
+
+func (h *UserBioModerationHook) Register(manager *Manager) {
+	manager.Register(&Hook{
+		Name:     h.name,
+		HookType: HookUserBioPreModerate,
+		Priority: PriorityHigh,
+		Func:     h.Execute,
+		Async:    false,
+		Timeout:  30 * time.Second,
+	})
+}
+
+func (h *UserBioModerationHook) Execute(ctx *HookContext) error {
+	data, ok := ctx.Data.(*UserBioModerateHookData)
+	if !ok {
+		return nil
+	}
+
+	result, ok := ctx.Metadata["result"].(*ModerationResult)
+	if !ok || result == nil {
+		return nil
+	}
+
+	if result.RejectReason != "" || result.NeedsReview {
+		return nil
+	}
+
+	if h.sensitiveService != nil {
+		hasSensitive, words := h.sensitiveService.Check(ctx.Ctx, data.Bio)
+		if hasSensitive {
+			result.Approved = false
+			result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
+			result.ReviewedBy = "sensitive_word"
+			zap.L().Info("User bio rejected by sensitive word filter",
+				zap.String("user_id", data.UserID),
+				zap.Strings("words", words),
+			)
+			return nil
+		}
+	}
+
+	if h.postAIService == nil || !h.postAIService.IsEnabled() {
+		result.Approved = true
+		result.ReviewedBy = "system"
+		return nil
+	}
+
+	err := h.postAIService.ModerateBio(ctx.Ctx, data.Bio)
+	if err != nil {
+		if userErr, ok := err.(interface{ UserMessage() string }); ok {
+			if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
+				result.Approved = false
+				result.NeedsReview = true
+				result.RejectReason = userErr.UserMessage()
+				result.ReviewedBy = "ai"
+				zap.L().Info("User bio flagged for manual review by AI",
+					zap.String("user_id", data.UserID),
+					zap.String("reason", result.RejectReason),
+				)
+				return nil
+			}
+			result.Approved = false
+			result.RejectReason = userErr.UserMessage()
+			result.ReviewedBy = "ai"
+			zap.L().Info("User bio rejected by AI moderation",
+				zap.String("user_id", data.UserID),
+				zap.String("reason", result.RejectReason),
+			)
+			return nil
+		}
+
+		if h.strictMode {
+			result.Approved = false
+			result.Error = err
+			result.ReviewedBy = "ai"
+			zap.L().Error("AI bio moderation failed in strict mode, reject",
+				zap.String("user_id", data.UserID),
+				zap.Error(err),
+			)
+			return nil
+		}
+
+		result.Approved = true
+		result.ReviewedBy = "system"
+		zap.L().Warn("AI moderation error, fallback approve bio",
+			zap.String("user_id", data.UserID),
+			zap.Error(err),
+		)
+		return nil
+	}
+
+	result.Approved = true
+	result.ReviewedBy = "ai"
+	return nil
+}