a971fb0e29
The moderation system now uses a "pass/review" strategy instead of "pass/review/block", treating hard violations the same as review (only routing to manual review rather than auto-rejecting). Add fallback-approve behavior in the hook system so slow moderation APIs never silently block normal content when they time out or get cancelled - timed-out moderation now approves with "ReviewedBy: system" instead of leaving the zero-value result that would reject content. Update post/comment/vote services and handlers to use the new unified behavior. Simplify moderation prompts to emphasize "宁可放过，不可误伤" (prefer false positives over false negatives).
452 lines
12 KiB
Go
452 lines
12 KiB
Go
package hook
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"with_you/internal/repository"
|
|
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type reviewableError interface {
|
|
IsReview() bool
|
|
}
|
|
|
|
type PostAIService interface {
|
|
IsEnabled() bool
|
|
ModeratePost(ctx context.Context, title, content string, images []string) error
|
|
ModerateComment(ctx context.Context, content string, images []string) error
|
|
ModerateImage(ctx context.Context, imageURL string) error
|
|
ModerateBio(ctx context.Context, bio string) error
|
|
}
|
|
|
|
type PostModerationHook struct {
|
|
name string
|
|
postAIService PostAIService
|
|
postRepo repository.PostRepository
|
|
strictMode bool
|
|
}
|
|
|
|
func NewPostModerationHook(postAIService PostAIService, postRepo repository.PostRepository, strictMode bool) *PostModerationHook {
|
|
return &PostModerationHook{
|
|
name: "moderation.post.ai",
|
|
postAIService: postAIService,
|
|
postRepo: postRepo,
|
|
strictMode: strictMode,
|
|
}
|
|
}
|
|
|
|
func (h *PostModerationHook) Register(manager *Manager) {
|
|
manager.Register(&Hook{
|
|
Name: h.name,
|
|
HookType: HookPostPreModerate,
|
|
Priority: PriorityHigh,
|
|
Func: h.Execute,
|
|
Async: false,
|
|
Timeout: 30 * time.Second,
|
|
})
|
|
}
|
|
|
|
func (h *PostModerationHook) Execute(ctx *HookContext) error {
|
|
data, ok := ctx.Data.(*PostModerateHookData)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
|
if !ok || result == nil {
|
|
return nil
|
|
}
|
|
|
|
if result.RejectReason != "" || result.NeedsReview {
|
|
return nil
|
|
}
|
|
|
|
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
|
result.Approved = true
|
|
result.ReviewedBy = "system"
|
|
return nil
|
|
}
|
|
|
|
err := h.postAIService.ModeratePost(ctx.Ctx, data.Title, data.Content, data.Images)
|
|
if err != nil {
|
|
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
|
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
|
result.Approved = false
|
|
result.NeedsReview = true
|
|
result.RejectReason = userErr.UserMessage()
|
|
result.ReviewedBy = "ai"
|
|
zap.L().Info("Post flagged for manual review by AI",
|
|
zap.String("post_id", data.PostID),
|
|
zap.String("reason", result.RejectReason),
|
|
)
|
|
return nil
|
|
}
|
|
result.Approved = false
|
|
result.RejectReason = userErr.UserMessage()
|
|
result.ReviewedBy = "ai"
|
|
zap.L().Info("Post rejected by AI moderation",
|
|
zap.String("post_id", data.PostID),
|
|
zap.String("reason", result.RejectReason),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
if h.strictMode {
|
|
result.Approved = false
|
|
result.Error = err
|
|
result.ReviewedBy = "ai"
|
|
zap.L().Error("AI post moderation failed in strict mode, reject post",
|
|
zap.String("post_id", data.PostID),
|
|
zap.Error(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
result.Approved = true
|
|
result.ReviewedBy = "system"
|
|
zap.L().Warn("AI moderation error, fallback approve",
|
|
zap.String("post_id", data.PostID),
|
|
zap.Error(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
result.Approved = true
|
|
result.ReviewedBy = "ai"
|
|
return nil
|
|
}
|
|
|
|
type CommentModerationHook struct {
|
|
name string
|
|
postAIService PostAIService
|
|
commentRepo repository.CommentRepository
|
|
strictMode bool
|
|
}
|
|
|
|
func NewCommentModerationHook(postAIService PostAIService, commentRepo repository.CommentRepository, strictMode bool) *CommentModerationHook {
|
|
return &CommentModerationHook{
|
|
name: "moderation.comment.ai",
|
|
postAIService: postAIService,
|
|
commentRepo: commentRepo,
|
|
strictMode: strictMode,
|
|
}
|
|
}
|
|
|
|
func (h *CommentModerationHook) Register(manager *Manager) {
|
|
manager.Register(&Hook{
|
|
Name: h.name,
|
|
HookType: HookCommentPreModerate,
|
|
Priority: PriorityHigh,
|
|
Func: h.Execute,
|
|
Async: false,
|
|
Timeout: 30 * time.Second,
|
|
})
|
|
}
|
|
|
|
func (h *CommentModerationHook) Execute(ctx *HookContext) error {
|
|
data, ok := ctx.Data.(*CommentModerateHookData)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
|
if !ok || result == nil {
|
|
return nil
|
|
}
|
|
|
|
if result.RejectReason != "" || result.NeedsReview {
|
|
return nil
|
|
}
|
|
|
|
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
|
result.Approved = true
|
|
result.ReviewedBy = "system"
|
|
return nil
|
|
}
|
|
|
|
err := h.postAIService.ModerateComment(ctx.Ctx, data.Content, data.Images)
|
|
if err != nil {
|
|
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
|
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
|
result.Approved = false
|
|
result.NeedsReview = true
|
|
result.RejectReason = userErr.UserMessage()
|
|
result.ReviewedBy = "ai"
|
|
zap.L().Info("Comment flagged for manual review by AI",
|
|
zap.String("comment_id", data.CommentID),
|
|
zap.String("reason", result.RejectReason),
|
|
)
|
|
return nil
|
|
}
|
|
result.Approved = false
|
|
result.RejectReason = userErr.UserMessage()
|
|
result.ReviewedBy = "ai"
|
|
zap.L().Info("Comment rejected by AI moderation",
|
|
zap.String("comment_id", data.CommentID),
|
|
zap.String("reason", result.RejectReason),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
if h.strictMode {
|
|
result.Approved = false
|
|
result.Error = err
|
|
result.ReviewedBy = "ai"
|
|
zap.L().Error("AI comment moderation failed in strict mode, reject comment",
|
|
zap.String("comment_id", data.CommentID),
|
|
zap.Error(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
result.Approved = true
|
|
result.ReviewedBy = "system"
|
|
zap.L().Warn("AI moderation error, fallback approve",
|
|
zap.String("comment_id", data.CommentID),
|
|
zap.Error(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
result.Approved = true
|
|
result.ReviewedBy = "ai"
|
|
return nil
|
|
}
|
|
|
|
type SensitiveService interface {
|
|
Check(ctx context.Context, text string) (bool, []string)
|
|
Replace(ctx context.Context, text string, repl string) string
|
|
}
|
|
|
|
type SensitiveWordHook struct {
|
|
name string
|
|
sensitive SensitiveService
|
|
replaceStr string
|
|
}
|
|
|
|
func NewSensitiveWordHook(sensitive SensitiveService, replaceStr string) *SensitiveWordHook {
|
|
return &SensitiveWordHook{
|
|
name: "moderation.sensitive_word",
|
|
sensitive: sensitive,
|
|
replaceStr: replaceStr,
|
|
}
|
|
}
|
|
|
|
func (h *SensitiveWordHook) RegisterPostHook(manager *Manager) {
|
|
manager.Register(&Hook{
|
|
Name: h.name + ".post",
|
|
HookType: HookPostPreModerate,
|
|
Priority: PriorityHighest,
|
|
Func: h.executePost,
|
|
Async: false,
|
|
})
|
|
}
|
|
|
|
func (h *SensitiveWordHook) RegisterCommentHook(manager *Manager) {
|
|
manager.Register(&Hook{
|
|
Name: h.name + ".comment",
|
|
HookType: HookCommentPreModerate,
|
|
Priority: PriorityHighest,
|
|
Func: h.executeComment,
|
|
Async: false,
|
|
})
|
|
}
|
|
|
|
func (h *SensitiveWordHook) executePost(ctx *HookContext) error {
|
|
if h.sensitive == nil {
|
|
return nil
|
|
}
|
|
|
|
data, ok := ctx.Data.(*PostModerateHookData)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
|
if !ok || result == nil {
|
|
return nil
|
|
}
|
|
|
|
if result.RejectReason != "" {
|
|
return nil
|
|
}
|
|
|
|
hasSensitive, words := h.sensitive.Check(ctx.Ctx, data.Title+" "+data.Content)
|
|
if hasSensitive {
|
|
result.Approved = false
|
|
result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
|
|
result.ReviewedBy = "sensitive_word"
|
|
zap.L().Info("Post rejected by sensitive word filter",
|
|
zap.String("post_id", data.PostID),
|
|
zap.Strings("words", words),
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (h *SensitiveWordHook) executeComment(ctx *HookContext) error {
|
|
if h.sensitive == nil {
|
|
return nil
|
|
}
|
|
|
|
data, ok := ctx.Data.(*CommentModerateHookData)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
|
if !ok || result == nil {
|
|
return nil
|
|
}
|
|
|
|
if result.RejectReason != "" {
|
|
return nil
|
|
}
|
|
|
|
hasSensitive, words := h.sensitive.Check(ctx.Ctx, data.Content)
|
|
if hasSensitive {
|
|
result.Approved = false
|
|
result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
|
|
result.ReviewedBy = "sensitive_word"
|
|
zap.L().Info("Comment rejected by sensitive word filter",
|
|
zap.String("comment_id", data.CommentID),
|
|
zap.Strings("words", words),
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type ModerationHooks struct {
|
|
postAIService PostAIService
|
|
sensitiveService SensitiveService
|
|
postRepo repository.PostRepository
|
|
commentRepo repository.CommentRepository
|
|
userRepo repository.UserRepository
|
|
strictMode bool
|
|
replaceStr string
|
|
}
|
|
|
|
func NewModerationHooks(
|
|
postAIService PostAIService,
|
|
sensitiveService SensitiveService,
|
|
postRepo repository.PostRepository,
|
|
commentRepo repository.CommentRepository,
|
|
userRepo repository.UserRepository,
|
|
strictMode bool,
|
|
replaceStr string,
|
|
) *ModerationHooks {
|
|
return &ModerationHooks{
|
|
postAIService: postAIService,
|
|
sensitiveService: sensitiveService,
|
|
postRepo: postRepo,
|
|
commentRepo: commentRepo,
|
|
userRepo: userRepo,
|
|
strictMode: strictMode,
|
|
replaceStr: replaceStr,
|
|
}
|
|
}
|
|
|
|
func (m *ModerationHooks) RegisterAll(manager *Manager) {
|
|
if m.sensitiveService != nil {
|
|
sensitiveHook := NewSensitiveWordHook(m.sensitiveService, m.replaceStr)
|
|
sensitiveHook.RegisterPostHook(manager)
|
|
sensitiveHook.RegisterCommentHook(manager)
|
|
}
|
|
|
|
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
|
|
postHook.Register(manager)
|
|
|
|
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
|
|
commentHook.Register(manager)
|
|
|
|
avatarHook := NewUserAvatarModerationHook(m.postAIService, m.userRepo, m.strictMode)
|
|
avatarHook.Register(manager)
|
|
|
|
coverHook := NewUserCoverModerationHook(m.postAIService, m.userRepo, m.strictMode)
|
|
coverHook.Register(manager)
|
|
|
|
bioHook := NewUserBioModerationHook(m.postAIService, m.sensitiveService, m.userRepo, m.strictMode)
|
|
bioHook.Register(manager)
|
|
|
|
zap.L().Info("Registered moderation hooks",
|
|
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
|
|
zap.Bool("sensitive_enabled", m.sensitiveService != nil),
|
|
zap.Bool("strict_mode", m.strictMode),
|
|
)
|
|
}
|
|
|
|
func (m *ModerationHooks) ModeratePost(ctx context.Context, manager *Manager, data *PostModerateHookData) *ModerationResult {
|
|
result := &ModerationResult{}
|
|
if err := manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{
|
|
"result": result,
|
|
}); err != nil {
|
|
applyModerationFallback(result, err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager, data *CommentModerateHookData) *ModerationResult {
|
|
result := &ModerationResult{}
|
|
if err := manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{
|
|
"result": result,
|
|
}); err != nil {
|
|
applyModerationFallback(result, err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
|
|
result := &ModerationResult{}
|
|
if err := manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
|
|
"result": result,
|
|
}); err != nil {
|
|
applyModerationFallback(result, err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
|
|
result := &ModerationResult{}
|
|
if err := manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
|
|
"result": result,
|
|
}); err != nil {
|
|
applyModerationFallback(result, err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
|
|
result := &ModerationResult{}
|
|
if err := manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
|
|
"result": result,
|
|
}); err != nil {
|
|
applyModerationFallback(result, err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// applyModerationFallback handles moderation hook timeouts/cancellations by
|
|
// falling back to "approved by system" instead of leaving the zero-value
|
|
// result (Approved=false) which would silently reject otherwise-fine content.
|
|
func applyModerationFallback(result *ModerationResult, err error) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
|
result.Approved = true
|
|
result.ReviewedBy = "system"
|
|
result.RejectReason = ""
|
|
result.NeedsReview = false
|
|
result.Error = err
|
|
zap.L().Warn("Moderation hook timed out or was cancelled, fallback approve",
|
|
zap.Error(err),
|
|
)
|
|
}
|
|
}
|