feat(moderation): add manual content review system with three-tier AI moderation
Add admin comment moderation endpoints (single and batch) and introduce a three-tier moderation system (pass/review/block) for content flagged by AI. Content with unclear violations is now held for manual review instead of being auto-rejected. Deleted the legacy audit_service.go in favor of the unified hook-based moderation system.
This commit is contained in:
@@ -10,11 +10,8 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type PostModerationHook struct {
|
||||
name string
|
||||
postAIService PostAIService
|
||||
postRepo repository.PostRepository
|
||||
strictMode bool
|
||||
type reviewableError interface {
|
||||
IsReview() bool
|
||||
}
|
||||
|
||||
type PostAIService interface {
|
||||
@@ -23,6 +20,13 @@ type PostAIService interface {
|
||||
ModerateComment(ctx context.Context, content string, images []string) error
|
||||
}
|
||||
|
||||
type PostModerationHook struct {
|
||||
name string
|
||||
postAIService PostAIService
|
||||
postRepo repository.PostRepository
|
||||
strictMode bool
|
||||
}
|
||||
|
||||
func NewPostModerationHook(postAIService PostAIService, postRepo repository.PostRepository, strictMode bool) *PostModerationHook {
|
||||
return &PostModerationHook{
|
||||
name: "moderation.post.ai",
|
||||
@@ -36,9 +40,9 @@ func (h *PostModerationHook) Register(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name,
|
||||
HookType: HookPostPreModerate,
|
||||
Priority: PriorityHighest,
|
||||
Priority: PriorityHigh,
|
||||
Func: h.Execute,
|
||||
Async: false, // 必须同步执行,因为调用方需要审核结果
|
||||
Async: false,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
@@ -54,20 +58,32 @@ func (h *PostModerationHook) Execute(ctx *HookContext) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if result.RejectReason != "" || result.NeedsReview {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
zap.L().Debug("AI moderation disabled, auto approve post",
|
||||
zap.String("post_id", data.PostID),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
err := h.postAIService.ModeratePost(ctx.Ctx, data.Title, data.Content, data.Images)
|
||||
if err != nil {
|
||||
if rejectErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
||||
result.Approved = false
|
||||
result.NeedsReview = true
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("Post flagged for manual review by AI",
|
||||
zap.String("post_id", data.PostID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
result.Approved = false
|
||||
result.RejectReason = rejectErr.UserMessage()
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("Post rejected by AI moderation",
|
||||
zap.String("post_id", data.PostID),
|
||||
@@ -98,9 +114,6 @@ func (h *PostModerationHook) Execute(ctx *HookContext) error {
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Debug("Post approved by AI moderation",
|
||||
zap.String("post_id", data.PostID),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -124,9 +137,9 @@ func (h *CommentModerationHook) Register(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name,
|
||||
HookType: HookCommentPreModerate,
|
||||
Priority: PriorityHighest,
|
||||
Priority: PriorityHigh,
|
||||
Func: h.Execute,
|
||||
Async: false, // 必须同步执行,因为调用方需要审核结果
|
||||
Async: false,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
@@ -142,20 +155,32 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if result.RejectReason != "" || result.NeedsReview {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
zap.L().Debug("AI moderation disabled, auto approve comment",
|
||||
zap.String("comment_id", data.CommentID),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
err := h.postAIService.ModerateComment(ctx.Ctx, data.Content, data.Images)
|
||||
if err != nil {
|
||||
if rejectErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
||||
result.Approved = false
|
||||
result.NeedsReview = true
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("Comment flagged for manual review by AI",
|
||||
zap.String("comment_id", data.CommentID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
result.Approved = false
|
||||
result.RejectReason = rejectErr.UserMessage()
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("Comment rejected by AI moderation",
|
||||
zap.String("comment_id", data.CommentID),
|
||||
@@ -177,7 +202,7 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
zap.L().Warn("AI comment moderation error, fallback approve",
|
||||
zap.L().Warn("AI moderation error, fallback approve",
|
||||
zap.String("comment_id", data.CommentID),
|
||||
zap.Error(err),
|
||||
)
|
||||
@@ -186,23 +211,20 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Debug("Comment approved by AI moderation",
|
||||
zap.String("comment_id", data.CommentID),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
type SensitiveService interface {
|
||||
Check(ctx context.Context, text string) (bool, []string)
|
||||
Replace(ctx context.Context, text string, repl string) string
|
||||
}
|
||||
|
||||
type SensitiveWordHook struct {
|
||||
name string
|
||||
sensitive SensitiveService
|
||||
replaceStr string
|
||||
}
|
||||
|
||||
type SensitiveService interface {
|
||||
Check(ctx context.Context, text string) (bool, []string)
|
||||
Replace(ctx context.Context, text string, repl string) string
|
||||
}
|
||||
|
||||
func NewSensitiveWordHook(sensitive SensitiveService, replaceStr string) *SensitiveWordHook {
|
||||
return &SensitiveWordHook{
|
||||
name: "moderation.sensitive_word",
|
||||
@@ -215,7 +237,7 @@ func (h *SensitiveWordHook) RegisterPostHook(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name + ".post",
|
||||
HookType: HookPostPreModerate,
|
||||
Priority: PriorityHigh,
|
||||
Priority: PriorityHighest,
|
||||
Func: h.executePost,
|
||||
Async: false,
|
||||
})
|
||||
@@ -225,7 +247,7 @@ func (h *SensitiveWordHook) RegisterCommentHook(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name + ".comment",
|
||||
HookType: HookCommentPreModerate,
|
||||
Priority: PriorityHigh,
|
||||
Priority: PriorityHighest,
|
||||
Func: h.executeComment,
|
||||
Async: false,
|
||||
})
|
||||
@@ -325,18 +347,18 @@ func NewModerationHooks(
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) RegisterAll(manager *Manager) {
|
||||
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
|
||||
postHook.Register(manager)
|
||||
|
||||
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
|
||||
commentHook.Register(manager)
|
||||
|
||||
if m.sensitiveService != nil {
|
||||
sensitiveHook := NewSensitiveWordHook(m.sensitiveService, m.replaceStr)
|
||||
sensitiveHook.RegisterPostHook(manager)
|
||||
sensitiveHook.RegisterCommentHook(manager)
|
||||
}
|
||||
|
||||
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
|
||||
postHook.Register(manager)
|
||||
|
||||
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
|
||||
commentHook.Register(manager)
|
||||
|
||||
zap.L().Info("Registered moderation hooks",
|
||||
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
|
||||
zap.Bool("sensitive_enabled", m.sensitiveService != nil),
|
||||
|
||||
Reference in New Issue
Block a user