feat(moderation): add manual content review system with three-tier AI moderation
All checks were successful
Build Backend / build (push) Successful in 2m43s
Build Backend / build-docker (push) Successful in 1m22s

Add admin comment moderation endpoints (single and batch) and introduce a three-tier moderation system (pass/review/block) for content flagged by AI. Content with unclear violations is now held for manual review instead of being auto-rejected. Deleted the legacy audit_service.go in favor of the unified hook-based moderation system.
This commit is contained in:
lafay
2026-04-11 04:23:24 +08:00
parent 6af6aaa9d0
commit 1bb82e1e2b
17 changed files with 432 additions and 732 deletions

View File

@@ -10,11 +10,8 @@ import (
"go.uber.org/zap"
)
type PostModerationHook struct {
name string
postAIService PostAIService
postRepo repository.PostRepository
strictMode bool
type reviewableError interface {
IsReview() bool
}
type PostAIService interface {
@@ -23,6 +20,13 @@ type PostAIService interface {
ModerateComment(ctx context.Context, content string, images []string) error
}
type PostModerationHook struct {
name string
postAIService PostAIService
postRepo repository.PostRepository
strictMode bool
}
func NewPostModerationHook(postAIService PostAIService, postRepo repository.PostRepository, strictMode bool) *PostModerationHook {
return &PostModerationHook{
name: "moderation.post.ai",
@@ -36,9 +40,9 @@ func (h *PostModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookPostPreModerate,
Priority: PriorityHighest,
Priority: PriorityHigh,
Func: h.Execute,
Async: false, // 必须同步执行,因为调用方需要审核结果
Async: false,
Timeout: 30 * time.Second,
})
}
@@ -54,20 +58,32 @@ func (h *PostModerationHook) Execute(ctx *HookContext) error {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
zap.L().Debug("AI moderation disabled, auto approve post",
zap.String("post_id", data.PostID),
)
return nil
}
err := h.postAIService.ModeratePost(ctx.Ctx, data.Title, data.Content, data.Images)
if err != nil {
if rejectErr, ok := err.(interface{ UserMessage() string }); ok {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Post flagged for manual review by AI",
zap.String("post_id", data.PostID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = rejectErr.UserMessage()
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Post rejected by AI moderation",
zap.String("post_id", data.PostID),
@@ -98,9 +114,6 @@ func (h *PostModerationHook) Execute(ctx *HookContext) error {
result.Approved = true
result.ReviewedBy = "ai"
zap.L().Debug("Post approved by AI moderation",
zap.String("post_id", data.PostID),
)
return nil
}
@@ -124,9 +137,9 @@ func (h *CommentModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookCommentPreModerate,
Priority: PriorityHighest,
Priority: PriorityHigh,
Func: h.Execute,
Async: false, // 必须同步执行,因为调用方需要审核结果
Async: false,
Timeout: 30 * time.Second,
})
}
@@ -142,20 +155,32 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
zap.L().Debug("AI moderation disabled, auto approve comment",
zap.String("comment_id", data.CommentID),
)
return nil
}
err := h.postAIService.ModerateComment(ctx.Ctx, data.Content, data.Images)
if err != nil {
if rejectErr, ok := err.(interface{ UserMessage() string }); ok {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Comment flagged for manual review by AI",
zap.String("comment_id", data.CommentID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = rejectErr.UserMessage()
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Comment rejected by AI moderation",
zap.String("comment_id", data.CommentID),
@@ -177,7 +202,7 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI comment moderation error, fallback approve",
zap.L().Warn("AI moderation error, fallback approve",
zap.String("comment_id", data.CommentID),
zap.Error(err),
)
@@ -186,23 +211,20 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
result.Approved = true
result.ReviewedBy = "ai"
zap.L().Debug("Comment approved by AI moderation",
zap.String("comment_id", data.CommentID),
)
return nil
}
type SensitiveService interface {
Check(ctx context.Context, text string) (bool, []string)
Replace(ctx context.Context, text string, repl string) string
}
type SensitiveWordHook struct {
name string
sensitive SensitiveService
replaceStr string
}
type SensitiveService interface {
Check(ctx context.Context, text string) (bool, []string)
Replace(ctx context.Context, text string, repl string) string
}
func NewSensitiveWordHook(sensitive SensitiveService, replaceStr string) *SensitiveWordHook {
return &SensitiveWordHook{
name: "moderation.sensitive_word",
@@ -215,7 +237,7 @@ func (h *SensitiveWordHook) RegisterPostHook(manager *Manager) {
manager.Register(&Hook{
Name: h.name + ".post",
HookType: HookPostPreModerate,
Priority: PriorityHigh,
Priority: PriorityHighest,
Func: h.executePost,
Async: false,
})
@@ -225,7 +247,7 @@ func (h *SensitiveWordHook) RegisterCommentHook(manager *Manager) {
manager.Register(&Hook{
Name: h.name + ".comment",
HookType: HookCommentPreModerate,
Priority: PriorityHigh,
Priority: PriorityHighest,
Func: h.executeComment,
Async: false,
})
@@ -325,18 +347,18 @@ func NewModerationHooks(
}
func (m *ModerationHooks) RegisterAll(manager *Manager) {
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
postHook.Register(manager)
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager)
if m.sensitiveService != nil {
sensitiveHook := NewSensitiveWordHook(m.sensitiveService, m.replaceStr)
sensitiveHook.RegisterPostHook(manager)
sensitiveHook.RegisterCommentHook(manager)
}
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
postHook.Register(manager)
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager)
zap.L().Info("Registered moderation hooks",
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
zap.Bool("sensitive_enabled", m.sensitiveService != nil),