feat(moderation): add manual content review system with three-tier AI moderation
Add admin comment moderation endpoints (single and batch) and introduce a three-tier moderation system (pass/review/block) for content flagged by AI. Content with unclear violations is now held for manual review instead of being auto-rejected. Deleted the legacy audit_service.go in favor of the unified hook-based moderation system.
This commit is contained in:
@@ -9,7 +9,6 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// PostModerationRejectedError 帖子审核拒绝错误
|
||||
type PostModerationRejectedError struct {
|
||||
Reason string
|
||||
}
|
||||
@@ -21,7 +20,6 @@ func (e *PostModerationRejectedError) Error() string {
|
||||
return "post rejected by moderation: " + e.Reason
|
||||
}
|
||||
|
||||
// UserMessage 返回给前端的用户可读文案
|
||||
func (e *PostModerationRejectedError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "内容未通过审核,请修改后重试"
|
||||
@@ -29,7 +27,28 @@ func (e *PostModerationRejectedError) UserMessage() string {
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
// CommentModerationRejectedError 评论审核拒绝错误
|
||||
type PostModerationReviewError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *PostModerationReviewError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "post needs manual review"
|
||||
}
|
||||
return "post needs manual review: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *PostModerationReviewError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "内容需要人工复审,请耐心等待"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
func (e *PostModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type CommentModerationRejectedError struct {
|
||||
Reason string
|
||||
}
|
||||
@@ -41,7 +60,6 @@ func (e *CommentModerationRejectedError) Error() string {
|
||||
return "comment rejected by moderation: " + e.Reason
|
||||
}
|
||||
|
||||
// UserMessage 返回给前端的用户可读文案
|
||||
func (e *CommentModerationRejectedError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "评论未通过审核,请修改后重试"
|
||||
@@ -49,6 +67,28 @@ func (e *CommentModerationRejectedError) UserMessage() string {
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
type CommentModerationReviewError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *CommentModerationReviewError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "comment needs manual review"
|
||||
}
|
||||
return "comment needs manual review: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *CommentModerationReviewError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "评论需要人工复审,请耐心等待"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
func (e *CommentModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type PostAIService struct {
|
||||
openAIClient openai.Client
|
||||
}
|
||||
@@ -63,13 +103,12 @@ func (s *PostAIService) IsEnabled() bool {
|
||||
return s != nil && s.openAIClient != nil && s.openAIClient.IsEnabled()
|
||||
}
|
||||
|
||||
// ModeratePost 审核帖子内容,返回 nil 表示通过
|
||||
func (s *PostAIService) ModeratePost(ctx context.Context, title, content string, images []string) error {
|
||||
if !s.IsEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
approved, reason, err := s.openAIClient.ModeratePost(ctx, title, content, images)
|
||||
resp, err := s.openAIClient.ModeratePost(ctx, title, content, images)
|
||||
if err != nil {
|
||||
if s.openAIClient.Config().StrictModeration {
|
||||
return err
|
||||
@@ -79,19 +118,23 @@ func (s *PostAIService) ModeratePost(ctx context.Context, title, content string,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
if !approved {
|
||||
return &PostModerationRejectedError{Reason: reason}
|
||||
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &PostModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
return &PostModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModerateComment 审核评论内容,返回 nil 表示通过
|
||||
func (s *PostAIService) ModerateComment(ctx context.Context, content string, images []string) error {
|
||||
if !s.IsEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
approved, reason, err := s.openAIClient.ModerateComment(ctx, content, images)
|
||||
resp, err := s.openAIClient.ModerateComment(ctx, content, images)
|
||||
if err != nil {
|
||||
if s.openAIClient.Config().StrictModeration {
|
||||
return err
|
||||
@@ -101,8 +144,13 @@ func (s *PostAIService) ModerateComment(ctx context.Context, content string, ima
|
||||
)
|
||||
return nil
|
||||
}
|
||||
if !approved {
|
||||
return &CommentModerationRejectedError{Reason: reason}
|
||||
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &CommentModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
return &CommentModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user