Files
backend/internal/service/post_ai_service.go
lafay 1bb82e1e2b
All checks were successful
Build Backend / build (push) Successful in 2m43s
Build Backend / build-docker (push) Successful in 1m22s
feat(moderation): add manual content review system with three-tier AI moderation
Add admin comment moderation endpoints (single and batch) and introduce a three-tier moderation system (pass/review/block) for content flagged by AI. Content with unclear violations is now held for manual review instead of being auto-rejected. Deleted the legacy audit_service.go in favor of the unified hook-based moderation system.
2026-04-11 04:23:24 +08:00

157 lines
3.5 KiB
Go

package service
import (
"context"
"strings"
"carrot_bbs/internal/pkg/openai"
"go.uber.org/zap"
)
type PostModerationRejectedError struct {
Reason string
}
func (e *PostModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "post rejected by moderation"
}
return "post rejected by moderation: " + e.Reason
}
func (e *PostModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "内容未通过审核,请修改后重试"
}
return strings.TrimSpace(e.Reason)
}
type PostModerationReviewError struct {
Reason string
}
func (e *PostModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "post needs manual review"
}
return "post needs manual review: " + e.Reason
}
func (e *PostModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "内容需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *PostModerationReviewError) IsReview() bool {
return true
}
type CommentModerationRejectedError struct {
Reason string
}
func (e *CommentModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "comment rejected by moderation"
}
return "comment rejected by moderation: " + e.Reason
}
func (e *CommentModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "评论未通过审核,请修改后重试"
}
return strings.TrimSpace(e.Reason)
}
type CommentModerationReviewError struct {
Reason string
}
func (e *CommentModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "comment needs manual review"
}
return "comment needs manual review: " + e.Reason
}
func (e *CommentModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "评论需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *CommentModerationReviewError) IsReview() bool {
return true
}
type PostAIService struct {
openAIClient openai.Client
}
func NewPostAIService(openAIClient openai.Client) *PostAIService {
return &PostAIService{
openAIClient: openAIClient,
}
}
func (s *PostAIService) IsEnabled() bool {
return s != nil && s.openAIClient != nil && s.openAIClient.IsEnabled()
}
func (s *PostAIService) ModeratePost(ctx context.Context, title, content string, images []string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModeratePost(ctx, title, content, images)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &PostModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &PostModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}
func (s *PostAIService) ModerateComment(ctx context.Context, content string, images []string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateComment(ctx, content, images)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI comment moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &CommentModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &CommentModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}