The moderation system now uses a "pass/review" strategy instead of "pass/review/block", treating hard violations the same as review (only routing to manual review rather than auto-rejecting). Add fallback-approve behavior in the hook system so slow moderation APIs never silently block normal content when they time out or get cancelled - timed-out moderation now approves with "ReviewedBy: system" instead of leaving the zero-value result that would reject content. Update post/comment/vote services and handlers to use the new unified behavior. Simplify moderation prompts to emphasize "宁可放过,不可误伤" (prefer false positives over false negatives).
366 lines
9.0 KiB
Go
366 lines
9.0 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"with_you/internal/pkg/openai"
|
|
"with_you/internal/pkg/tencent"
|
|
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type PostModerationRejectedError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *PostModerationRejectedError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "post rejected by moderation"
|
|
}
|
|
return "post rejected by moderation: " + e.Reason
|
|
}
|
|
|
|
func (e *PostModerationRejectedError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "内容未通过审核,请修改后重试"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
type PostModerationReviewError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *PostModerationReviewError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "post needs manual review"
|
|
}
|
|
return "post needs manual review: " + e.Reason
|
|
}
|
|
|
|
func (e *PostModerationReviewError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "内容需要人工复审,请耐心等待"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
func (e *PostModerationReviewError) IsReview() bool {
|
|
return true
|
|
}
|
|
|
|
type CommentModerationRejectedError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *CommentModerationRejectedError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "comment rejected by moderation"
|
|
}
|
|
return "comment rejected by moderation: " + e.Reason
|
|
}
|
|
|
|
func (e *CommentModerationRejectedError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "评论未通过审核,请修改后重试"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
type CommentModerationReviewError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *CommentModerationReviewError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "comment needs manual review"
|
|
}
|
|
return "comment needs manual review: " + e.Reason
|
|
}
|
|
|
|
func (e *CommentModerationReviewError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "评论需要人工复审,请耐心等待"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
func (e *CommentModerationReviewError) IsReview() bool {
|
|
return true
|
|
}
|
|
|
|
type ImageModerationRejectedError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *ImageModerationRejectedError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "image rejected by moderation"
|
|
}
|
|
return "image rejected by moderation: " + e.Reason
|
|
}
|
|
|
|
func (e *ImageModerationRejectedError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "图片未通过审核,请更换后重试"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
type ImageModerationReviewError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *ImageModerationReviewError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "image needs manual review"
|
|
}
|
|
return "image needs manual review: " + e.Reason
|
|
}
|
|
|
|
func (e *ImageModerationReviewError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "图片需要人工复审,请耐心等待"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
func (e *ImageModerationReviewError) IsReview() bool {
|
|
return true
|
|
}
|
|
|
|
type BioModerationRejectedError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *BioModerationRejectedError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "bio rejected by moderation"
|
|
}
|
|
return "bio rejected by moderation: " + e.Reason
|
|
}
|
|
|
|
func (e *BioModerationRejectedError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "个性签名未通过审核,请修改后重试"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
type BioModerationReviewError struct {
|
|
Reason string
|
|
}
|
|
|
|
func (e *BioModerationReviewError) Error() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "bio needs manual review"
|
|
}
|
|
return "bio needs manual review: " + e.Reason
|
|
}
|
|
|
|
func (e *BioModerationReviewError) UserMessage() string {
|
|
if strings.TrimSpace(e.Reason) == "" {
|
|
return "个性签名需要人工复审,请耐心等待"
|
|
}
|
|
return strings.TrimSpace(e.Reason)
|
|
}
|
|
|
|
func (e *BioModerationReviewError) IsReview() bool {
|
|
return true
|
|
}
|
|
|
|
// PostAIService 帖子 AI 审核服务接口
|
|
type PostAIService interface {
|
|
IsEnabled() bool
|
|
ModeratePost(ctx context.Context, title, content string, images []string) error
|
|
ModerateComment(ctx context.Context, content string, images []string) error
|
|
ModerateImage(ctx context.Context, imageURL string) error
|
|
ModerateBio(ctx context.Context, bio string) error
|
|
}
|
|
|
|
// postAIService 帖子 AI 审核服务实现
|
|
type postAIService struct {
|
|
openAIClient openai.Client
|
|
tencentClient tencent.Client
|
|
strictMode bool
|
|
}
|
|
|
|
func NewPostAIService(openAIClient openai.Client, tencentClient tencent.Client, strictMode bool) PostAIService {
|
|
return &postAIService{
|
|
openAIClient: openAIClient,
|
|
tencentClient: tencentClient,
|
|
strictMode: strictMode,
|
|
}
|
|
}
|
|
|
|
func (s *postAIService) IsEnabled() bool {
|
|
if s == nil {
|
|
return false
|
|
}
|
|
return (s.openAIClient != nil && s.openAIClient.IsEnabled()) ||
|
|
(s.tencentClient != nil && s.tencentClient.IsEnabled())
|
|
}
|
|
|
|
func (s *postAIService) isOpenAIEnabled() bool {
|
|
return s != nil && s.openAIClient != nil && s.openAIClient.IsEnabled()
|
|
}
|
|
|
|
func (s *postAIService) isTencentEnabled() bool {
|
|
return s != nil && s.tencentClient != nil && s.tencentClient.IsEnabled()
|
|
}
|
|
|
|
func (s *postAIService) ModeratePost(ctx context.Context, title, content string, images []string) error {
|
|
if !s.IsEnabled() {
|
|
return nil
|
|
}
|
|
|
|
if s.isOpenAIEnabled() {
|
|
resp, err := s.openAIClient.ModeratePost(ctx, title, content, images)
|
|
if err != nil {
|
|
zap.L().Warn("AI post moderation failed, trying Tencent fallback",
|
|
zap.Error(err),
|
|
)
|
|
return s.moderateTextWithTencent(ctx, title+" "+content,
|
|
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
|
)
|
|
}
|
|
|
|
// block 与 review 同等处理:均仅送人工复审。
|
|
switch resp.Result {
|
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
|
return &PostModerationReviewError{Reason: resp.Reason}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return s.moderateTextWithTencent(ctx, title+" "+content,
|
|
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
|
)
|
|
}
|
|
|
|
func (s *postAIService) ModerateComment(ctx context.Context, content string, images []string) error {
|
|
if !s.IsEnabled() {
|
|
return nil
|
|
}
|
|
|
|
if s.isOpenAIEnabled() {
|
|
resp, err := s.openAIClient.ModerateComment(ctx, content, images)
|
|
if err != nil {
|
|
zap.L().Warn("AI comment moderation failed, trying Tencent fallback",
|
|
zap.Error(err),
|
|
)
|
|
return s.moderateTextWithTencent(ctx, content,
|
|
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
|
)
|
|
}
|
|
|
|
// block 与 review 同等处理:均仅送人工复审。
|
|
switch resp.Result {
|
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
|
return &CommentModerationReviewError{Reason: resp.Reason}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return s.moderateTextWithTencent(ctx, content,
|
|
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
|
)
|
|
}
|
|
|
|
func (s *postAIService) ModerateImage(ctx context.Context, imageURL string) error {
|
|
if !s.isOpenAIEnabled() {
|
|
return nil
|
|
}
|
|
|
|
resp, err := s.openAIClient.ModerateImage(ctx, imageURL)
|
|
if err != nil {
|
|
if s.strictMode {
|
|
return err
|
|
}
|
|
zap.L().Warn("AI image moderation failed, fallback allow",
|
|
zap.Error(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
// block 与 review 同等处理:均仅送人工复审。
|
|
switch resp.Result {
|
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
|
return &ImageModerationReviewError{Reason: resp.Reason}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
|
if !s.IsEnabled() {
|
|
return nil
|
|
}
|
|
|
|
if s.isOpenAIEnabled() {
|
|
resp, err := s.openAIClient.ModerateBio(ctx, bio)
|
|
if err != nil {
|
|
zap.L().Warn("AI bio moderation failed, trying Tencent fallback",
|
|
zap.Error(err),
|
|
)
|
|
return s.moderateTextWithTencent(ctx, bio,
|
|
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
|
)
|
|
}
|
|
|
|
// block 与 review 同等处理:均仅送人工复审。
|
|
switch resp.Result {
|
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
|
return &BioModerationReviewError{Reason: resp.Reason}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return s.moderateTextWithTencent(ctx, bio,
|
|
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
|
)
|
|
}
|
|
|
|
func (s *postAIService) moderateTextWithTencent(
|
|
ctx context.Context,
|
|
text string,
|
|
reviewFactory func(string) error,
|
|
) error {
|
|
if !s.isTencentEnabled() {
|
|
if s.strictMode {
|
|
return fmt.Errorf("no moderation service available")
|
|
}
|
|
zap.L().Warn("No moderation service available, fallback allow")
|
|
return nil
|
|
}
|
|
|
|
resp, err := s.tencentClient.TextModeration(ctx, text)
|
|
if err != nil {
|
|
if s.strictMode {
|
|
return err
|
|
}
|
|
zap.L().Warn("Tencent TMS moderation failed, fallback allow",
|
|
zap.Error(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
reason := resp.Label
|
|
if len(resp.Keywords) > 0 {
|
|
reason = resp.Label + ": " + strings.Join(resp.Keywords, ", ")
|
|
}
|
|
|
|
// 腾讯云 block 也仅送审,与 AI 同语义。
|
|
switch resp.Suggestion {
|
|
case tencent.SuggestionBlock, tencent.SuggestionReview:
|
|
return reviewFactory(reason)
|
|
default:
|
|
return nil
|
|
}
|
|
}
|