feat(moderation): add user profile content moderation for avatars, covers, and bios
This commit is contained in:
@@ -89,6 +89,86 @@ func (e *CommentModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type ImageModerationRejectedError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ImageModerationRejectedError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "image rejected by moderation"
|
||||
}
|
||||
return "image rejected by moderation: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *ImageModerationRejectedError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "图片未通过审核,请更换后重试"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
type ImageModerationReviewError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ImageModerationReviewError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "image needs manual review"
|
||||
}
|
||||
return "image needs manual review: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *ImageModerationReviewError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "图片需要人工复审,请耐心等待"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
func (e *ImageModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type BioModerationRejectedError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *BioModerationRejectedError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "bio rejected by moderation"
|
||||
}
|
||||
return "bio rejected by moderation: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *BioModerationRejectedError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "个性签名未通过审核,请修改后重试"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
type BioModerationReviewError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *BioModerationReviewError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "bio needs manual review"
|
||||
}
|
||||
return "bio needs manual review: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *BioModerationReviewError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "个性签名需要人工复审,请耐心等待"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
func (e *BioModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type PostAIService struct {
|
||||
openAIClient openai.Client
|
||||
}
|
||||
@@ -154,3 +234,55 @@ func (s *PostAIService) ModerateComment(ctx context.Context, content string, ima
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostAIService) ModerateImage(ctx context.Context, imageURL string) error {
|
||||
if !s.IsEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
resp, err := s.openAIClient.ModerateImage(ctx, imageURL)
|
||||
if err != nil {
|
||||
if s.openAIClient.Config().StrictModeration {
|
||||
return err
|
||||
}
|
||||
zap.L().Warn("AI image moderation failed, fallback allow",
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &ImageModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
return &ImageModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostAIService) ModerateBio(ctx context.Context, bio string) error {
|
||||
if !s.IsEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
resp, err := s.openAIClient.ModerateBio(ctx, bio)
|
||||
if err != nil {
|
||||
if s.openAIClient.Config().StrictModeration {
|
||||
return err
|
||||
}
|
||||
zap.L().Warn("AI bio moderation failed, fallback allow",
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &BioModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
return &BioModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user