feat(moderation): add user profile content moderation for avatars, covers, and bios
All checks were successful
Build Backend / build (push) Successful in 4m24s
Build Backend / build-docker (push) Successful in 1m14s

This commit is contained in:
lafay
2026-04-15 11:50:47 +08:00
parent b6f2df87c4
commit 90495385cd
23 changed files with 1561 additions and 55 deletions

View File

@@ -89,6 +89,86 @@ func (e *CommentModerationReviewError) IsReview() bool {
return true
}
type ImageModerationRejectedError struct {
Reason string
}
func (e *ImageModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "image rejected by moderation"
}
return "image rejected by moderation: " + e.Reason
}
func (e *ImageModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "图片未通过审核,请更换后重试"
}
return strings.TrimSpace(e.Reason)
}
type ImageModerationReviewError struct {
Reason string
}
func (e *ImageModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "image needs manual review"
}
return "image needs manual review: " + e.Reason
}
func (e *ImageModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "图片需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *ImageModerationReviewError) IsReview() bool {
return true
}
type BioModerationRejectedError struct {
Reason string
}
func (e *BioModerationRejectedError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "bio rejected by moderation"
}
return "bio rejected by moderation: " + e.Reason
}
func (e *BioModerationRejectedError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "个性签名未通过审核,请修改后重试"
}
return strings.TrimSpace(e.Reason)
}
type BioModerationReviewError struct {
Reason string
}
func (e *BioModerationReviewError) Error() string {
if strings.TrimSpace(e.Reason) == "" {
return "bio needs manual review"
}
return "bio needs manual review: " + e.Reason
}
func (e *BioModerationReviewError) UserMessage() string {
if strings.TrimSpace(e.Reason) == "" {
return "个性签名需要人工复审,请耐心等待"
}
return strings.TrimSpace(e.Reason)
}
func (e *BioModerationReviewError) IsReview() bool {
return true
}
type PostAIService struct {
openAIClient openai.Client
}
@@ -154,3 +234,55 @@ func (s *PostAIService) ModerateComment(ctx context.Context, content string, ima
return nil
}
}
func (s *PostAIService) ModerateImage(ctx context.Context, imageURL string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateImage(ctx, imageURL)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI image moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &ImageModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &ImageModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}
func (s *PostAIService) ModerateBio(ctx context.Context, bio string) error {
if !s.IsEnabled() {
return nil
}
resp, err := s.openAIClient.ModerateBio(ctx, bio)
if err != nil {
if s.openAIClient.Config().StrictModeration {
return err
}
zap.L().Warn("AI bio moderation failed, fallback allow",
zap.Error(err),
)
return nil
}
switch resp.Result {
case openai.ModerationResultBlock:
return &BioModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &BioModerationReviewError{Reason: resp.Reason}
default:
return nil
}
}