feat(moderation): add user profile content moderation for avatars, covers, and bios
This commit is contained in:
@@ -11,31 +11,37 @@ import (
|
||||
type HookType string
|
||||
|
||||
const (
|
||||
HookPostCreated HookType = "post.created"
|
||||
HookPostUpdated HookType = "post.updated"
|
||||
HookPostDeleted HookType = "post.deleted"
|
||||
HookPostPreModerate HookType = "post.pre_moderate"
|
||||
HookPostModerated HookType = "post.moderated"
|
||||
HookCommentCreated HookType = "comment.created"
|
||||
HookCommentDeleted HookType = "comment.deleted"
|
||||
HookCommentPreModerate HookType = "comment.pre_moderate"
|
||||
HookCommentModerated HookType = "comment.moderated"
|
||||
HookUserCreated HookType = "user.created"
|
||||
HookUserUpdated HookType = "user.updated"
|
||||
HookUserDeleted HookType = "user.deleted"
|
||||
HookUserLogin HookType = "user.login"
|
||||
HookUserLogout HookType = "user.logout"
|
||||
HookGroupCreated HookType = "group.created"
|
||||
HookGroupUpdated HookType = "group.updated"
|
||||
HookGroupDeleted HookType = "group.deleted"
|
||||
HookGroupJoined HookType = "group.joined"
|
||||
HookGroupLeft HookType = "group.left"
|
||||
HookMessageSent HookType = "message.sent"
|
||||
HookMessageRead HookType = "message.read"
|
||||
HookNotificationSent HookType = "notification.sent"
|
||||
HookVoteCreated HookType = "vote.created"
|
||||
HookVoteUpdated HookType = "vote.updated"
|
||||
HookFileUploaded HookType = "file.uploaded"
|
||||
HookPostCreated HookType = "post.created"
|
||||
HookPostUpdated HookType = "post.updated"
|
||||
HookPostDeleted HookType = "post.deleted"
|
||||
HookPostPreModerate HookType = "post.pre_moderate"
|
||||
HookPostModerated HookType = "post.moderated"
|
||||
HookCommentCreated HookType = "comment.created"
|
||||
HookCommentDeleted HookType = "comment.deleted"
|
||||
HookCommentPreModerate HookType = "comment.pre_moderate"
|
||||
HookCommentModerated HookType = "comment.moderated"
|
||||
HookUserCreated HookType = "user.created"
|
||||
HookUserUpdated HookType = "user.updated"
|
||||
HookUserDeleted HookType = "user.deleted"
|
||||
HookUserLogin HookType = "user.login"
|
||||
HookUserLogout HookType = "user.logout"
|
||||
HookGroupCreated HookType = "group.created"
|
||||
HookGroupUpdated HookType = "group.updated"
|
||||
HookGroupDeleted HookType = "group.deleted"
|
||||
HookGroupJoined HookType = "group.joined"
|
||||
HookGroupLeft HookType = "group.left"
|
||||
HookMessageSent HookType = "message.sent"
|
||||
HookMessageRead HookType = "message.read"
|
||||
HookNotificationSent HookType = "notification.sent"
|
||||
HookVoteCreated HookType = "vote.created"
|
||||
HookVoteUpdated HookType = "vote.updated"
|
||||
HookFileUploaded HookType = "file.uploaded"
|
||||
HookUserAvatarPreModerate HookType = "user.avatar.pre_moderate"
|
||||
HookUserAvatarModerated HookType = "user.avatar.moderated"
|
||||
HookUserCoverPreModerate HookType = "user.cover.pre_moderate"
|
||||
HookUserCoverModerated HookType = "user.cover.moderated"
|
||||
HookUserBioPreModerate HookType = "user.bio.pre_moderate"
|
||||
HookUserBioModerated HookType = "user.bio.moderated"
|
||||
)
|
||||
|
||||
type Priority int
|
||||
|
||||
@@ -18,6 +18,8 @@ type PostAIService interface {
|
||||
IsEnabled() bool
|
||||
ModeratePost(ctx context.Context, title, content string, images []string) error
|
||||
ModerateComment(ctx context.Context, content string, images []string) error
|
||||
ModerateImage(ctx context.Context, imageURL string) error
|
||||
ModerateBio(ctx context.Context, bio string) error
|
||||
}
|
||||
|
||||
type PostModerationHook struct {
|
||||
@@ -324,6 +326,7 @@ type ModerationHooks struct {
|
||||
sensitiveService SensitiveService
|
||||
postRepo repository.PostRepository
|
||||
commentRepo repository.CommentRepository
|
||||
userRepo repository.UserRepository
|
||||
strictMode bool
|
||||
replaceStr string
|
||||
}
|
||||
@@ -333,6 +336,7 @@ func NewModerationHooks(
|
||||
sensitiveService SensitiveService,
|
||||
postRepo repository.PostRepository,
|
||||
commentRepo repository.CommentRepository,
|
||||
userRepo repository.UserRepository,
|
||||
strictMode bool,
|
||||
replaceStr string,
|
||||
) *ModerationHooks {
|
||||
@@ -341,6 +345,7 @@ func NewModerationHooks(
|
||||
sensitiveService: sensitiveService,
|
||||
postRepo: postRepo,
|
||||
commentRepo: commentRepo,
|
||||
userRepo: userRepo,
|
||||
strictMode: strictMode,
|
||||
replaceStr: replaceStr,
|
||||
}
|
||||
@@ -359,6 +364,15 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
|
||||
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
|
||||
commentHook.Register(manager)
|
||||
|
||||
avatarHook := NewUserAvatarModerationHook(m.postAIService, m.userRepo, m.strictMode)
|
||||
avatarHook.Register(manager)
|
||||
|
||||
coverHook := NewUserCoverModerationHook(m.postAIService, m.userRepo, m.strictMode)
|
||||
coverHook.Register(manager)
|
||||
|
||||
bioHook := NewUserBioModerationHook(m.postAIService, m.sensitiveService, m.userRepo, m.strictMode)
|
||||
bioHook.Register(manager)
|
||||
|
||||
zap.L().Info("Registered moderation hooks",
|
||||
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
|
||||
zap.Bool("sensitive_enabled", m.sensitiveService != nil),
|
||||
@@ -381,3 +395,27 @@ func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -101,6 +101,42 @@ type CommentModeratedHookData struct {
|
||||
ReviewedBy string
|
||||
}
|
||||
|
||||
type UserAvatarModerateHookData struct {
|
||||
UserID string
|
||||
AvatarURL string
|
||||
}
|
||||
|
||||
type UserCoverModerateHookData struct {
|
||||
UserID string
|
||||
CoverURL string
|
||||
}
|
||||
|
||||
type UserBioModerateHookData struct {
|
||||
UserID string
|
||||
Bio string
|
||||
}
|
||||
|
||||
type UserAvatarModeratedHookData struct {
|
||||
UserID string
|
||||
Approved bool
|
||||
RejectReason string
|
||||
ReviewedBy string
|
||||
}
|
||||
|
||||
type UserCoverModeratedHookData struct {
|
||||
UserID string
|
||||
Approved bool
|
||||
RejectReason string
|
||||
ReviewedBy string
|
||||
}
|
||||
|
||||
type UserBioModeratedHookData struct {
|
||||
UserID string
|
||||
Approved bool
|
||||
RejectReason string
|
||||
ReviewedBy string
|
||||
}
|
||||
|
||||
type ModerationResult struct {
|
||||
Approved bool
|
||||
NeedsReview bool
|
||||
@@ -271,3 +307,33 @@ func (h *HookTriggerHelper) TriggerCommentPreModerate(ctx context.Context, userI
|
||||
func (h *HookTriggerHelper) TriggerCommentModerated(ctx context.Context, userID uint, data *CommentModeratedHookData) {
|
||||
h.manager.Trigger(ctx, HookCommentModerated, userID, data)
|
||||
}
|
||||
|
||||
func (h *HookTriggerHelper) TriggerUserAvatarPreModerate(ctx context.Context, userID uint, data *UserAvatarModerateHookData, result *ModerationResult) {
|
||||
h.manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, userID, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *HookTriggerHelper) TriggerUserAvatarModerated(ctx context.Context, userID uint, data *UserAvatarModeratedHookData) {
|
||||
h.manager.Trigger(ctx, HookUserAvatarModerated, userID, data)
|
||||
}
|
||||
|
||||
func (h *HookTriggerHelper) TriggerUserCoverPreModerate(ctx context.Context, userID uint, data *UserCoverModerateHookData, result *ModerationResult) {
|
||||
h.manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, userID, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *HookTriggerHelper) TriggerUserCoverModerated(ctx context.Context, userID uint, data *UserCoverModeratedHookData) {
|
||||
h.manager.Trigger(ctx, HookUserCoverModerated, userID, data)
|
||||
}
|
||||
|
||||
func (h *HookTriggerHelper) TriggerUserBioPreModerate(ctx context.Context, userID uint, data *UserBioModerateHookData, result *ModerationResult) {
|
||||
h.manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, userID, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *HookTriggerHelper) TriggerUserBioModerated(ctx context.Context, userID uint, data *UserBioModeratedHookData) {
|
||||
h.manager.Trigger(ctx, HookUserBioModerated, userID, data)
|
||||
}
|
||||
|
||||
317
internal/pkg/hook/user_profile_moderation_hooks.go
Normal file
317
internal/pkg/hook/user_profile_moderation_hooks.go
Normal file
@@ -0,0 +1,317 @@
|
||||
package hook
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"carrot_bbs/internal/repository"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type UserAvatarModerationHook struct {
|
||||
name string
|
||||
postAIService PostAIService
|
||||
userRepo repository.UserRepository
|
||||
strictMode bool
|
||||
}
|
||||
|
||||
func NewUserAvatarModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserAvatarModerationHook {
|
||||
return &UserAvatarModerationHook{
|
||||
name: "moderation.user.avatar.ai",
|
||||
postAIService: postAIService,
|
||||
userRepo: userRepo,
|
||||
strictMode: strictMode,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *UserAvatarModerationHook) Register(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name,
|
||||
HookType: HookUserAvatarPreModerate,
|
||||
Priority: PriorityHigh,
|
||||
Func: h.Execute,
|
||||
Async: false,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *UserAvatarModerationHook) Execute(ctx *HookContext) error {
|
||||
data, ok := ctx.Data.(*UserAvatarModerateHookData)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
||||
if !ok || result == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if result.RejectReason != "" || result.NeedsReview {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
return nil
|
||||
}
|
||||
|
||||
err := h.postAIService.ModerateImage(ctx.Ctx, data.AvatarURL)
|
||||
if err != nil {
|
||||
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
||||
result.Approved = false
|
||||
result.NeedsReview = true
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("User avatar flagged for manual review by AI",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
result.Approved = false
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("User avatar rejected by AI moderation",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.strictMode {
|
||||
result.Approved = false
|
||||
result.Error = err
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Error("AI avatar moderation failed in strict mode, reject",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
zap.L().Warn("AI moderation error, fallback approve avatar",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "ai"
|
||||
return nil
|
||||
}
|
||||
|
||||
type UserCoverModerationHook struct {
|
||||
name string
|
||||
postAIService PostAIService
|
||||
userRepo repository.UserRepository
|
||||
strictMode bool
|
||||
}
|
||||
|
||||
func NewUserCoverModerationHook(postAIService PostAIService, userRepo repository.UserRepository, strictMode bool) *UserCoverModerationHook {
|
||||
return &UserCoverModerationHook{
|
||||
name: "moderation.user.cover.ai",
|
||||
postAIService: postAIService,
|
||||
userRepo: userRepo,
|
||||
strictMode: strictMode,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *UserCoverModerationHook) Register(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name,
|
||||
HookType: HookUserCoverPreModerate,
|
||||
Priority: PriorityHigh,
|
||||
Func: h.Execute,
|
||||
Async: false,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *UserCoverModerationHook) Execute(ctx *HookContext) error {
|
||||
data, ok := ctx.Data.(*UserCoverModerateHookData)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
||||
if !ok || result == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if result.RejectReason != "" || result.NeedsReview {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
return nil
|
||||
}
|
||||
|
||||
err := h.postAIService.ModerateImage(ctx.Ctx, data.CoverURL)
|
||||
if err != nil {
|
||||
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
||||
result.Approved = false
|
||||
result.NeedsReview = true
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("User cover flagged for manual review by AI",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
result.Approved = false
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("User cover rejected by AI moderation",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.strictMode {
|
||||
result.Approved = false
|
||||
result.Error = err
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Error("AI cover moderation failed in strict mode, reject",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
zap.L().Warn("AI moderation error, fallback approve cover",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "ai"
|
||||
return nil
|
||||
}
|
||||
|
||||
type UserBioModerationHook struct {
|
||||
name string
|
||||
postAIService PostAIService
|
||||
sensitiveService SensitiveService
|
||||
userRepo repository.UserRepository
|
||||
strictMode bool
|
||||
}
|
||||
|
||||
func NewUserBioModerationHook(postAIService PostAIService, sensitiveService SensitiveService, userRepo repository.UserRepository, strictMode bool) *UserBioModerationHook {
|
||||
return &UserBioModerationHook{
|
||||
name: "moderation.user.bio.ai",
|
||||
postAIService: postAIService,
|
||||
sensitiveService: sensitiveService,
|
||||
userRepo: userRepo,
|
||||
strictMode: strictMode,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *UserBioModerationHook) Register(manager *Manager) {
|
||||
manager.Register(&Hook{
|
||||
Name: h.name,
|
||||
HookType: HookUserBioPreModerate,
|
||||
Priority: PriorityHigh,
|
||||
Func: h.Execute,
|
||||
Async: false,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *UserBioModerationHook) Execute(ctx *HookContext) error {
|
||||
data, ok := ctx.Data.(*UserBioModerateHookData)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := ctx.Metadata["result"].(*ModerationResult)
|
||||
if !ok || result == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if result.RejectReason != "" || result.NeedsReview {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.sensitiveService != nil {
|
||||
hasSensitive, words := h.sensitiveService.Check(ctx.Ctx, data.Bio)
|
||||
if hasSensitive {
|
||||
result.Approved = false
|
||||
result.RejectReason = "内容包含敏感词: " + strings.Join(words, ", ")
|
||||
result.ReviewedBy = "sensitive_word"
|
||||
zap.L().Info("User bio rejected by sensitive word filter",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Strings("words", words),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if h.postAIService == nil || !h.postAIService.IsEnabled() {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
return nil
|
||||
}
|
||||
|
||||
err := h.postAIService.ModerateBio(ctx.Ctx, data.Bio)
|
||||
if err != nil {
|
||||
if userErr, ok := err.(interface{ UserMessage() string }); ok {
|
||||
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
|
||||
result.Approved = false
|
||||
result.NeedsReview = true
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("User bio flagged for manual review by AI",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
result.Approved = false
|
||||
result.RejectReason = userErr.UserMessage()
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Info("User bio rejected by AI moderation",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.strictMode {
|
||||
result.Approved = false
|
||||
result.Error = err
|
||||
result.ReviewedBy = "ai"
|
||||
zap.L().Error("AI bio moderation failed in strict mode, reject",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
zap.L().Warn("AI moderation error, fallback approve bio",
|
||||
zap.String("user_id", data.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "ai"
|
||||
return nil
|
||||
}
|
||||
@@ -93,6 +93,8 @@ type Client interface {
|
||||
Config() Config
|
||||
ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error)
|
||||
ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error)
|
||||
ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error)
|
||||
ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error)
|
||||
}
|
||||
|
||||
type clientImpl struct {
|
||||
@@ -163,6 +165,157 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
|
||||
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
||||
}
|
||||
|
||||
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
- 色情低俗、裸露、性暗示内容
|
||||
- 暴力血腥、恐怖画面
|
||||
- 违法犯罪相关内容
|
||||
- 明确的政治敏感宣传
|
||||
- 恶意广告、诈骗信息
|
||||
- 仇恨歧视符号或标志
|
||||
|
||||
可疑内容(result=review):
|
||||
- 图片含义模糊,无法确定是否存在违规
|
||||
- 轻微争议性内容但不完全违规
|
||||
- AI无法准确判断的内容
|
||||
|
||||
放行规则(result=pass):
|
||||
- 正常风景、人物、动物、动漫等图片
|
||||
- 表情包、梗图
|
||||
- 正常的艺术创作、摄影作品
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
|
||||
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
- 明确的违法犯罪、暴力威胁内容
|
||||
- 色情低俗、性暗示描述
|
||||
- 诈骗引流、恶意广告
|
||||
- 明确煽动群体对立、仇恨
|
||||
- 恶意公开他人隐私信息
|
||||
|
||||
可疑内容(result=review):
|
||||
- 内容含义模糊,无法确定违规意图
|
||||
- 涉及敏感话题但未明确违规
|
||||
- AI无法准确判断的表达
|
||||
|
||||
放行规则(result=pass):
|
||||
- 正常的自我介绍、心情表达
|
||||
- 玩梗、幽默、吐槽
|
||||
- 引用、歌词、诗句等
|
||||
- 理性的观点表达
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
|
||||
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
|
||||
if !c.IsEnabled() {
|
||||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||||
}
|
||||
|
||||
if strings.TrimSpace(imageURL) == "" {
|
||||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||||
}
|
||||
|
||||
userPrompt := "请审核以下用户图片"
|
||||
cleanImages := normalizeImageURLs([]string{imageURL})
|
||||
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
||||
}
|
||||
|
||||
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
|
||||
if !c.IsEnabled() {
|
||||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||||
}
|
||||
|
||||
if strings.TrimSpace(bio) == "" {
|
||||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||||
}
|
||||
|
||||
userPrompt := "个性签名内容:" + bio
|
||||
return c.moderateWithCustomPrompt(ctx, bioModerationSystemPrompt, userPrompt)
|
||||
}
|
||||
|
||||
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
||||
}
|
||||
|
||||
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
|
||||
cleanImages := normalizeImageURLs(images)
|
||||
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
||||
|
||||
Reference in New Issue
Block a user