feat(moderation): add user profile content moderation for avatars, covers, and bios
This commit is contained in:
lafay
@@ -164,8 +164,16 @@ func (s *adminDashboardServiceImpl) GetStats(ctx context.Context) (*dto.Dashboar
|
||||
}
|
||||
stats.PendingComments = pendingComments
|
||||
|
||||
var pendingAvatars, pendingCovers, pendingBios int64
|
||||
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeAvatar, model.UserProfileAuditStatusPending).Count(&pendingAvatars)
|
||||
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeCover, model.UserProfileAuditStatusPending).Count(&pendingCovers)
|
||||
s.db.Model(&model.UserProfileAudit{}).Where("content_type = ? AND status = ?", model.UserProfileContentTypeBio, model.UserProfileAuditStatusPending).Count(&pendingBios)
|
||||
stats.PendingAvatars = pendingAvatars
|
||||
stats.PendingCovers = pendingCovers
|
||||
stats.PendingBios = pendingBios
|
||||
|
||||
// 计算待审核总数
|
||||
stats.PendingReview = stats.PendingPosts + stats.PendingComments
|
||||
stats.PendingReview = stats.PendingPosts + stats.PendingComments + stats.PendingAvatars + stats.PendingCovers + stats.PendingBios
|
||||
stats.PendingPostsCount = stats.PendingPosts
|
||||
|
||||
return &stats, nil
|
||||
|
||||
@@ -89,6 +89,86 @@ func (e *CommentModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type ImageModerationRejectedError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ImageModerationRejectedError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "image rejected by moderation"
|
||||
}
|
||||
return "image rejected by moderation: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *ImageModerationRejectedError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "图片未通过审核，请更换后重试"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
type ImageModerationReviewError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ImageModerationReviewError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "image needs manual review"
|
||||
}
|
||||
return "image needs manual review: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *ImageModerationReviewError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "图片需要人工复审，请耐心等待"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
func (e *ImageModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type BioModerationRejectedError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *BioModerationRejectedError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "bio rejected by moderation"
|
||||
}
|
||||
return "bio rejected by moderation: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *BioModerationRejectedError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "个性签名未通过审核，请修改后重试"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
type BioModerationReviewError struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *BioModerationReviewError) Error() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "bio needs manual review"
|
||||
}
|
||||
return "bio needs manual review: " + e.Reason
|
||||
}
|
||||
|
||||
func (e *BioModerationReviewError) UserMessage() string {
|
||||
if strings.TrimSpace(e.Reason) == "" {
|
||||
return "个性签名需要人工复审，请耐心等待"
|
||||
}
|
||||
return strings.TrimSpace(e.Reason)
|
||||
}
|
||||
|
||||
func (e *BioModerationReviewError) IsReview() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type PostAIService struct {
|
||||
openAIClient openai.Client
|
||||
}
|
||||
@@ -154,3 +234,55 @@ func (s *PostAIService) ModerateComment(ctx context.Context, content string, ima
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostAIService) ModerateImage(ctx context.Context, imageURL string) error {
|
||||
if !s.IsEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
resp, err := s.openAIClient.ModerateImage(ctx, imageURL)
|
||||
if err != nil {
|
||||
if s.openAIClient.Config().StrictModeration {
|
||||
return err
|
||||
}
|
||||
zap.L().Warn("AI image moderation failed, fallback allow",
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &ImageModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
return &ImageModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostAIService) ModerateBio(ctx context.Context, bio string) error {
|
||||
if !s.IsEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
resp, err := s.openAIClient.ModerateBio(ctx, bio)
|
||||
if err != nil {
|
||||
if s.openAIClient.Config().StrictModeration {
|
||||
return err
|
||||
}
|
||||
zap.L().Warn("AI bio moderation failed, fallback allow",
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &BioModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
return &BioModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
395
internal/service/user_profile_audit_service.go
Normal file
395
internal/service/user_profile_audit_service.go
Normal file
@@ -0,0 +1,395 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"carrot_bbs/internal/model"
|
||||
"carrot_bbs/internal/pkg/hook"
|
||||
"carrot_bbs/internal/repository"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type UserProfileAuditService interface {
|
||||
SubmitAvatarAudit(ctx context.Context, userID, avatarURL string) (*model.UserProfileAudit, error)
|
||||
SubmitCoverAudit(ctx context.Context, userID, coverURL string) (*model.UserProfileAudit, error)
|
||||
SubmitBioAudit(ctx context.Context, userID, bio string) (*model.UserProfileAudit, error)
|
||||
GetPendingAudit(ctx context.Context, userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error)
|
||||
GetAuditStatus(ctx context.Context, userID string) (*ProfileAuditStatusResponse, error)
|
||||
|
||||
ModerateAudit(ctx context.Context, auditID string, status model.UserProfileAuditStatus, reason, reviewedBy string) error
|
||||
BatchModerateAudits(ctx context.Context, ids []string, status model.UserProfileAuditStatus, reason, reviewedBy string) error
|
||||
GetPendingList(ctx context.Context, contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error)
|
||||
GetPendingCounts(ctx context.Context) (int64, int64, int64, error)
|
||||
}
|
||||
|
||||
type ProfileAuditStatusResponse struct {
|
||||
AvatarStatus string `json:"avatar_status"`
|
||||
CoverStatus string `json:"cover_status"`
|
||||
BioStatus string `json:"bio_status"`
|
||||
}
|
||||
|
||||
type userProfileAuditServiceImpl struct {
|
||||
auditRepo repository.UserProfileAuditRepository
|
||||
userRepo repository.UserRepository
|
||||
hookManager *hook.Manager
|
||||
moderationHooks *hook.ModerationHooks
|
||||
}
|
||||
|
||||
func NewUserProfileAuditService(
|
||||
auditRepo repository.UserProfileAuditRepository,
|
||||
userRepo repository.UserRepository,
|
||||
hookManager *hook.Manager,
|
||||
moderationHooks *hook.ModerationHooks,
|
||||
) UserProfileAuditService {
|
||||
return &userProfileAuditServiceImpl{
|
||||
auditRepo: auditRepo,
|
||||
userRepo: userRepo,
|
||||
hookManager: hookManager,
|
||||
moderationHooks: moderationHooks,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) SubmitAvatarAudit(ctx context.Context, userID, avatarURL string) (*model.UserProfileAudit, error) {
|
||||
audit := &model.UserProfileAudit{
|
||||
UserID: userID,
|
||||
ContentType: model.UserProfileContentTypeAvatar,
|
||||
Content: avatarURL,
|
||||
Status: model.UserProfileAuditStatusPending,
|
||||
}
|
||||
|
||||
if err := s.auditRepo.Create(audit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
go s.reviewAvatarAsync(audit.ID, userID, avatarURL)
|
||||
|
||||
return audit, nil
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) SubmitCoverAudit(ctx context.Context, userID, coverURL string) (*model.UserProfileAudit, error) {
|
||||
audit := &model.UserProfileAudit{
|
||||
UserID: userID,
|
||||
ContentType: model.UserProfileContentTypeCover,
|
||||
Content: coverURL,
|
||||
Status: model.UserProfileAuditStatusPending,
|
||||
}
|
||||
|
||||
if err := s.auditRepo.Create(audit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
go s.reviewCoverAsync(audit.ID, userID, coverURL)
|
||||
|
||||
return audit, nil
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) SubmitBioAudit(ctx context.Context, userID, bio string) (*model.UserProfileAudit, error) {
|
||||
audit := &model.UserProfileAudit{
|
||||
UserID: userID,
|
||||
ContentType: model.UserProfileContentTypeBio,
|
||||
Content: bio,
|
||||
Status: model.UserProfileAuditStatusPending,
|
||||
}
|
||||
|
||||
if err := s.auditRepo.Create(audit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
go s.reviewBioAsync(audit.ID, userID, bio)
|
||||
|
||||
return audit, nil
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) GetPendingAudit(ctx context.Context, userID string, contentType model.UserProfileContentType) (*model.UserProfileAudit, error) {
|
||||
return s.auditRepo.GetPendingByUserID(userID, contentType)
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) GetAuditStatus(ctx context.Context, userID string) (*ProfileAuditStatusResponse, error) {
|
||||
resp := &ProfileAuditStatusResponse{
|
||||
AvatarStatus: string(model.UserProfileAuditStatusApproved),
|
||||
CoverStatus: string(model.UserProfileAuditStatusApproved),
|
||||
BioStatus: string(model.UserProfileAuditStatusApproved),
|
||||
}
|
||||
|
||||
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeAvatar); err == nil {
|
||||
resp.AvatarStatus = string(audit.Status)
|
||||
}
|
||||
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeCover); err == nil {
|
||||
resp.CoverStatus = string(audit.Status)
|
||||
}
|
||||
if audit, err := s.auditRepo.GetByUserID(userID, model.UserProfileContentTypeBio); err == nil {
|
||||
resp.BioStatus = string(audit.Status)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) ModerateAudit(ctx context.Context, auditID string, status model.UserProfileAuditStatus, reason, reviewedBy string) error {
|
||||
audit, err := s.auditRepo.GetByID(auditID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
audit.Status = status
|
||||
audit.ReviewedAt = &now
|
||||
audit.ReviewedBy = reviewedBy
|
||||
audit.RejectReason = reason
|
||||
|
||||
if err := s.auditRepo.Update(audit); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if status == model.UserProfileAuditStatusApproved {
|
||||
if err := s.applyAuditContent(audit); err != nil {
|
||||
zap.L().Error("Failed to apply approved audit content",
|
||||
zap.String("audit_id", audit.ID),
|
||||
zap.String("user_id", audit.UserID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) BatchModerateAudits(ctx context.Context, ids []string, status model.UserProfileAuditStatus, reason string, reviewedBy string) error {
|
||||
for _, id := range ids {
|
||||
if err := s.ModerateAudit(ctx, id, status, reason, reviewedBy); err != nil {
|
||||
zap.L().Warn("Failed to moderate audit in batch",
|
||||
zap.String("audit_id", id),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) GetPendingList(ctx context.Context, contentType model.UserProfileContentType, page, pageSize int) ([]*model.UserProfileAudit, int64, error) {
|
||||
return s.auditRepo.GetPendingList(contentType, page, pageSize)
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) GetPendingCounts(ctx context.Context) (int64, int64, int64, error) {
|
||||
return s.auditRepo.CountPendingAll()
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) reviewAvatarAsync(auditID, userID, avatarURL string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[ERROR] Panic in avatar moderation async flow, audit=%s panic=%v", auditID, r)
|
||||
}
|
||||
}()
|
||||
|
||||
result := &hook.ModerationResult{}
|
||||
if s.moderationHooks != nil && s.hookManager != nil {
|
||||
result = s.moderationHooks.ModerateAvatar(context.Background(), s.hookManager, &hook.UserAvatarModerateHookData{
|
||||
UserID: userID,
|
||||
AvatarURL: avatarURL,
|
||||
})
|
||||
} else {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
}
|
||||
|
||||
audit, err := s.auditRepo.GetByID(auditID)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get audit for avatar moderation result",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if !result.Approved {
|
||||
if result.NeedsReview {
|
||||
audit.ReviewedBy = "ai"
|
||||
zap.L().Info("User avatar flagged for manual review",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.String("user_id", userID),
|
||||
)
|
||||
} else {
|
||||
audit.Status = model.UserProfileAuditStatusRejected
|
||||
audit.RejectReason = result.RejectReason
|
||||
audit.ReviewedBy = result.ReviewedBy
|
||||
audit.ReviewedAt = &now
|
||||
zap.L().Info("User avatar rejected by moderation",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.String("user_id", userID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
audit.Status = model.UserProfileAuditStatusApproved
|
||||
audit.ReviewedBy = result.ReviewedBy
|
||||
audit.ReviewedAt = &now
|
||||
if err := s.applyAuditContent(audit); err != nil {
|
||||
zap.L().Error("Failed to apply approved avatar",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.auditRepo.Update(audit); err != nil {
|
||||
zap.L().Error("Failed to update avatar audit status",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) reviewCoverAsync(auditID, userID, coverURL string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[ERROR] Panic in cover moderation async flow, audit=%s panic=%v", auditID, r)
|
||||
}
|
||||
}()
|
||||
|
||||
result := &hook.ModerationResult{}
|
||||
if s.moderationHooks != nil && s.hookManager != nil {
|
||||
result = s.moderationHooks.ModerateCover(context.Background(), s.hookManager, &hook.UserCoverModerateHookData{
|
||||
UserID: userID,
|
||||
CoverURL: coverURL,
|
||||
})
|
||||
} else {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
}
|
||||
|
||||
audit, err := s.auditRepo.GetByID(auditID)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get audit for cover moderation result",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if !result.Approved {
|
||||
if result.NeedsReview {
|
||||
audit.ReviewedBy = "ai"
|
||||
zap.L().Info("User cover flagged for manual review",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.String("user_id", userID),
|
||||
)
|
||||
} else {
|
||||
audit.Status = model.UserProfileAuditStatusRejected
|
||||
audit.RejectReason = result.RejectReason
|
||||
audit.ReviewedBy = result.ReviewedBy
|
||||
audit.ReviewedAt = &now
|
||||
zap.L().Info("User cover rejected by moderation",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.String("user_id", userID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
audit.Status = model.UserProfileAuditStatusApproved
|
||||
audit.ReviewedBy = result.ReviewedBy
|
||||
audit.ReviewedAt = &now
|
||||
if err := s.applyAuditContent(audit); err != nil {
|
||||
zap.L().Error("Failed to apply approved cover",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.auditRepo.Update(audit); err != nil {
|
||||
zap.L().Error("Failed to update cover audit status",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) reviewBioAsync(auditID, userID, bio string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[ERROR] Panic in bio moderation async flow, audit=%s panic=%v", auditID, r)
|
||||
}
|
||||
}()
|
||||
|
||||
result := &hook.ModerationResult{}
|
||||
if s.moderationHooks != nil && s.hookManager != nil {
|
||||
result = s.moderationHooks.ModerateBio(context.Background(), s.hookManager, &hook.UserBioModerateHookData{
|
||||
UserID: userID,
|
||||
Bio: bio,
|
||||
})
|
||||
} else {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
}
|
||||
|
||||
audit, err := s.auditRepo.GetByID(auditID)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get audit for bio moderation result",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if !result.Approved {
|
||||
if result.NeedsReview {
|
||||
audit.ReviewedBy = "ai"
|
||||
zap.L().Info("User bio flagged for manual review",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.String("user_id", userID),
|
||||
)
|
||||
} else {
|
||||
audit.Status = model.UserProfileAuditStatusRejected
|
||||
audit.RejectReason = result.RejectReason
|
||||
audit.ReviewedBy = result.ReviewedBy
|
||||
audit.ReviewedAt = &now
|
||||
zap.L().Info("User bio rejected by moderation",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.String("user_id", userID),
|
||||
zap.String("reason", result.RejectReason),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
audit.Status = model.UserProfileAuditStatusApproved
|
||||
audit.ReviewedBy = result.ReviewedBy
|
||||
audit.ReviewedAt = &now
|
||||
if err := s.applyAuditContent(audit); err != nil {
|
||||
zap.L().Error("Failed to apply approved bio",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.auditRepo.Update(audit); err != nil {
|
||||
zap.L().Error("Failed to update bio audit status",
|
||||
zap.String("audit_id", auditID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *userProfileAuditServiceImpl) applyAuditContent(audit *model.UserProfileAudit) error {
|
||||
user, err := s.userRepo.GetByID(audit.UserID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch audit.ContentType {
|
||||
case model.UserProfileContentTypeAvatar:
|
||||
user.Avatar = audit.Content
|
||||
case model.UserProfileContentTypeCover:
|
||||
user.CoverURL = audit.Content
|
||||
case model.UserProfileContentTypeBio:
|
||||
user.Bio = audit.Content
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.userRepo.Update(user)
|
||||
}
|
||||
Reference in New Issue
Block a user