carrot_bbs/backend
4
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(moderation): add manual content review system with three-tier AI moderation
All checks were successful
Build Backend / build (push) Successful in 2m43s
Details
Build Backend / build-docker (push) Successful in 1m22s
Details

Browse Source 
Add admin comment moderation endpoints (single and batch) and introduce a three-tier moderation system (pass/review/block) for content flagged by AI. Content with unclear violations is now held for manual review instead of being auto-rejected. Deleted the legacy audit_service.go in favor of the unified hook-based moderation system.
This commit is contained in:
lafay
2026-04-11 04:23:24 +08:00
parent 6af6aaa9d0
commit 1bb82e1e2b
17 changed files with 432 additions and 732 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
internal/pkg/hook/moderation_hooks.go
View File

@@ -10,11 +10,8 @@ import (
"go.uber.org/zap"
)
type PostModerationHook struct {
name string
postAIService PostAIService
postRepo repository.PostRepository
strictMode bool
type reviewableError interface {
IsReview() bool
}
type PostAIService interface {
@@ -23,6 +20,13 @@ type PostAIService interface {
ModerateComment(ctx context.Context, content string, images []string) error
}
type PostModerationHook struct {
name string
postAIService PostAIService
postRepo repository.PostRepository
strictMode bool
}
func NewPostModerationHook(postAIService PostAIService, postRepo repository.PostRepository, strictMode bool) *PostModerationHook {
return &PostModerationHook{
name: "moderation.post.ai",
@@ -36,9 +40,9 @@ func (h *PostModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookPostPreModerate,
Priority: PriorityHighest,
Priority: PriorityHigh,
Func: h.Execute,
Async: false, // 必须同步执行，因为调用方需要审核结果
Async: false,
Timeout: 30 * time.Second,
})
}
@@ -54,20 +58,32 @@ func (h *PostModerationHook) Execute(ctx *HookContext) error {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
zap.L().Debug("AI moderation disabled, auto approve post",
zap.String("post_id", data.PostID),
)
return nil
}
err := h.postAIService.ModeratePost(ctx.Ctx, data.Title, data.Content, data.Images)
if err != nil {
if rejectErr, ok := err.(interface{ UserMessage() string }); ok {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Post flagged for manual review by AI",
zap.String("post_id", data.PostID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = rejectErr.UserMessage()
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Post rejected by AI moderation",
zap.String("post_id", data.PostID),
@@ -98,9 +114,6 @@ func (h *PostModerationHook) Execute(ctx *HookContext) error {
result.Approved = true
result.ReviewedBy = "ai"
zap.L().Debug("Post approved by AI moderation",
zap.String("post_id", data.PostID),
)
return nil
}
@@ -124,9 +137,9 @@ func (h *CommentModerationHook) Register(manager *Manager) {
manager.Register(&Hook{
Name: h.name,
HookType: HookCommentPreModerate,
Priority: PriorityHighest,
Priority: PriorityHigh,
Func: h.Execute,
Async: false, // 必须同步执行，因为调用方需要审核结果
Async: false,
Timeout: 30 * time.Second,
})
}
@@ -142,20 +155,32 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
return nil
}
if result.RejectReason != "" || result.NeedsReview {
return nil
}
if h.postAIService == nil || !h.postAIService.IsEnabled() {
result.Approved = true
result.ReviewedBy = "system"
zap.L().Debug("AI moderation disabled, auto approve comment",
zap.String("comment_id", data.CommentID),
)
return nil
}
err := h.postAIService.ModerateComment(ctx.Ctx, data.Content, data.Images)
if err != nil {
if rejectErr, ok := err.(interface{ UserMessage() string }); ok {
if userErr, ok := err.(interface{ UserMessage() string }); ok {
if revErr, ok := err.(reviewableError); ok && revErr.IsReview() {
result.Approved = false
result.NeedsReview = true
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Comment flagged for manual review by AI",
zap.String("comment_id", data.CommentID),
zap.String("reason", result.RejectReason),
)
return nil
}
result.Approved = false
result.RejectReason = rejectErr.UserMessage()
result.RejectReason = userErr.UserMessage()
result.ReviewedBy = "ai"
zap.L().Info("Comment rejected by AI moderation",
zap.String("comment_id", data.CommentID),
@@ -177,7 +202,7 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
result.Approved = true
result.ReviewedBy = "system"
zap.L().Warn("AI comment moderation error, fallback approve",
zap.L().Warn("AI moderation error, fallback approve",
zap.String("comment_id", data.CommentID),
zap.Error(err),
)
@@ -186,23 +211,20 @@ func (h *CommentModerationHook) Execute(ctx *HookContext) error {
result.Approved = true
result.ReviewedBy = "ai"
zap.L().Debug("Comment approved by AI moderation",
zap.String("comment_id", data.CommentID),
)
return nil
}
type SensitiveService interface {
Check(ctx context.Context, text string) (bool, []string)
Replace(ctx context.Context, text string, repl string) string
}
type SensitiveWordHook struct {
name string
sensitive SensitiveService
replaceStr string
}
type SensitiveService interface {
Check(ctx context.Context, text string) (bool, []string)
Replace(ctx context.Context, text string, repl string) string
}
func NewSensitiveWordHook(sensitive SensitiveService, replaceStr string) *SensitiveWordHook {
return &SensitiveWordHook{
name: "moderation.sensitive_word",
@@ -215,7 +237,7 @@ func (h *SensitiveWordHook) RegisterPostHook(manager *Manager) {
manager.Register(&Hook{
Name: h.name + ".post",
HookType: HookPostPreModerate,
Priority: PriorityHigh,
Priority: PriorityHighest,
Func: h.executePost,
Async: false,
})
@@ -225,7 +247,7 @@ func (h *SensitiveWordHook) RegisterCommentHook(manager *Manager) {
manager.Register(&Hook{
Name: h.name + ".comment",
HookType: HookCommentPreModerate,
Priority: PriorityHigh,
Priority: PriorityHighest,
Func: h.executeComment,
Async: false,
})
@@ -325,18 +347,18 @@ func NewModerationHooks(
}
func (m *ModerationHooks) RegisterAll(manager *Manager) {
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
postHook.Register(manager)
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager)
if m.sensitiveService != nil {
sensitiveHook := NewSensitiveWordHook(m.sensitiveService, m.replaceStr)
sensitiveHook.RegisterPostHook(manager)
sensitiveHook.RegisterCommentHook(manager)
}
postHook := NewPostModerationHook(m.postAIService, m.postRepo, m.strictMode)
postHook.Register(manager)
commentHook := NewCommentModerationHook(m.postAIService, m.commentRepo, m.strictMode)
commentHook.Register(manager)
zap.L().Info("Registered moderation hooks",
zap.Bool("ai_enabled", m.postAIService != nil && m.postAIService.IsEnabled()),
zap.Bool("sensitive_enabled", m.sensitiveService != nil),

1
internal/pkg/hook/types.go
View File

@@ -103,6 +103,7 @@ type CommentModeratedHookData struct {
type ModerationResult struct {
Approved bool
NeedsReview bool
RejectReason string
ReviewedBy string
Error error

125
internal/pkg/openai/client.go
View File

@@ -19,7 +19,50 @@ import (
xdraw "golang.org/x/image/draw"
)
const moderationSystemPrompt = "你是中文社区的内容审核助手，负责对“帖子标题、正文、配图”做联合审核。目标是平衡社区安全与正常交流：必须拦截高风险违规内容，但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。\n\n审核流程：\n1) 先判断是否命中硬性违规；\n2) 再判断语境（玩笑/自嘲/朋友间互动/作品讨论）；\n3) 做文图交叉判断（文本+图片合并理解）；\n4) 给出 approved 与简短 reason。\n\n硬性违规（命中任一项必须 approved=false）：\nA. 宣传对立与煽动撕裂：\n- 明确煽动群体对立、地域对立、性别对立、民族宗教对立，鼓动仇恨、排斥、报复。\nB. 严重人身攻击与网暴引导：\n- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突。\nC. 开盒/人肉/隐私暴露：\n- 故意公开、拼接、索取他人可识别隐私信息（姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等）；\n- 图片/截图中出现可识别隐私信息并伴随曝光意图，也按违规处理。\nD. 其他高危违规：\n- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。\n\n放行规则（以下通常 approved=true）：\n- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽；\n- 非定向、轻度口语化吐槽（无明确攻击对象、无网暴号召、无隐私暴露）；\n- 对社会事件/作品的理性讨论、观点争论（即使语气尖锐，但未煽动对立或人身攻击）。\n\n边界判定：\n- 若只是“梗文化表达”且不指向现实伤害，优先通过；\n- 若存在明确伤害意图（煽动、围攻、曝光隐私），必须拒绝；\n- 对模糊内容不因个别粗口直接拒绝，需结合对象、意图、号召性和可执行性综合判断。\n\nreason 要求：\n- approved=false 时：中文10-30字，说明核心违规点；\n- approved=true 时：reason 为空字符串。\n\n输出格式（严格）：\n仅输出一行JSON对象，不要Markdown，不要额外解释：\n{\"approved\": true/false, \"reason\": \"...\"}"
const moderationSystemPrompt = `你是中文社区的内容审核助手，负责对"帖子标题、正文、配图"做联合审核。目标是平衡社区安全与正常交流：必须拦截高风险违规内容，但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。
审核流程：
1) 先判断是否命中硬性违规；
2) 再判断语境（玩笑/自嘲/朋友间互动/作品讨论）；
3) 做文图交叉判断（文本+图片合并理解）；
4) 给出 result 与简短 reason。
硬性违规（命中任一项必须 result=block）：
A. 宣传对立与煽动撕裂：
- 明确煽动群体对立、地域对立、性别对立、民族宗教对立，鼓动仇恨、排斥、报复。
B. 严重人身攻击与网暴引导：
- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突。
C. 开盒/人肉/隐私暴露：
- 故意公开、拼接、索取他人可识别隐私信息（姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等）；
- 图片/截图中出现可识别隐私信息并伴随曝光意图，也按违规处理。
D. 其他高危违规：
- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。
可疑内容（需要人工复审，result=review）：
- 内容含义模糊，无法确定是否存在违规意图；
- 涉及敏感话题但未明确违规（如政治人物、社会热点事件等）；
- 存在轻微争议性表达，但不完全符合硬性违规标准；
- AI无法准确判断的文化/地域特定表达。
放行规则（以下通常 result=pass）：
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽；
- 非定向、轻度口语化吐槽（无明确攻击对象、无网暴号召、无隐私暴露）；
- 对社会事件/作品的理性讨论、观点争论（即使语气尖锐，但未煽动对立或人身攻击）。
边界判定：
- 若只是"梗文化表达"且不指向现实伤害，优先通过；
- 若存在明确伤害意图（煽动、围攻、曝光隐私），必须拒绝；
- 对模糊内容不因个别粗口直接拒绝，需结合对象、意图、号召性和可执行性综合判断；
- 如果实在无法确定是否违规，标记为 review 状态交由人工复审。
reason 要求：
- result=block 时：中文10-30字，说明核心违规点；
- result=review 时：中文10-30字，说明可疑原因；
- result=pass 时：reason 为空字符串。
输出格式（严格）：
仅输出一行JSON对象，不要Markdown，不要额外解释：
{"result": "pass"/"review"/"block", "reason": "..."}`
const (
defaultMaxImagesPerModerationRequest = 1
@@ -32,11 +75,24 @@ const (
maxCompressedPayloadBytes = 1536 * 1024
)
type ModerationResult string
const (
ModerationResultPass ModerationResult = "pass"
ModerationResultReview ModerationResult = "review"
ModerationResultBlock ModerationResult = "block"
)
type ModerationResponse struct {
Result ModerationResult `json:"result"`
Reason string `json:"reason"`
}
type Client interface {
IsEnabled() bool
Config() Config
ModeratePost(ctx context.Context, title, content string, images []string) (bool, string, error)
ModerateComment(ctx context.Context, content string, images []string) (bool, string, error)
ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error)
ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error)
}
type clientImpl struct {
@@ -65,12 +121,11 @@ func (c *clientImpl) Config() Config {
return c.cfg
}
func (c *clientImpl) ModeratePost(ctx context.Context, title, content string, images []string) (bool, string, error) {
func (c *clientImpl) ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return true, "", nil
return &ModerationResponse{Result: ModerationResultPass}, nil
}
// 构建帖子提示词，处理标题或内容为空的情况
var prompt strings.Builder
if strings.TrimSpace(title) != "" {
prompt.WriteString("帖子标题：" + title)
@@ -81,7 +136,6 @@ func (c *clientImpl) ModeratePost(ctx context.Context, title, content string, im
}
prompt.WriteString("帖子内容：" + content)
}
// 如果标题和内容都为空但有图片，说明是纯图片帖子
if prompt.Len() == 0 && len(normalizeImageURLs(images)) > 0 {
prompt.WriteString("帖子内容：[仅包含图片]")
}
@@ -92,17 +146,15 @@ func (c *clientImpl) ModeratePost(ctx context.Context, title, content string, im
return c.moderateContentInBatches(ctx, prompt.String(), images)
}
func (c *clientImpl) ModerateComment(ctx context.Context, content string, images []string) (bool, string, error) {
func (c *clientImpl) ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error) {
if !c.IsEnabled() {
return true, "", nil
return &ModerationResponse{Result: ModerationResultPass}, nil
}
// 构建评论提示词，处理纯图片评论的情况
var prompt strings.Builder
if strings.TrimSpace(content) != "" {
prompt.WriteString("评论内容：" + content)
} else if len(normalizeImageURLs(images)) > 0 {
// 评论只有图片没有文字，说明是纯图片评论
prompt.WriteString("评论内容：[仅包含图片]")
} else {
prompt.WriteString("评论内容：[空]")
@@ -111,7 +163,7 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
return c.moderateContentInBatches(ctx, prompt.String(), images)
}
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (bool, string, error) {
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
cleanImages := normalizeImageURLs(images)
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
maxImagesPerRequest := c.maxImagesPerModerationRequest()
@@ -120,7 +172,6 @@ func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt
totalBatches = (len(optimizedImages) + maxImagesPerRequest - 1) / maxImagesPerRequest
}
// 图片超过单批上限时分批审核，任一批次拒绝即整体拒绝
for i := 0; i < totalBatches; i++ {
start := i * maxImagesPerRequest
end := start + maxImagesPerRequest
@@ -133,19 +184,19 @@ func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt
batchImages = optimizedImages[start:end]
}
approved, reason, err := c.moderateSingleBatch(ctx, contentPrompt, batchImages, i+1, totalBatches)
resp, err := c.moderateSingleBatch(ctx, contentPrompt, batchImages, i+1, totalBatches)
if err != nil {
return false, "", err
return nil, err
}
if !approved {
if strings.TrimSpace(reason) != "" && totalBatches > 1 {
reason = fmt.Sprintf("第%d/%d批图片未通过：%s", i+1, totalBatches, reason)
if resp.Result != ModerationResultPass {
if totalBatches > 1 && strings.TrimSpace(resp.Reason) != "" {
resp.Reason = fmt.Sprintf("第%d/%d批图片未通过：%s", i+1, totalBatches, resp.Reason)
}
return false, reason, nil
return resp, nil
}
}
return true, "", nil
return &ModerationResponse{Result: ModerationResultPass}, nil
}
func (c *clientImpl) moderateSingleBatch(
@@ -153,7 +204,7 @@ func (c *clientImpl) moderateSingleBatch(
contentPrompt string,
images []string,
batchNo, totalBatches int,
) (bool, string, error) {
) (*ModerationResponse, error) {
userPrompt := fmt.Sprintf(
"%s\n图片批次：%d/%d（本次仅提供当前批次图片）",
contentPrompt,
@@ -168,13 +219,23 @@ func (c *clientImpl) moderateSingleBatch(
lastErr = err
} else {
parsed := struct {
Approved bool `json:"approved"`
Reason string `json:"reason"`
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
return parsed.Approved, parsed.Reason, nil
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
@@ -182,11 +243,11 @@ func (c *clientImpl) moderateSingleBatch(
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return false, "", sleepErr
return nil, sleepErr
}
}
return false, "", fmt.Errorf(
return nil, fmt.Errorf(
"moderation batch %d/%d failed after %d attempts: %w",
batchNo,
totalBatches,
@@ -200,13 +261,13 @@ type chatCompletionsRequest struct {
Messages []chatMessage `json:"messages"`
Temperature float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
EnableThinking *bool `json:"enable_thinking,omitempty"` // qwen3.5思考模式控制
ThinkingBudget *int `json:"thinking_budget,omitempty"` // 思考过程最大token数
ResponseFormat *responseFormatConfig `json:"response_format,omitempty"` // 响应格式
EnableThinking *bool `json:"enable_thinking,omitempty"`
ThinkingBudget *int `json:"thinking_budget,omitempty"`
ResponseFormat *responseFormatConfig `json:"response_format,omitempty"`
}
type responseFormatConfig struct {
Type string `json:"type"` // "text" or "json_object"
Type string `json:"type"`
}
type chatMessage struct {
@@ -266,12 +327,10 @@ func (c *clientImpl) chatCompletion(
Temperature: temperature,
MaxTokens: maxTokens,
}
// 禁用qwen3.5的思考模式，避免产生大量不必要的token消耗
falseVal := false
reqBody.EnableThinking = &falseVal
zero := 0
reqBody.ThinkingBudget = &zero
// 使用JSON输出格式
reqBody.ResponseFormat = &responseFormatConfig{Type: "json_object"}
data, err := json.Marshal(reqBody)
@@ -381,7 +440,6 @@ func extractJSONObject(raw string) string {
}
func (c *clientImpl) maxImagesPerModerationRequest() int {
// 审核固定单图请求，降低单次payload体积，减少超时风险。
if c.cfg.ModerationMaxImagesPerRequest <= 0 {
return defaultMaxImagesPerModerationRequest
}
@@ -447,7 +505,6 @@ func (c *clientImpl) tryCompressImageForModeration(ctx context.Context, imageURL
if len(compressed) == 0 || len(compressed) > maxCompressedPayloadBytes {
return imageURL
}
// 压缩效果不明显时直接使用原图URL，避免增大请求体。
if len(compressed) >= int(float64(len(originBytes))*0.95) {
return imageURL
}