feat(moderation): make moderation more lenient and add timeout fallback behavior
All checks were successful
Build Backend / build (push) Successful in 2m20s
Build Backend / build-docker (push) Successful in 1m7s

The moderation system now uses a "pass/review" strategy instead of "pass/review/block", treating hard violations the same as review (only routing to manual review rather than auto-rejecting). Add fallback-approve behavior in the hook system so slow moderation APIs never silently block normal content when they time out or get cancelled - timed-out moderation now approves with "ReviewedBy: system" instead of leaving the zero-value result that would reject content. Update post/comment/vote services and handlers to use the new unified behavior. Simplify moderation prompts to emphasize "宁可放过,不可误伤" (prefer false positives over false negatives).
This commit is contained in:
lafay
2026-07-08 01:47:48 +08:00
parent eb931bf1c6
commit a971fb0e29
9 changed files with 324 additions and 267 deletions

View File

@@ -108,11 +108,6 @@ func (h *PostHandler) Create(c *gin.Context) {
post, err := h.postService.Create(c.Request.Context(), userID, req.Title, content, segments, req.Images, req.ChannelID, req.ClientRequestID) post, err := h.postService.Create(c.Request.Context(), userID, req.Title, content, segments, req.Images, req.ChannelID, req.ClientRequestID)
if err != nil { if err != nil {
var moderationErr *service.PostModerationRejectedError
if errors.As(err, &moderationErr) {
response.BadRequest(c, moderationErr.UserMessage())
return
}
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交 // 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
if errors.Is(err, service.ErrDuplicatePostRequest) { if errors.Is(err, service.ErrDuplicatePostRequest) {
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error()) response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())

View File

@@ -42,11 +42,6 @@ func (h *VoteHandler) CreateVotePost(c *gin.Context) {
post, err := h.voteService.CreateVotePost(c.Request.Context(), userID, &req) post, err := h.voteService.CreateVotePost(c.Request.Context(), userID, &req)
if err != nil { if err != nil {
var moderationErr *service.PostModerationRejectedError
if errors.As(err, &moderationErr) {
response.BadRequest(c, moderationErr.UserMessage())
return
}
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交 // 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
if errors.Is(err, service.ErrDuplicatePostRequest) { if errors.Is(err, service.ErrDuplicatePostRequest) {
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error()) response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())

View File

@@ -215,6 +215,7 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
var asyncHooks []*hookEntry var asyncHooks []*hookEntry
var firstSyncErr error
for _, entry := range entries { for _, entry := range entries {
if !entry.hook.Enabled { if !entry.hook.Enabled {
continue continue
@@ -226,6 +227,9 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
} }
if err := m.executeHook(entry, hookCtx); err != nil { if err := m.executeHook(entry, hookCtx); err != nil {
if firstSyncErr == nil {
firstSyncErr = err
}
zap.L().Error("Hook execution failed", zap.L().Error("Hook execution failed",
zap.String("name", entry.hook.Name), zap.String("name", entry.hook.Name),
zap.String("type", string(hookType)), zap.String("type", string(hookType)),
@@ -256,12 +260,29 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
}() }()
} }
return nil return firstSyncErr
} }
func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error { func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
start := time.Now() start := time.Now()
// Derive a cancellable context bounded by the hook's Timeout so that
// long-running hook work (e.g. HTTP calls to the moderation API) is
// actually cancelled when the timeout fires, instead of leaking and
// racing on shared metadata after executeHook returns.
if entry.hook.Timeout > 0 {
hookCtx, cancel := context.WithTimeout(ctx.Ctx, entry.hook.Timeout)
defer cancel()
ctx = &HookContext{
Ctx: hookCtx,
HookType: ctx.HookType,
UserID: ctx.UserID,
Data: ctx.Data,
Metadata: ctx.Metadata,
}
}
var err error var err error
if entry.hook.Timeout > 0 { if entry.hook.Timeout > 0 {
done := make(chan error, 1) done := make(chan error, 1)
@@ -280,13 +301,17 @@ func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
select { select {
case err = <-done: case err = <-done:
case <-time.After(entry.hook.Timeout): case <-ctx.Ctx.Done():
err = context.DeadlineExceeded err = ctx.Ctx.Err()
zap.L().Warn("Hook timeout", zap.L().Warn("Hook timeout",
zap.String("name", entry.hook.Name), zap.String("name", entry.hook.Name),
zap.Duration("timeout", entry.hook.Timeout), zap.Duration("timeout", entry.hook.Timeout),
zap.Error(err),
) )
} }
// Wait for the spawned goroutine to observe cancellation and return,
// so it cannot keep writing to ctx.Metadata after executeHook returns.
<-done
} else { } else {
err = entry.hook.Func(ctx) err = entry.hook.Func(ctx)
} }

View File

@@ -2,6 +2,7 @@
import ( import (
"context" "context"
"errors"
"strings" "strings"
"time" "time"
@@ -382,40 +383,69 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
func (m *ModerationHooks) ModeratePost(ctx context.Context, manager *Manager, data *PostModerateHookData) *ModerationResult { func (m *ModerationHooks) ModeratePost(ctx context.Context, manager *Manager, data *PostModerateHookData) *ModerationResult {
result := &ModerationResult{} result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{ if err := manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{
"result": result, "result": result,
}) }); err != nil {
applyModerationFallback(result, err)
}
return result return result
} }
func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager, data *CommentModerateHookData) *ModerationResult { func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager, data *CommentModerateHookData) *ModerationResult {
result := &ModerationResult{} result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{ if err := manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{
"result": result, "result": result,
}) }); err != nil {
applyModerationFallback(result, err)
}
return result return result
} }
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult { func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
result := &ModerationResult{} result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{ if err := manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
"result": result, "result": result,
}) }); err != nil {
applyModerationFallback(result, err)
}
return result return result
} }
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult { func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
result := &ModerationResult{} result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{ if err := manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
"result": result, "result": result,
}) }); err != nil {
applyModerationFallback(result, err)
}
return result return result
} }
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult { func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
result := &ModerationResult{} result := &ModerationResult{}
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{ if err := manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
"result": result, "result": result,
}) }); err != nil {
applyModerationFallback(result, err)
}
return result return result
} }
// applyModerationFallback handles moderation hook timeouts/cancellations by
// falling back to "approved by system" instead of leaving the zero-value
// result (Approved=false) which would silently reject otherwise-fine content.
func applyModerationFallback(result *ModerationResult, err error) {
if err == nil {
return
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
result.Approved = true
result.ReviewedBy = "system"
result.RejectReason = ""
result.NeedsReview = false
result.Error = err
zap.L().Warn("Moderation hook timed out or was cancelled, fallback approve",
zap.Error(err),
)
}
}

View File

@@ -5,6 +5,7 @@ import (
"context" "context"
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"image" "image"
_ "image/gif" _ "image/gif"
@@ -21,55 +22,41 @@ import (
xdraw "golang.org/x/image/draw" xdraw "golang.org/x/image/draw"
) )
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。目标是平衡社区安全与正常交流:必须拦截高风险违规内容,但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。 const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
审核流程 核心原则
1) 先判断是否命中硬性违规 - 除明确违法违规外,不要拦截或送审正常内容,包括尖锐吐槽、玩梗、二创、争论
2) 再判断语境(玩笑/自嘲/朋友间互动/作品讨论); - 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
3) 做文图交叉判断(文本+图片合并理解);
4) 给出 result 与简短 reason。
硬性违规(命中任一项必须 result=block 只有命中**明确违法违规**时才输出 result=review
A. 宣传对立与煽动撕裂: A. 违法犯罪:明确宣传违法犯罪行为、教唆或指导他人犯罪(如毒品、枪支、诈骗教程)。
- 明确煽动群体对立、地域对立、性别对立、民族宗教对立,鼓动仇恨、排斥、报复 B. 真实暴力威胁:针对具体个人或群体的实质性暴力威胁、人身伤害恐吓
B. 严重人身攻击与网暴引导: C. 色情低俗:明确色情、露骨性描写或诱导性内容;正常人体艺术、二次创作用感形象不在此列。
- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突 D. 诈骗引流:明确的诈骗信息、钓鱼链接、黑产引流
C. 开盒/人肉/隐私暴露: E. 未成年保护:涉及未成年人的色情或剥削内容。
- 故意公开、拼接、索取他人可识别隐私信息(姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等); F. 开盒/人肉:故意公开他人可识别隐私信息(身份证号、住址、联系方式、定位轨迹)并伴有曝光意图。
- 图片/截图中出现可识别隐私信息并伴随曝光意图,也按违规处理 G. 极端仇恨:鼓动对特定族裔/宗教/残障群体的仇恨与暴力,含纳粹等极端符号
D. 其他高危违规:
- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。
可疑内容需要人工复审result=review 以下情形即使存在也**一律 pass**(不送审、不拦截
- 内容含义模糊,无法确定是否存在违规意图;
- 涉及敏感话题但未明确违规(如政治人物、社会热点事件等);
- 存在轻微争议性表达,但不完全符合硬性违规标准;
- AI无法准确判断的文化/地域特定表达。
放行规则(以下通常 result=pass
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽; - 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽;
- 非定向、轻度口语化吐槽(无明确攻击对象、无网暴号召、无隐私暴露) - 非定向、轻度口语化吐槽、玩笑、自嘲、朋友间互动
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐,但未煽动对立或人身攻击)。 - 对社会事件/作品的理性讨论、观点争论(即使语气尖锐、用词激烈);
- 一般性粗口、调侃、讽刺,未煽动对立或人身攻击的;
边界判定: - 对政治人物/社会热点的评论、争议性观点表达;
- 若只是"梗文化表达"且不指向现实伤害,优先通过; - 含义模糊但无明显违法意图的内容。
- 若存在明确伤害意图(煽动、围攻、曝光隐私),必须拒绝;
- 对模糊内容不因个别粗口直接拒绝,需结合对象、意图、号召性和可执行性综合判断;
- 如果实在无法确定是否违规,标记为 review 状态交由人工复审。
reason 要求: reason 要求:
- result=block中文10-30字说明核心违规点; - result=review中文10-30字说明疑似违法点;
- result=review 时中文10-30字说明可疑原因
- result=pass 时reason 为空字符串。 - result=pass 时reason 为空字符串。
输出格式(严格): 输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释 仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}` {"result": "pass"/"review", "reason": "..."}`
const ( const (
defaultMaxImagesPerModerationRequest = 1 defaultMaxImagesPerModerationRequest = 1
maxAllowedImagesPerModerationRequest = 4
maxModerationResultRetries = 3 maxModerationResultRetries = 3
maxChatCompletionRetries = 3
initialRetryBackoff = 500 * time.Millisecond initialRetryBackoff = 500 * time.Millisecond
maxDownloadImageBytes = 10 * 1024 * 1024 maxDownloadImageBytes = 10 * 1024 * 1024
maxModerationImageSide = 1280 maxModerationImageSide = 1280
@@ -165,53 +152,61 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
return c.moderateContentInBatches(ctx, prompt.String(), images) return c.moderateContentInBatches(ctx, prompt.String(), images)
} }
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。 const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。原则是**宁可放过,不可误伤**:只有明确违法违规的图片才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
硬性违规(命中任一项必须 result=block 核心原则
- 色情低俗、裸露、性暗示内容 - 除明确违法违规外,不要拦截或送审正常图片;
- 暴力血腥、恐怖画面 - 判定从严,倾向放行:宁可放过十张可疑,不可误伤一张正常。
- 违法犯罪相关内容
- 明确的政治敏感宣传
- 恶意广告、诈骗信息
- 仇恨歧视符号或标志
可疑内容(result=review 只有命中**明确违法违规**时才输出 result=review
- 图片含义模糊,无法确定是否存在违规 - 色情低俗:明确色情、露骨性描写或诱导性图片;正常人体艺术、二次创作用感形象不在此列;
- 轻微争议性内容但不完全违规 - 真实暴力/血腥:明确展示严重暴力、凶杀、虐待等违法犯罪画面;
- AI无法准确判断的内容 - 违法犯罪相关:明确涉及毒品、枪支、诈骗、黑产等内容
- 未成年保护:涉及未成年人的色情或剥削图片;
- 诈骗引流:明确的诈骗二维码、钓鱼链接、黑产广告;
- 极端仇恨:纳粹等极端仇恨符号或标志。
放行规则result=pass 以下图片即使存在也**一律 pass**(不送审、不拦截
- 正常风景、人物、动物、动漫等图片 - 正常风景、人物、动物、动漫等图片
- 表情包、梗图 - 表情包、梗图、二次创作;
- 正常的艺术创作、摄影作品 - 正常的艺术创作、摄影作品
- 含义模糊但无明显违法意图的图片。
reason 要求:
- result=review 时中文10-30字说明疑似违法点
- result=pass 时reason 为空字符串。
输出格式(严格): 输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释 仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}` {"result": "pass"/"review", "reason": "..."}`
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。 const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
硬性违规(命中任一项必须 result=block 核心原则
- 明确违法犯罪、暴力威胁内容 - 明确违法违规外,不要拦截或送审正常签名;
- 色情低俗、性暗示描述 - 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
- 诈骗引流、恶意广告
- 明确煽动群体对立、仇恨
- 恶意公开他人隐私信息
可疑内容(result=review 只有命中**明确违法违规**时才输出 result=review
- 内容含义模糊,无法确定违规意图 - 明确违法犯罪、暴力威胁内容;
- 涉及敏感话题但未明确违规 - 色情低俗、露骨性暗示描述;
- AI无法准确判断的表达 - 诈骗引流、黑产广告;
- 恶意公开他人可识别隐私信息;
- 涉及未成年人的色情或剥削内容。
放行规则result=pass 以下签名即使存在也**一律 pass**(不送审、不拦截
- 正常的自我介绍、心情表达 - 正常的自我介绍、心情表达
- 玩梗、幽默、吐槽 - 玩梗、幽默、吐槽、谐音梗;
- 引用、歌词、诗句等 - 引用、歌词、诗句等
- 理性的观点表达 - 理性的观点表达、对社会事件的评论;
- 含义模糊但无明显违法意图的内容。
reason 要求:
- result=review 时中文10-30字说明疑似违法点
- result=pass 时reason 为空字符串。
输出格式(严格): 输出格式(严格):
仅输出一行JSON对象不要Markdown不要额外解释 仅输出一行JSON对象不要Markdown不要额外解释
{"result": "pass"/"review"/"block", "reason": "..."}` {"result": "pass"/"review", "reason": "..."}`
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) { func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
if !c.IsEnabled() { if !c.IsEnabled() {
@@ -226,42 +221,11 @@ func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*Moder
cleanImages := normalizeImageURLs([]string{imageURL}) cleanImages := normalizeImageURLs([]string{imageURL})
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages) optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
var lastErr error resp, err := c.moderateWithRetry(ctx, imageModerationSystemPrompt, userPrompt, optimizedImages)
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
if err != nil { if err != nil {
lastErr = err return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, err)
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
} }
return &ModerationResponse{ return resp, nil
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
} }
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) { func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
@@ -278,42 +242,11 @@ func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationRe
} }
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) { func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
var lastErr error resp, err := c.moderateWithRetry(ctx, systemPrompt, userPrompt, nil)
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
if err != nil { if err != nil {
lastErr = err return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, err)
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
} }
return &ModerationResponse{ return resp, nil
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
} }
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) { func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
@@ -365,49 +298,18 @@ func (c *clientImpl) moderateSingleBatch(
totalBatches, totalBatches,
) )
var lastErr error resp, err := c.moderateWithRetry(ctx, moderationSystemPrompt, userPrompt, images)
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, moderationSystemPrompt, userPrompt, images, 0.1, 220)
if err != nil { if err != nil {
lastErr = err
} else {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
} else {
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, fmt.Errorf( return nil, fmt.Errorf(
"moderation batch %d/%d failed after %d attempts: %w", "moderation batch %d/%d failed after %d attempts: %w",
batchNo, batchNo,
totalBatches, totalBatches,
maxModerationResultRetries, maxModerationResultRetries,
lastErr, err,
) )
} }
return resp, nil
}
type chatCompletionsRequest struct { type chatCompletionsRequest struct {
Model string `json:"model"` Model string `json:"model"`
@@ -497,36 +399,69 @@ func (c *clientImpl) chatCompletion(
endpoint = baseURL + "/chat/completions" endpoint = baseURL + "/chat/completions"
} }
var lastErr error
for attempt := 0; attempt < maxChatCompletionRetries; attempt++ {
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data) body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
if err != nil { if err != nil {
lastErr = err return "", &chatCompletionError{StatusCode: 0, Err: err}
} else if statusCode >= 400 {
lastErr = fmt.Errorf("openai error status=%d body=%s", statusCode, string(body))
if !isRetryableStatusCode(statusCode) {
return "", lastErr
} }
} else { if statusCode >= 400 {
return "", &chatCompletionError{
StatusCode: statusCode,
Err: fmt.Errorf("openai error status=%d body=%s", statusCode, string(body)),
}
}
var parsed chatCompletionsResponse var parsed chatCompletionsResponse
if err := json.Unmarshal(body, &parsed); err != nil { if err := json.Unmarshal(body, &parsed); err != nil {
return "", fmt.Errorf("decode response: %w", err) return "", &chatCompletionError{
StatusCode: statusCode,
Err: fmt.Errorf("decode response: %w", err),
}
} }
if len(parsed.Choices) == 0 { if len(parsed.Choices) == 0 {
return "", fmt.Errorf("empty response choices") return "", &chatCompletionError{
StatusCode: statusCode,
Err: fmt.Errorf("empty response choices"),
}
} }
return parsed.Choices[0].Message.Content, nil return parsed.Choices[0].Message.Content, nil
} }
if attempt == maxChatCompletionRetries-1 { // chatCompletionError wraps errors from chatCompletion so callers can decide
break // retryability based on the upstream HTTP status code. StatusCode == 0 means
} // the request never reached the server (transport failure).
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil { type chatCompletionError struct {
return "", sleepErr StatusCode int
} Err error
} }
return "", lastErr func (e *chatCompletionError) Error() string {
if e.Err == nil {
return "chat completion error"
}
return e.Err.Error()
}
func (e *chatCompletionError) Unwrap() error {
return e.Err
}
// isRetryableModerationErr reports whether a moderation caller should retry
// after the given chatCompletion error. Context cancellation and non-429 4xx
// responses are not retryable; transport, 429, 5xx, and parsing errors are.
func isRetryableModerationErr(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
var ce *chatCompletionError
if errors.As(err, &ce) {
if ce.StatusCode >= 400 && ce.StatusCode != http.StatusTooManyRequests && ce.StatusCode < 500 {
return false
}
}
return true
} }
func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) { func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) {
@@ -550,13 +485,6 @@ func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint strin
return body, resp.StatusCode, nil return body, resp.StatusCode, nil
} }
func isRetryableStatusCode(statusCode int) bool {
if statusCode == http.StatusTooManyRequests {
return true
}
return statusCode >= 500 && statusCode <= 599
}
func sleepWithBackoff(ctx context.Context, attempt int) error { func sleepWithBackoff(ctx context.Context, attempt int) error {
delay := initialRetryBackoff * time.Duration(1<<attempt) delay := initialRetryBackoff * time.Duration(1<<attempt)
timer := time.NewTimer(delay) timer := time.NewTimer(delay)
@@ -570,6 +498,63 @@ func sleepWithBackoff(ctx context.Context, attempt int) error {
} }
} }
// moderateWithRetry runs a single moderation request up to maxModerationResultRetries
// times, retrying only on retryable chatCompletion errors. The underlying chatCompletion
// no longer retries on its own, so the total number of upstream calls is bounded by
// maxModerationResultRetries (previously 3x3=9).
func (c *clientImpl) moderateWithRetry(
ctx context.Context,
systemPrompt, userPrompt string,
images []string,
) (*ModerationResponse, error) {
var lastErr error
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, images, 0.1, 220)
if err != nil {
lastErr = err
} else {
resp, parseErr := parseModerationReply(replyText)
if parseErr != nil {
lastErr = parseErr
} else {
return resp, nil
}
}
if attempt == maxModerationResultRetries-1 {
break
}
if !isRetryableModerationErr(lastErr) {
break
}
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
return nil, sleepErr
}
}
return nil, lastErr
}
func parseModerationReply(replyText string) (*ModerationResponse, error) {
parsed := struct {
Result string `json:"result"`
Reason string `json:"reason"`
}{}
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
return nil, fmt.Errorf("failed to parse moderation result: %w", err)
}
result := ModerationResultPass
switch parsed.Result {
case "review":
result = ModerationResultReview
case "block":
result = ModerationResultBlock
}
return &ModerationResponse{
Result: result,
Reason: parsed.Reason,
}, nil
}
func normalizeImageURLs(images []string) []string { func normalizeImageURLs(images []string) []string {
clean := make([]string, 0, len(images)) clean := make([]string, 0, len(images))
for _, image := range images { for _, image := range images {
@@ -593,13 +578,14 @@ func extractJSONObject(raw string) string {
} }
func (c *clientImpl) maxImagesPerModerationRequest() int { func (c *clientImpl) maxImagesPerModerationRequest() int {
if c.cfg.ModerationMaxImagesPerRequest <= 0 { configured := c.cfg.ModerationMaxImagesPerRequest
if configured <= 0 {
return defaultMaxImagesPerModerationRequest return defaultMaxImagesPerModerationRequest
} }
if c.cfg.ModerationMaxImagesPerRequest > 1 { if configured > maxAllowedImagesPerModerationRequest {
return 1 return maxAllowedImagesPerModerationRequest
} }
return c.cfg.ModerationMaxImagesPerRequest return configured
} }
func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string { func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string {

View File

@@ -2,6 +2,7 @@ package service
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"log" "log"
"strings" "strings"
@@ -170,7 +171,7 @@ func (s *commentService) reviewCommentAsync(
fmt.Sscanf(userID, "%d", &authorID) fmt.Sscanf(userID, "%d", &authorID)
result := &hook.ModerationResult{} result := &hook.ModerationResult{}
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookCommentPreModerate, authorID, &hook.CommentModerateHookData{ triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookCommentPreModerate, authorID, &hook.CommentModerateHookData{
CommentID: commentID, CommentID: commentID,
PostID: postID, PostID: postID,
Content: content, Content: content,
@@ -181,6 +182,19 @@ func (s *commentService) reviewCommentAsync(
"result": result, "result": result,
}) })
// Fallback-approve when a moderation hook times out or is cancelled, so a
// slow moderation API never silently rejects an otherwise-fine comment.
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
zap.L().Warn("Comment moderation hook timed out or was cancelled, fallback approve",
zap.String("comment_id", commentID),
zap.Error(triggerErr),
)
result.Approved = true
result.ReviewedBy = "system"
result.RejectReason = ""
result.NeedsReview = false
}
s.hookManager.Trigger(context.Background(), hook.HookCommentModerated, authorID, &hook.CommentModeratedHookData{ s.hookManager.Trigger(context.Background(), hook.HookCommentModerated, authorID, &hook.CommentModeratedHookData{
CommentID: commentID, CommentID: commentID,
PostID: postID, PostID: postID,

View File

@@ -223,15 +223,13 @@ func (s *postAIService) ModeratePost(ctx context.Context, title, content string,
zap.Error(err), zap.Error(err),
) )
return s.moderateTextWithTencent(ctx, title+" "+content, return s.moderateTextWithTencent(ctx, title+" "+content,
func(reason string) error { return &PostModerationRejectedError{Reason: reason} },
func(reason string) error { return &PostModerationReviewError{Reason: reason} }, func(reason string) error { return &PostModerationReviewError{Reason: reason} },
) )
} }
// block 与 review 同等处理:均仅送人工复审。
switch resp.Result { switch resp.Result {
case openai.ModerationResultBlock: case openai.ModerationResultBlock, openai.ModerationResultReview:
return &PostModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &PostModerationReviewError{Reason: resp.Reason} return &PostModerationReviewError{Reason: resp.Reason}
default: default:
return nil return nil
@@ -239,7 +237,6 @@ func (s *postAIService) ModeratePost(ctx context.Context, title, content string,
} }
return s.moderateTextWithTencent(ctx, title+" "+content, return s.moderateTextWithTencent(ctx, title+" "+content,
func(reason string) error { return &PostModerationRejectedError{Reason: reason} },
func(reason string) error { return &PostModerationReviewError{Reason: reason} }, func(reason string) error { return &PostModerationReviewError{Reason: reason} },
) )
} }
@@ -256,15 +253,13 @@ func (s *postAIService) ModerateComment(ctx context.Context, content string, ima
zap.Error(err), zap.Error(err),
) )
return s.moderateTextWithTencent(ctx, content, return s.moderateTextWithTencent(ctx, content,
func(reason string) error { return &CommentModerationRejectedError{Reason: reason} },
func(reason string) error { return &CommentModerationReviewError{Reason: reason} }, func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
) )
} }
// block 与 review 同等处理:均仅送人工复审。
switch resp.Result { switch resp.Result {
case openai.ModerationResultBlock: case openai.ModerationResultBlock, openai.ModerationResultReview:
return &CommentModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &CommentModerationReviewError{Reason: resp.Reason} return &CommentModerationReviewError{Reason: resp.Reason}
default: default:
return nil return nil
@@ -272,7 +267,6 @@ func (s *postAIService) ModerateComment(ctx context.Context, content string, ima
} }
return s.moderateTextWithTencent(ctx, content, return s.moderateTextWithTencent(ctx, content,
func(reason string) error { return &CommentModerationRejectedError{Reason: reason} },
func(reason string) error { return &CommentModerationReviewError{Reason: reason} }, func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
) )
} }
@@ -293,10 +287,9 @@ func (s *postAIService) ModerateImage(ctx context.Context, imageURL string) erro
return nil return nil
} }
// block 与 review 同等处理:均仅送人工复审。
switch resp.Result { switch resp.Result {
case openai.ModerationResultBlock: case openai.ModerationResultBlock, openai.ModerationResultReview:
return &ImageModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &ImageModerationReviewError{Reason: resp.Reason} return &ImageModerationReviewError{Reason: resp.Reason}
default: default:
return nil return nil
@@ -315,15 +308,13 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
zap.Error(err), zap.Error(err),
) )
return s.moderateTextWithTencent(ctx, bio, return s.moderateTextWithTencent(ctx, bio,
func(reason string) error { return &BioModerationRejectedError{Reason: reason} },
func(reason string) error { return &BioModerationReviewError{Reason: reason} }, func(reason string) error { return &BioModerationReviewError{Reason: reason} },
) )
} }
// block 与 review 同等处理:均仅送人工复审。
switch resp.Result { switch resp.Result {
case openai.ModerationResultBlock: case openai.ModerationResultBlock, openai.ModerationResultReview:
return &BioModerationRejectedError{Reason: resp.Reason}
case openai.ModerationResultReview:
return &BioModerationReviewError{Reason: resp.Reason} return &BioModerationReviewError{Reason: resp.Reason}
default: default:
return nil return nil
@@ -331,7 +322,6 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
} }
return s.moderateTextWithTencent(ctx, bio, return s.moderateTextWithTencent(ctx, bio,
func(reason string) error { return &BioModerationRejectedError{Reason: reason} },
func(reason string) error { return &BioModerationReviewError{Reason: reason} }, func(reason string) error { return &BioModerationReviewError{Reason: reason} },
) )
} }
@@ -339,7 +329,6 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
func (s *postAIService) moderateTextWithTencent( func (s *postAIService) moderateTextWithTencent(
ctx context.Context, ctx context.Context,
text string, text string,
rejectFactory func(string) error,
reviewFactory func(string) error, reviewFactory func(string) error,
) error { ) error {
if !s.isTencentEnabled() { if !s.isTencentEnabled() {
@@ -366,10 +355,9 @@ func (s *postAIService) moderateTextWithTencent(
reason = resp.Label + ": " + strings.Join(resp.Keywords, ", ") reason = resp.Label + ": " + strings.Join(resp.Keywords, ", ")
} }
// 腾讯云 block 也仅送审,与 AI 同语义。
switch resp.Suggestion { switch resp.Suggestion {
case tencent.SuggestionBlock: case tencent.SuggestionBlock, tencent.SuggestionReview:
return rejectFactory(reason)
case tencent.SuggestionReview:
return reviewFactory(reason) return reviewFactory(reason)
default: default:
return nil return nil

View File

@@ -2,6 +2,7 @@ package service
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"log" "log"
"strconv" "strconv"
@@ -216,7 +217,7 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
fmt.Sscanf(userID, "%d", &authorID) fmt.Sscanf(userID, "%d", &authorID)
result := &hook.ModerationResult{} result := &hook.ModerationResult{}
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{ triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
PostID: postID, PostID: postID,
Title: title, Title: title,
Content: content, Content: content,
@@ -226,6 +227,19 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
"result": result, "result": result,
}) })
// Trigger can return context.DeadlineExceeded/Canceled if a moderation hook
// times out. In that case the result may not have been written at all
// (result.Approved stays false). Treat an unfinished moderation as
// "approved + system" rather than silently rejecting the post, so a slow
// moderation API never blocks normal content.
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
log.Printf("[WARN] Post moderation hook timed out or was cancelled, fallback approve post=%s err=%v", postID, triggerErr)
result.Approved = true
result.ReviewedBy = "system"
result.RejectReason = ""
result.NeedsReview = false
}
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{ s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
PostID: postID, PostID: postID,
AuthorID: authorID, AuthorID: authorID,

View File

@@ -177,7 +177,7 @@ func (s *voteService) reviewVotePostAsync(postID, userID, title, content string,
} }
result := &hook.ModerationResult{} result := &hook.ModerationResult{}
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{ triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
PostID: postID, PostID: postID,
Title: title, Title: title,
Content: content, Content: content,
@@ -187,6 +187,16 @@ func (s *voteService) reviewVotePostAsync(postID, userID, title, content string,
"result": result, "result": result,
}) })
// Fallback-approve when a moderation hook times out or is cancelled, so a
// slow moderation API never silently rejects an otherwise-fine vote post.
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
log.Printf("[WARN] Vote post moderation hook timed out or was cancelled, fallback approve post=%s err=%v", postID, triggerErr)
result.Approved = true
result.ReviewedBy = "system"
result.RejectReason = ""
result.NeedsReview = false
}
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{ s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
PostID: postID, PostID: postID,
AuthorID: authorID, AuthorID: authorID,