feat(moderation): make moderation more lenient and add timeout fallback behavior
The moderation system now uses a "pass/review" strategy instead of "pass/review/block", treating hard violations the same as review (only routing to manual review rather than auto-rejecting). Add fallback-approve behavior in the hook system so slow moderation APIs never silently block normal content when they time out or get cancelled - timed-out moderation now approves with "ReviewedBy: system" instead of leaving the zero-value result that would reject content. Update post/comment/vote services and handlers to use the new unified behavior. Simplify moderation prompts to emphasize "宁可放过,不可误伤" (prefer false positives over false negatives).
This commit is contained in:
@@ -106,13 +106,8 @@ func (h *PostHandler) Create(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
post, err := h.postService.Create(c.Request.Context(), userID, req.Title, content, segments, req.Images, req.ChannelID, req.ClientRequestID)
|
||||
post, err := h.postService.Create(c.Request.Context(), userID, req.Title, content, segments, req.Images, req.ChannelID, req.ClientRequestID)
|
||||
if err != nil {
|
||||
var moderationErr *service.PostModerationRejectedError
|
||||
if errors.As(err, &moderationErr) {
|
||||
response.BadRequest(c, moderationErr.UserMessage())
|
||||
return
|
||||
}
|
||||
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
|
||||
if errors.Is(err, service.ErrDuplicatePostRequest) {
|
||||
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())
|
||||
|
||||
@@ -42,11 +42,6 @@ func (h *VoteHandler) CreateVotePost(c *gin.Context) {
|
||||
|
||||
post, err := h.voteService.CreateVotePost(c.Request.Context(), userID, &req)
|
||||
if err != nil {
|
||||
var moderationErr *service.PostModerationRejectedError
|
||||
if errors.As(err, &moderationErr) {
|
||||
response.BadRequest(c, moderationErr.UserMessage())
|
||||
return
|
||||
}
|
||||
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
|
||||
if errors.Is(err, service.ErrDuplicatePostRequest) {
|
||||
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())
|
||||
|
||||
@@ -215,6 +215,7 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
|
||||
|
||||
var asyncHooks []*hookEntry
|
||||
|
||||
var firstSyncErr error
|
||||
for _, entry := range entries {
|
||||
if !entry.hook.Enabled {
|
||||
continue
|
||||
@@ -226,6 +227,9 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
|
||||
}
|
||||
|
||||
if err := m.executeHook(entry, hookCtx); err != nil {
|
||||
if firstSyncErr == nil {
|
||||
firstSyncErr = err
|
||||
}
|
||||
zap.L().Error("Hook execution failed",
|
||||
zap.String("name", entry.hook.Name),
|
||||
zap.String("type", string(hookType)),
|
||||
@@ -256,12 +260,29 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
|
||||
}()
|
||||
}
|
||||
|
||||
return nil
|
||||
return firstSyncErr
|
||||
}
|
||||
|
||||
func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
|
||||
start := time.Now()
|
||||
|
||||
// Derive a cancellable context bounded by the hook's Timeout so that
|
||||
// long-running hook work (e.g. HTTP calls to the moderation API) is
|
||||
// actually cancelled when the timeout fires, instead of leaking and
|
||||
// racing on shared metadata after executeHook returns.
|
||||
if entry.hook.Timeout > 0 {
|
||||
hookCtx, cancel := context.WithTimeout(ctx.Ctx, entry.hook.Timeout)
|
||||
defer cancel()
|
||||
|
||||
ctx = &HookContext{
|
||||
Ctx: hookCtx,
|
||||
HookType: ctx.HookType,
|
||||
UserID: ctx.UserID,
|
||||
Data: ctx.Data,
|
||||
Metadata: ctx.Metadata,
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
if entry.hook.Timeout > 0 {
|
||||
done := make(chan error, 1)
|
||||
@@ -280,13 +301,17 @@ func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
|
||||
|
||||
select {
|
||||
case err = <-done:
|
||||
case <-time.After(entry.hook.Timeout):
|
||||
err = context.DeadlineExceeded
|
||||
case <-ctx.Ctx.Done():
|
||||
err = ctx.Ctx.Err()
|
||||
zap.L().Warn("Hook timeout",
|
||||
zap.String("name", entry.hook.Name),
|
||||
zap.Duration("timeout", entry.hook.Timeout),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
// Wait for the spawned goroutine to observe cancellation and return,
|
||||
// so it cannot keep writing to ctx.Metadata after executeHook returns.
|
||||
<-done
|
||||
} else {
|
||||
err = entry.hook.Func(ctx)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -382,40 +383,69 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
|
||||
|
||||
func (m *ModerationHooks) ModeratePost(ctx context.Context, manager *Manager, data *PostModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{
|
||||
if err := manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}); err != nil {
|
||||
applyModerationFallback(result, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager, data *CommentModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{
|
||||
if err := manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}); err != nil {
|
||||
applyModerationFallback(result, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
|
||||
if err := manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}); err != nil {
|
||||
applyModerationFallback(result, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
|
||||
if err := manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}); err != nil {
|
||||
applyModerationFallback(result, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
|
||||
result := &ModerationResult{}
|
||||
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
|
||||
if err := manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
|
||||
"result": result,
|
||||
})
|
||||
}); err != nil {
|
||||
applyModerationFallback(result, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// applyModerationFallback handles moderation hook timeouts/cancellations by
|
||||
// falling back to "approved by system" instead of leaving the zero-value
|
||||
// result (Approved=false) which would silently reject otherwise-fine content.
|
||||
func applyModerationFallback(result *ModerationResult, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
result.RejectReason = ""
|
||||
result.NeedsReview = false
|
||||
result.Error = err
|
||||
zap.L().Warn("Moderation hook timed out or was cancelled, fallback approve",
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
@@ -21,55 +22,41 @@ import (
|
||||
xdraw "golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。目标是平衡社区安全与正常交流:必须拦截高风险违规内容,但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。
|
||||
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||
|
||||
审核流程:
|
||||
1) 先判断是否命中硬性违规;
|
||||
2) 再判断语境(玩笑/自嘲/朋友间互动/作品讨论);
|
||||
3) 做文图交叉判断(文本+图片合并理解);
|
||||
4) 给出 result 与简短 reason。
|
||||
核心原则:
|
||||
- 除明确违法违规外,不要拦截或送审正常内容,包括尖锐吐槽、玩梗、二创、争论;
|
||||
- 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
A. 宣传对立与煽动撕裂:
|
||||
- 明确煽动群体对立、地域对立、性别对立、民族宗教对立,鼓动仇恨、排斥、报复。
|
||||
B. 严重人身攻击与网暴引导:
|
||||
- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突。
|
||||
C. 开盒/人肉/隐私暴露:
|
||||
- 故意公开、拼接、索取他人可识别隐私信息(姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等);
|
||||
- 图片/截图中出现可识别隐私信息并伴随曝光意图,也按违规处理。
|
||||
D. 其他高危违规:
|
||||
- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。
|
||||
只有命中**明确违法违规**时才输出 result=review:
|
||||
A. 违法犯罪:明确宣传违法犯罪行为、教唆或指导他人犯罪(如毒品、枪支、诈骗教程)。
|
||||
B. 真实暴力威胁:针对具体个人或群体的实质性暴力威胁、人身伤害恐吓。
|
||||
C. 色情低俗:明确色情、露骨性描写或诱导性内容;正常人体艺术、二次创作用感形象不在此列。
|
||||
D. 诈骗引流:明确的诈骗信息、钓鱼链接、黑产引流。
|
||||
E. 未成年保护:涉及未成年人的色情或剥削内容。
|
||||
F. 开盒/人肉:故意公开他人可识别隐私信息(身份证号、住址、联系方式、定位轨迹)并伴有曝光意图。
|
||||
G. 极端仇恨:鼓动对特定族裔/宗教/残障群体的仇恨与暴力,含纳粹等极端符号。
|
||||
|
||||
可疑内容(需要人工复审,result=review):
|
||||
- 内容含义模糊,无法确定是否存在违规意图;
|
||||
- 涉及敏感话题但未明确违规(如政治人物、社会热点事件等);
|
||||
- 存在轻微争议性表达,但不完全符合硬性违规标准;
|
||||
- AI无法准确判断的文化/地域特定表达。
|
||||
|
||||
放行规则(以下通常 result=pass):
|
||||
以下情形即使存在也**一律 pass**(不送审、不拦截):
|
||||
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽;
|
||||
- 非定向、轻度口语化吐槽(无明确攻击对象、无网暴号召、无隐私暴露);
|
||||
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐,但未煽动对立或人身攻击)。
|
||||
|
||||
边界判定:
|
||||
- 若只是"梗文化表达"且不指向现实伤害,优先通过;
|
||||
- 若存在明确伤害意图(煽动、围攻、曝光隐私),必须拒绝;
|
||||
- 对模糊内容不因个别粗口直接拒绝,需结合对象、意图、号召性和可执行性综合判断;
|
||||
- 如果实在无法确定是否违规,标记为 review 状态交由人工复审。
|
||||
- 非定向、轻度口语化吐槽、玩笑、自嘲、朋友间互动;
|
||||
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐、用词激烈);
|
||||
- 一般性粗口、调侃、讽刺,未煽动对立或人身攻击的;
|
||||
- 对政治人物/社会热点的评论、争议性观点表达;
|
||||
- 含义模糊但无明显违法意图的内容。
|
||||
|
||||
reason 要求:
|
||||
- result=block 时:中文10-30字,说明核心违规点;
|
||||
- result=review 时:中文10-30字,说明可疑原因;
|
||||
- result=review 时:中文10-30字,说明疑似违法点;
|
||||
- result=pass 时:reason 为空字符串。
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
{"result": "pass"/"review", "reason": "..."}`
|
||||
|
||||
const (
|
||||
defaultMaxImagesPerModerationRequest = 1
|
||||
maxAllowedImagesPerModerationRequest = 4
|
||||
maxModerationResultRetries = 3
|
||||
maxChatCompletionRetries = 3
|
||||
initialRetryBackoff = 500 * time.Millisecond
|
||||
maxDownloadImageBytes = 10 * 1024 * 1024
|
||||
maxModerationImageSide = 1280
|
||||
@@ -165,53 +152,61 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
|
||||
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
||||
}
|
||||
|
||||
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
|
||||
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。原则是**宁可放过,不可误伤**:只有明确违法违规的图片才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
- 色情低俗、裸露、性暗示内容
|
||||
- 暴力血腥、恐怖画面
|
||||
- 违法犯罪相关内容
|
||||
- 明确的政治敏感宣传
|
||||
- 恶意广告、诈骗信息
|
||||
- 仇恨歧视符号或标志
|
||||
核心原则:
|
||||
- 除明确违法违规外,不要拦截或送审正常图片;
|
||||
- 判定从严,倾向放行:宁可放过十张可疑,不可误伤一张正常。
|
||||
|
||||
可疑内容(result=review):
|
||||
- 图片含义模糊,无法确定是否存在违规
|
||||
- 轻微争议性内容但不完全违规
|
||||
- AI无法准确判断的内容
|
||||
只有命中**明确违法违规**时才输出 result=review:
|
||||
- 色情低俗:明确色情、露骨性描写或诱导性图片;正常人体艺术、二次创作用感形象不在此列;
|
||||
- 真实暴力/血腥:明确展示严重暴力、凶杀、虐待等违法犯罪画面;
|
||||
- 违法犯罪相关:明确涉及毒品、枪支、诈骗、黑产等内容;
|
||||
- 未成年保护:涉及未成年人的色情或剥削图片;
|
||||
- 诈骗引流:明确的诈骗二维码、钓鱼链接、黑产广告;
|
||||
- 极端仇恨:纳粹等极端仇恨符号或标志。
|
||||
|
||||
放行规则(result=pass):
|
||||
- 正常风景、人物、动物、动漫等图片
|
||||
- 表情包、梗图
|
||||
- 正常的艺术创作、摄影作品
|
||||
以下图片即使存在也**一律 pass**(不送审、不拦截):
|
||||
- 正常风景、人物、动物、动漫等图片;
|
||||
- 表情包、梗图、二次创作;
|
||||
- 正常的艺术创作、摄影作品;
|
||||
- 含义模糊但无明显违法意图的图片。
|
||||
|
||||
reason 要求:
|
||||
- result=review 时:中文10-30字,说明疑似违法点;
|
||||
- result=pass 时:reason 为空字符串。
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
{"result": "pass"/"review", "reason": "..."}`
|
||||
|
||||
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
|
||||
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
- 明确的违法犯罪、暴力威胁内容
|
||||
- 色情低俗、性暗示描述
|
||||
- 诈骗引流、恶意广告
|
||||
- 明确煽动群体对立、仇恨
|
||||
- 恶意公开他人隐私信息
|
||||
核心原则:
|
||||
- 除明确违法违规外,不要拦截或送审正常签名;
|
||||
- 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
|
||||
|
||||
可疑内容(result=review):
|
||||
- 内容含义模糊,无法确定违规意图
|
||||
- 涉及敏感话题但未明确违规
|
||||
- AI无法准确判断的表达
|
||||
只有命中**明确违法违规**时才输出 result=review:
|
||||
- 明确违法犯罪、暴力威胁内容;
|
||||
- 色情低俗、露骨性暗示描述;
|
||||
- 诈骗引流、黑产广告;
|
||||
- 恶意公开他人可识别隐私信息;
|
||||
- 涉及未成年人的色情或剥削内容。
|
||||
|
||||
放行规则(result=pass):
|
||||
- 正常的自我介绍、心情表达
|
||||
- 玩梗、幽默、吐槽
|
||||
- 引用、歌词、诗句等
|
||||
- 理性的观点表达
|
||||
以下签名即使存在也**一律 pass**(不送审、不拦截):
|
||||
- 正常的自我介绍、心情表达;
|
||||
- 玩梗、幽默、吐槽、谐音梗;
|
||||
- 引用、歌词、诗句等;
|
||||
- 理性的观点表达、对社会事件的评论;
|
||||
- 含义模糊但无明显违法意图的内容。
|
||||
|
||||
reason 要求:
|
||||
- result=review 时:中文10-30字,说明疑似违法点;
|
||||
- result=pass 时:reason 为空字符串。
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
{"result": "pass"/"review", "reason": "..."}`
|
||||
|
||||
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
|
||||
if !c.IsEnabled() {
|
||||
@@ -226,42 +221,11 @@ func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*Moder
|
||||
cleanImages := normalizeImageURLs([]string{imageURL})
|
||||
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
resp, err := c.moderateWithRetry(ctx, imageModerationSystemPrompt, userPrompt, optimizedImages)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
|
||||
@@ -278,42 +242,11 @@ func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationRe
|
||||
}
|
||||
|
||||
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
resp, err := c.moderateWithRetry(ctx, systemPrompt, userPrompt, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
|
||||
@@ -365,48 +298,17 @@ func (c *clientImpl) moderateSingleBatch(
|
||||
totalBatches,
|
||||
)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, moderationSystemPrompt, userPrompt, images, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
resp, err := c.moderateWithRetry(ctx, moderationSystemPrompt, userPrompt, images)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"moderation batch %d/%d failed after %d attempts: %w",
|
||||
batchNo,
|
||||
totalBatches,
|
||||
maxModerationResultRetries,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"moderation batch %d/%d failed after %d attempts: %w",
|
||||
batchNo,
|
||||
totalBatches,
|
||||
maxModerationResultRetries,
|
||||
lastErr,
|
||||
)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
type chatCompletionsRequest struct {
|
||||
@@ -497,36 +399,69 @@ func (c *clientImpl) chatCompletion(
|
||||
endpoint = baseURL + "/chat/completions"
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxChatCompletionRetries; attempt++ {
|
||||
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else if statusCode >= 400 {
|
||||
lastErr = fmt.Errorf("openai error status=%d body=%s", statusCode, string(body))
|
||||
if !isRetryableStatusCode(statusCode) {
|
||||
return "", lastErr
|
||||
}
|
||||
} else {
|
||||
var parsed chatCompletionsResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return "", fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return "", fmt.Errorf("empty response choices")
|
||||
}
|
||||
return parsed.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
if attempt == maxChatCompletionRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return "", sleepErr
|
||||
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
||||
if err != nil {
|
||||
return "", &chatCompletionError{StatusCode: 0, Err: err}
|
||||
}
|
||||
if statusCode >= 400 {
|
||||
return "", &chatCompletionError{
|
||||
StatusCode: statusCode,
|
||||
Err: fmt.Errorf("openai error status=%d body=%s", statusCode, string(body)),
|
||||
}
|
||||
}
|
||||
|
||||
return "", lastErr
|
||||
var parsed chatCompletionsResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return "", &chatCompletionError{
|
||||
StatusCode: statusCode,
|
||||
Err: fmt.Errorf("decode response: %w", err),
|
||||
}
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return "", &chatCompletionError{
|
||||
StatusCode: statusCode,
|
||||
Err: fmt.Errorf("empty response choices"),
|
||||
}
|
||||
}
|
||||
return parsed.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
// chatCompletionError wraps errors from chatCompletion so callers can decide
|
||||
// retryability based on the upstream HTTP status code. StatusCode == 0 means
|
||||
// the request never reached the server (transport failure).
|
||||
type chatCompletionError struct {
|
||||
StatusCode int
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *chatCompletionError) Error() string {
|
||||
if e.Err == nil {
|
||||
return "chat completion error"
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *chatCompletionError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// isRetryableModerationErr reports whether a moderation caller should retry
|
||||
// after the given chatCompletion error. Context cancellation and non-429 4xx
|
||||
// responses are not retryable; transport, 429, 5xx, and parsing errors are.
|
||||
func isRetryableModerationErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
var ce *chatCompletionError
|
||||
if errors.As(err, &ce) {
|
||||
if ce.StatusCode >= 400 && ce.StatusCode != http.StatusTooManyRequests && ce.StatusCode < 500 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) {
|
||||
@@ -550,13 +485,6 @@ func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint strin
|
||||
return body, resp.StatusCode, nil
|
||||
}
|
||||
|
||||
func isRetryableStatusCode(statusCode int) bool {
|
||||
if statusCode == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
return statusCode >= 500 && statusCode <= 599
|
||||
}
|
||||
|
||||
func sleepWithBackoff(ctx context.Context, attempt int) error {
|
||||
delay := initialRetryBackoff * time.Duration(1<<attempt)
|
||||
timer := time.NewTimer(delay)
|
||||
@@ -570,6 +498,63 @@ func sleepWithBackoff(ctx context.Context, attempt int) error {
|
||||
}
|
||||
}
|
||||
|
||||
// moderateWithRetry runs a single moderation request up to maxModerationResultRetries
|
||||
// times, retrying only on retryable chatCompletion errors. The underlying chatCompletion
|
||||
// no longer retries on its own, so the total number of upstream calls is bounded by
|
||||
// maxModerationResultRetries (previously 3x3=9).
|
||||
func (c *clientImpl) moderateWithRetry(
|
||||
ctx context.Context,
|
||||
systemPrompt, userPrompt string,
|
||||
images []string,
|
||||
) (*ModerationResponse, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, images, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
resp, parseErr := parseModerationReply(replyText)
|
||||
if parseErr != nil {
|
||||
lastErr = parseErr
|
||||
} else {
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if !isRetryableModerationErr(lastErr) {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func parseModerationReply(replyText string) (*ModerationResponse, error) {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
}
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeImageURLs(images []string) []string {
|
||||
clean := make([]string, 0, len(images))
|
||||
for _, image := range images {
|
||||
@@ -593,13 +578,14 @@ func extractJSONObject(raw string) string {
|
||||
}
|
||||
|
||||
func (c *clientImpl) maxImagesPerModerationRequest() int {
|
||||
if c.cfg.ModerationMaxImagesPerRequest <= 0 {
|
||||
configured := c.cfg.ModerationMaxImagesPerRequest
|
||||
if configured <= 0 {
|
||||
return defaultMaxImagesPerModerationRequest
|
||||
}
|
||||
if c.cfg.ModerationMaxImagesPerRequest > 1 {
|
||||
return 1
|
||||
if configured > maxAllowedImagesPerModerationRequest {
|
||||
return maxAllowedImagesPerModerationRequest
|
||||
}
|
||||
return c.cfg.ModerationMaxImagesPerRequest
|
||||
return configured
|
||||
}
|
||||
|
||||
func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string {
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
@@ -170,7 +171,7 @@ func (s *commentService) reviewCommentAsync(
|
||||
fmt.Sscanf(userID, "%d", &authorID)
|
||||
|
||||
result := &hook.ModerationResult{}
|
||||
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookCommentPreModerate, authorID, &hook.CommentModerateHookData{
|
||||
triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookCommentPreModerate, authorID, &hook.CommentModerateHookData{
|
||||
CommentID: commentID,
|
||||
PostID: postID,
|
||||
Content: content,
|
||||
@@ -181,6 +182,19 @@ func (s *commentService) reviewCommentAsync(
|
||||
"result": result,
|
||||
})
|
||||
|
||||
// Fallback-approve when a moderation hook times out or is cancelled, so a
|
||||
// slow moderation API never silently rejects an otherwise-fine comment.
|
||||
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
|
||||
zap.L().Warn("Comment moderation hook timed out or was cancelled, fallback approve",
|
||||
zap.String("comment_id", commentID),
|
||||
zap.Error(triggerErr),
|
||||
)
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
result.RejectReason = ""
|
||||
result.NeedsReview = false
|
||||
}
|
||||
|
||||
s.hookManager.Trigger(context.Background(), hook.HookCommentModerated, authorID, &hook.CommentModeratedHookData{
|
||||
CommentID: commentID,
|
||||
PostID: postID,
|
||||
|
||||
@@ -223,15 +223,13 @@ func (s *postAIService) ModeratePost(ctx context.Context, title, content string,
|
||||
zap.Error(err),
|
||||
)
|
||||
return s.moderateTextWithTencent(ctx, title+" "+content,
|
||||
func(reason string) error { return &PostModerationRejectedError{Reason: reason} },
|
||||
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
||||
)
|
||||
}
|
||||
|
||||
// block 与 review 同等处理:均仅送人工复审。
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &PostModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||
return &PostModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
@@ -239,7 +237,6 @@ func (s *postAIService) ModeratePost(ctx context.Context, title, content string,
|
||||
}
|
||||
|
||||
return s.moderateTextWithTencent(ctx, title+" "+content,
|
||||
func(reason string) error { return &PostModerationRejectedError{Reason: reason} },
|
||||
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
||||
)
|
||||
}
|
||||
@@ -256,15 +253,13 @@ func (s *postAIService) ModerateComment(ctx context.Context, content string, ima
|
||||
zap.Error(err),
|
||||
)
|
||||
return s.moderateTextWithTencent(ctx, content,
|
||||
func(reason string) error { return &CommentModerationRejectedError{Reason: reason} },
|
||||
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
||||
)
|
||||
}
|
||||
|
||||
// block 与 review 同等处理:均仅送人工复审。
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &CommentModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||
return &CommentModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
@@ -272,7 +267,6 @@ func (s *postAIService) ModerateComment(ctx context.Context, content string, ima
|
||||
}
|
||||
|
||||
return s.moderateTextWithTencent(ctx, content,
|
||||
func(reason string) error { return &CommentModerationRejectedError{Reason: reason} },
|
||||
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
||||
)
|
||||
}
|
||||
@@ -293,10 +287,9 @@ func (s *postAIService) ModerateImage(ctx context.Context, imageURL string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// block 与 review 同等处理:均仅送人工复审。
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &ImageModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||
return &ImageModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
@@ -315,15 +308,13 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
||||
zap.Error(err),
|
||||
)
|
||||
return s.moderateTextWithTencent(ctx, bio,
|
||||
func(reason string) error { return &BioModerationRejectedError{Reason: reason} },
|
||||
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
||||
)
|
||||
}
|
||||
|
||||
// block 与 review 同等处理:均仅送人工复审。
|
||||
switch resp.Result {
|
||||
case openai.ModerationResultBlock:
|
||||
return &BioModerationRejectedError{Reason: resp.Reason}
|
||||
case openai.ModerationResultReview:
|
||||
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||
return &BioModerationReviewError{Reason: resp.Reason}
|
||||
default:
|
||||
return nil
|
||||
@@ -331,7 +322,6 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
||||
}
|
||||
|
||||
return s.moderateTextWithTencent(ctx, bio,
|
||||
func(reason string) error { return &BioModerationRejectedError{Reason: reason} },
|
||||
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
||||
)
|
||||
}
|
||||
@@ -339,7 +329,6 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
||||
func (s *postAIService) moderateTextWithTencent(
|
||||
ctx context.Context,
|
||||
text string,
|
||||
rejectFactory func(string) error,
|
||||
reviewFactory func(string) error,
|
||||
) error {
|
||||
if !s.isTencentEnabled() {
|
||||
@@ -366,10 +355,9 @@ func (s *postAIService) moderateTextWithTencent(
|
||||
reason = resp.Label + ": " + strings.Join(resp.Keywords, ", ")
|
||||
}
|
||||
|
||||
// 腾讯云 block 也仅送审,与 AI 同语义。
|
||||
switch resp.Suggestion {
|
||||
case tencent.SuggestionBlock:
|
||||
return rejectFactory(reason)
|
||||
case tencent.SuggestionReview:
|
||||
case tencent.SuggestionBlock, tencent.SuggestionReview:
|
||||
return reviewFactory(reason)
|
||||
default:
|
||||
return nil
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
@@ -216,7 +217,7 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
|
||||
fmt.Sscanf(userID, "%d", &authorID)
|
||||
|
||||
result := &hook.ModerationResult{}
|
||||
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
||||
triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
||||
PostID: postID,
|
||||
Title: title,
|
||||
Content: content,
|
||||
@@ -226,6 +227,19 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
|
||||
"result": result,
|
||||
})
|
||||
|
||||
// Trigger can return context.DeadlineExceeded/Canceled if a moderation hook
|
||||
// times out. In that case the result may not have been written at all
|
||||
// (result.Approved stays false). Treat an unfinished moderation as
|
||||
// "approved + system" rather than silently rejecting the post, so a slow
|
||||
// moderation API never blocks normal content.
|
||||
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
|
||||
log.Printf("[WARN] Post moderation hook timed out or was cancelled, fallback approve post=%s err=%v", postID, triggerErr)
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
result.RejectReason = ""
|
||||
result.NeedsReview = false
|
||||
}
|
||||
|
||||
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
|
||||
PostID: postID,
|
||||
AuthorID: authorID,
|
||||
|
||||
@@ -177,7 +177,7 @@ func (s *voteService) reviewVotePostAsync(postID, userID, title, content string,
|
||||
}
|
||||
|
||||
result := &hook.ModerationResult{}
|
||||
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
||||
triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
||||
PostID: postID,
|
||||
Title: title,
|
||||
Content: content,
|
||||
@@ -187,6 +187,16 @@ func (s *voteService) reviewVotePostAsync(postID, userID, title, content string,
|
||||
"result": result,
|
||||
})
|
||||
|
||||
// Fallback-approve when a moderation hook times out or is cancelled, so a
|
||||
// slow moderation API never silently rejects an otherwise-fine vote post.
|
||||
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
|
||||
log.Printf("[WARN] Vote post moderation hook timed out or was cancelled, fallback approve post=%s err=%v", postID, triggerErr)
|
||||
result.Approved = true
|
||||
result.ReviewedBy = "system"
|
||||
result.RejectReason = ""
|
||||
result.NeedsReview = false
|
||||
}
|
||||
|
||||
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
|
||||
PostID: postID,
|
||||
AuthorID: authorID,
|
||||
|
||||
Reference in New Issue
Block a user