feat(moderation): make moderation more lenient and add timeout fallback behavior
The moderation system now uses a "pass/review" strategy instead of "pass/review/block", treating hard violations the same as review (only routing to manual review rather than auto-rejecting). Add fallback-approve behavior in the hook system so slow moderation APIs never silently block normal content when they time out or get cancelled - timed-out moderation now approves with "ReviewedBy: system" instead of leaving the zero-value result that would reject content. Update post/comment/vote services and handlers to use the new unified behavior. Simplify moderation prompts to emphasize "宁可放过,不可误伤" (prefer false positives over false negatives).
This commit is contained in:
@@ -106,13 +106,8 @@ func (h *PostHandler) Create(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
post, err := h.postService.Create(c.Request.Context(), userID, req.Title, content, segments, req.Images, req.ChannelID, req.ClientRequestID)
|
post, err := h.postService.Create(c.Request.Context(), userID, req.Title, content, segments, req.Images, req.ChannelID, req.ClientRequestID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var moderationErr *service.PostModerationRejectedError
|
|
||||||
if errors.As(err, &moderationErr) {
|
|
||||||
response.BadRequest(c, moderationErr.UserMessage())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
|
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
|
||||||
if errors.Is(err, service.ErrDuplicatePostRequest) {
|
if errors.Is(err, service.ErrDuplicatePostRequest) {
|
||||||
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())
|
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())
|
||||||
|
|||||||
@@ -42,11 +42,6 @@ func (h *VoteHandler) CreateVotePost(c *gin.Context) {
|
|||||||
|
|
||||||
post, err := h.voteService.CreateVotePost(c.Request.Context(), userID, &req)
|
post, err := h.voteService.CreateVotePost(c.Request.Context(), userID, &req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var moderationErr *service.PostModerationRejectedError
|
|
||||||
if errors.As(err, &moderationErr) {
|
|
||||||
response.BadRequest(c, moderationErr.UserMessage())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
|
// 幂等命中"进行中"占位:提示客户端稍候,避免重复提交
|
||||||
if errors.Is(err, service.ErrDuplicatePostRequest) {
|
if errors.Is(err, service.ErrDuplicatePostRequest) {
|
||||||
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())
|
response.ErrorWithStatusCode(c, http.StatusTooManyRequests, 429, err.Error())
|
||||||
|
|||||||
@@ -215,6 +215,7 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
|
|||||||
|
|
||||||
var asyncHooks []*hookEntry
|
var asyncHooks []*hookEntry
|
||||||
|
|
||||||
|
var firstSyncErr error
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if !entry.hook.Enabled {
|
if !entry.hook.Enabled {
|
||||||
continue
|
continue
|
||||||
@@ -226,6 +227,9 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := m.executeHook(entry, hookCtx); err != nil {
|
if err := m.executeHook(entry, hookCtx); err != nil {
|
||||||
|
if firstSyncErr == nil {
|
||||||
|
firstSyncErr = err
|
||||||
|
}
|
||||||
zap.L().Error("Hook execution failed",
|
zap.L().Error("Hook execution failed",
|
||||||
zap.String("name", entry.hook.Name),
|
zap.String("name", entry.hook.Name),
|
||||||
zap.String("type", string(hookType)),
|
zap.String("type", string(hookType)),
|
||||||
@@ -256,12 +260,29 @@ func (m *Manager) TriggerWithMetadata(ctx context.Context, hookType HookType, us
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return firstSyncErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
|
func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
|
// Derive a cancellable context bounded by the hook's Timeout so that
|
||||||
|
// long-running hook work (e.g. HTTP calls to the moderation API) is
|
||||||
|
// actually cancelled when the timeout fires, instead of leaking and
|
||||||
|
// racing on shared metadata after executeHook returns.
|
||||||
|
if entry.hook.Timeout > 0 {
|
||||||
|
hookCtx, cancel := context.WithTimeout(ctx.Ctx, entry.hook.Timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
ctx = &HookContext{
|
||||||
|
Ctx: hookCtx,
|
||||||
|
HookType: ctx.HookType,
|
||||||
|
UserID: ctx.UserID,
|
||||||
|
Data: ctx.Data,
|
||||||
|
Metadata: ctx.Metadata,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
if entry.hook.Timeout > 0 {
|
if entry.hook.Timeout > 0 {
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
@@ -280,13 +301,17 @@ func (m *Manager) executeHook(entry *hookEntry, ctx *HookContext) error {
|
|||||||
|
|
||||||
select {
|
select {
|
||||||
case err = <-done:
|
case err = <-done:
|
||||||
case <-time.After(entry.hook.Timeout):
|
case <-ctx.Ctx.Done():
|
||||||
err = context.DeadlineExceeded
|
err = ctx.Ctx.Err()
|
||||||
zap.L().Warn("Hook timeout",
|
zap.L().Warn("Hook timeout",
|
||||||
zap.String("name", entry.hook.Name),
|
zap.String("name", entry.hook.Name),
|
||||||
zap.Duration("timeout", entry.hook.Timeout),
|
zap.Duration("timeout", entry.hook.Timeout),
|
||||||
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
// Wait for the spawned goroutine to observe cancellation and return,
|
||||||
|
// so it cannot keep writing to ctx.Metadata after executeHook returns.
|
||||||
|
<-done
|
||||||
} else {
|
} else {
|
||||||
err = entry.hook.Func(ctx)
|
err = entry.hook.Func(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -382,40 +383,69 @@ func (m *ModerationHooks) RegisterAll(manager *Manager) {
|
|||||||
|
|
||||||
func (m *ModerationHooks) ModeratePost(ctx context.Context, manager *Manager, data *PostModerateHookData) *ModerationResult {
|
func (m *ModerationHooks) ModeratePost(ctx context.Context, manager *Manager, data *PostModerateHookData) *ModerationResult {
|
||||||
result := &ModerationResult{}
|
result := &ModerationResult{}
|
||||||
manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{
|
if err := manager.TriggerWithMetadata(ctx, HookPostPreModerate, data.AuthorID, data, map[string]any{
|
||||||
"result": result,
|
"result": result,
|
||||||
})
|
}); err != nil {
|
||||||
|
applyModerationFallback(result, err)
|
||||||
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager, data *CommentModerateHookData) *ModerationResult {
|
func (m *ModerationHooks) ModerateComment(ctx context.Context, manager *Manager, data *CommentModerateHookData) *ModerationResult {
|
||||||
result := &ModerationResult{}
|
result := &ModerationResult{}
|
||||||
manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{
|
if err := manager.TriggerWithMetadata(ctx, HookCommentPreModerate, data.AuthorID, data, map[string]any{
|
||||||
"result": result,
|
"result": result,
|
||||||
})
|
}); err != nil {
|
||||||
|
applyModerationFallback(result, err)
|
||||||
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
|
func (m *ModerationHooks) ModerateAvatar(ctx context.Context, manager *Manager, data *UserAvatarModerateHookData) *ModerationResult {
|
||||||
result := &ModerationResult{}
|
result := &ModerationResult{}
|
||||||
manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
|
if err := manager.TriggerWithMetadata(ctx, HookUserAvatarPreModerate, 0, data, map[string]any{
|
||||||
"result": result,
|
"result": result,
|
||||||
})
|
}); err != nil {
|
||||||
|
applyModerationFallback(result, err)
|
||||||
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
|
func (m *ModerationHooks) ModerateCover(ctx context.Context, manager *Manager, data *UserCoverModerateHookData) *ModerationResult {
|
||||||
result := &ModerationResult{}
|
result := &ModerationResult{}
|
||||||
manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
|
if err := manager.TriggerWithMetadata(ctx, HookUserCoverPreModerate, 0, data, map[string]any{
|
||||||
"result": result,
|
"result": result,
|
||||||
})
|
}); err != nil {
|
||||||
|
applyModerationFallback(result, err)
|
||||||
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
|
func (m *ModerationHooks) ModerateBio(ctx context.Context, manager *Manager, data *UserBioModerateHookData) *ModerationResult {
|
||||||
result := &ModerationResult{}
|
result := &ModerationResult{}
|
||||||
manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
|
if err := manager.TriggerWithMetadata(ctx, HookUserBioPreModerate, 0, data, map[string]any{
|
||||||
"result": result,
|
"result": result,
|
||||||
})
|
}); err != nil {
|
||||||
|
applyModerationFallback(result, err)
|
||||||
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// applyModerationFallback handles moderation hook timeouts/cancellations by
|
||||||
|
// falling back to "approved by system" instead of leaving the zero-value
|
||||||
|
// result (Approved=false) which would silently reject otherwise-fine content.
|
||||||
|
func applyModerationFallback(result *ModerationResult, err error) {
|
||||||
|
if err == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||||
|
result.Approved = true
|
||||||
|
result.ReviewedBy = "system"
|
||||||
|
result.RejectReason = ""
|
||||||
|
result.NeedsReview = false
|
||||||
|
result.Error = err
|
||||||
|
zap.L().Warn("Moderation hook timed out or was cancelled, fallback approve",
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"image"
|
"image"
|
||||||
_ "image/gif"
|
_ "image/gif"
|
||||||
@@ -21,55 +22,41 @@ import (
|
|||||||
xdraw "golang.org/x/image/draw"
|
xdraw "golang.org/x/image/draw"
|
||||||
)
|
)
|
||||||
|
|
||||||
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。目标是平衡社区安全与正常交流:必须拦截高风险违规内容,但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。
|
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||||
|
|
||||||
审核流程:
|
核心原则:
|
||||||
1) 先判断是否命中硬性违规;
|
- 除明确违法违规外,不要拦截或送审正常内容,包括尖锐吐槽、玩梗、二创、争论;
|
||||||
2) 再判断语境(玩笑/自嘲/朋友间互动/作品讨论);
|
- 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
|
||||||
3) 做文图交叉判断(文本+图片合并理解);
|
|
||||||
4) 给出 result 与简短 reason。
|
|
||||||
|
|
||||||
硬性违规(命中任一项必须 result=block):
|
只有命中**明确违法违规**时才输出 result=review:
|
||||||
A. 宣传对立与煽动撕裂:
|
A. 违法犯罪:明确宣传违法犯罪行为、教唆或指导他人犯罪(如毒品、枪支、诈骗教程)。
|
||||||
- 明确煽动群体对立、地域对立、性别对立、民族宗教对立,鼓动仇恨、排斥、报复。
|
B. 真实暴力威胁:针对具体个人或群体的实质性暴力威胁、人身伤害恐吓。
|
||||||
B. 严重人身攻击与网暴引导:
|
C. 色情低俗:明确色情、露骨性描写或诱导性内容;正常人体艺术、二次创作用感形象不在此列。
|
||||||
- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突。
|
D. 诈骗引流:明确的诈骗信息、钓鱼链接、黑产引流。
|
||||||
C. 开盒/人肉/隐私暴露:
|
E. 未成年保护:涉及未成年人的色情或剥削内容。
|
||||||
- 故意公开、拼接、索取他人可识别隐私信息(姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等);
|
F. 开盒/人肉:故意公开他人可识别隐私信息(身份证号、住址、联系方式、定位轨迹)并伴有曝光意图。
|
||||||
- 图片/截图中出现可识别隐私信息并伴随曝光意图,也按违规处理。
|
G. 极端仇恨:鼓动对特定族裔/宗教/残障群体的仇恨与暴力,含纳粹等极端符号。
|
||||||
D. 其他高危违规:
|
|
||||||
- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。
|
|
||||||
|
|
||||||
可疑内容(需要人工复审,result=review):
|
以下情形即使存在也**一律 pass**(不送审、不拦截):
|
||||||
- 内容含义模糊,无法确定是否存在违规意图;
|
|
||||||
- 涉及敏感话题但未明确违规(如政治人物、社会热点事件等);
|
|
||||||
- 存在轻微争议性表达,但不完全符合硬性违规标准;
|
|
||||||
- AI无法准确判断的文化/地域特定表达。
|
|
||||||
|
|
||||||
放行规则(以下通常 result=pass):
|
|
||||||
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽;
|
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽;
|
||||||
- 非定向、轻度口语化吐槽(无明确攻击对象、无网暴号召、无隐私暴露);
|
- 非定向、轻度口语化吐槽、玩笑、自嘲、朋友间互动;
|
||||||
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐,但未煽动对立或人身攻击)。
|
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐、用词激烈);
|
||||||
|
- 一般性粗口、调侃、讽刺,未煽动对立或人身攻击的;
|
||||||
边界判定:
|
- 对政治人物/社会热点的评论、争议性观点表达;
|
||||||
- 若只是"梗文化表达"且不指向现实伤害,优先通过;
|
- 含义模糊但无明显违法意图的内容。
|
||||||
- 若存在明确伤害意图(煽动、围攻、曝光隐私),必须拒绝;
|
|
||||||
- 对模糊内容不因个别粗口直接拒绝,需结合对象、意图、号召性和可执行性综合判断;
|
|
||||||
- 如果实在无法确定是否违规,标记为 review 状态交由人工复审。
|
|
||||||
|
|
||||||
reason 要求:
|
reason 要求:
|
||||||
- result=block 时:中文10-30字,说明核心违规点;
|
- result=review 时:中文10-30字,说明疑似违法点;
|
||||||
- result=review 时:中文10-30字,说明可疑原因;
|
|
||||||
- result=pass 时:reason 为空字符串。
|
- result=pass 时:reason 为空字符串。
|
||||||
|
|
||||||
输出格式(严格):
|
输出格式(严格):
|
||||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
{"result": "pass"/"review", "reason": "..."}`
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultMaxImagesPerModerationRequest = 1
|
defaultMaxImagesPerModerationRequest = 1
|
||||||
|
maxAllowedImagesPerModerationRequest = 4
|
||||||
maxModerationResultRetries = 3
|
maxModerationResultRetries = 3
|
||||||
maxChatCompletionRetries = 3
|
|
||||||
initialRetryBackoff = 500 * time.Millisecond
|
initialRetryBackoff = 500 * time.Millisecond
|
||||||
maxDownloadImageBytes = 10 * 1024 * 1024
|
maxDownloadImageBytes = 10 * 1024 * 1024
|
||||||
maxModerationImageSide = 1280
|
maxModerationImageSide = 1280
|
||||||
@@ -165,53 +152,61 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
|
|||||||
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
||||||
}
|
}
|
||||||
|
|
||||||
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
|
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。原则是**宁可放过,不可误伤**:只有明确违法违规的图片才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||||
|
|
||||||
硬性违规(命中任一项必须 result=block):
|
核心原则:
|
||||||
- 色情低俗、裸露、性暗示内容
|
- 除明确违法违规外,不要拦截或送审正常图片;
|
||||||
- 暴力血腥、恐怖画面
|
- 判定从严,倾向放行:宁可放过十张可疑,不可误伤一张正常。
|
||||||
- 违法犯罪相关内容
|
|
||||||
- 明确的政治敏感宣传
|
|
||||||
- 恶意广告、诈骗信息
|
|
||||||
- 仇恨歧视符号或标志
|
|
||||||
|
|
||||||
可疑内容(result=review):
|
只有命中**明确违法违规**时才输出 result=review:
|
||||||
- 图片含义模糊,无法确定是否存在违规
|
- 色情低俗:明确色情、露骨性描写或诱导性图片;正常人体艺术、二次创作用感形象不在此列;
|
||||||
- 轻微争议性内容但不完全违规
|
- 真实暴力/血腥:明确展示严重暴力、凶杀、虐待等违法犯罪画面;
|
||||||
- AI无法准确判断的内容
|
- 违法犯罪相关:明确涉及毒品、枪支、诈骗、黑产等内容;
|
||||||
|
- 未成年保护:涉及未成年人的色情或剥削图片;
|
||||||
|
- 诈骗引流:明确的诈骗二维码、钓鱼链接、黑产广告;
|
||||||
|
- 极端仇恨:纳粹等极端仇恨符号或标志。
|
||||||
|
|
||||||
放行规则(result=pass):
|
以下图片即使存在也**一律 pass**(不送审、不拦截):
|
||||||
- 正常风景、人物、动物、动漫等图片
|
- 正常风景、人物、动物、动漫等图片;
|
||||||
- 表情包、梗图
|
- 表情包、梗图、二次创作;
|
||||||
- 正常的艺术创作、摄影作品
|
- 正常的艺术创作、摄影作品;
|
||||||
|
- 含义模糊但无明显违法意图的图片。
|
||||||
|
|
||||||
|
reason 要求:
|
||||||
|
- result=review 时:中文10-30字,说明疑似违法点;
|
||||||
|
- result=pass 时:reason 为空字符串。
|
||||||
|
|
||||||
输出格式(严格):
|
输出格式(严格):
|
||||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
{"result": "pass"/"review", "reason": "..."}`
|
||||||
|
|
||||||
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
|
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||||
|
|
||||||
硬性违规(命中任一项必须 result=block):
|
核心原则:
|
||||||
- 明确的违法犯罪、暴力威胁内容
|
- 除明确违法违规外,不要拦截或送审正常签名;
|
||||||
- 色情低俗、性暗示描述
|
- 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
|
||||||
- 诈骗引流、恶意广告
|
|
||||||
- 明确煽动群体对立、仇恨
|
|
||||||
- 恶意公开他人隐私信息
|
|
||||||
|
|
||||||
可疑内容(result=review):
|
只有命中**明确违法违规**时才输出 result=review:
|
||||||
- 内容含义模糊,无法确定违规意图
|
- 明确违法犯罪、暴力威胁内容;
|
||||||
- 涉及敏感话题但未明确违规
|
- 色情低俗、露骨性暗示描述;
|
||||||
- AI无法准确判断的表达
|
- 诈骗引流、黑产广告;
|
||||||
|
- 恶意公开他人可识别隐私信息;
|
||||||
|
- 涉及未成年人的色情或剥削内容。
|
||||||
|
|
||||||
放行规则(result=pass):
|
以下签名即使存在也**一律 pass**(不送审、不拦截):
|
||||||
- 正常的自我介绍、心情表达
|
- 正常的自我介绍、心情表达;
|
||||||
- 玩梗、幽默、吐槽
|
- 玩梗、幽默、吐槽、谐音梗;
|
||||||
- 引用、歌词、诗句等
|
- 引用、歌词、诗句等;
|
||||||
- 理性的观点表达
|
- 理性的观点表达、对社会事件的评论;
|
||||||
|
- 含义模糊但无明显违法意图的内容。
|
||||||
|
|
||||||
|
reason 要求:
|
||||||
|
- result=review 时:中文10-30字,说明疑似违法点;
|
||||||
|
- result=pass 时:reason 为空字符串。
|
||||||
|
|
||||||
输出格式(严格):
|
输出格式(严格):
|
||||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
{"result": "pass"/"review", "reason": "..."}`
|
||||||
|
|
||||||
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
|
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
|
||||||
if !c.IsEnabled() {
|
if !c.IsEnabled() {
|
||||||
@@ -226,42 +221,11 @@ func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*Moder
|
|||||||
cleanImages := normalizeImageURLs([]string{imageURL})
|
cleanImages := normalizeImageURLs([]string{imageURL})
|
||||||
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
||||||
|
|
||||||
var lastErr error
|
resp, err := c.moderateWithRetry(ctx, imageModerationSystemPrompt, userPrompt, optimizedImages)
|
||||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
if err != nil {
|
||||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
|
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, err)
|
||||||
if err != nil {
|
|
||||||
lastErr = err
|
|
||||||
} else {
|
|
||||||
parsed := struct {
|
|
||||||
Result string `json:"result"`
|
|
||||||
Reason string `json:"reason"`
|
|
||||||
}{}
|
|
||||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
|
||||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
|
||||||
} else {
|
|
||||||
result := ModerationResultPass
|
|
||||||
switch parsed.Result {
|
|
||||||
case "review":
|
|
||||||
result = ModerationResultReview
|
|
||||||
case "block":
|
|
||||||
result = ModerationResultBlock
|
|
||||||
}
|
|
||||||
return &ModerationResponse{
|
|
||||||
Result: result,
|
|
||||||
Reason: parsed.Reason,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if attempt == maxModerationResultRetries-1 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
|
||||||
return nil, sleepErr
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return resp, nil
|
||||||
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
|
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
|
||||||
@@ -278,42 +242,11 @@ func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
|
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
|
||||||
var lastErr error
|
resp, err := c.moderateWithRetry(ctx, systemPrompt, userPrompt, nil)
|
||||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
if err != nil {
|
||||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
|
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, err)
|
||||||
if err != nil {
|
|
||||||
lastErr = err
|
|
||||||
} else {
|
|
||||||
parsed := struct {
|
|
||||||
Result string `json:"result"`
|
|
||||||
Reason string `json:"reason"`
|
|
||||||
}{}
|
|
||||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
|
||||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
|
||||||
} else {
|
|
||||||
result := ModerationResultPass
|
|
||||||
switch parsed.Result {
|
|
||||||
case "review":
|
|
||||||
result = ModerationResultReview
|
|
||||||
case "block":
|
|
||||||
result = ModerationResultBlock
|
|
||||||
}
|
|
||||||
return &ModerationResponse{
|
|
||||||
Result: result,
|
|
||||||
Reason: parsed.Reason,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if attempt == maxModerationResultRetries-1 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
|
||||||
return nil, sleepErr
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return resp, nil
|
||||||
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
|
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
|
||||||
@@ -365,48 +298,17 @@ func (c *clientImpl) moderateSingleBatch(
|
|||||||
totalBatches,
|
totalBatches,
|
||||||
)
|
)
|
||||||
|
|
||||||
var lastErr error
|
resp, err := c.moderateWithRetry(ctx, moderationSystemPrompt, userPrompt, images)
|
||||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
if err != nil {
|
||||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, moderationSystemPrompt, userPrompt, images, 0.1, 220)
|
return nil, fmt.Errorf(
|
||||||
if err != nil {
|
"moderation batch %d/%d failed after %d attempts: %w",
|
||||||
lastErr = err
|
batchNo,
|
||||||
} else {
|
totalBatches,
|
||||||
parsed := struct {
|
maxModerationResultRetries,
|
||||||
Result string `json:"result"`
|
err,
|
||||||
Reason string `json:"reason"`
|
)
|
||||||
}{}
|
|
||||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
|
||||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
|
||||||
} else {
|
|
||||||
result := ModerationResultPass
|
|
||||||
switch parsed.Result {
|
|
||||||
case "review":
|
|
||||||
result = ModerationResultReview
|
|
||||||
case "block":
|
|
||||||
result = ModerationResultBlock
|
|
||||||
}
|
|
||||||
return &ModerationResponse{
|
|
||||||
Result: result,
|
|
||||||
Reason: parsed.Reason,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if attempt == maxModerationResultRetries-1 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
|
||||||
return nil, sleepErr
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return resp, nil
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"moderation batch %d/%d failed after %d attempts: %w",
|
|
||||||
batchNo,
|
|
||||||
totalBatches,
|
|
||||||
maxModerationResultRetries,
|
|
||||||
lastErr,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type chatCompletionsRequest struct {
|
type chatCompletionsRequest struct {
|
||||||
@@ -497,36 +399,69 @@ func (c *clientImpl) chatCompletion(
|
|||||||
endpoint = baseURL + "/chat/completions"
|
endpoint = baseURL + "/chat/completions"
|
||||||
}
|
}
|
||||||
|
|
||||||
var lastErr error
|
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
||||||
for attempt := 0; attempt < maxChatCompletionRetries; attempt++ {
|
if err != nil {
|
||||||
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
return "", &chatCompletionError{StatusCode: 0, Err: err}
|
||||||
if err != nil {
|
}
|
||||||
lastErr = err
|
if statusCode >= 400 {
|
||||||
} else if statusCode >= 400 {
|
return "", &chatCompletionError{
|
||||||
lastErr = fmt.Errorf("openai error status=%d body=%s", statusCode, string(body))
|
StatusCode: statusCode,
|
||||||
if !isRetryableStatusCode(statusCode) {
|
Err: fmt.Errorf("openai error status=%d body=%s", statusCode, string(body)),
|
||||||
return "", lastErr
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
var parsed chatCompletionsResponse
|
|
||||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
||||||
return "", fmt.Errorf("decode response: %w", err)
|
|
||||||
}
|
|
||||||
if len(parsed.Choices) == 0 {
|
|
||||||
return "", fmt.Errorf("empty response choices")
|
|
||||||
}
|
|
||||||
return parsed.Choices[0].Message.Content, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if attempt == maxChatCompletionRetries-1 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
|
||||||
return "", sleepErr
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return "", lastErr
|
var parsed chatCompletionsResponse
|
||||||
|
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||||
|
return "", &chatCompletionError{
|
||||||
|
StatusCode: statusCode,
|
||||||
|
Err: fmt.Errorf("decode response: %w", err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(parsed.Choices) == 0 {
|
||||||
|
return "", &chatCompletionError{
|
||||||
|
StatusCode: statusCode,
|
||||||
|
Err: fmt.Errorf("empty response choices"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return parsed.Choices[0].Message.Content, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// chatCompletionError wraps errors from chatCompletion so callers can decide
|
||||||
|
// retryability based on the upstream HTTP status code. StatusCode == 0 means
|
||||||
|
// the request never reached the server (transport failure).
|
||||||
|
type chatCompletionError struct {
|
||||||
|
StatusCode int
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *chatCompletionError) Error() string {
|
||||||
|
if e.Err == nil {
|
||||||
|
return "chat completion error"
|
||||||
|
}
|
||||||
|
return e.Err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *chatCompletionError) Unwrap() error {
|
||||||
|
return e.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRetryableModerationErr reports whether a moderation caller should retry
|
||||||
|
// after the given chatCompletion error. Context cancellation and non-429 4xx
|
||||||
|
// responses are not retryable; transport, 429, 5xx, and parsing errors are.
|
||||||
|
func isRetryableModerationErr(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var ce *chatCompletionError
|
||||||
|
if errors.As(err, &ce) {
|
||||||
|
if ce.StatusCode >= 400 && ce.StatusCode != http.StatusTooManyRequests && ce.StatusCode < 500 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) {
|
func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) {
|
||||||
@@ -550,13 +485,6 @@ func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint strin
|
|||||||
return body, resp.StatusCode, nil
|
return body, resp.StatusCode, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isRetryableStatusCode(statusCode int) bool {
|
|
||||||
if statusCode == http.StatusTooManyRequests {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return statusCode >= 500 && statusCode <= 599
|
|
||||||
}
|
|
||||||
|
|
||||||
func sleepWithBackoff(ctx context.Context, attempt int) error {
|
func sleepWithBackoff(ctx context.Context, attempt int) error {
|
||||||
delay := initialRetryBackoff * time.Duration(1<<attempt)
|
delay := initialRetryBackoff * time.Duration(1<<attempt)
|
||||||
timer := time.NewTimer(delay)
|
timer := time.NewTimer(delay)
|
||||||
@@ -570,6 +498,63 @@ func sleepWithBackoff(ctx context.Context, attempt int) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// moderateWithRetry runs a single moderation request up to maxModerationResultRetries
|
||||||
|
// times, retrying only on retryable chatCompletion errors. The underlying chatCompletion
|
||||||
|
// no longer retries on its own, so the total number of upstream calls is bounded by
|
||||||
|
// maxModerationResultRetries (previously 3x3=9).
|
||||||
|
func (c *clientImpl) moderateWithRetry(
|
||||||
|
ctx context.Context,
|
||||||
|
systemPrompt, userPrompt string,
|
||||||
|
images []string,
|
||||||
|
) (*ModerationResponse, error) {
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||||
|
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, images, 0.1, 220)
|
||||||
|
if err != nil {
|
||||||
|
lastErr = err
|
||||||
|
} else {
|
||||||
|
resp, parseErr := parseModerationReply(replyText)
|
||||||
|
if parseErr != nil {
|
||||||
|
lastErr = parseErr
|
||||||
|
} else {
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if attempt == maxModerationResultRetries-1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !isRetryableModerationErr(lastErr) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||||
|
return nil, sleepErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseModerationReply(replyText string) (*ModerationResponse, error) {
|
||||||
|
parsed := struct {
|
||||||
|
Result string `json:"result"`
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
}{}
|
||||||
|
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse moderation result: %w", err)
|
||||||
|
}
|
||||||
|
result := ModerationResultPass
|
||||||
|
switch parsed.Result {
|
||||||
|
case "review":
|
||||||
|
result = ModerationResultReview
|
||||||
|
case "block":
|
||||||
|
result = ModerationResultBlock
|
||||||
|
}
|
||||||
|
return &ModerationResponse{
|
||||||
|
Result: result,
|
||||||
|
Reason: parsed.Reason,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeImageURLs(images []string) []string {
|
func normalizeImageURLs(images []string) []string {
|
||||||
clean := make([]string, 0, len(images))
|
clean := make([]string, 0, len(images))
|
||||||
for _, image := range images {
|
for _, image := range images {
|
||||||
@@ -593,13 +578,14 @@ func extractJSONObject(raw string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *clientImpl) maxImagesPerModerationRequest() int {
|
func (c *clientImpl) maxImagesPerModerationRequest() int {
|
||||||
if c.cfg.ModerationMaxImagesPerRequest <= 0 {
|
configured := c.cfg.ModerationMaxImagesPerRequest
|
||||||
|
if configured <= 0 {
|
||||||
return defaultMaxImagesPerModerationRequest
|
return defaultMaxImagesPerModerationRequest
|
||||||
}
|
}
|
||||||
if c.cfg.ModerationMaxImagesPerRequest > 1 {
|
if configured > maxAllowedImagesPerModerationRequest {
|
||||||
return 1
|
return maxAllowedImagesPerModerationRequest
|
||||||
}
|
}
|
||||||
return c.cfg.ModerationMaxImagesPerRequest
|
return configured
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string {
|
func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package service
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -170,7 +171,7 @@ func (s *commentService) reviewCommentAsync(
|
|||||||
fmt.Sscanf(userID, "%d", &authorID)
|
fmt.Sscanf(userID, "%d", &authorID)
|
||||||
|
|
||||||
result := &hook.ModerationResult{}
|
result := &hook.ModerationResult{}
|
||||||
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookCommentPreModerate, authorID, &hook.CommentModerateHookData{
|
triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookCommentPreModerate, authorID, &hook.CommentModerateHookData{
|
||||||
CommentID: commentID,
|
CommentID: commentID,
|
||||||
PostID: postID,
|
PostID: postID,
|
||||||
Content: content,
|
Content: content,
|
||||||
@@ -181,6 +182,19 @@ func (s *commentService) reviewCommentAsync(
|
|||||||
"result": result,
|
"result": result,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Fallback-approve when a moderation hook times out or is cancelled, so a
|
||||||
|
// slow moderation API never silently rejects an otherwise-fine comment.
|
||||||
|
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
|
||||||
|
zap.L().Warn("Comment moderation hook timed out or was cancelled, fallback approve",
|
||||||
|
zap.String("comment_id", commentID),
|
||||||
|
zap.Error(triggerErr),
|
||||||
|
)
|
||||||
|
result.Approved = true
|
||||||
|
result.ReviewedBy = "system"
|
||||||
|
result.RejectReason = ""
|
||||||
|
result.NeedsReview = false
|
||||||
|
}
|
||||||
|
|
||||||
s.hookManager.Trigger(context.Background(), hook.HookCommentModerated, authorID, &hook.CommentModeratedHookData{
|
s.hookManager.Trigger(context.Background(), hook.HookCommentModerated, authorID, &hook.CommentModeratedHookData{
|
||||||
CommentID: commentID,
|
CommentID: commentID,
|
||||||
PostID: postID,
|
PostID: postID,
|
||||||
|
|||||||
@@ -223,15 +223,13 @@ func (s *postAIService) ModeratePost(ctx context.Context, title, content string,
|
|||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
return s.moderateTextWithTencent(ctx, title+" "+content,
|
return s.moderateTextWithTencent(ctx, title+" "+content,
|
||||||
func(reason string) error { return &PostModerationRejectedError{Reason: reason} },
|
|
||||||
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// block 与 review 同等处理:均仅送人工复审。
|
||||||
switch resp.Result {
|
switch resp.Result {
|
||||||
case openai.ModerationResultBlock:
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||||
return &PostModerationRejectedError{Reason: resp.Reason}
|
|
||||||
case openai.ModerationResultReview:
|
|
||||||
return &PostModerationReviewError{Reason: resp.Reason}
|
return &PostModerationReviewError{Reason: resp.Reason}
|
||||||
default:
|
default:
|
||||||
return nil
|
return nil
|
||||||
@@ -239,7 +237,6 @@ func (s *postAIService) ModeratePost(ctx context.Context, title, content string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
return s.moderateTextWithTencent(ctx, title+" "+content,
|
return s.moderateTextWithTencent(ctx, title+" "+content,
|
||||||
func(reason string) error { return &PostModerationRejectedError{Reason: reason} },
|
|
||||||
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
func(reason string) error { return &PostModerationReviewError{Reason: reason} },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -256,15 +253,13 @@ func (s *postAIService) ModerateComment(ctx context.Context, content string, ima
|
|||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
return s.moderateTextWithTencent(ctx, content,
|
return s.moderateTextWithTencent(ctx, content,
|
||||||
func(reason string) error { return &CommentModerationRejectedError{Reason: reason} },
|
|
||||||
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// block 与 review 同等处理:均仅送人工复审。
|
||||||
switch resp.Result {
|
switch resp.Result {
|
||||||
case openai.ModerationResultBlock:
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||||
return &CommentModerationRejectedError{Reason: resp.Reason}
|
|
||||||
case openai.ModerationResultReview:
|
|
||||||
return &CommentModerationReviewError{Reason: resp.Reason}
|
return &CommentModerationReviewError{Reason: resp.Reason}
|
||||||
default:
|
default:
|
||||||
return nil
|
return nil
|
||||||
@@ -272,7 +267,6 @@ func (s *postAIService) ModerateComment(ctx context.Context, content string, ima
|
|||||||
}
|
}
|
||||||
|
|
||||||
return s.moderateTextWithTencent(ctx, content,
|
return s.moderateTextWithTencent(ctx, content,
|
||||||
func(reason string) error { return &CommentModerationRejectedError{Reason: reason} },
|
|
||||||
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
func(reason string) error { return &CommentModerationReviewError{Reason: reason} },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -293,10 +287,9 @@ func (s *postAIService) ModerateImage(ctx context.Context, imageURL string) erro
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// block 与 review 同等处理:均仅送人工复审。
|
||||||
switch resp.Result {
|
switch resp.Result {
|
||||||
case openai.ModerationResultBlock:
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||||
return &ImageModerationRejectedError{Reason: resp.Reason}
|
|
||||||
case openai.ModerationResultReview:
|
|
||||||
return &ImageModerationReviewError{Reason: resp.Reason}
|
return &ImageModerationReviewError{Reason: resp.Reason}
|
||||||
default:
|
default:
|
||||||
return nil
|
return nil
|
||||||
@@ -315,15 +308,13 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
|||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
return s.moderateTextWithTencent(ctx, bio,
|
return s.moderateTextWithTencent(ctx, bio,
|
||||||
func(reason string) error { return &BioModerationRejectedError{Reason: reason} },
|
|
||||||
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// block 与 review 同等处理:均仅送人工复审。
|
||||||
switch resp.Result {
|
switch resp.Result {
|
||||||
case openai.ModerationResultBlock:
|
case openai.ModerationResultBlock, openai.ModerationResultReview:
|
||||||
return &BioModerationRejectedError{Reason: resp.Reason}
|
|
||||||
case openai.ModerationResultReview:
|
|
||||||
return &BioModerationReviewError{Reason: resp.Reason}
|
return &BioModerationReviewError{Reason: resp.Reason}
|
||||||
default:
|
default:
|
||||||
return nil
|
return nil
|
||||||
@@ -331,7 +322,6 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return s.moderateTextWithTencent(ctx, bio,
|
return s.moderateTextWithTencent(ctx, bio,
|
||||||
func(reason string) error { return &BioModerationRejectedError{Reason: reason} },
|
|
||||||
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
func(reason string) error { return &BioModerationReviewError{Reason: reason} },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -339,7 +329,6 @@ func (s *postAIService) ModerateBio(ctx context.Context, bio string) error {
|
|||||||
func (s *postAIService) moderateTextWithTencent(
|
func (s *postAIService) moderateTextWithTencent(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
text string,
|
text string,
|
||||||
rejectFactory func(string) error,
|
|
||||||
reviewFactory func(string) error,
|
reviewFactory func(string) error,
|
||||||
) error {
|
) error {
|
||||||
if !s.isTencentEnabled() {
|
if !s.isTencentEnabled() {
|
||||||
@@ -366,10 +355,9 @@ func (s *postAIService) moderateTextWithTencent(
|
|||||||
reason = resp.Label + ": " + strings.Join(resp.Keywords, ", ")
|
reason = resp.Label + ": " + strings.Join(resp.Keywords, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 腾讯云 block 也仅送审,与 AI 同语义。
|
||||||
switch resp.Suggestion {
|
switch resp.Suggestion {
|
||||||
case tencent.SuggestionBlock:
|
case tencent.SuggestionBlock, tencent.SuggestionReview:
|
||||||
return rejectFactory(reason)
|
|
||||||
case tencent.SuggestionReview:
|
|
||||||
return reviewFactory(reason)
|
return reviewFactory(reason)
|
||||||
default:
|
default:
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package service
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -216,7 +217,7 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
|
|||||||
fmt.Sscanf(userID, "%d", &authorID)
|
fmt.Sscanf(userID, "%d", &authorID)
|
||||||
|
|
||||||
result := &hook.ModerationResult{}
|
result := &hook.ModerationResult{}
|
||||||
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
||||||
PostID: postID,
|
PostID: postID,
|
||||||
Title: title,
|
Title: title,
|
||||||
Content: content,
|
Content: content,
|
||||||
@@ -226,6 +227,19 @@ func (s *postServiceImpl) reviewPostAsync(postID, userID, title, content string,
|
|||||||
"result": result,
|
"result": result,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Trigger can return context.DeadlineExceeded/Canceled if a moderation hook
|
||||||
|
// times out. In that case the result may not have been written at all
|
||||||
|
// (result.Approved stays false). Treat an unfinished moderation as
|
||||||
|
// "approved + system" rather than silently rejecting the post, so a slow
|
||||||
|
// moderation API never blocks normal content.
|
||||||
|
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
|
||||||
|
log.Printf("[WARN] Post moderation hook timed out or was cancelled, fallback approve post=%s err=%v", postID, triggerErr)
|
||||||
|
result.Approved = true
|
||||||
|
result.ReviewedBy = "system"
|
||||||
|
result.RejectReason = ""
|
||||||
|
result.NeedsReview = false
|
||||||
|
}
|
||||||
|
|
||||||
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
|
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
|
||||||
PostID: postID,
|
PostID: postID,
|
||||||
AuthorID: authorID,
|
AuthorID: authorID,
|
||||||
|
|||||||
@@ -177,7 +177,7 @@ func (s *voteService) reviewVotePostAsync(postID, userID, title, content string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
result := &hook.ModerationResult{}
|
result := &hook.ModerationResult{}
|
||||||
s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
triggerErr := s.hookManager.TriggerWithMetadata(context.Background(), hook.HookPostPreModerate, authorID, &hook.PostModerateHookData{
|
||||||
PostID: postID,
|
PostID: postID,
|
||||||
Title: title,
|
Title: title,
|
||||||
Content: content,
|
Content: content,
|
||||||
@@ -187,6 +187,16 @@ func (s *voteService) reviewVotePostAsync(postID, userID, title, content string,
|
|||||||
"result": result,
|
"result": result,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Fallback-approve when a moderation hook times out or is cancelled, so a
|
||||||
|
// slow moderation API never silently rejects an otherwise-fine vote post.
|
||||||
|
if triggerErr != nil && (errors.Is(triggerErr, context.DeadlineExceeded) || errors.Is(triggerErr, context.Canceled)) {
|
||||||
|
log.Printf("[WARN] Vote post moderation hook timed out or was cancelled, fallback approve post=%s err=%v", postID, triggerErr)
|
||||||
|
result.Approved = true
|
||||||
|
result.ReviewedBy = "system"
|
||||||
|
result.RejectReason = ""
|
||||||
|
result.NeedsReview = false
|
||||||
|
}
|
||||||
|
|
||||||
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
|
s.hookManager.Trigger(context.Background(), hook.HookPostModerated, authorID, &hook.PostModeratedHookData{
|
||||||
PostID: postID,
|
PostID: postID,
|
||||||
AuthorID: authorID,
|
AuthorID: authorID,
|
||||||
|
|||||||
Reference in New Issue
Block a user