feat(moderation): make moderation more lenient and add timeout fallback behavior
The moderation system now uses a "pass/review" strategy instead of "pass/review/block", treating hard violations the same as review (only routing to manual review rather than auto-rejecting). Add fallback-approve behavior in the hook system so slow moderation APIs never silently block normal content when they time out or get cancelled - timed-out moderation now approves with "ReviewedBy: system" instead of leaving the zero-value result that would reject content. Update post/comment/vote services and handlers to use the new unified behavior. Simplify moderation prompts to emphasize "宁可放过,不可误伤" (prefer false positives over false negatives).
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
@@ -21,55 +22,41 @@ import (
|
||||
xdraw "golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。目标是平衡社区安全与正常交流:必须拦截高风险违规内容,但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。
|
||||
const moderationSystemPrompt = `你是中文社区的内容审核助手,负责对"帖子标题、正文、配图"做联合审核。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||
|
||||
审核流程:
|
||||
1) 先判断是否命中硬性违规;
|
||||
2) 再判断语境(玩笑/自嘲/朋友间互动/作品讨论);
|
||||
3) 做文图交叉判断(文本+图片合并理解);
|
||||
4) 给出 result 与简短 reason。
|
||||
核心原则:
|
||||
- 除明确违法违规外,不要拦截或送审正常内容,包括尖锐吐槽、玩梗、二创、争论;
|
||||
- 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
A. 宣传对立与煽动撕裂:
|
||||
- 明确煽动群体对立、地域对立、性别对立、民族宗教对立,鼓动仇恨、排斥、报复。
|
||||
B. 严重人身攻击与网暴引导:
|
||||
- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突。
|
||||
C. 开盒/人肉/隐私暴露:
|
||||
- 故意公开、拼接、索取他人可识别隐私信息(姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等);
|
||||
- 图片/截图中出现可识别隐私信息并伴随曝光意图,也按违规处理。
|
||||
D. 其他高危违规:
|
||||
- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。
|
||||
只有命中**明确违法违规**时才输出 result=review:
|
||||
A. 违法犯罪:明确宣传违法犯罪行为、教唆或指导他人犯罪(如毒品、枪支、诈骗教程)。
|
||||
B. 真实暴力威胁:针对具体个人或群体的实质性暴力威胁、人身伤害恐吓。
|
||||
C. 色情低俗:明确色情、露骨性描写或诱导性内容;正常人体艺术、二次创作用感形象不在此列。
|
||||
D. 诈骗引流:明确的诈骗信息、钓鱼链接、黑产引流。
|
||||
E. 未成年保护:涉及未成年人的色情或剥削内容。
|
||||
F. 开盒/人肉:故意公开他人可识别隐私信息(身份证号、住址、联系方式、定位轨迹)并伴有曝光意图。
|
||||
G. 极端仇恨:鼓动对特定族裔/宗教/残障群体的仇恨与暴力,含纳粹等极端符号。
|
||||
|
||||
可疑内容(需要人工复审,result=review):
|
||||
- 内容含义模糊,无法确定是否存在违规意图;
|
||||
- 涉及敏感话题但未明确违规(如政治人物、社会热点事件等);
|
||||
- 存在轻微争议性表达,但不完全符合硬性违规标准;
|
||||
- AI无法准确判断的文化/地域特定表达。
|
||||
|
||||
放行规则(以下通常 result=pass):
|
||||
以下情形即使存在也**一律 pass**(不送审、不拦截):
|
||||
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽;
|
||||
- 非定向、轻度口语化吐槽(无明确攻击对象、无网暴号召、无隐私暴露);
|
||||
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐,但未煽动对立或人身攻击)。
|
||||
|
||||
边界判定:
|
||||
- 若只是"梗文化表达"且不指向现实伤害,优先通过;
|
||||
- 若存在明确伤害意图(煽动、围攻、曝光隐私),必须拒绝;
|
||||
- 对模糊内容不因个别粗口直接拒绝,需结合对象、意图、号召性和可执行性综合判断;
|
||||
- 如果实在无法确定是否违规,标记为 review 状态交由人工复审。
|
||||
- 非定向、轻度口语化吐槽、玩笑、自嘲、朋友间互动;
|
||||
- 对社会事件/作品的理性讨论、观点争论(即使语气尖锐、用词激烈);
|
||||
- 一般性粗口、调侃、讽刺,未煽动对立或人身攻击的;
|
||||
- 对政治人物/社会热点的评论、争议性观点表达;
|
||||
- 含义模糊但无明显违法意图的内容。
|
||||
|
||||
reason 要求:
|
||||
- result=block 时:中文10-30字,说明核心违规点;
|
||||
- result=review 时:中文10-30字,说明可疑原因;
|
||||
- result=review 时:中文10-30字,说明疑似违法点;
|
||||
- result=pass 时:reason 为空字符串。
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
{"result": "pass"/"review", "reason": "..."}`
|
||||
|
||||
const (
|
||||
defaultMaxImagesPerModerationRequest = 1
|
||||
maxAllowedImagesPerModerationRequest = 4
|
||||
maxModerationResultRetries = 3
|
||||
maxChatCompletionRetries = 3
|
||||
initialRetryBackoff = 500 * time.Millisecond
|
||||
maxDownloadImageBytes = 10 * 1024 * 1024
|
||||
maxModerationImageSide = 1280
|
||||
@@ -165,53 +152,61 @@ func (c *clientImpl) ModerateComment(ctx context.Context, content string, images
|
||||
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
||||
}
|
||||
|
||||
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。目标是确保图片内容符合社区规范。请只输出指定JSON。
|
||||
const imageModerationSystemPrompt = `你是图片内容审核助手,负责审核用户头像和背景图片。原则是**宁可放过,不可误伤**:只有明确违法违规的图片才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
- 色情低俗、裸露、性暗示内容
|
||||
- 暴力血腥、恐怖画面
|
||||
- 违法犯罪相关内容
|
||||
- 明确的政治敏感宣传
|
||||
- 恶意广告、诈骗信息
|
||||
- 仇恨歧视符号或标志
|
||||
核心原则:
|
||||
- 除明确违法违规外,不要拦截或送审正常图片;
|
||||
- 判定从严,倾向放行:宁可放过十张可疑,不可误伤一张正常。
|
||||
|
||||
可疑内容(result=review):
|
||||
- 图片含义模糊,无法确定是否存在违规
|
||||
- 轻微争议性内容但不完全违规
|
||||
- AI无法准确判断的内容
|
||||
只有命中**明确违法违规**时才输出 result=review:
|
||||
- 色情低俗:明确色情、露骨性描写或诱导性图片;正常人体艺术、二次创作用感形象不在此列;
|
||||
- 真实暴力/血腥:明确展示严重暴力、凶杀、虐待等违法犯罪画面;
|
||||
- 违法犯罪相关:明确涉及毒品、枪支、诈骗、黑产等内容;
|
||||
- 未成年保护:涉及未成年人的色情或剥削图片;
|
||||
- 诈骗引流:明确的诈骗二维码、钓鱼链接、黑产广告;
|
||||
- 极端仇恨:纳粹等极端仇恨符号或标志。
|
||||
|
||||
放行规则(result=pass):
|
||||
- 正常风景、人物、动物、动漫等图片
|
||||
- 表情包、梗图
|
||||
- 正常的艺术创作、摄影作品
|
||||
以下图片即使存在也**一律 pass**(不送审、不拦截):
|
||||
- 正常风景、人物、动物、动漫等图片;
|
||||
- 表情包、梗图、二次创作;
|
||||
- 正常的艺术创作、摄影作品;
|
||||
- 含义模糊但无明显违法意图的图片。
|
||||
|
||||
reason 要求:
|
||||
- result=review 时:中文10-30字,说明疑似违法点;
|
||||
- result=pass 时:reason 为空字符串。
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
{"result": "pass"/"review", "reason": "..."}`
|
||||
|
||||
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。目标是平衡社区安全与正常自我表达。请只输出指定JSON。
|
||||
const bioModerationSystemPrompt = `你是用户个性签名内容审核助手,负责审核用户个人简介/签名。原则是**宁可放过,不可误伤**:只有明确违法违规内容才标记 review 交人工复核,其余一律 pass。请只输出指定JSON。
|
||||
|
||||
硬性违规(命中任一项必须 result=block):
|
||||
- 明确的违法犯罪、暴力威胁内容
|
||||
- 色情低俗、性暗示描述
|
||||
- 诈骗引流、恶意广告
|
||||
- 明确煽动群体对立、仇恨
|
||||
- 恶意公开他人隐私信息
|
||||
核心原则:
|
||||
- 除明确违法违规外,不要拦截或送审正常签名;
|
||||
- 判定从严,倾向放行:宁可放过十个可疑,不可误伤一个正常。
|
||||
|
||||
可疑内容(result=review):
|
||||
- 内容含义模糊,无法确定违规意图
|
||||
- 涉及敏感话题但未明确违规
|
||||
- AI无法准确判断的表达
|
||||
只有命中**明确违法违规**时才输出 result=review:
|
||||
- 明确违法犯罪、暴力威胁内容;
|
||||
- 色情低俗、露骨性暗示描述;
|
||||
- 诈骗引流、黑产广告;
|
||||
- 恶意公开他人可识别隐私信息;
|
||||
- 涉及未成年人的色情或剥削内容。
|
||||
|
||||
放行规则(result=pass):
|
||||
- 正常的自我介绍、心情表达
|
||||
- 玩梗、幽默、吐槽
|
||||
- 引用、歌词、诗句等
|
||||
- 理性的观点表达
|
||||
以下签名即使存在也**一律 pass**(不送审、不拦截):
|
||||
- 正常的自我介绍、心情表达;
|
||||
- 玩梗、幽默、吐槽、谐音梗;
|
||||
- 引用、歌词、诗句等;
|
||||
- 理性的观点表达、对社会事件的评论;
|
||||
- 含义模糊但无明显违法意图的内容。
|
||||
|
||||
reason 要求:
|
||||
- result=review 时:中文10-30字,说明疑似违法点;
|
||||
- result=pass 时:reason 为空字符串。
|
||||
|
||||
输出格式(严格):
|
||||
仅输出一行JSON对象,不要Markdown,不要额外解释:
|
||||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||||
{"result": "pass"/"review", "reason": "..."}`
|
||||
|
||||
func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*ModerationResponse, error) {
|
||||
if !c.IsEnabled() {
|
||||
@@ -226,42 +221,11 @@ func (c *clientImpl) ModerateImage(ctx context.Context, imageURL string) (*Moder
|
||||
cleanImages := normalizeImageURLs([]string{imageURL})
|
||||
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, imageModerationSystemPrompt, userPrompt, optimizedImages, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
resp, err := c.moderateWithRetry(ctx, imageModerationSystemPrompt, userPrompt, optimizedImages)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("image moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationResponse, error) {
|
||||
@@ -278,42 +242,11 @@ func (c *clientImpl) ModerateBio(ctx context.Context, bio string) (*ModerationRe
|
||||
}
|
||||
|
||||
func (c *clientImpl) moderateWithCustomPrompt(ctx context.Context, systemPrompt string, userPrompt string) (*ModerationResponse, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, nil, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
resp, err := c.moderateWithRetry(ctx, systemPrompt, userPrompt, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("moderation failed after %d attempts: %w", maxModerationResultRetries, lastErr)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
|
||||
@@ -365,48 +298,17 @@ func (c *clientImpl) moderateSingleBatch(
|
||||
totalBatches,
|
||||
)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, moderationSystemPrompt, userPrompt, images, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
} else {
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
resp, err := c.moderateWithRetry(ctx, moderationSystemPrompt, userPrompt, images)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"moderation batch %d/%d failed after %d attempts: %w",
|
||||
batchNo,
|
||||
totalBatches,
|
||||
maxModerationResultRetries,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"moderation batch %d/%d failed after %d attempts: %w",
|
||||
batchNo,
|
||||
totalBatches,
|
||||
maxModerationResultRetries,
|
||||
lastErr,
|
||||
)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
type chatCompletionsRequest struct {
|
||||
@@ -497,36 +399,69 @@ func (c *clientImpl) chatCompletion(
|
||||
endpoint = baseURL + "/chat/completions"
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxChatCompletionRetries; attempt++ {
|
||||
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else if statusCode >= 400 {
|
||||
lastErr = fmt.Errorf("openai error status=%d body=%s", statusCode, string(body))
|
||||
if !isRetryableStatusCode(statusCode) {
|
||||
return "", lastErr
|
||||
}
|
||||
} else {
|
||||
var parsed chatCompletionsResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return "", fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return "", fmt.Errorf("empty response choices")
|
||||
}
|
||||
return parsed.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
if attempt == maxChatCompletionRetries-1 {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return "", sleepErr
|
||||
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
||||
if err != nil {
|
||||
return "", &chatCompletionError{StatusCode: 0, Err: err}
|
||||
}
|
||||
if statusCode >= 400 {
|
||||
return "", &chatCompletionError{
|
||||
StatusCode: statusCode,
|
||||
Err: fmt.Errorf("openai error status=%d body=%s", statusCode, string(body)),
|
||||
}
|
||||
}
|
||||
|
||||
return "", lastErr
|
||||
var parsed chatCompletionsResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return "", &chatCompletionError{
|
||||
StatusCode: statusCode,
|
||||
Err: fmt.Errorf("decode response: %w", err),
|
||||
}
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return "", &chatCompletionError{
|
||||
StatusCode: statusCode,
|
||||
Err: fmt.Errorf("empty response choices"),
|
||||
}
|
||||
}
|
||||
return parsed.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
// chatCompletionError wraps errors from chatCompletion so callers can decide
|
||||
// retryability based on the upstream HTTP status code. StatusCode == 0 means
|
||||
// the request never reached the server (transport failure).
|
||||
type chatCompletionError struct {
|
||||
StatusCode int
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *chatCompletionError) Error() string {
|
||||
if e.Err == nil {
|
||||
return "chat completion error"
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *chatCompletionError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// isRetryableModerationErr reports whether a moderation caller should retry
|
||||
// after the given chatCompletion error. Context cancellation and non-429 4xx
|
||||
// responses are not retryable; transport, 429, 5xx, and parsing errors are.
|
||||
func isRetryableModerationErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
var ce *chatCompletionError
|
||||
if errors.As(err, &ce) {
|
||||
if ce.StatusCode >= 400 && ce.StatusCode != http.StatusTooManyRequests && ce.StatusCode < 500 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) {
|
||||
@@ -550,13 +485,6 @@ func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint strin
|
||||
return body, resp.StatusCode, nil
|
||||
}
|
||||
|
||||
func isRetryableStatusCode(statusCode int) bool {
|
||||
if statusCode == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
return statusCode >= 500 && statusCode <= 599
|
||||
}
|
||||
|
||||
func sleepWithBackoff(ctx context.Context, attempt int) error {
|
||||
delay := initialRetryBackoff * time.Duration(1<<attempt)
|
||||
timer := time.NewTimer(delay)
|
||||
@@ -570,6 +498,63 @@ func sleepWithBackoff(ctx context.Context, attempt int) error {
|
||||
}
|
||||
}
|
||||
|
||||
// moderateWithRetry runs a single moderation request up to maxModerationResultRetries
|
||||
// times, retrying only on retryable chatCompletion errors. The underlying chatCompletion
|
||||
// no longer retries on its own, so the total number of upstream calls is bounded by
|
||||
// maxModerationResultRetries (previously 3x3=9).
|
||||
func (c *clientImpl) moderateWithRetry(
|
||||
ctx context.Context,
|
||||
systemPrompt, userPrompt string,
|
||||
images []string,
|
||||
) (*ModerationResponse, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, systemPrompt, userPrompt, images, 0.1, 220)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
resp, parseErr := parseModerationReply(replyText)
|
||||
if parseErr != nil {
|
||||
lastErr = parseErr
|
||||
} else {
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
if attempt == maxModerationResultRetries-1 {
|
||||
break
|
||||
}
|
||||
if !isRetryableModerationErr(lastErr) {
|
||||
break
|
||||
}
|
||||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func parseModerationReply(replyText string) (*ModerationResponse, error) {
|
||||
parsed := struct {
|
||||
Result string `json:"result"`
|
||||
Reason string `json:"reason"`
|
||||
}{}
|
||||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse moderation result: %w", err)
|
||||
}
|
||||
result := ModerationResultPass
|
||||
switch parsed.Result {
|
||||
case "review":
|
||||
result = ModerationResultReview
|
||||
case "block":
|
||||
result = ModerationResultBlock
|
||||
}
|
||||
return &ModerationResponse{
|
||||
Result: result,
|
||||
Reason: parsed.Reason,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeImageURLs(images []string) []string {
|
||||
clean := make([]string, 0, len(images))
|
||||
for _, image := range images {
|
||||
@@ -593,13 +578,14 @@ func extractJSONObject(raw string) string {
|
||||
}
|
||||
|
||||
func (c *clientImpl) maxImagesPerModerationRequest() int {
|
||||
if c.cfg.ModerationMaxImagesPerRequest <= 0 {
|
||||
configured := c.cfg.ModerationMaxImagesPerRequest
|
||||
if configured <= 0 {
|
||||
return defaultMaxImagesPerModerationRequest
|
||||
}
|
||||
if c.cfg.ModerationMaxImagesPerRequest > 1 {
|
||||
return 1
|
||||
if configured > maxAllowedImagesPerModerationRequest {
|
||||
return maxAllowedImagesPerModerationRequest
|
||||
}
|
||||
return c.cfg.ModerationMaxImagesPerRequest
|
||||
return configured
|
||||
}
|
||||
|
||||
func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string {
|
||||
|
||||
Reference in New Issue
Block a user