1bb82e1e2b
Add admin comment moderation endpoints (single and batch) and introduce a three-tier moderation system (pass/review/block) for content flagged by AI. Content with unclear violations is now held for manual review instead of being auto-rejected. Deleted the legacy audit_service.go in favor of the unified hook-based moderation system.
542 lines
15 KiB
Go
542 lines
15 KiB
Go
package openai
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"fmt"
|
||
"image"
|
||
_ "image/gif"
|
||
"image/jpeg"
|
||
_ "image/png"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"strings"
|
||
"time"
|
||
|
||
xdraw "golang.org/x/image/draw"
|
||
)
|
||
|
||
const moderationSystemPrompt = `你是中文社区的内容审核助手，负责对"帖子标题、正文、配图"做联合审核。目标是平衡社区安全与正常交流：必须拦截高风险违规内容，但不要误伤正常玩梗、二创、吐槽和轻度调侃。请只输出指定JSON。
|
||
|
||
审核流程：
|
||
1) 先判断是否命中硬性违规；
|
||
2) 再判断语境（玩笑/自嘲/朋友间互动/作品讨论）；
|
||
3) 做文图交叉判断（文本+图片合并理解）；
|
||
4) 给出 result 与简短 reason。
|
||
|
||
硬性违规（命中任一项必须 result=block）：
|
||
A. 宣传对立与煽动撕裂：
|
||
- 明确煽动群体对立、地域对立、性别对立、民族宗教对立，鼓动仇恨、排斥、报复。
|
||
B. 严重人身攻击与网暴引导：
|
||
- 持续性侮辱贬损、羞辱人格、号召围攻/骚扰/挂人/线下冲突。
|
||
C. 开盒/人肉/隐私暴露：
|
||
- 故意公开、拼接、索取他人可识别隐私信息（姓名+联系方式、身份证号、住址、学校单位、车牌、定位轨迹等）；
|
||
- 图片/截图中出现可识别隐私信息并伴随曝光意图，也按违规处理。
|
||
D. 其他高危违规：
|
||
- 违法犯罪、暴力威胁、极端仇恨、色情低俗、诈骗引流、恶意广告等。
|
||
|
||
可疑内容（需要人工复审，result=review）：
|
||
- 内容含义模糊，无法确定是否存在违规意图；
|
||
- 涉及敏感话题但未明确违规（如政治人物、社会热点事件等）；
|
||
- 存在轻微争议性表达，但不完全符合硬性违规标准；
|
||
- AI无法准确判断的文化/地域特定表达。
|
||
|
||
放行规则（以下通常 result=pass）：
|
||
- 正常玩梗、表情包、谐音梗、二次创作、无恶意的吐槽；
|
||
- 非定向、轻度口语化吐槽（无明确攻击对象、无网暴号召、无隐私暴露）；
|
||
- 对社会事件/作品的理性讨论、观点争论（即使语气尖锐，但未煽动对立或人身攻击）。
|
||
|
||
边界判定：
|
||
- 若只是"梗文化表达"且不指向现实伤害，优先通过；
|
||
- 若存在明确伤害意图（煽动、围攻、曝光隐私），必须拒绝；
|
||
- 对模糊内容不因个别粗口直接拒绝，需结合对象、意图、号召性和可执行性综合判断；
|
||
- 如果实在无法确定是否违规，标记为 review 状态交由人工复审。
|
||
|
||
reason 要求：
|
||
- result=block 时：中文10-30字，说明核心违规点；
|
||
- result=review 时：中文10-30字，说明可疑原因；
|
||
- result=pass 时：reason 为空字符串。
|
||
|
||
输出格式（严格）：
|
||
仅输出一行JSON对象，不要Markdown，不要额外解释：
|
||
{"result": "pass"/"review"/"block", "reason": "..."}`
|
||
|
||
const (
|
||
defaultMaxImagesPerModerationRequest = 1
|
||
maxModerationResultRetries = 3
|
||
maxChatCompletionRetries = 3
|
||
initialRetryBackoff = 500 * time.Millisecond
|
||
maxDownloadImageBytes = 10 * 1024 * 1024
|
||
maxModerationImageSide = 1280
|
||
compressedJPEGQuality = 72
|
||
maxCompressedPayloadBytes = 1536 * 1024
|
||
)
|
||
|
||
type ModerationResult string
|
||
|
||
const (
|
||
ModerationResultPass ModerationResult = "pass"
|
||
ModerationResultReview ModerationResult = "review"
|
||
ModerationResultBlock ModerationResult = "block"
|
||
)
|
||
|
||
type ModerationResponse struct {
|
||
Result ModerationResult `json:"result"`
|
||
Reason string `json:"reason"`
|
||
}
|
||
|
||
type Client interface {
|
||
IsEnabled() bool
|
||
Config() Config
|
||
ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error)
|
||
ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error)
|
||
}
|
||
|
||
type clientImpl struct {
|
||
cfg Config
|
||
httpClient *http.Client
|
||
}
|
||
|
||
func NewClient(cfg Config) Client {
|
||
timeout := cfg.RequestTimeoutSeconds
|
||
if timeout <= 0 {
|
||
timeout = 30
|
||
}
|
||
return &clientImpl{
|
||
cfg: cfg,
|
||
httpClient: &http.Client{
|
||
Timeout: time.Duration(timeout) * time.Second,
|
||
},
|
||
}
|
||
}
|
||
|
||
func (c *clientImpl) IsEnabled() bool {
|
||
return c.cfg.Enabled && c.cfg.APIKey != "" && c.cfg.BaseURL != ""
|
||
}
|
||
|
||
func (c *clientImpl) Config() Config {
|
||
return c.cfg
|
||
}
|
||
|
||
func (c *clientImpl) ModeratePost(ctx context.Context, title, content string, images []string) (*ModerationResponse, error) {
|
||
if !c.IsEnabled() {
|
||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||
}
|
||
|
||
var prompt strings.Builder
|
||
if strings.TrimSpace(title) != "" {
|
||
prompt.WriteString("帖子标题：" + title)
|
||
}
|
||
if strings.TrimSpace(content) != "" {
|
||
if prompt.Len() > 0 {
|
||
prompt.WriteString("\n")
|
||
}
|
||
prompt.WriteString("帖子内容：" + content)
|
||
}
|
||
if prompt.Len() == 0 && len(normalizeImageURLs(images)) > 0 {
|
||
prompt.WriteString("帖子内容：[仅包含图片]")
|
||
}
|
||
if prompt.Len() == 0 {
|
||
prompt.WriteString("帖子内容：[空]")
|
||
}
|
||
|
||
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
||
}
|
||
|
||
func (c *clientImpl) ModerateComment(ctx context.Context, content string, images []string) (*ModerationResponse, error) {
|
||
if !c.IsEnabled() {
|
||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||
}
|
||
|
||
var prompt strings.Builder
|
||
if strings.TrimSpace(content) != "" {
|
||
prompt.WriteString("评论内容：" + content)
|
||
} else if len(normalizeImageURLs(images)) > 0 {
|
||
prompt.WriteString("评论内容：[仅包含图片]")
|
||
} else {
|
||
prompt.WriteString("评论内容：[空]")
|
||
}
|
||
|
||
return c.moderateContentInBatches(ctx, prompt.String(), images)
|
||
}
|
||
|
||
func (c *clientImpl) moderateContentInBatches(ctx context.Context, contentPrompt string, images []string) (*ModerationResponse, error) {
|
||
cleanImages := normalizeImageURLs(images)
|
||
optimizedImages := c.optimizeImagesForModeration(ctx, cleanImages)
|
||
maxImagesPerRequest := c.maxImagesPerModerationRequest()
|
||
totalBatches := 1
|
||
if len(optimizedImages) > 0 {
|
||
totalBatches = (len(optimizedImages) + maxImagesPerRequest - 1) / maxImagesPerRequest
|
||
}
|
||
|
||
for i := 0; i < totalBatches; i++ {
|
||
start := i * maxImagesPerRequest
|
||
end := start + maxImagesPerRequest
|
||
if end > len(optimizedImages) {
|
||
end = len(optimizedImages)
|
||
}
|
||
|
||
batchImages := []string{}
|
||
if len(optimizedImages) > 0 {
|
||
batchImages = optimizedImages[start:end]
|
||
}
|
||
|
||
resp, err := c.moderateSingleBatch(ctx, contentPrompt, batchImages, i+1, totalBatches)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if resp.Result != ModerationResultPass {
|
||
if totalBatches > 1 && strings.TrimSpace(resp.Reason) != "" {
|
||
resp.Reason = fmt.Sprintf("第%d/%d批图片未通过：%s", i+1, totalBatches, resp.Reason)
|
||
}
|
||
return resp, nil
|
||
}
|
||
}
|
||
|
||
return &ModerationResponse{Result: ModerationResultPass}, nil
|
||
}
|
||
|
||
func (c *clientImpl) moderateSingleBatch(
|
||
ctx context.Context,
|
||
contentPrompt string,
|
||
images []string,
|
||
batchNo, totalBatches int,
|
||
) (*ModerationResponse, error) {
|
||
userPrompt := fmt.Sprintf(
|
||
"%s\n图片批次：%d/%d（本次仅提供当前批次图片）",
|
||
contentPrompt,
|
||
batchNo,
|
||
totalBatches,
|
||
)
|
||
|
||
var lastErr error
|
||
for attempt := 0; attempt < maxModerationResultRetries; attempt++ {
|
||
replyText, err := c.chatCompletion(ctx, c.cfg.ModerationModel, moderationSystemPrompt, userPrompt, images, 0.1, 220)
|
||
if err != nil {
|
||
lastErr = err
|
||
} else {
|
||
parsed := struct {
|
||
Result string `json:"result"`
|
||
Reason string `json:"reason"`
|
||
}{}
|
||
if err := json.Unmarshal([]byte(extractJSONObject(replyText)), &parsed); err != nil {
|
||
lastErr = fmt.Errorf("failed to parse moderation result: %w", err)
|
||
} else {
|
||
result := ModerationResultPass
|
||
switch parsed.Result {
|
||
case "review":
|
||
result = ModerationResultReview
|
||
case "block":
|
||
result = ModerationResultBlock
|
||
}
|
||
return &ModerationResponse{
|
||
Result: result,
|
||
Reason: parsed.Reason,
|
||
}, nil
|
||
}
|
||
}
|
||
|
||
if attempt == maxModerationResultRetries-1 {
|
||
break
|
||
}
|
||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||
return nil, sleepErr
|
||
}
|
||
}
|
||
|
||
return nil, fmt.Errorf(
|
||
"moderation batch %d/%d failed after %d attempts: %w",
|
||
batchNo,
|
||
totalBatches,
|
||
maxModerationResultRetries,
|
||
lastErr,
|
||
)
|
||
}
|
||
|
||
type chatCompletionsRequest struct {
|
||
Model string `json:"model"`
|
||
Messages []chatMessage `json:"messages"`
|
||
Temperature float64 `json:"temperature,omitempty"`
|
||
MaxTokens int `json:"max_tokens,omitempty"`
|
||
EnableThinking *bool `json:"enable_thinking,omitempty"`
|
||
ThinkingBudget *int `json:"thinking_budget,omitempty"`
|
||
ResponseFormat *responseFormatConfig `json:"response_format,omitempty"`
|
||
}
|
||
|
||
type responseFormatConfig struct {
|
||
Type string `json:"type"`
|
||
}
|
||
|
||
type chatMessage struct {
|
||
Role string `json:"role"`
|
||
Content any `json:"content"`
|
||
}
|
||
|
||
type contentPart struct {
|
||
Type string `json:"type"`
|
||
Text string `json:"text,omitempty"`
|
||
ImageURL *imageURLPart `json:"image_url,omitempty"`
|
||
}
|
||
|
||
type imageURLPart struct {
|
||
URL string `json:"url"`
|
||
}
|
||
|
||
type chatCompletionsResponse struct {
|
||
Choices []struct {
|
||
Message struct {
|
||
Content string `json:"content"`
|
||
} `json:"message"`
|
||
} `json:"choices"`
|
||
}
|
||
|
||
func (c *clientImpl) chatCompletion(
|
||
ctx context.Context,
|
||
model string,
|
||
systemPrompt string,
|
||
userPrompt string,
|
||
images []string,
|
||
temperature float64,
|
||
maxTokens int,
|
||
) (string, error) {
|
||
if model == "" {
|
||
return "", fmt.Errorf("model is empty")
|
||
}
|
||
|
||
cleanImages := normalizeImageURLs(images)
|
||
|
||
userParts := []contentPart{
|
||
{Type: "text", Text: userPrompt},
|
||
}
|
||
for _, image := range cleanImages {
|
||
userParts = append(userParts, contentPart{
|
||
Type: "image_url",
|
||
ImageURL: &imageURLPart{URL: image},
|
||
})
|
||
}
|
||
|
||
reqBody := chatCompletionsRequest{
|
||
Model: model,
|
||
Messages: []chatMessage{
|
||
{Role: "system", Content: systemPrompt},
|
||
{Role: "user", Content: userParts},
|
||
},
|
||
Temperature: temperature,
|
||
MaxTokens: maxTokens,
|
||
}
|
||
falseVal := false
|
||
reqBody.EnableThinking = &falseVal
|
||
zero := 0
|
||
reqBody.ThinkingBudget = &zero
|
||
reqBody.ResponseFormat = &responseFormatConfig{Type: "json_object"}
|
||
|
||
data, err := json.Marshal(reqBody)
|
||
if err != nil {
|
||
return "", fmt.Errorf("marshal request: %w", err)
|
||
}
|
||
|
||
baseURL := strings.TrimRight(c.cfg.BaseURL, "/")
|
||
endpoint := baseURL + "/v1/chat/completions"
|
||
if strings.HasSuffix(baseURL, "/v1") {
|
||
endpoint = baseURL + "/chat/completions"
|
||
}
|
||
|
||
var lastErr error
|
||
for attempt := 0; attempt < maxChatCompletionRetries; attempt++ {
|
||
body, statusCode, err := c.doChatCompletionRequest(ctx, endpoint, data)
|
||
if err != nil {
|
||
lastErr = err
|
||
} else if statusCode >= 400 {
|
||
lastErr = fmt.Errorf("openai error status=%d body=%s", statusCode, string(body))
|
||
if !isRetryableStatusCode(statusCode) {
|
||
return "", lastErr
|
||
}
|
||
} else {
|
||
var parsed chatCompletionsResponse
|
||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||
return "", fmt.Errorf("decode response: %w", err)
|
||
}
|
||
if len(parsed.Choices) == 0 {
|
||
return "", fmt.Errorf("empty response choices")
|
||
}
|
||
return parsed.Choices[0].Message.Content, nil
|
||
}
|
||
|
||
if attempt == maxChatCompletionRetries-1 {
|
||
break
|
||
}
|
||
if sleepErr := sleepWithBackoff(ctx, attempt); sleepErr != nil {
|
||
return "", sleepErr
|
||
}
|
||
}
|
||
|
||
return "", lastErr
|
||
}
|
||
|
||
func (c *clientImpl) doChatCompletionRequest(ctx context.Context, endpoint string, data []byte) ([]byte, int, error) {
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(data))
|
||
if err != nil {
|
||
return nil, 0, fmt.Errorf("create request: %w", err)
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||
|
||
resp, err := c.httpClient.Do(req)
|
||
if err != nil {
|
||
return nil, 0, fmt.Errorf("request openai: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
body, err := io.ReadAll(resp.Body)
|
||
if err != nil {
|
||
return nil, 0, fmt.Errorf("read response: %w", err)
|
||
}
|
||
return body, resp.StatusCode, nil
|
||
}
|
||
|
||
func isRetryableStatusCode(statusCode int) bool {
|
||
if statusCode == http.StatusTooManyRequests {
|
||
return true
|
||
}
|
||
return statusCode >= 500 && statusCode <= 599
|
||
}
|
||
|
||
func sleepWithBackoff(ctx context.Context, attempt int) error {
|
||
delay := initialRetryBackoff * time.Duration(1<<attempt)
|
||
timer := time.NewTimer(delay)
|
||
defer timer.Stop()
|
||
|
||
select {
|
||
case <-ctx.Done():
|
||
return fmt.Errorf("request cancelled: %w", ctx.Err())
|
||
case <-timer.C:
|
||
return nil
|
||
}
|
||
}
|
||
|
||
func normalizeImageURLs(images []string) []string {
|
||
clean := make([]string, 0, len(images))
|
||
for _, image := range images {
|
||
trimmed := strings.TrimSpace(image)
|
||
if trimmed == "" {
|
||
continue
|
||
}
|
||
clean = append(clean, trimmed)
|
||
}
|
||
return clean
|
||
}
|
||
|
||
func extractJSONObject(raw string) string {
|
||
text := strings.TrimSpace(raw)
|
||
start := strings.Index(text, "{")
|
||
end := strings.LastIndex(text, "}")
|
||
if start >= 0 && end > start {
|
||
return text[start : end+1]
|
||
}
|
||
return text
|
||
}
|
||
|
||
func (c *clientImpl) maxImagesPerModerationRequest() int {
|
||
if c.cfg.ModerationMaxImagesPerRequest <= 0 {
|
||
return defaultMaxImagesPerModerationRequest
|
||
}
|
||
if c.cfg.ModerationMaxImagesPerRequest > 1 {
|
||
return 1
|
||
}
|
||
return c.cfg.ModerationMaxImagesPerRequest
|
||
}
|
||
|
||
func (c *clientImpl) optimizeImagesForModeration(ctx context.Context, images []string) []string {
|
||
if len(images) == 0 {
|
||
return images
|
||
}
|
||
|
||
optimized := make([]string, 0, len(images))
|
||
for _, imageURL := range images {
|
||
optimized = append(optimized, c.tryCompressImageForModeration(ctx, imageURL))
|
||
}
|
||
return optimized
|
||
}
|
||
|
||
func (c *clientImpl) tryCompressImageForModeration(ctx context.Context, imageURL string) string {
|
||
parsed, err := url.Parse(imageURL)
|
||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||
return imageURL
|
||
}
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||
if err != nil {
|
||
return imageURL
|
||
}
|
||
|
||
resp, err := c.httpClient.Do(req)
|
||
if err != nil {
|
||
return imageURL
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode >= 400 {
|
||
return imageURL
|
||
}
|
||
if !strings.HasPrefix(strings.ToLower(resp.Header.Get("Content-Type")), "image/") {
|
||
return imageURL
|
||
}
|
||
|
||
originBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxDownloadImageBytes))
|
||
if err != nil || len(originBytes) == 0 {
|
||
return imageURL
|
||
}
|
||
|
||
srcImg, _, err := image.Decode(bytes.NewReader(originBytes))
|
||
if err != nil {
|
||
return imageURL
|
||
}
|
||
|
||
dstImg := resizeIfNeeded(srcImg, maxModerationImageSide)
|
||
var buf bytes.Buffer
|
||
if err := jpeg.Encode(&buf, dstImg, &jpeg.Options{Quality: compressedJPEGQuality}); err != nil {
|
||
return imageURL
|
||
}
|
||
|
||
compressed := buf.Bytes()
|
||
if len(compressed) == 0 || len(compressed) > maxCompressedPayloadBytes {
|
||
return imageURL
|
||
}
|
||
if len(compressed) >= int(float64(len(originBytes))*0.95) {
|
||
return imageURL
|
||
}
|
||
|
||
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(compressed)
|
||
}
|
||
|
||
func resizeIfNeeded(src image.Image, maxSide int) image.Image {
|
||
bounds := src.Bounds()
|
||
w := bounds.Dx()
|
||
h := bounds.Dy()
|
||
if w <= 0 || h <= 0 || max(w, h) <= maxSide {
|
||
return src
|
||
}
|
||
|
||
newW, newH := w, h
|
||
if w >= h {
|
||
newW = maxSide
|
||
newH = int(float64(h) * (float64(maxSide) / float64(w)))
|
||
} else {
|
||
newH = maxSide
|
||
newW = int(float64(w) * (float64(maxSide) / float64(h)))
|
||
}
|
||
if newW < 1 {
|
||
newW = 1
|
||
}
|
||
if newH < 1 {
|
||
newH = 1
|
||
}
|
||
|
||
dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
|
||
xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, bounds, xdraw.Over, nil)
|
||
return dst
|
||
}
|