Files
backend/internal/service/upload_service.go
lafay a0e210feab
All checks were successful
Build Backend / build (push) Successful in 3m1s
Build Backend / build-docker (push) Successful in 2m10s
feat(server): implement chat file TTL cleanup and enhance JPush vendor channel support
Add FileCleanupWorker for automatic expiration of chat files with configurable retention period and batch processing. Files uploaded via `/api/v1/uploads/files` are tracked in `uploaded_files` table with expiration timestamps, then deleted from S3 and database upon expiry. Expired file URLs are injected as `"expired": true` in message responses.

Extend JPush push notification configuration with vendor-specific channel_id support (xiaomi, huawei, oppo, vivo, meizu, honor, fcm) for differentiating system vs chat notifications. Add iOS APNs thread-id grouping configuration for notification categorization.
2026-06-17 20:41:55 +08:00

619 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"bytes"
"cmp"
"context"
"crypto/sha256"
"fmt"
"image"
"image/jpeg"
"image/png"
"io"
"mime"
"mime/multipart"
"net/http"
"path/filepath"
"strings"
"with_you/internal/model"
"with_you/internal/pkg/s3"
"with_you/internal/repository"
"github.com/disintegration/imaging"
"go.uber.org/zap"
_ "golang.org/x/image/bmp"
_ "golang.org/x/image/tiff"
)
// UploadFileResult 通用文件上传结果
type UploadFileResult struct {
URL string `json:"url"`
Name string `json:"name"`
Size int64 `json:"size"`
MimeType string `json:"mime_type"`
}
// UploadService 上传服务接口
type UploadService interface {
UploadImage(ctx context.Context, file *multipart.FileHeader) (string, string, string, error)
UploadImageBytes(ctx context.Context, raw []byte, filename string) (url, previewURL, previewURLLarge string, err error)
UploadAvatar(ctx context.Context, userID string, file *multipart.FileHeader) (string, error)
UploadCover(ctx context.Context, userID string, file *multipart.FileHeader) (string, error)
UploadFile(ctx context.Context, file *multipart.FileHeader, folder string, uploaderID string) (*UploadFileResult, error)
GetURL(ctx context.Context, objectName string) (string, error)
Delete(ctx context.Context, objectName string) error
GeneratePreviewImages(ctx context.Context, originalData []byte, hash string) (previewURL, previewURLLarge string, err error)
ValidateChatMessageImageSegments(segments model.MessageSegments) error
// GetExpiredURLSet 返回所有已从 S3 清理的文件 URL 集合,供 DTO 层标记失效
GetExpiredURLSet(ctx context.Context) (map[string]struct{}, error)
}
// uploadService 上传服务实现
type uploadService struct {
s3Client *s3.Client
userService UserService
uploadedRepo repository.UploadedFileRepository
}
// 预览图配置
const (
PreviewMaxWidth = 300 // 列表/网格模式最大宽度
PreviewMaxHeight = 800 // 详情页最大高度
PreviewQuality = 70 // WebP 质量
)
// NewUploadService 创建上传服务
func NewUploadService(s3Client *s3.Client, userService UserService, uploadedRepo repository.UploadedFileRepository) UploadService {
return &uploadService{
s3Client: s3Client,
userService: userService,
uploadedRepo: uploadedRepo,
}
}
// UploadImage 上传图片返回原图URL和预览图URL
func (s *uploadService) UploadImage(ctx context.Context, file *multipart.FileHeader) (string, string, string, error) {
processedData, contentType, ext, err := prepareImageForUpload(file)
if err != nil {
return "", "", "", err
}
// 压缩后再计算哈希,确保同一压缩结果映射同一对象名
hash := sha256.Sum256(processedData)
hashStr := fmt.Sprintf("%x", hash)
// 上传原图
objectName := fmt.Sprintf("images/%s%s", hashStr, ext)
url, err := s.s3Client.UploadData(ctx, objectName, processedData, contentType)
if err != nil {
return "", "", "", fmt.Errorf("failed to upload to S3: %w", err)
}
// 生成预览图
previewURL, previewURLLarge, err := s.GeneratePreviewImages(ctx, processedData, hashStr)
if err != nil {
zap.L().Warn("Failed to generate preview images",
zap.Error(err),
)
}
return url, previewURL, previewURLLarge, nil
}
const maxUploadImageBytesFromBuffer = 15 << 20 // 与聊天内联解码上限一致,略大于 upload.max_file_size 时可接受
// UploadImageBytes 从内存字节上传图片(与 multipart 上传走同一套压缩与存储逻辑)。
func (s *uploadService) UploadImageBytes(ctx context.Context, raw []byte, filename string) (url, previewURL, previewURLLarge string, err error) {
if len(raw) == 0 {
return "", "", "", fmt.Errorf("empty image data")
}
if len(raw) > maxUploadImageBytesFromBuffer {
return "", "", "", fmt.Errorf("image data too large")
}
detectedType := normalizeImageContentType(http.DetectContentType(raw))
compressedData, compressedType, cerr := compressImageData(raw, detectedType)
if cerr != nil {
compressedData = raw
compressedType = detectedType
}
compressedType = cmp.Or(compressedType, http.DetectContentType(compressedData))
ext := cmp.Or(getExtFromContentType(compressedType), strings.ToLower(filepath.Ext(filename)), ".jpg")
hash := sha256.Sum256(compressedData)
hashStr := fmt.Sprintf("%x", hash)
objectName := fmt.Sprintf("images/%s%s", hashStr, ext)
url, err = s.s3Client.UploadData(ctx, objectName, compressedData, compressedType)
if err != nil {
return "", "", "", fmt.Errorf("failed to upload to S3: %w", err)
}
previewURL, previewURLLarge, prevErr := s.GeneratePreviewImages(ctx, compressedData, hashStr)
if prevErr != nil {
zap.L().Warn("UploadImageBytes: preview generation failed", zap.Error(prevErr))
}
return url, previewURL, previewURLLarge, nil
}
// getExtFromContentType 根据Content-Type获取文件扩展名
func getExtFromContentType(contentType string) string {
baseType, _, err := mime.ParseMediaType(contentType)
if err == nil && baseType != "" {
contentType = baseType
}
switch contentType {
case "image/jpg", "image/jpeg":
return ".jpg"
case "image/png":
return ".png"
case "image/gif":
return ".gif"
case "image/webp":
return ".webp"
case "image/bmp", "image/x-ms-bmp":
return ".bmp"
case "image/tiff":
return ".tiff"
default:
return ""
}
}
// UploadAvatar 上传头像
func (s *uploadService) UploadAvatar(ctx context.Context, userID string, file *multipart.FileHeader) (string, error) {
processedData, contentType, ext, err := prepareImageForUpload(file)
if err != nil {
return "", err
}
// 压缩后再计算哈希
hash := sha256.Sum256(processedData)
hashStr := fmt.Sprintf("%x", hash)
objectName := fmt.Sprintf("avatars/%s%s", hashStr, ext)
url, err := s.s3Client.UploadData(ctx, objectName, processedData, contentType)
if err != nil {
return "", fmt.Errorf("failed to upload to S3: %w", err)
}
// 更新用户头像
if s.userService != nil {
user, err := s.userService.GetUserByID(ctx, userID)
if err == nil && user != nil {
user.Avatar = url
err = s.userService.UpdateUser(ctx, user)
if err != nil {
// 更新失败不影响上传结果,只记录日志
fmt.Printf("[UploadAvatar] failed to update user avatar: %v\n", err)
}
}
}
return url, nil
}
// UploadCover 上传头图(个人主页封面)
func (s *uploadService) UploadCover(ctx context.Context, userID string, file *multipart.FileHeader) (string, error) {
processedData, contentType, ext, err := prepareImageForUpload(file)
if err != nil {
return "", err
}
// 压缩后再计算哈希
hash := sha256.Sum256(processedData)
hashStr := fmt.Sprintf("%x", hash)
objectName := fmt.Sprintf("covers/%s%s", hashStr, ext)
url, err := s.s3Client.UploadData(ctx, objectName, processedData, contentType)
if err != nil {
return "", fmt.Errorf("failed to upload to S3: %w", err)
}
// 更新用户头图
if s.userService != nil {
user, err := s.userService.GetUserByID(ctx, userID)
if err == nil && user != nil {
user.CoverURL = url
err = s.userService.UpdateUser(ctx, user)
if err != nil {
// 更新失败不影响上传结果,只记录日志
fmt.Printf("[UploadCover] failed to update user cover: %v\n", err)
}
}
}
return url, nil
}
// 聊天文件上传限制
const (
MaxChatFileSize = 50 << 20 // 50MB
chatFileFolderLimit = 32 // folder 名长度上限,避免路径注入
)
// chatFileAllowedMime 允许上传的文件 MIME 白名单(按类型前缀匹配)。
// 不在白名单内的类型一律拒绝,防止上传可执行脚本等高危文件。
var chatFileAllowedMime = map[string]bool{
// 文档
"application/pdf": true,
"application/msword": true,
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
"application/vnd.ms-excel": true,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
"application/vnd.ms-powerpoint": true,
"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
"application/rtf": true,
"text/plain": true,
"text/csv": true,
"text/markdown": true,
"application/json": true,
"application/xml": true,
"text/xml": true,
"application/zip": true,
"application/x-zip-compressed": true,
"application/x-rar-compressed": true,
"application/x-7z-compressed": true,
"application/gzip": true,
"application/x-tar": true,
// 音视频/图片(聊天文件入口)
"audio/mpeg": true,
"audio/mp3": true,
"audio/wav": true,
"audio/x-wav": true,
"audio/aac": true,
"audio/x-m4a": true,
"audio/flac": true,
"video/mp4": true,
"video/quicktime": true,
"video/x-msvideo": true,
"video/webm": true,
"image/jpeg": true,
"image/png": true,
"image/gif": true,
"image/webp": true,
"image/bmp": true,
// 其他常用类型
"application/octet-stream": true, // 部分客户端(如 Windows上传无明确类型需结合扩展名二次校验
}
// chatFileExtFallback 当 MIME 为 octet-stream 时,按扩展名推断的真实类型
var chatFileExtFallback = map[string]string{
".pdf": "application/pdf",
".doc": "application/msword",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".xls": "application/vnd.ms-excel",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".ppt": "application/vnd.ms-powerpoint",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".zip": "application/zip",
".rar": "application/x-rar-compressed",
".7z": "application/x-7z-compressed",
".gz": "application/gzip",
".tar": "application/x-tar",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".mov": "video/quicktime",
".avi": "video/x-msvideo",
".json": "application/json",
".csv": "text/csv",
".md": "text/markdown",
".txt": "text/plain",
}
// normalizeChatFileMime 校正文件 MIME 类型:
// 1. 解析 multipart header 的 Content-Type
// 2. 若为通用 octet-stream依据文件名扩展名回退到更具体的类型
// 3. 全部兜底为 application/octet-stream。
func normalizeChatFileMime(headerType, filename string) string {
baseType, _, err := mime.ParseMediaType(strings.TrimSpace(headerType))
if err != nil || baseType == "" {
baseType = "application/octet-stream"
}
baseType = strings.ToLower(baseType)
if baseType == "application/octet-stream" {
ext := strings.ToLower(filepath.Ext(filename))
if real, ok := chatFileExtFallback[ext]; ok {
return real
}
}
return baseType
}
// validateChatFile 校验聊天文件大小与 MIME 白名单
func validateChatFile(size int64, mimeType string) error {
if size <= 0 {
return fmt.Errorf("文件为空")
}
if size > MaxChatFileSize {
return fmt.Errorf("文件大小超过限制(最大 %d MB", MaxChatFileSize>>20)
}
if !chatFileAllowedMime[mimeType] {
return fmt.Errorf("不支持的文件类型: %s", mimeType)
}
return nil
}
// UploadFile 上传聊天文件(非图片,按文件类型原样存储到 S3
// folder 用于隔离不同来源(如 chat/post仅允许字母数字与下划线。
// uploaderID 为上传者用户 ID用于审计。
func (s *uploadService) UploadFile(ctx context.Context, file *multipart.FileHeader, folder string, uploaderID string) (*UploadFileResult, error) {
if file == nil {
return nil, fmt.Errorf("文件为空")
}
if file.Size > MaxChatFileSize {
return nil, fmt.Errorf("文件大小超过限制(最大 %d MB", MaxChatFileSize>>20)
}
// 校验与规范化 folder避免路径穿越
folder = strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
return r
}
return -1
}, folder)
if folder == "" || len(folder) > chatFileFolderLimit {
folder = "chat"
}
// 校验 MIME 类型
mimeType := normalizeChatFileMime(file.Header.Get("Content-Type"), file.Filename)
if err := validateChatFile(file.Size, mimeType); err != nil {
return nil, err
}
// 读取文件内容
f, err := file.Open()
if err != nil {
return nil, fmt.Errorf("打开文件失败: %w", err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
return nil, fmt.Errorf("读取文件失败: %w", err)
}
if err := validateChatFile(int64(len(data)), mimeType); err != nil {
return nil, err
}
// 以内容哈希命名,避免重复存储;保留原始扩展名(便于浏览器识别)
hash := sha256.Sum256(data)
hashStr := fmt.Sprintf("%x", hash)
ext := strings.ToLower(filepath.Ext(file.Filename))
if ext == "" {
// 从 MIME 推断扩展名mime.ExtensionsByType 在新版本返回 (exts, err)
if es, _ := mime.ExtensionsByType(mimeType); len(es) > 0 {
ext = es[0]
} else {
ext = ".bin"
}
}
objectName := fmt.Sprintf("files/%s/%s%s", folder, hashStr, ext)
url, err := s.s3Client.UploadData(ctx, objectName, data, mimeType)
if err != nil {
return nil, fmt.Errorf("上传到 S3 失败: %w", err)
}
// 写入上传记录,用于 7 天 TTL 过期清理(失败仅记日志,不影响上传)
if s.uploadedRepo != nil {
record := &model.UploadedFile{
URL: url,
ObjectName: objectName,
Name: file.Filename,
Size: file.Size,
MimeType: mimeType,
Folder: folder,
UploaderID: uploaderID,
}
if rerr := s.uploadedRepo.Create(ctx, record); rerr != nil {
zap.L().Warn("写入文件上传记录失败", zap.String("url", url), zap.Error(rerr))
}
}
return &UploadFileResult{
URL: url,
Name: file.Filename,
Size: file.Size,
MimeType: mimeType,
}, nil
}
// GetExpiredURLSet 返回所有已从 S3 清理的文件 URL 集合,供 DTO 层标记失效
func (s *uploadService) GetExpiredURLSet(ctx context.Context) (map[string]struct{}, error) {
if s.uploadedRepo == nil {
return nil, nil
}
return s.uploadedRepo.GetExpiredURLSet(ctx)
}
// GetURL 获取文件URL
func (s *uploadService) GetURL(ctx context.Context, objectName string) (string, error) {
return s.s3Client.GetURL(ctx, objectName)
}
// Delete 删除文件
func (s *uploadService) Delete(ctx context.Context, objectName string) error {
return s.s3Client.Delete(ctx, objectName)
}
func prepareImageForUpload(file *multipart.FileHeader) ([]byte, string, string, error) {
f, err := file.Open()
if err != nil {
return nil, "", "", fmt.Errorf("failed to open file: %w", err)
}
defer f.Close()
originalData, err := io.ReadAll(f)
if err != nil {
return nil, "", "", fmt.Errorf("failed to read file: %w", err)
}
// 优先从文件字节探测真实类型,避免前端压缩/转码后 header 与实际格式不一致
detectedType := normalizeImageContentType(http.DetectContentType(originalData))
headerType := normalizeImageContentType(file.Header.Get("Content-Type"))
contentType := cmp.Or(detectedType, headerType, "application/octet-stream")
compressedData, compressedType, err := compressImageData(originalData, contentType)
if err != nil {
// 压缩失败时回退到原图,保证上传可用性
compressedData = originalData
compressedType = contentType
}
compressedType = cmp.Or(compressedType, http.DetectContentType(compressedData))
ext := cmp.Or(getExtFromContentType(compressedType), strings.ToLower(filepath.Ext(file.Filename)), ".jpg")
return compressedData, compressedType, ext, nil
}
func compressImageData(data []byte, contentType string) ([]byte, string, error) {
contentType = normalizeImageContentType(contentType)
// GIF/WebP 等格式先保留原图,避免动画和透明通道丢失
if contentType == "image/gif" || contentType == "image/webp" {
return data, contentType, nil
}
if contentType != "image/jpeg" &&
contentType != "image/png" &&
contentType != "image/bmp" &&
contentType != "image/x-ms-bmp" &&
contentType != "image/tiff" {
return data, contentType, nil
}
img, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return nil, "", fmt.Errorf("failed to decode image: %w", err)
}
var buf bytes.Buffer
switch contentType {
case "image/png":
encoder := png.Encoder{CompressionLevel: png.BestCompression}
if err := encoder.Encode(&buf, img); err != nil {
return nil, "", fmt.Errorf("failed to encode png: %w", err)
}
return buf.Bytes(), "image/png", nil
default:
// BMP/TIFF 等无损大图统一压缩为 JPEG控制体积
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 82}); err != nil {
return nil, "", fmt.Errorf("failed to encode jpeg: %w", err)
}
return buf.Bytes(), "image/jpeg", nil
}
}
func normalizeImageContentType(contentType string) string {
if contentType == "" {
return ""
}
baseType, _, err := mime.ParseMediaType(contentType)
if err == nil && baseType != "" {
contentType = baseType
}
switch strings.ToLower(contentType) {
case "image/jpg":
return "image/jpeg"
case "image/jpeg":
return "image/jpeg"
case "image/png":
return "image/png"
case "image/gif":
return "image/gif"
case "image/webp":
return "image/webp"
case "image/bmp", "image/x-ms-bmp":
return "image/bmp"
case "image/tiff":
return "image/tiff"
default:
return contentType
}
}
// GeneratePreviewImages 生成预览图
func (s *uploadService) GeneratePreviewImages(ctx context.Context, originalData []byte, hash string) (previewURL, previewURLLarge string, err error) {
img, _, err := image.Decode(bytes.NewReader(originalData))
if err != nil {
return "", "", fmt.Errorf("failed to decode image: %w", err)
}
bounds := img.Bounds()
originalWidth := bounds.Dx()
originalHeight := bounds.Dy()
// 生成普通预览图(列表/网格模式)
previewURL, err = s.generatePreview(ctx, img, originalWidth, originalHeight, "preview", PreviewMaxWidth, hash)
if err != nil {
zap.L().Warn("Failed to generate preview",
zap.Error(err),
)
}
// 生成大预览图(详情页)
previewURLLarge, err = s.generatePreview(ctx, img, originalWidth, originalHeight, "large", PreviewMaxHeight, hash)
if err != nil {
zap.L().Warn("Failed to generate large preview",
zap.Error(err),
)
}
return previewURL, previewURLLarge, nil
}
// generatePreview 生成单个预览图
func (s *uploadService) generatePreview(ctx context.Context, img image.Image, originalWidth, originalHeight int, mode string, maxDim int, hash string) (string, error) {
var targetWidth, targetHeight int
// 计算目标尺寸
if originalWidth > originalHeight {
// 宽图:按宽度缩放
if originalWidth > maxDim {
targetWidth = maxDim
targetHeight = int(float64(originalHeight) * float64(maxDim) / float64(originalWidth))
} else {
return "", nil // 图片足够小,不需要生成预览图
}
} else {
// 高图:按高度缩放
if originalHeight > maxDim {
targetHeight = maxDim
targetWidth = int(float64(originalWidth) * float64(maxDim) / float64(originalHeight))
} else {
return "", nil // 图片足够小,不需要生成预览图
}
}
// 缩放图片(使用高质量 Lanczos 算法)
scaledImg := imaging.Resize(img, targetWidth, targetHeight, imaging.Lanczos)
// 编码为 JPEGWebP 在某些设备上可能不支持,使用 JPEG 保证兼容性)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, scaledImg, &jpeg.Options{Quality: PreviewQuality}); err != nil {
return "", fmt.Errorf("failed to encode jpeg: %w", err)
}
// 构建对象名thumbnails/{hash}_{mode}.jpg
objectName := fmt.Sprintf("thumbnails/%s_%s.jpg", hash, mode)
// 上传到 S3
url, err := s.s3Client.UploadData(ctx, objectName, buf.Bytes(), "image/jpeg")
if err != nil {
return "", fmt.Errorf("failed to upload preview: %w", err)
}
return url, nil
}