feat(server): implement chat file TTL cleanup and enhance JPush vendor channel support
Add FileCleanupWorker for automatic expiration of chat files with configurable retention period and batch processing. Files uploaded via `/api/v1/uploads/files` are tracked in `uploaded_files` table with expiration timestamps, then deleted from S3 and database upon expiry. Expired file URLs are injected as `"expired": true` in message responses. Extend JPush push notification configuration with vendor-specific channel_id support (xiaomi, huawei, oppo, vivo, meizu, honor, fcm) for differentiating system vs chat notifications. Add iOS APNs thread-id grouping configuration for notification categorization.
This commit is contained in:
lafay
@@ -18,6 +18,7 @@ import (
|
||||
|
||||
"with_you/internal/model"
|
||||
"with_you/internal/pkg/s3"
|
||||
"with_you/internal/repository"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"go.uber.org/zap"
|
||||
@@ -26,22 +27,34 @@ import (
|
||||
_ "golang.org/x/image/tiff"
|
||||
)
|
||||
|
||||
// UploadFileResult 通用文件上传结果
|
||||
type UploadFileResult struct {
|
||||
URL string `json:"url"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
}
|
||||
|
||||
// UploadService 上传服务接口
|
||||
type UploadService interface {
|
||||
UploadImage(ctx context.Context, file *multipart.FileHeader) (string, string, string, error)
|
||||
UploadImageBytes(ctx context.Context, raw []byte, filename string) (url, previewURL, previewURLLarge string, err error)
|
||||
UploadAvatar(ctx context.Context, userID string, file *multipart.FileHeader) (string, error)
|
||||
UploadCover(ctx context.Context, userID string, file *multipart.FileHeader) (string, error)
|
||||
UploadFile(ctx context.Context, file *multipart.FileHeader, folder string, uploaderID string) (*UploadFileResult, error)
|
||||
GetURL(ctx context.Context, objectName string) (string, error)
|
||||
Delete(ctx context.Context, objectName string) error
|
||||
GeneratePreviewImages(ctx context.Context, originalData []byte, hash string) (previewURL, previewURLLarge string, err error)
|
||||
ValidateChatMessageImageSegments(segments model.MessageSegments) error
|
||||
// GetExpiredURLSet 返回所有已从 S3 清理的文件 URL 集合，供 DTO 层标记失效
|
||||
GetExpiredURLSet(ctx context.Context) (map[string]struct{}, error)
|
||||
}
|
||||
|
||||
// uploadService 上传服务实现
|
||||
type uploadService struct {
|
||||
s3Client *s3.Client
|
||||
userService UserService
|
||||
s3Client *s3.Client
|
||||
userService UserService
|
||||
uploadedRepo repository.UploadedFileRepository
|
||||
}
|
||||
|
||||
// 预览图配置
|
||||
@@ -52,10 +65,11 @@ const (
|
||||
)
|
||||
|
||||
// NewUploadService 创建上传服务
|
||||
func NewUploadService(s3Client *s3.Client, userService UserService) UploadService {
|
||||
func NewUploadService(s3Client *s3.Client, userService UserService, uploadedRepo repository.UploadedFileRepository) UploadService {
|
||||
return &uploadService{
|
||||
s3Client: s3Client,
|
||||
userService: userService,
|
||||
s3Client: s3Client,
|
||||
userService: userService,
|
||||
uploadedRepo: uploadedRepo,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,6 +230,210 @@ func (s *uploadService) UploadCover(ctx context.Context, userID string, file *mu
|
||||
return url, nil
|
||||
}
|
||||
|
||||
// 聊天文件上传限制
|
||||
const (
|
||||
MaxChatFileSize = 50 << 20 // 50MB
|
||||
chatFileFolderLimit = 32 // folder 名长度上限，避免路径注入
|
||||
)
|
||||
|
||||
// chatFileAllowedMime 允许上传的文件 MIME 白名单（按类型前缀匹配）。
|
||||
// 不在白名单内的类型一律拒绝，防止上传可执行脚本等高危文件。
|
||||
var chatFileAllowedMime = map[string]bool{
|
||||
// 文档
|
||||
"application/pdf": true,
|
||||
"application/msword": true,
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
|
||||
"application/vnd.ms-excel": true,
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
|
||||
"application/vnd.ms-powerpoint": true,
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
|
||||
"application/rtf": true,
|
||||
"text/plain": true,
|
||||
"text/csv": true,
|
||||
"text/markdown": true,
|
||||
"application/json": true,
|
||||
"application/xml": true,
|
||||
"text/xml": true,
|
||||
"application/zip": true,
|
||||
"application/x-zip-compressed": true,
|
||||
"application/x-rar-compressed": true,
|
||||
"application/x-7z-compressed": true,
|
||||
"application/gzip": true,
|
||||
"application/x-tar": true,
|
||||
// 音视频/图片（聊天文件入口）
|
||||
"audio/mpeg": true,
|
||||
"audio/mp3": true,
|
||||
"audio/wav": true,
|
||||
"audio/x-wav": true,
|
||||
"audio/aac": true,
|
||||
"audio/x-m4a": true,
|
||||
"audio/flac": true,
|
||||
"video/mp4": true,
|
||||
"video/quicktime": true,
|
||||
"video/x-msvideo": true,
|
||||
"video/webm": true,
|
||||
"image/jpeg": true,
|
||||
"image/png": true,
|
||||
"image/gif": true,
|
||||
"image/webp": true,
|
||||
"image/bmp": true,
|
||||
// 其他常用类型
|
||||
"application/octet-stream": true, // 部分客户端（如 Windows）上传无明确类型，需结合扩展名二次校验
|
||||
}
|
||||
|
||||
// chatFileExtFallback 当 MIME 为 octet-stream 时，按扩展名推断的真实类型
|
||||
var chatFileExtFallback = map[string]string{
|
||||
".pdf": "application/pdf",
|
||||
".doc": "application/msword",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".xls": "application/vnd.ms-excel",
|
||||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".ppt": "application/vnd.ms-powerpoint",
|
||||
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
".zip": "application/zip",
|
||||
".rar": "application/x-rar-compressed",
|
||||
".7z": "application/x-7z-compressed",
|
||||
".gz": "application/gzip",
|
||||
".tar": "application/x-tar",
|
||||
".mp3": "audio/mpeg",
|
||||
".mp4": "video/mp4",
|
||||
".mov": "video/quicktime",
|
||||
".avi": "video/x-msvideo",
|
||||
".json": "application/json",
|
||||
".csv": "text/csv",
|
||||
".md": "text/markdown",
|
||||
".txt": "text/plain",
|
||||
}
|
||||
|
||||
// normalizeChatFileMime 校正文件 MIME 类型：
|
||||
// 1. 解析 multipart header 的 Content-Type；
|
||||
// 2. 若为通用 octet-stream，依据文件名扩展名回退到更具体的类型；
|
||||
// 3. 全部兜底为 application/octet-stream。
|
||||
func normalizeChatFileMime(headerType, filename string) string {
|
||||
baseType, _, err := mime.ParseMediaType(strings.TrimSpace(headerType))
|
||||
if err != nil || baseType == "" {
|
||||
baseType = "application/octet-stream"
|
||||
}
|
||||
baseType = strings.ToLower(baseType)
|
||||
|
||||
if baseType == "application/octet-stream" {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if real, ok := chatFileExtFallback[ext]; ok {
|
||||
return real
|
||||
}
|
||||
}
|
||||
return baseType
|
||||
}
|
||||
|
||||
// validateChatFile 校验聊天文件大小与 MIME 白名单
|
||||
func validateChatFile(size int64, mimeType string) error {
|
||||
if size <= 0 {
|
||||
return fmt.Errorf("文件为空")
|
||||
}
|
||||
if size > MaxChatFileSize {
|
||||
return fmt.Errorf("文件大小超过限制（最大 %d MB）", MaxChatFileSize>>20)
|
||||
}
|
||||
if !chatFileAllowedMime[mimeType] {
|
||||
return fmt.Errorf("不支持的文件类型: %s", mimeType)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadFile 上传聊天文件（非图片，按文件类型原样存储到 S3）
|
||||
// folder 用于隔离不同来源（如 chat/post），仅允许字母数字与下划线。
|
||||
// uploaderID 为上传者用户 ID，用于审计。
|
||||
func (s *uploadService) UploadFile(ctx context.Context, file *multipart.FileHeader, folder string, uploaderID string) (*UploadFileResult, error) {
|
||||
if file == nil {
|
||||
return nil, fmt.Errorf("文件为空")
|
||||
}
|
||||
if file.Size > MaxChatFileSize {
|
||||
return nil, fmt.Errorf("文件大小超过限制（最大 %d MB）", MaxChatFileSize>>20)
|
||||
}
|
||||
|
||||
// 校验与规范化 folder，避免路径穿越
|
||||
folder = strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, folder)
|
||||
if folder == "" || len(folder) > chatFileFolderLimit {
|
||||
folder = "chat"
|
||||
}
|
||||
|
||||
// 校验 MIME 类型
|
||||
mimeType := normalizeChatFileMime(file.Header.Get("Content-Type"), file.Filename)
|
||||
if err := validateChatFile(file.Size, mimeType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 读取文件内容
|
||||
f, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("打开文件失败: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取文件失败: %w", err)
|
||||
}
|
||||
|
||||
if err := validateChatFile(int64(len(data)), mimeType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 以内容哈希命名，避免重复存储；保留原始扩展名（便于浏览器识别）
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := fmt.Sprintf("%x", hash)
|
||||
ext := strings.ToLower(filepath.Ext(file.Filename))
|
||||
if ext == "" {
|
||||
// 从 MIME 推断扩展名（mime.ExtensionsByType 在新版本返回 (exts, err)）
|
||||
if es, _ := mime.ExtensionsByType(mimeType); len(es) > 0 {
|
||||
ext = es[0]
|
||||
} else {
|
||||
ext = ".bin"
|
||||
}
|
||||
}
|
||||
|
||||
objectName := fmt.Sprintf("files/%s/%s%s", folder, hashStr, ext)
|
||||
url, err := s.s3Client.UploadData(ctx, objectName, data, mimeType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("上传到 S3 失败: %w", err)
|
||||
}
|
||||
|
||||
// 写入上传记录，用于 7 天 TTL 过期清理（失败仅记日志，不影响上传）
|
||||
if s.uploadedRepo != nil {
|
||||
record := &model.UploadedFile{
|
||||
URL: url,
|
||||
ObjectName: objectName,
|
||||
Name: file.Filename,
|
||||
Size: file.Size,
|
||||
MimeType: mimeType,
|
||||
Folder: folder,
|
||||
UploaderID: uploaderID,
|
||||
}
|
||||
if rerr := s.uploadedRepo.Create(ctx, record); rerr != nil {
|
||||
zap.L().Warn("写入文件上传记录失败", zap.String("url", url), zap.Error(rerr))
|
||||
}
|
||||
}
|
||||
|
||||
return &UploadFileResult{
|
||||
URL: url,
|
||||
Name: file.Filename,
|
||||
Size: file.Size,
|
||||
MimeType: mimeType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetExpiredURLSet 返回所有已从 S3 清理的文件 URL 集合，供 DTO 层标记失效
|
||||
func (s *uploadService) GetExpiredURLSet(ctx context.Context) (map[string]struct{}, error) {
|
||||
if s.uploadedRepo == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return s.uploadedRepo.GetExpiredURLSet(ctx)
|
||||
}
|
||||
|
||||
// GetURL 获取文件URL
|
||||
func (s *uploadService) GetURL(ctx context.Context, objectName string) (string, error) {
|
||||
return s.s3Client.GetURL(ctx, objectName)
|
||||
|
||||
Reference in New Issue
Block a user