feat(server): implement chat file TTL cleanup and enhance JPush vendor channel support
Add FileCleanupWorker for automatic expiration of chat files with configurable retention period and batch processing. Files uploaded via `/api/v1/uploads/files` are tracked in `uploaded_files` table with expiration timestamps, then deleted from S3 and database upon expiry. Expired file URLs are injected as `"expired": true` in message responses. Extend JPush push notification configuration with vendor-specific channel_id support (xiaomi, huawei, oppo, vivo, meizu, honor, fcm) for differentiating system vs chat notifications. Add iOS APNs thread-id grouping configuration for notification categorization.
This commit is contained in:
261
internal/service/file_cleanup_worker.go
Normal file
261
internal/service/file_cleanup_worker.go
Normal file
@@ -0,0 +1,261 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"with_you/internal/config"
|
||||
"with_you/internal/pkg/s3"
|
||||
"with_you/internal/repository"
|
||||
|
||||
redislib "github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// FileCleanupWorker 聊天文件 TTL 过期清理 worker
|
||||
// 周期性扫描 uploaded_files 表,将超过保留期的文件从 S3 删除并标记 expired。
|
||||
// 多实例部署时通过 Redis 分布式锁保证仅主实例执行清理。
|
||||
type FileCleanupWorker struct {
|
||||
cfg *config.Config
|
||||
uploadedRepo repository.UploadedFileRepository
|
||||
s3Client *s3.Client
|
||||
rdb *redislib.Client // 用于分布式锁(可为 nil,降级为本地运行)
|
||||
stopOnce sync.Once
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
}
|
||||
|
||||
const (
|
||||
// fileCleanupLockKey Redis 分布式锁键
|
||||
fileCleanupLockKey = "file_cleanup:leader_lock"
|
||||
// fileCleanupLockTTL 锁持有时间(略大于扫描间隔,确保单实例持有)
|
||||
fileCleanupLockTTL = 7 * time.Hour
|
||||
// fileCleanupLockRetryInterval 竞争锁的重试间隔
|
||||
fileCleanupLockRetryInterval = 1 * time.Minute
|
||||
)
|
||||
|
||||
// NewFileCleanupWorker 创建文件清理 worker
|
||||
func NewFileCleanupWorker(
|
||||
cfg *config.Config,
|
||||
uploadedRepo repository.UploadedFileRepository,
|
||||
s3Client *s3.Client,
|
||||
rdb *redislib.Client,
|
||||
) *FileCleanupWorker {
|
||||
return &FileCleanupWorker{
|
||||
cfg: cfg,
|
||||
uploadedRepo: uploadedRepo,
|
||||
s3Client: s3Client,
|
||||
rdb: rdb,
|
||||
stopCh: make(chan struct{}),
|
||||
doneCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// tryAcquireLock 尝试获取分布式锁,返回是否成功
|
||||
func (w *FileCleanupWorker) tryAcquireLock(ctx context.Context) bool {
|
||||
if w.rdb == nil {
|
||||
return true // 无 Redis 时降级为本地运行
|
||||
}
|
||||
ok, err := w.rdb.SetNX(ctx, fileCleanupLockKey, "1", fileCleanupLockTTL).Result()
|
||||
if err != nil {
|
||||
zap.L().Warn("file cleanup: failed to acquire leader lock, running locally", zap.Error(err))
|
||||
return true
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// keepLockAlive 周期性续期分布式锁
|
||||
func (w *FileCleanupWorker) keepLockAlive(ctx context.Context) {
|
||||
if w.rdb == nil {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(fileCleanupLockTTL / 2)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-w.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
w.rdb.Expire(ctx, fileCleanupLockKey, fileCleanupLockTTL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// releaseLock 释放分布式锁
|
||||
func (w *FileCleanupWorker) releaseLock() {
|
||||
if w.rdb == nil {
|
||||
return
|
||||
}
|
||||
w.rdb.Del(context.Background(), fileCleanupLockKey)
|
||||
}
|
||||
|
||||
// Start 启动清理 worker
|
||||
func (w *FileCleanupWorker) Start(ctx context.Context) {
|
||||
if w == nil || w.cfg == nil || !w.cfg.FileCleanup.Enabled {
|
||||
close(w.doneCh)
|
||||
return
|
||||
}
|
||||
if w.uploadedRepo == nil || w.s3Client == nil {
|
||||
zap.L().Warn("file cleanup: missing dependencies, worker disabled")
|
||||
close(w.doneCh)
|
||||
return
|
||||
}
|
||||
|
||||
// 扫描间隔,默认 6 小时,最小 5 分钟
|
||||
interval := time.Duration(w.cfg.FileCleanup.IntervalMinutes) * time.Minute
|
||||
if interval < 5*time.Minute {
|
||||
interval = 6 * time.Hour
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(w.doneCh)
|
||||
|
||||
isLeader := false
|
||||
lockCtx, lockCancel := context.WithCancel(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
if isLeader {
|
||||
w.releaseLock()
|
||||
}
|
||||
lockCancel()
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
acquired := w.tryAcquireLock(ctx)
|
||||
if acquired {
|
||||
if !isLeader {
|
||||
zap.L().Info("file cleanup: acquired leader lock, becoming primary")
|
||||
isLeader = true
|
||||
go w.keepLockAlive(lockCtx)
|
||||
|
||||
// 主实例启动后立即执行一次清理
|
||||
w.runOnce(ctx)
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
for {
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
ticker.Stop()
|
||||
lockCancel()
|
||||
w.releaseLock()
|
||||
return
|
||||
case <-ticker.C:
|
||||
w.runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
zap.L().Debug("file cleanup: another instance holds leader lock, retrying later")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
lockCancel()
|
||||
return
|
||||
case <-time.After(fileCleanupLockRetryInterval):
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop 停止 worker
|
||||
func (w *FileCleanupWorker) Stop() {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
w.stopOnce.Do(func() {
|
||||
close(w.stopCh)
|
||||
})
|
||||
<-w.doneCh
|
||||
w.releaseLock()
|
||||
}
|
||||
|
||||
// runOnce 执行一次清理扫描
|
||||
func (w *FileCleanupWorker) runOnce(ctx context.Context) {
|
||||
cfg := w.cfg.FileCleanup
|
||||
retentionDays := cfg.RetentionDays
|
||||
if retentionDays <= 0 {
|
||||
retentionDays = 7
|
||||
}
|
||||
batchSize := cfg.BatchSize
|
||||
if batchSize <= 0 {
|
||||
batchSize = 100
|
||||
}
|
||||
|
||||
before := time.Now().Add(-time.Duration(retentionDays) * 24 * time.Hour)
|
||||
totalDeleted := 0
|
||||
totalFailed := 0
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
files, err := w.uploadedRepo.ListPendingCleanup(ctx, before, batchSize)
|
||||
if err != nil {
|
||||
zap.L().Warn("file cleanup: list pending failed", zap.Error(err))
|
||||
break
|
||||
}
|
||||
if len(files) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// 删除 S3 对象
|
||||
if derr := w.s3Client.Delete(ctx, f.ObjectName); derr != nil {
|
||||
// 删除失败不标记,下次重试
|
||||
totalFailed++
|
||||
zap.L().Warn("file cleanup: delete object failed, will retry",
|
||||
zap.String("object", f.ObjectName),
|
||||
zap.String("url", f.URL),
|
||||
zap.Error(derr),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// 标记已清理
|
||||
now := time.Now()
|
||||
if merr := w.uploadedRepo.MarkExpired(ctx, f.ID, now); merr != nil {
|
||||
zap.L().Warn("file cleanup: mark expired failed",
|
||||
zap.Uint("id", f.ID),
|
||||
zap.String("url", f.URL),
|
||||
zap.Error(merr),
|
||||
)
|
||||
totalFailed++
|
||||
} else {
|
||||
totalDeleted++
|
||||
}
|
||||
}
|
||||
|
||||
// 本批次不足 batchSize,说明已无待清理记录
|
||||
if len(files) < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if totalDeleted > 0 || totalFailed > 0 {
|
||||
zap.L().Info("file cleanup sweep done",
|
||||
zap.Int("deleted", totalDeleted),
|
||||
zap.Int("failed", totalFailed),
|
||||
zap.Int("retention_days", retentionDays),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// 兜底:保证 errors 包被引用(若未来 runOnce 增加错误判断)
|
||||
var _ = errors.Is
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"with_you/internal/config"
|
||||
"with_you/internal/dto"
|
||||
"with_you/internal/model"
|
||||
"with_you/internal/pkg/jpush"
|
||||
"with_you/internal/pkg/jpush/vendor"
|
||||
"with_you/internal/pkg/ws"
|
||||
"with_you/internal/repository"
|
||||
|
||||
@@ -85,6 +87,11 @@ type pushServiceImpl struct {
|
||||
wsHub ws.MessagePublisher
|
||||
jpushClient *jpush.Client
|
||||
|
||||
// channelConfig 厂商通道 channel_id 配置(系统消息/私聊消息分别注册)
|
||||
channelConfig config.ChannelConfig
|
||||
// vendorFactory 厂商转换器工厂,按统一参数产出各厂商推送字段
|
||||
vendorFactory *vendor.Factory
|
||||
|
||||
pushQueue chan *pushTask
|
||||
stopChan chan struct{}
|
||||
}
|
||||
@@ -104,15 +111,18 @@ func NewPushService(
|
||||
messageRepo repository.MessageRepository,
|
||||
publisher ws.MessagePublisher,
|
||||
jpushClient *jpush.Client,
|
||||
channelConfig config.ChannelConfig,
|
||||
) PushService {
|
||||
return &pushServiceImpl{
|
||||
pushRepo: pushRepo,
|
||||
deviceRepo: deviceRepo,
|
||||
messageRepo: messageRepo,
|
||||
wsHub: publisher,
|
||||
jpushClient: jpushClient,
|
||||
pushQueue: make(chan *pushTask, PushQueueSize),
|
||||
stopChan: make(chan struct{}),
|
||||
pushRepo: pushRepo,
|
||||
deviceRepo: deviceRepo,
|
||||
messageRepo: messageRepo,
|
||||
wsHub: publisher,
|
||||
jpushClient: jpushClient,
|
||||
channelConfig: channelConfig,
|
||||
vendorFactory: vendor.NewFactory(),
|
||||
pushQueue: make(chan *pushTask, PushQueueSize),
|
||||
stopChan: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,8 +153,8 @@ func (s *pushServiceImpl) PushToUser(ctx context.Context, userID string, message
|
||||
if err == nil && len(devices) > 0 {
|
||||
mobileDevices := getMobileDevices(devices)
|
||||
if len(mobileDevices) > 0 {
|
||||
payload := buildMessagePayload(message)
|
||||
pushErr := s.pushViaJPushBatch(mobileDevices, payload.Title, payload.Content, payload.Extras)
|
||||
params := s.messageToParams(message)
|
||||
pushErr := s.pushViaJPushBatch(mobileDevices, params)
|
||||
if pushErr == nil {
|
||||
s.batchCreatePushedRecords(ctx, userID, message.ID, mobileDevices)
|
||||
return nil
|
||||
@@ -296,7 +306,8 @@ func buildMessagePayload(message *model.Message) messagePayload {
|
||||
}
|
||||
|
||||
// pushViaJPush 通过极光推送发送通知
|
||||
func (s *pushServiceImpl) pushViaJPush(ctx context.Context, device *model.DeviceToken, title, content string, extras map[string]any) error {
|
||||
// params 为厂商无关统一参数,由 vendor 工厂转换为各厂商推送字段
|
||||
func (s *pushServiceImpl) pushViaJPush(ctx context.Context, device *model.DeviceToken, params vendor.MessageParams) error {
|
||||
if !s.isJPushAvailable() {
|
||||
return errors.New("jpush client not configured")
|
||||
}
|
||||
@@ -305,8 +316,8 @@ func (s *pushServiceImpl) pushViaJPush(ctx context.Context, device *model.Device
|
||||
return errors.New("device has no registration ID")
|
||||
}
|
||||
|
||||
notification := jpush.BuildNotification(title, content, "message", extras)
|
||||
_, err := s.jpushClient.PushByRegistrationIDs([]string{device.PushToken}, notification)
|
||||
payload := s.buildPayload(params)
|
||||
_, err := s.jpushClient.PushByRegistrationIDs([]string{device.PushToken}, payload.Notification, payload.TPC)
|
||||
if err != nil {
|
||||
return fmt.Errorf("jpush push failed: %w", err)
|
||||
}
|
||||
@@ -343,7 +354,9 @@ func (s *pushServiceImpl) batchCreatePushedRecords(ctx context.Context, userID s
|
||||
}
|
||||
|
||||
// pushViaJPushBatch 批量通过极光推送发送通知
|
||||
func (s *pushServiceImpl) pushViaJPushBatch(devices []*model.DeviceToken, title, content string, extras map[string]any) error {
|
||||
// pushViaJPushBatch 批量通过极光推送发送通知
|
||||
// params 为厂商无关统一参数,由 vendor 工厂转换为各厂商推送字段
|
||||
func (s *pushServiceImpl) pushViaJPushBatch(devices []*model.DeviceToken, params vendor.MessageParams) error {
|
||||
if !s.isJPushAvailable() {
|
||||
return errors.New("jpush client not configured")
|
||||
}
|
||||
@@ -359,8 +372,8 @@ func (s *pushServiceImpl) pushViaJPushBatch(devices []*model.DeviceToken, title,
|
||||
return errors.New("no valid registration IDs")
|
||||
}
|
||||
|
||||
notification := jpush.BuildNotification(title, content, "message", extras)
|
||||
_, err := s.jpushClient.PushByRegistrationIDs(regIDs, notification)
|
||||
payload := s.buildPayload(params)
|
||||
_, err := s.jpushClient.PushByRegistrationIDs(regIDs, payload.Notification, payload.TPC)
|
||||
if err != nil {
|
||||
return fmt.Errorf("jpush batch push failed: %w", err)
|
||||
}
|
||||
@@ -520,7 +533,7 @@ func (s *pushServiceImpl) processPushTask(task *pushTask) {
|
||||
return
|
||||
}
|
||||
|
||||
payload := buildMessagePayload(task.message)
|
||||
params := s.messageToParams(task.message)
|
||||
|
||||
// 尝试使用极光推送(统一推送到所有移动设备)
|
||||
mobileDevices := getMobileDevices(devices)
|
||||
@@ -528,7 +541,7 @@ func (s *pushServiceImpl) processPushTask(task *pushTask) {
|
||||
if len(mobileDevices) > 0 {
|
||||
// 优先使用极光推送
|
||||
if s.isJPushAvailable() {
|
||||
err := s.pushViaJPushBatch(mobileDevices, payload.Title, payload.Content, payload.Extras)
|
||||
err := s.pushViaJPushBatch(mobileDevices, params)
|
||||
if err == nil {
|
||||
s.batchCreatePushedRecords(ctx, task.userID, task.message.ID, mobileDevices)
|
||||
// 原始 pending 记录标记为已推送(被批量记录替代)
|
||||
@@ -549,7 +562,7 @@ func (s *pushServiceImpl) processPushTask(task *pushTask) {
|
||||
|
||||
var pushErr error
|
||||
if s.isJPushAvailable() {
|
||||
pushErr = s.pushViaJPush(ctx, device, payload.Title, payload.Content, payload.Extras)
|
||||
pushErr = s.pushViaJPush(ctx, device, params)
|
||||
} else {
|
||||
pushErr = errors.New("jpush not configured")
|
||||
}
|
||||
@@ -583,6 +596,46 @@ func (s *pushServiceImpl) isJPushAvailable() bool {
|
||||
return s.jpushClient != nil
|
||||
}
|
||||
|
||||
// resolveChannelScene 根据消息类型判断 channel 场景
|
||||
// 系统消息/通知/公告 -> vendor.SceneSystem,其余(普通聊天)-> vendor.SceneChat
|
||||
func (s *pushServiceImpl) resolveChannelScene(message *model.Message) string {
|
||||
if message == nil {
|
||||
return vendor.SceneChat
|
||||
}
|
||||
if message.IsSystemMessage() ||
|
||||
message.Category == model.CategoryNotification ||
|
||||
message.Category == model.CategoryAnnouncement {
|
||||
return vendor.SceneSystem
|
||||
}
|
||||
return vendor.SceneChat
|
||||
}
|
||||
|
||||
// buildPayload 根据统一参数,通过厂商工厂产出完整推送片段(Notification + ThirdPartyChannel)
|
||||
func (s *pushServiceImpl) buildPayload(params vendor.MessageParams) *vendor.PushPayload {
|
||||
return s.vendorFactory.Build(params, s.channelConfig)
|
||||
}
|
||||
|
||||
// messageToParams 从 model.Message 提取厂商无关的统一参数
|
||||
// 用于 PushToUser / worker / retry 等通用路径
|
||||
func (s *pushServiceImpl) messageToParams(message *model.Message) vendor.MessageParams {
|
||||
_ = message.Decrypt()
|
||||
payload := buildMessagePayload(message)
|
||||
scene := s.resolveChannelScene(message)
|
||||
|
||||
senderName := ""
|
||||
if message.ExtraData != nil {
|
||||
senderName = message.ExtraData.ActorName
|
||||
}
|
||||
return vendor.MessageParams{
|
||||
Scene: scene,
|
||||
Title: payload.Title,
|
||||
Body: payload.Content,
|
||||
SenderName: senderName,
|
||||
MessageType: "message",
|
||||
Extra: payload.Extras,
|
||||
}
|
||||
}
|
||||
|
||||
// getMobileDevices 从设备列表中筛选出支持手机推送的活跃设备
|
||||
func getMobileDevices(devices []*model.DeviceToken) []*model.DeviceToken {
|
||||
mobileDevices := make([]*model.DeviceToken, 0, len(devices))
|
||||
@@ -668,8 +721,8 @@ func (s *pushServiceImpl) doRetry() int {
|
||||
|
||||
var pushErr error
|
||||
if s.isJPushAvailable() && device.SupportsMobilePush() && device.PushToken != "" {
|
||||
payload := buildMessagePayload(message)
|
||||
pushErr = s.pushViaJPush(ctx, device, payload.Title, payload.Content, payload.Extras)
|
||||
params := s.messageToParams(message)
|
||||
pushErr = s.pushViaJPush(ctx, device, params)
|
||||
}
|
||||
|
||||
if pushErr != nil {
|
||||
@@ -760,14 +813,25 @@ func (s *pushServiceImpl) PushChatMessage(ctx context.Context, userID string, co
|
||||
extras["target_type"] = message.ExtraData.TargetType
|
||||
}
|
||||
|
||||
notification := jpush.BuildNotification(title, content, "chat_message", extras)
|
||||
// 群聊用群头像,私聊用发送者头像
|
||||
largeIcon := sender.Avatar
|
||||
if convType == model.ConversationTypeGroup && groupAvatar != "" {
|
||||
notification.Android.LargeIcon = groupAvatar
|
||||
} else if sender.Avatar != "" {
|
||||
notification.Android.LargeIcon = sender.Avatar
|
||||
largeIcon = groupAvatar
|
||||
}
|
||||
if _, pushErr := s.jpushClient.PushByRegistrationIDs(regIDs, notification); pushErr != nil {
|
||||
params := vendor.MessageParams{
|
||||
Scene: s.resolveChannelScene(message),
|
||||
Title: title,
|
||||
Body: content,
|
||||
SenderName: sender.Name,
|
||||
SenderID: message.SenderID,
|
||||
ConversationName: convName,
|
||||
ConversationID: message.ConversationID,
|
||||
MessageType: "chat_message",
|
||||
LargeIcon: largeIcon,
|
||||
Extra: extras,
|
||||
}
|
||||
payload := s.buildPayload(params)
|
||||
if _, pushErr := s.jpushClient.PushByRegistrationIDs(regIDs, payload.Notification, payload.TPC); pushErr != nil {
|
||||
zap.L().Error("jpush push chat message failed",
|
||||
zap.String("userID", userID),
|
||||
zap.String("conversationID", conversationID),
|
||||
@@ -847,10 +911,19 @@ func (s *pushServiceImpl) PushSystemNotification(ctx context.Context, userID str
|
||||
extras["target_type"] = notification.ExtraData.TargetType
|
||||
}
|
||||
notifType := string(notification.Type)
|
||||
jpushNotif := jpush.BuildNotification(title, content, notifType, extras)
|
||||
largeIcon := ""
|
||||
if notification.ExtraData != nil && notification.ExtraData.AvatarURL != "" {
|
||||
jpushNotif.Android.LargeIcon = notification.ExtraData.AvatarURL
|
||||
largeIcon = notification.ExtraData.AvatarURL
|
||||
}
|
||||
params := vendor.MessageParams{
|
||||
Scene: vendor.SceneSystem,
|
||||
Title: title,
|
||||
Body: content,
|
||||
MessageType: notifType,
|
||||
LargeIcon: largeIcon,
|
||||
Extra: extras,
|
||||
}
|
||||
payload := s.buildPayload(params)
|
||||
_, pushErr := s.jpushClient.PushByRegistrationIDs(
|
||||
func() []string {
|
||||
ids := make([]string, 0, len(mobileDevices))
|
||||
@@ -859,7 +932,8 @@ func (s *pushServiceImpl) PushSystemNotification(ctx context.Context, userID str
|
||||
}
|
||||
return ids
|
||||
}(),
|
||||
jpushNotif,
|
||||
payload.Notification,
|
||||
payload.TPC,
|
||||
)
|
||||
if pushErr == nil {
|
||||
return nil
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
|
||||
"with_you/internal/model"
|
||||
"with_you/internal/pkg/s3"
|
||||
"with_you/internal/repository"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"go.uber.org/zap"
|
||||
@@ -26,22 +27,34 @@ import (
|
||||
_ "golang.org/x/image/tiff"
|
||||
)
|
||||
|
||||
// UploadFileResult 通用文件上传结果
|
||||
type UploadFileResult struct {
|
||||
URL string `json:"url"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
}
|
||||
|
||||
// UploadService 上传服务接口
|
||||
type UploadService interface {
|
||||
UploadImage(ctx context.Context, file *multipart.FileHeader) (string, string, string, error)
|
||||
UploadImageBytes(ctx context.Context, raw []byte, filename string) (url, previewURL, previewURLLarge string, err error)
|
||||
UploadAvatar(ctx context.Context, userID string, file *multipart.FileHeader) (string, error)
|
||||
UploadCover(ctx context.Context, userID string, file *multipart.FileHeader) (string, error)
|
||||
UploadFile(ctx context.Context, file *multipart.FileHeader, folder string, uploaderID string) (*UploadFileResult, error)
|
||||
GetURL(ctx context.Context, objectName string) (string, error)
|
||||
Delete(ctx context.Context, objectName string) error
|
||||
GeneratePreviewImages(ctx context.Context, originalData []byte, hash string) (previewURL, previewURLLarge string, err error)
|
||||
ValidateChatMessageImageSegments(segments model.MessageSegments) error
|
||||
// GetExpiredURLSet 返回所有已从 S3 清理的文件 URL 集合,供 DTO 层标记失效
|
||||
GetExpiredURLSet(ctx context.Context) (map[string]struct{}, error)
|
||||
}
|
||||
|
||||
// uploadService 上传服务实现
|
||||
type uploadService struct {
|
||||
s3Client *s3.Client
|
||||
userService UserService
|
||||
s3Client *s3.Client
|
||||
userService UserService
|
||||
uploadedRepo repository.UploadedFileRepository
|
||||
}
|
||||
|
||||
// 预览图配置
|
||||
@@ -52,10 +65,11 @@ const (
|
||||
)
|
||||
|
||||
// NewUploadService 创建上传服务
|
||||
func NewUploadService(s3Client *s3.Client, userService UserService) UploadService {
|
||||
func NewUploadService(s3Client *s3.Client, userService UserService, uploadedRepo repository.UploadedFileRepository) UploadService {
|
||||
return &uploadService{
|
||||
s3Client: s3Client,
|
||||
userService: userService,
|
||||
s3Client: s3Client,
|
||||
userService: userService,
|
||||
uploadedRepo: uploadedRepo,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,6 +230,210 @@ func (s *uploadService) UploadCover(ctx context.Context, userID string, file *mu
|
||||
return url, nil
|
||||
}
|
||||
|
||||
// 聊天文件上传限制
|
||||
const (
|
||||
MaxChatFileSize = 50 << 20 // 50MB
|
||||
chatFileFolderLimit = 32 // folder 名长度上限,避免路径注入
|
||||
)
|
||||
|
||||
// chatFileAllowedMime 允许上传的文件 MIME 白名单(按类型前缀匹配)。
|
||||
// 不在白名单内的类型一律拒绝,防止上传可执行脚本等高危文件。
|
||||
var chatFileAllowedMime = map[string]bool{
|
||||
// 文档
|
||||
"application/pdf": true,
|
||||
"application/msword": true,
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
|
||||
"application/vnd.ms-excel": true,
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
|
||||
"application/vnd.ms-powerpoint": true,
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
|
||||
"application/rtf": true,
|
||||
"text/plain": true,
|
||||
"text/csv": true,
|
||||
"text/markdown": true,
|
||||
"application/json": true,
|
||||
"application/xml": true,
|
||||
"text/xml": true,
|
||||
"application/zip": true,
|
||||
"application/x-zip-compressed": true,
|
||||
"application/x-rar-compressed": true,
|
||||
"application/x-7z-compressed": true,
|
||||
"application/gzip": true,
|
||||
"application/x-tar": true,
|
||||
// 音视频/图片(聊天文件入口)
|
||||
"audio/mpeg": true,
|
||||
"audio/mp3": true,
|
||||
"audio/wav": true,
|
||||
"audio/x-wav": true,
|
||||
"audio/aac": true,
|
||||
"audio/x-m4a": true,
|
||||
"audio/flac": true,
|
||||
"video/mp4": true,
|
||||
"video/quicktime": true,
|
||||
"video/x-msvideo": true,
|
||||
"video/webm": true,
|
||||
"image/jpeg": true,
|
||||
"image/png": true,
|
||||
"image/gif": true,
|
||||
"image/webp": true,
|
||||
"image/bmp": true,
|
||||
// 其他常用类型
|
||||
"application/octet-stream": true, // 部分客户端(如 Windows)上传无明确类型,需结合扩展名二次校验
|
||||
}
|
||||
|
||||
// chatFileExtFallback 当 MIME 为 octet-stream 时,按扩展名推断的真实类型
|
||||
var chatFileExtFallback = map[string]string{
|
||||
".pdf": "application/pdf",
|
||||
".doc": "application/msword",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".xls": "application/vnd.ms-excel",
|
||||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".ppt": "application/vnd.ms-powerpoint",
|
||||
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
".zip": "application/zip",
|
||||
".rar": "application/x-rar-compressed",
|
||||
".7z": "application/x-7z-compressed",
|
||||
".gz": "application/gzip",
|
||||
".tar": "application/x-tar",
|
||||
".mp3": "audio/mpeg",
|
||||
".mp4": "video/mp4",
|
||||
".mov": "video/quicktime",
|
||||
".avi": "video/x-msvideo",
|
||||
".json": "application/json",
|
||||
".csv": "text/csv",
|
||||
".md": "text/markdown",
|
||||
".txt": "text/plain",
|
||||
}
|
||||
|
||||
// normalizeChatFileMime 校正文件 MIME 类型:
|
||||
// 1. 解析 multipart header 的 Content-Type;
|
||||
// 2. 若为通用 octet-stream,依据文件名扩展名回退到更具体的类型;
|
||||
// 3. 全部兜底为 application/octet-stream。
|
||||
func normalizeChatFileMime(headerType, filename string) string {
|
||||
baseType, _, err := mime.ParseMediaType(strings.TrimSpace(headerType))
|
||||
if err != nil || baseType == "" {
|
||||
baseType = "application/octet-stream"
|
||||
}
|
||||
baseType = strings.ToLower(baseType)
|
||||
|
||||
if baseType == "application/octet-stream" {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
if real, ok := chatFileExtFallback[ext]; ok {
|
||||
return real
|
||||
}
|
||||
}
|
||||
return baseType
|
||||
}
|
||||
|
||||
// validateChatFile 校验聊天文件大小与 MIME 白名单
|
||||
func validateChatFile(size int64, mimeType string) error {
|
||||
if size <= 0 {
|
||||
return fmt.Errorf("文件为空")
|
||||
}
|
||||
if size > MaxChatFileSize {
|
||||
return fmt.Errorf("文件大小超过限制(最大 %d MB)", MaxChatFileSize>>20)
|
||||
}
|
||||
if !chatFileAllowedMime[mimeType] {
|
||||
return fmt.Errorf("不支持的文件类型: %s", mimeType)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadFile 上传聊天文件(非图片,按文件类型原样存储到 S3)
|
||||
// folder 用于隔离不同来源(如 chat/post),仅允许字母数字与下划线。
|
||||
// uploaderID 为上传者用户 ID,用于审计。
|
||||
func (s *uploadService) UploadFile(ctx context.Context, file *multipart.FileHeader, folder string, uploaderID string) (*UploadFileResult, error) {
|
||||
if file == nil {
|
||||
return nil, fmt.Errorf("文件为空")
|
||||
}
|
||||
if file.Size > MaxChatFileSize {
|
||||
return nil, fmt.Errorf("文件大小超过限制(最大 %d MB)", MaxChatFileSize>>20)
|
||||
}
|
||||
|
||||
// 校验与规范化 folder,避免路径穿越
|
||||
folder = strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, folder)
|
||||
if folder == "" || len(folder) > chatFileFolderLimit {
|
||||
folder = "chat"
|
||||
}
|
||||
|
||||
// 校验 MIME 类型
|
||||
mimeType := normalizeChatFileMime(file.Header.Get("Content-Type"), file.Filename)
|
||||
if err := validateChatFile(file.Size, mimeType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 读取文件内容
|
||||
f, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("打开文件失败: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取文件失败: %w", err)
|
||||
}
|
||||
|
||||
if err := validateChatFile(int64(len(data)), mimeType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 以内容哈希命名,避免重复存储;保留原始扩展名(便于浏览器识别)
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := fmt.Sprintf("%x", hash)
|
||||
ext := strings.ToLower(filepath.Ext(file.Filename))
|
||||
if ext == "" {
|
||||
// 从 MIME 推断扩展名(mime.ExtensionsByType 在新版本返回 (exts, err))
|
||||
if es, _ := mime.ExtensionsByType(mimeType); len(es) > 0 {
|
||||
ext = es[0]
|
||||
} else {
|
||||
ext = ".bin"
|
||||
}
|
||||
}
|
||||
|
||||
objectName := fmt.Sprintf("files/%s/%s%s", folder, hashStr, ext)
|
||||
url, err := s.s3Client.UploadData(ctx, objectName, data, mimeType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("上传到 S3 失败: %w", err)
|
||||
}
|
||||
|
||||
// 写入上传记录,用于 7 天 TTL 过期清理(失败仅记日志,不影响上传)
|
||||
if s.uploadedRepo != nil {
|
||||
record := &model.UploadedFile{
|
||||
URL: url,
|
||||
ObjectName: objectName,
|
||||
Name: file.Filename,
|
||||
Size: file.Size,
|
||||
MimeType: mimeType,
|
||||
Folder: folder,
|
||||
UploaderID: uploaderID,
|
||||
}
|
||||
if rerr := s.uploadedRepo.Create(ctx, record); rerr != nil {
|
||||
zap.L().Warn("写入文件上传记录失败", zap.String("url", url), zap.Error(rerr))
|
||||
}
|
||||
}
|
||||
|
||||
return &UploadFileResult{
|
||||
URL: url,
|
||||
Name: file.Filename,
|
||||
Size: file.Size,
|
||||
MimeType: mimeType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetExpiredURLSet 返回所有已从 S3 清理的文件 URL 集合,供 DTO 层标记失效
|
||||
func (s *uploadService) GetExpiredURLSet(ctx context.Context) (map[string]struct{}, error) {
|
||||
if s.uploadedRepo == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return s.uploadedRepo.GetExpiredURLSet(ctx)
|
||||
}
|
||||
|
||||
// GetURL 获取文件URL
|
||||
func (s *uploadService) GetURL(ctx context.Context, objectName string) (string, error) {
|
||||
return s.s3Client.GetURL(ctx, objectName)
|
||||
|
||||
Reference in New Issue
Block a user