feat(runner): implement cluster mode with Redis-based task dispatching and registry
All checks were successful
Build Backend / build (push) Successful in 2m22s
Build Backend / build-docker (push) Successful in 1m20s

Introduce a distributed task execution system for gRPC runners. This includes:
- A `TaskBus` for cluster-mode task dispatching via Redis Pub/Sub.
- A `RunnerRegistry` to track active runner instances in Redis.
- Support for both standalone and cluster modes via configuration.
- Refactored `RunnerHub` and `TaskManager` to use a `TaskDispatcher` interface, decoupling task submission from local execution.
- Added distributed locking to `HotRankWorker` to ensure only one instance performs periodic hot rank recalculations in a multi-instance deployment.
- Updated dependency injection (Wire) to support the new runner components and registry.
This commit is contained in:
2026-05-07 01:08:39 +08:00
parent dc27eb5cde
commit 8aa85ca4b2
15 changed files with 1125 additions and 86 deletions

View File

@@ -18,15 +18,16 @@ import (
// App 应用程序结构体 // App 应用程序结构体
type App struct { type App struct {
Config *config.Config Config *config.Config
DB *gorm.DB DB *gorm.DB
Router *router.Router Router *router.Router
PushService service.PushService PushService service.PushService
HotRankWorker *service.HotRankWorker HotRankWorker *service.HotRankWorker
GRPCServer *runner.Server GRPCServer *runner.Server
Server *http.Server Server *http.Server
Publisher ws.MessagePublisher Publisher ws.MessagePublisher
Logger *zap.Logger TaskDispatcher runner.TaskDispatcher
Logger *zap.Logger
} }
// NewApp 创建应用程序 // NewApp 创建应用程序
@@ -38,17 +39,19 @@ func NewApp(
hotRankWorker *service.HotRankWorker, hotRankWorker *service.HotRankWorker,
grpcServer *runner.Server, grpcServer *runner.Server,
publisher ws.MessagePublisher, publisher ws.MessagePublisher,
taskDispatcher runner.TaskDispatcher,
logger *zap.Logger, logger *zap.Logger,
) *App { ) *App {
return &App{ return &App{
Config: cfg, Config: cfg,
DB: db, DB: db,
Router: r, Router: r,
PushService: pushService, PushService: pushService,
HotRankWorker: hotRankWorker, HotRankWorker: hotRankWorker,
GRPCServer: grpcServer, GRPCServer: grpcServer,
Publisher: publisher, Publisher: publisher,
Logger: logger, TaskDispatcher: taskDispatcher,
Logger: logger,
} }
} }
@@ -128,6 +131,14 @@ func (a *App) Stop(ctx context.Context) error {
a.GRPCServer.Stop() a.GRPCServer.Stop()
} }
// 4.1 停止 TaskBus集群模式
if a.TaskDispatcher != nil {
if bus, ok := a.TaskDispatcher.(*runner.TaskBus); ok {
a.Logger.Info("Stopping TaskBus")
bus.Stop()
}
}
// 5. 停止推送 Worker // 5. 停止推送 Worker
a.Logger.Info("Stopping push worker") a.Logger.Info("Stopping push worker")
a.PushService.StopPushWorker() a.PushService.StopPushWorker()

View File

@@ -101,8 +101,8 @@ func InitializeApp() (*App, error) {
channelHandler := handler.NewChannelHandler(channelService) channelHandler := handler.NewChannelHandler(channelService)
scheduleRepository := repository.NewScheduleRepository(db) scheduleRepository := repository.NewScheduleRepository(db)
scheduleService := wire.ProvideScheduleService(scheduleRepository) scheduleService := wire.ProvideScheduleService(scheduleRepository)
server := wire.ProvideGRPCServer(config, logger) runnerHub := wire.ProvideRunnerHub(logger)
taskManager := wire.ProvideTaskManager(server) taskManager := wire.ProvideTaskManager(runnerHub, logger)
scheduleSyncService := wire.ProvideScheduleSyncService(taskManager, scheduleRepository, logger) scheduleSyncService := wire.ProvideScheduleSyncService(taskManager, scheduleRepository, logger)
scheduleHandler := wire.ProvideScheduleHandler(scheduleService, scheduleSyncService) scheduleHandler := wire.ProvideScheduleHandler(scheduleService, scheduleSyncService)
roleRepository := wire.ProvideRoleRepository(db) roleRepository := wire.ProvideRoleRepository(db)
@@ -148,8 +148,11 @@ func InitializeApp() (*App, error) {
tradeHandler := handler.NewTradeHandler(tradeService) tradeHandler := handler.NewTradeHandler(tradeService)
wsHandler := wire.ProvideWSHandler(messagePublisher, chatService, groupService, jwtService, callService, userRepository) wsHandler := wire.ProvideWSHandler(messagePublisher, chatService, groupService, jwtService, callService, userRepository)
router := ProvideRouter(userRepository, userHandler, postHandler, commentHandler, messageHandler, notificationHandler, uploadHandler, jwtService, pushHandler, systemMessageHandler, groupHandler, stickerHandler, voteHandler, channelHandler, scheduleHandler, roleHandler, adminUserHandler, adminPostHandler, adminCommentHandler, adminGroupHandler, adminDashboardHandler, adminLogHandler, qrCodeHandler, materialHandler, callHandler, reportHandler, adminReportHandler, verificationHandler, adminVerificationHandler, adminProfileAuditHandler, setupHandler, tradeHandler, logService, userActivityService, casbinService, wsHandler) router := ProvideRouter(userRepository, userHandler, postHandler, commentHandler, messageHandler, notificationHandler, uploadHandler, jwtService, pushHandler, systemMessageHandler, groupHandler, stickerHandler, voteHandler, channelHandler, scheduleHandler, roleHandler, adminUserHandler, adminPostHandler, adminCommentHandler, adminGroupHandler, adminDashboardHandler, adminLogHandler, qrCodeHandler, materialHandler, callHandler, reportHandler, adminReportHandler, verificationHandler, adminVerificationHandler, adminProfileAuditHandler, setupHandler, tradeHandler, logService, userActivityService, casbinService, wsHandler)
hotRankWorker := wire.ProvideHotRankWorker(config, postRepository, channelRepository, cache) hotRankWorker := wire.ProvideHotRankWorker(config, postRepository, channelRepository, cache, client)
app := NewApp(config, db, router, pushService, hotRankWorker, server, messagePublisher, logger) server := wire.ProvideGRPCServer(config, logger, runnerHub)
runnerRegistry := wire.ProvideRunnerRegistry(config, client)
taskDispatcher := wire.ProvideTaskDispatcher(runnerHub, runnerRegistry, config, client, logger)
app := NewApp(config, db, router, pushService, hotRankWorker, server, messagePublisher, taskDispatcher, logger)
return app, nil return app, nil
} }

View File

@@ -33,6 +33,7 @@ type Config struct {
JPush JPushConfig `mapstructure:"jpush"` JPush JPushConfig `mapstructure:"jpush"`
SetupSecret string `mapstructure:"setup_secret"` SetupSecret string `mapstructure:"setup_secret"`
WebSocket WSConfig `mapstructure:"websocket"` WebSocket WSConfig `mapstructure:"websocket"`
Runner RunnerConfig `mapstructure:"runner"`
} }
// Load 加载配置文件 // Load 加载配置文件
@@ -180,6 +181,15 @@ func Load(configPath string) (*Config, error) {
viper.SetDefault("websocket.cluster.msg_channel", "ws:msg") viper.SetDefault("websocket.cluster.msg_channel", "ws:msg")
viper.SetDefault("websocket.cluster.online_ttl", 60) viper.SetDefault("websocket.cluster.online_ttl", 60)
viper.SetDefault("websocket.cluster.heartbeat_interval", 20) viper.SetDefault("websocket.cluster.heartbeat_interval", 20)
// Runner 集群默认值mode 和 instance_id 空则跟随 websocket
viper.SetDefault("runner.mode", "")
viper.SetDefault("runner.cluster.instance_id", "")
viper.SetDefault("runner.cluster.task_channel", "runner:task")
viper.SetDefault("runner.cluster.result_channel", "runner:result")
viper.SetDefault("runner.cluster.registry_prefix", "runner:reg:")
viper.SetDefault("runner.cluster.cap_prefix", "runner:cap:")
viper.SetDefault("runner.cluster.registry_ttl", 90)
viper.SetDefault("runner.cluster.heartbeat_interval", 30)
if err := viper.ReadInConfig(); err != nil { if err := viper.ReadInConfig(); err != nil {
return nil, fmt.Errorf("failed to read config: %w", err) return nil, fmt.Errorf("failed to read config: %w", err)
@@ -190,6 +200,9 @@ func Load(configPath string) (*Config, error) {
return nil, fmt.Errorf("failed to unmarshal config: %w", err) return nil, fmt.Errorf("failed to unmarshal config: %w", err)
} }
// Runner mode 和 instance_id 空则跟随 WebSocket 配置
cfg.Runner.ApplyWSDefaults(cfg.WebSocket)
// Convert seconds to duration // Convert seconds to duration
cfg.JWT.AccessTokenExpire = time.Duration(viper.GetInt("jwt.access_token_expire")) * time.Second cfg.JWT.AccessTokenExpire = time.Duration(viper.GetInt("jwt.access_token_expire")) * time.Second
cfg.JWT.RefreshTokenExpire = time.Duration(viper.GetInt("jwt.refresh_token_expire")) * time.Second cfg.JWT.RefreshTokenExpire = time.Duration(viper.GetInt("jwt.refresh_token_expire")) * time.Second

28
internal/config/runner.go Normal file
View File

@@ -0,0 +1,28 @@
package config
// RunnerConfig Runner 集群配置
type RunnerConfig struct {
Mode string `mapstructure:"mode"` // "standalone" | "cluster",空则跟随 websocket.mode
Cluster RunnerClusterConfig `mapstructure:"cluster"`
}
// RunnerClusterConfig Runner 集群模式配置
type RunnerClusterConfig struct {
InstanceID string `mapstructure:"instance_id"` // 空则跟随 websocket.cluster.instance_id
TaskChannel string `mapstructure:"task_channel"`
ResultChannel string `mapstructure:"result_channel"`
RegistryPrefix string `mapstructure:"registry_prefix"`
CapPrefix string `mapstructure:"cap_prefix"`
RegistryTTL int `mapstructure:"registry_ttl"`
HeartbeatInterval int `mapstructure:"heartbeat_interval"`
}
// ApplyWSDefaults 用 WebSocket 配置填充 Runner 中未设置的共享字段
func (r *RunnerConfig) ApplyWSDefaults(ws WSConfig) {
if r.Mode == "" {
r.Mode = ws.Mode
}
if r.Cluster.InstanceID == "" {
r.Cluster.InstanceID = ws.Cluster.InstanceID
}
}

View File

@@ -778,6 +778,12 @@ type SetGroupAvatarParams struct {
Avatar string `json:"avatar"` // 头像URL Avatar string `json:"avatar"` // 头像URL
} }
// SetGroupDescriptionParams 设置群描述参数
type SetGroupDescriptionParams struct {
GroupID string `json:"group_id"` // 群组ID
Description string `json:"description"` // 群描述
}
// SetGroupLeaveParams 退出群组参数 // SetGroupLeaveParams 退出群组参数
type SetGroupLeaveParams struct { type SetGroupLeaveParams struct {
GroupID string `json:"group_id"` // 群组ID GroupID string `json:"group_id"` // 群组ID

365
internal/grpc/runner/bus.go Normal file
View File

@@ -0,0 +1,365 @@
package runner
import (
"context"
"encoding/json"
"fmt"
"sync"
"with_you/proto/runner"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
)
// taskBusMsg Redis 任务频道消息格式
type taskBusMsg struct {
OriginInstanceID string `json:"origin_instance_id"`
TargetRunnerID string `json:"target_runner_id"`
TaskID string `json:"task_id"`
TaskType string `json:"task_type"`
TaskPayload json.RawMessage `json:"task_payload"`
CreatedAt int64 `json:"created_at"`
TimeoutSeconds int32 `json:"timeout_seconds"`
}
// resultBusMsg Redis 结果频道消息格式
type resultBusMsg struct {
OriginInstanceID string `json:"origin_instance_id"`
TaskID string `json:"task_id"`
Status string `json:"status"`
CompletedAt int64 `json:"completed_at"`
Data json.RawMessage `json:"data"`
ErrorCode string `json:"error_code"`
ErrorMessage string `json:"error_message"`
}
// TaskBus 组合 RunnerHub + Redis Pub/Sub实现跨实例任务分发和结果路由
type TaskBus struct {
hub *RunnerHub
registry *RunnerRegistry
rdb *redis.Client
instanceID string
taskChan string
resultChan string
logger *zap.Logger
remoteTasks sync.Map // taskID → originInstanceID从其他实例接收的任务
sub *redis.PubSub
cancel context.CancelFunc
mu sync.Mutex
closed bool
}
// NewTaskBus 创建 TaskBus 实例
func NewTaskBus(hub *RunnerHub, rdb *redis.Client, registry *RunnerRegistry, instanceID, taskChan, resultChan string, logger *zap.Logger) *TaskBus {
return &TaskBus{
hub: hub,
registry: registry,
rdb: rdb,
instanceID: instanceID,
taskChan: taskChan,
resultChan: resultChan,
logger: logger,
}
}
// Start 启动 Redis 订阅
func (b *TaskBus) Start() error {
if b.rdb == nil {
b.logger.Info("TaskBus: no Redis client, running in standalone mode")
return nil
}
ctx, cancel := context.WithCancel(context.Background())
b.cancel = cancel
sub := b.rdb.Subscribe(ctx, b.taskChan, b.resultChan)
b.sub = sub
if _, err := sub.Receive(ctx); err != nil {
cancel()
return err
}
go b.subscribeLoop(ctx, sub)
b.logger.Info("TaskBus started",
zap.String("instance_id", b.instanceID),
zap.String("task_channel", b.taskChan),
zap.String("result_channel", b.resultChan),
)
return nil
}
// Stop 关闭 TaskBus
func (b *TaskBus) Stop() error {
b.mu.Lock()
defer b.mu.Unlock()
if b.closed {
return nil
}
b.closed = true
if b.cancel != nil {
b.cancel()
}
if b.sub != nil {
b.sub.Close()
}
if b.registry != nil {
b.registry.Stop()
}
b.logger.Info("TaskBus stopped", zap.String("instance_id", b.instanceID))
return nil
}
// InstanceID 返回实例 ID
func (b *TaskBus) InstanceID() string {
return b.instanceID
}
// DispatchTask 实现 TaskDispatcher 接口
// 本地优先 → Redis 注册中心查找 → 远程发布
func (b *TaskBus) DispatchTask(task *runner.Task) error {
taskTypeStr := task.Type.String()
// 1. 尝试本地查找
conn, err := b.hub.GetAvailableRunner(task.Type)
if err == nil && conn != nil {
return b.hub.SendTask(conn.ID, task)
}
// 2. 查询 Redis 注册中心
if b.registry != nil {
runnerID, instanceID, err := b.registry.GetAvailableRunner(taskTypeStr)
if err != nil {
return fmt.Errorf("no available runner: %w", err)
}
// 如果 runner 在本实例(可能本地 map 还没更新),直接发送
if instanceID == b.instanceID {
return b.hub.SendTask(runnerID, task)
}
// 发布到远程实例
return b.publishTask(task, runnerID)
}
return fmt.Errorf("no available runner for task type: %s", taskTypeStr)
}
// HandleResult 处理任务结果
// 如果任务来自远程实例,转发结果;否则交给 TaskManager
func (b *TaskBus) HandleResult(result *runner.TaskResult) {
originInstance, ok := b.remoteTasks.Load(result.TaskId)
if ok {
b.remoteTasks.Delete(result.TaskId)
// 任务来自远程实例,转发结果
if originStr, ok := originInstance.(string); ok && originStr != b.instanceID {
b.publishResult(result, originStr)
return
}
}
// 本地任务,交给 TaskManager
if b.hub.taskManager != nil {
b.hub.taskManager.HandleResult(result)
}
}
// publishTask 发布任务到 Redis 频道
func (b *TaskBus) publishTask(task *runner.Task, targetRunnerID string) error {
msg := taskBusMsg{
OriginInstanceID: b.instanceID,
TargetRunnerID: targetRunnerID,
TaskID: task.TaskId,
TaskType: task.Type.String(),
TaskPayload: task.Payload,
CreatedAt: task.CreatedAt,
TimeoutSeconds: task.TimeoutSeconds,
}
data, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("failed to marshal task message: %w", err)
}
if err := b.rdb.Publish(context.Background(), b.taskChan, data).Err(); err != nil {
return fmt.Errorf("failed to publish task to Redis: %w", err)
}
return nil
}
// publishResult 发布结果到 Redis 频道
func (b *TaskBus) publishResult(result *runner.TaskResult, originInstanceID string) error {
var data json.RawMessage
if len(result.Data) > 0 {
data = result.Data
} else {
data = json.RawMessage("{}")
}
msg := resultBusMsg{
OriginInstanceID: originInstanceID,
TaskID: result.TaskId,
Status: result.Status.String(),
CompletedAt: result.CompletedAt,
Data: data,
ErrorCode: result.ErrorCode,
ErrorMessage: result.ErrorMessage,
}
raw, err := json.Marshal(msg)
if err != nil {
b.logger.Error("Failed to marshal result message", zap.Error(err))
return err
}
if err := b.rdb.Publish(context.Background(), b.resultChan, raw).Err(); err != nil {
b.logger.Error("Failed to publish result to Redis", zap.Error(err))
return err
}
return nil
}
// subscribeLoop 处理 Redis 订阅消息
func (b *TaskBus) subscribeLoop(ctx context.Context, sub *redis.PubSub) {
ch := sub.Channel()
for {
select {
case <-ctx.Done():
return
case msg, ok := <-ch:
if !ok {
return
}
switch msg.Channel {
case b.taskChan:
b.handleRemoteTask(msg)
case b.resultChan:
b.handleRemoteResult(msg)
}
}
}
}
// handleRemoteTask 处理从其他实例发来的任务
func (b *TaskBus) handleRemoteTask(msg *redis.Message) {
var taskMsg taskBusMsg
if err := json.Unmarshal([]byte(msg.Payload), &taskMsg); err != nil {
b.logger.Error("TaskBus: failed to unmarshal task message", zap.Error(err))
return
}
// 过滤自己发送的消息
if taskMsg.OriginInstanceID == b.instanceID {
return
}
// 检查目标 Runner 是否在本实例
conn, err := b.hub.GetRunnerByID(taskMsg.TargetRunnerID)
if err != nil {
b.logger.Warn("TaskBus: target runner not found locally",
zap.String("runner_id", taskMsg.TargetRunnerID),
zap.String("task_id", taskMsg.TaskID))
return
}
// 记录任务的来源实例,用于结果回传
b.remoteTasks.Store(taskMsg.TaskID, taskMsg.OriginInstanceID)
// 构造 Task 并发送
task := &runner.Task{
TaskId: taskMsg.TaskID,
Type: parseTaskType(taskMsg.TaskType),
CreatedAt: taskMsg.CreatedAt,
TimeoutSeconds: taskMsg.TimeoutSeconds,
Payload: taskMsg.TaskPayload,
}
if err := b.hub.SendTask(conn.ID, task); err != nil {
b.logger.Error("TaskBus: failed to send remote task to runner",
zap.String("runner_id", conn.ID),
zap.String("task_id", taskMsg.TaskID),
zap.Error(err))
b.remoteTasks.Delete(taskMsg.TaskID)
}
}
// handleRemoteResult 处理从其他实例发来的任务结果
func (b *TaskBus) handleRemoteResult(msg *redis.Message) {
var resultMsg resultBusMsg
if err := json.Unmarshal([]byte(msg.Payload), &resultMsg); err != nil {
b.logger.Error("TaskBus: failed to unmarshal result message", zap.Error(err))
return
}
// 过滤自己发送的消息
if resultMsg.OriginInstanceID == b.instanceID {
return
}
// 只有 origin 匹配本实例才需要处理
// originInstanceID 标识的是发起任务的实例)
// 收到其他实例转发的结果,且 origin 是我们 → 交给 TaskManager
// 但这里收到的是其他实例广播的结果,我们只是中继
// 实际上 result 是发到 runner:result channel 的,所有实例都会收到
// 我们只关心 origin 是本实例的结果
if resultMsg.OriginInstanceID != b.instanceID {
return
}
// 交给 TaskManager 处理
result := &runner.TaskResult{
TaskId: resultMsg.TaskID,
Status: parseTaskStatus(resultMsg.Status),
CompletedAt: resultMsg.CompletedAt,
Data: resultMsg.Data,
ErrorCode: resultMsg.ErrorCode,
ErrorMessage: resultMsg.ErrorMessage,
}
if b.hub.taskManager != nil {
b.hub.taskManager.HandleResult(result)
}
}
// parseTaskType 将字符串转换为 runner.TaskType 枚举
func parseTaskType(s string) runner.TaskType {
switch s {
case "TASK_TYPE_LOGIN":
return runner.TaskType_TASK_TYPE_LOGIN
case "TASK_TYPE_GET_SCHEDULE":
return runner.TaskType_TASK_TYPE_GET_SCHEDULE
case "TASK_TYPE_GET_GRADES":
return runner.TaskType_TASK_TYPE_GET_GRADES
case "TASK_TYPE_GET_EXAMS":
return runner.TaskType_TASK_TYPE_GET_EXAMS
case "TASK_TYPE_GET_USER_INFO":
return runner.TaskType_TASK_TYPE_GET_USER_INFO
default:
return runner.TaskType_TASK_TYPE_UNKNOWN
}
}
// parseTaskStatus 将字符串转换为 runner.TaskStatus 枚举
func parseTaskStatus(s string) runner.TaskStatus {
switch s {
case "TASK_STATUS_SUCCESS":
return runner.TaskStatus_TASK_STATUS_SUCCESS
case "TASK_STATUS_FAILED":
return runner.TaskStatus_TASK_STATUS_FAILED
case "TASK_STATUS_TIMEOUT":
return runner.TaskStatus_TASK_STATUS_TIMEOUT
case "TASK_STATUS_CANCELLED":
return runner.TaskStatus_TASK_STATUS_CANCELLED
default:
return runner.TaskStatus_TASK_STATUS_UNKNOWN
}
}

View File

@@ -26,22 +26,20 @@ type RunnerConnection struct {
type RunnerHub struct { type RunnerHub struct {
runner.UnimplementedRunnerHubServer // 嵌入以实现 gRPC 接口 runner.UnimplementedRunnerHubServer // 嵌入以实现 gRPC 接口
mu sync.RWMutex mu sync.RWMutex
runners map[string]*RunnerConnection runners map[string]*RunnerConnection
taskManager *TaskManager taskManager *TaskManager
registerChan chan *RunnerConnection bus *TaskBus
unregisterChan chan string registry *RunnerRegistry
logger *zap.Logger logger *zap.Logger
} }
// NewRunnerHub 创建新的 RunnerHub // NewRunnerHub 创建新的 RunnerHub
func NewRunnerHub(taskManager *TaskManager, logger *zap.Logger) *RunnerHub { func NewRunnerHub(taskManager *TaskManager, logger *zap.Logger) *RunnerHub {
return &RunnerHub{ return &RunnerHub{
runners: make(map[string]*RunnerConnection), runners: make(map[string]*RunnerConnection),
taskManager: taskManager, taskManager: taskManager,
registerChan: make(chan *RunnerConnection, 100), logger: logger,
unregisterChan: make(chan string, 100),
logger: logger,
} }
} }
@@ -137,6 +135,15 @@ func (h *RunnerHub) handleRegister(req *runner.RegisterRequest, stream runner.Ru
h.runners[req.RunnerId] = conn h.runners[req.RunnerId] = conn
h.mu.Unlock() h.mu.Unlock()
// 同步到 Redis 注册中心
if h.registry != nil {
if err := h.registry.Register(req.RunnerId, req.Version, req.Capabilities); err != nil {
h.logger.Error("Failed to register runner in registry",
zap.String("runner_id", req.RunnerId),
zap.Error(err))
}
}
// 发送注册响应 // 发送注册响应
if err := conn.send(&runner.StreamMessage{ if err := conn.send(&runner.StreamMessage{
Message: &runner.StreamMessage_RegisterResponse{ Message: &runner.StreamMessage_RegisterResponse{
@@ -161,6 +168,15 @@ func (h *RunnerHub) handleHeartbeat(conn *RunnerConnection, hb *runner.Heartbeat
} }
conn.LastSeen = time.Now() conn.LastSeen = time.Now()
// 刷新 Redis 注册中心的 TTL
if h.registry != nil {
if err := h.registry.RefreshHeartbeat(conn.ID); err != nil {
h.logger.Debug("Failed to refresh runner heartbeat in registry",
zap.String("runner_id", conn.ID),
zap.Error(err))
}
}
if err := conn.send(&runner.StreamMessage{ if err := conn.send(&runner.StreamMessage{
Message: &runner.StreamMessage_HeartbeatAck{ Message: &runner.StreamMessage_HeartbeatAck{
HeartbeatAck: &runner.HeartbeatAck{ HeartbeatAck: &runner.HeartbeatAck{
@@ -179,6 +195,12 @@ func (h *RunnerHub) handleTaskResult(result *runner.TaskResult) {
zap.String("task_id", result.TaskId), zap.String("task_id", result.TaskId),
zap.String("status", result.Status.String())) zap.String("status", result.Status.String()))
// 如果有 Bus委托给 Bus 处理(可能需要转发到远程实例)
if h.bus != nil {
h.bus.HandleResult(result)
return
}
if h.taskManager != nil { if h.taskManager != nil {
h.taskManager.HandleResult(result) h.taskManager.HandleResult(result)
} }
@@ -187,12 +209,19 @@ func (h *RunnerHub) handleTaskResult(result *runner.TaskResult) {
// unregisterRunner 注销 Runner // unregisterRunner 注销 Runner
func (h *RunnerHub) unregisterRunner(runnerID string) { func (h *RunnerHub) unregisterRunner(runnerID string) {
h.mu.Lock() h.mu.Lock()
var capabilities map[string]string
if conn, ok := h.runners[runnerID]; ok { if conn, ok := h.runners[runnerID]; ok {
capabilities = conn.Capabilities
close(conn.sendChan) close(conn.sendChan)
delete(h.runners, runnerID) delete(h.runners, runnerID)
} }
h.mu.Unlock() h.mu.Unlock()
// 从 Redis 注册中心移除
if h.registry != nil && capabilities != nil {
h.registry.Unregister(runnerID, capabilities)
}
h.logger.Info("Runner unregistered", zap.String("runner_id", runnerID)) h.logger.Info("Runner unregistered", zap.String("runner_id", runnerID))
} }
@@ -257,6 +286,37 @@ func (h *RunnerHub) SetTaskManager(tm *TaskManager) {
h.taskManager = tm h.taskManager = tm
} }
// SetBus 设置 TaskBus集群模式
func (h *RunnerHub) SetBus(bus *TaskBus) {
h.bus = bus
}
// SetRegistry 设置 RunnerRegistry集群模式
func (h *RunnerHub) SetRegistry(registry *RunnerRegistry) {
h.registry = registry
}
// DispatchTask 实现 TaskDispatcher 接口standalone 模式)
func (h *RunnerHub) DispatchTask(task *runner.Task) error {
conn, err := h.GetAvailableRunner(task.Type)
if err != nil {
return err
}
return h.SendTask(conn.ID, task)
}
// GetRunnerByID 根据 ID 获取 Runner 连接
func (h *RunnerHub) GetRunnerByID(runnerID string) (*RunnerConnection, error) {
h.mu.RLock()
defer h.mu.RUnlock()
conn, ok := h.runners[runnerID]
if !ok {
return nil, fmt.Errorf("runner not found: %s", runnerID)
}
return conn, nil
}
// send 向 RunnerConnection 发送消息 // send 向 RunnerConnection 发送消息
func (c *RunnerConnection) send(msg *runner.StreamMessage) error { func (c *RunnerConnection) send(msg *runner.StreamMessage) error {
select { select {
@@ -270,8 +330,11 @@ func (c *RunnerConnection) send(msg *runner.StreamMessage) error {
// CheckStaleRunners 检查并清理超时的 Runner 连接 // CheckStaleRunners 检查并清理超时的 Runner 连接
func (h *RunnerHub) CheckStaleRunners(timeout time.Duration) { func (h *RunnerHub) CheckStaleRunners(timeout time.Duration) {
h.mu.Lock() h.mu.Lock()
defer h.mu.Unlock()
var toRemove []struct {
id string
capabilities map[string]string
}
now := time.Now() now := time.Now()
for id, conn := range h.runners { for id, conn := range h.runners {
if now.Sub(conn.LastSeen) > timeout { if now.Sub(conn.LastSeen) > timeout {
@@ -280,6 +343,18 @@ func (h *RunnerHub) CheckStaleRunners(timeout time.Duration) {
zap.Time("last_seen", conn.LastSeen)) zap.Time("last_seen", conn.LastSeen))
close(conn.sendChan) close(conn.sendChan)
delete(h.runners, id) delete(h.runners, id)
toRemove = append(toRemove, struct {
id string
capabilities map[string]string
}{id: id, capabilities: conn.Capabilities})
}
}
h.mu.Unlock()
// 从 Redis 注册中心批量移除
if h.registry != nil {
for _, r := range toRemove {
h.registry.Unregister(r.id, r.capabilities)
} }
} }
} }

View File

@@ -0,0 +1,305 @@
package runner
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
)
// RunnerRegistry 分布式 Runner 注册中心
// 使用 Redis Hash 存储 Runner 信息Redis Set 按能力索引
type RunnerRegistry struct {
rdb *redis.Client
instanceID string
registryPrefix string
capPrefix string
onlineTTL time.Duration
heartbeatInterval time.Duration
localRunners sync.Map // runnerID → map[string]string (capabilities)
logger *zap.Logger
cancel context.CancelFunc
}
// runnerInfo 存储在 Redis Hash 中的 Runner 信息
type runnerInfo struct {
InstanceID string `json:"instance_id"`
Version string `json:"version"`
Capabilities string `json:"capabilities"` // JSON-encoded map
}
// NewRunnerRegistry 创建 Runner 注册中心
func NewRunnerRegistry(rdb *redis.Client, instanceID, registryPrefix, capPrefix string, onlineTTL, heartbeatInterval time.Duration, logger *zap.Logger) *RunnerRegistry {
return &RunnerRegistry{
rdb: rdb,
instanceID: instanceID,
registryPrefix: registryPrefix,
capPrefix: capPrefix,
onlineTTL: onlineTTL,
heartbeatInterval: heartbeatInterval,
logger: logger,
}
}
// Register 注册 Runner 到分布式注册中心
func (r *RunnerRegistry) Register(runnerID, version string, capabilities map[string]string) error {
if r.rdb == nil {
return nil
}
ctx := context.Background()
capJSON, _ := json.Marshal(capabilities)
key := r.registryPrefix + runnerID
info := runnerInfo{
InstanceID: r.instanceID,
Version: version,
Capabilities: string(capJSON),
}
infoJSON, _ := json.Marshal(info)
pipe := r.rdb.Pipeline()
pipe.Set(ctx, key, infoJSON, r.onlineTTL)
// 为每个能力创建索引
taskTypeStr := parseTaskTypes(capabilities)
for _, tt := range taskTypeStr {
capKey := r.capPrefix + tt
pipe.SAdd(ctx, capKey, runnerID)
pipe.Expire(ctx, capKey, r.onlineTTL*2)
}
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("failed to register runner in redis: %w", err)
}
r.localRunners.Store(runnerID, capabilities)
return nil
}
// Unregister 从分布式注册中心移除 Runner
func (r *RunnerRegistry) Unregister(runnerID string, capabilities map[string]string) {
if r.rdb == nil {
return
}
ctx := context.Background()
key := r.registryPrefix + runnerID
pipe := r.rdb.Pipeline()
pipe.Del(ctx, key)
taskTypeStr := parseTaskTypes(capabilities)
for _, tt := range taskTypeStr {
capKey := r.capPrefix + tt
pipe.SRem(ctx, capKey, runnerID)
}
if _, err := pipe.Exec(ctx); err != nil {
r.logger.Error("Failed to unregister runner from redis",
zap.String("runner_id", runnerID),
zap.Error(err))
}
r.localRunners.Delete(runnerID)
}
// RefreshHeartbeat 刷新 Runner 的 TTL
func (r *RunnerRegistry) RefreshHeartbeat(runnerID string) error {
if r.rdb == nil {
return nil
}
ctx := context.Background()
key := r.registryPrefix + runnerID
return r.rdb.Expire(ctx, key, r.onlineTTL).Err()
}
// GetAvailableRunner 查找支持指定任务类型的 Runner
// 返回 runnerID 和所在 instanceID
func (r *RunnerRegistry) GetAvailableRunner(taskType string) (runnerID, instanceID string, err error) {
if r.rdb == nil {
return "", "", fmt.Errorf("registry not available")
}
ctx := context.Background()
capKey := r.capPrefix + taskType
members, err := r.rdb.SMembers(ctx, capKey).Result()
if err != nil {
return "", "", fmt.Errorf("failed to query runner capabilities: %w", err)
}
if len(members) == 0 {
return "", "", fmt.Errorf("no runner available for task type: %s", taskType)
}
// 随机选一个伪随机map遍历本身随机性
for _, rid := range members {
key := r.registryPrefix + rid
val, err := r.rdb.Get(ctx, key).Bytes()
if err != nil {
continue // 已过期,跳过
}
var info runnerInfo
if err := json.Unmarshal(val, &info); err != nil {
continue
}
return rid, info.InstanceID, nil
}
return "", "", fmt.Errorf("no runner available for task type: %s", taskType)
}
// GetRunnerInstance 获取 Runner 所在的实例 ID
func (r *RunnerRegistry) GetRunnerInstance(runnerID string) (string, error) {
if r.rdb == nil {
return "", fmt.Errorf("registry not available")
}
ctx := context.Background()
key := r.registryPrefix + runnerID
val, err := r.rdb.Get(ctx, key).Bytes()
if err != nil {
return "", fmt.Errorf("runner not found: %s", runnerID)
}
var info runnerInfo
if err := json.Unmarshal(val, &info); err != nil {
return "", fmt.Errorf("failed to parse runner info: %w", err)
}
return info.InstanceID, nil
}
// GetAllRunners 获取所有注册的 Runner 信息
func (r *RunnerRegistry) GetAllRunners() ([]map[string]any, error) {
if r.rdb == nil {
return nil, nil
}
ctx := context.Background()
pattern := r.registryPrefix + "*"
keys, err := r.rdb.Keys(ctx, pattern).Result()
if err != nil {
return nil, err
}
result := make([]map[string]any, 0, len(keys))
for _, key := range keys {
val, err := r.rdb.Get(ctx, key).Bytes()
if err != nil {
continue
}
var info runnerInfo
if err := json.Unmarshal(val, &info); err != nil {
continue
}
var caps map[string]string
json.Unmarshal([]byte(info.Capabilities), &caps)
runnerID := key[len(r.registryPrefix):]
result = append(result, map[string]any{
"id": runnerID,
"instance_id": info.InstanceID,
"version": info.Version,
"capabilities": caps,
})
}
return result, nil
}
// StartHeartbeat 启动心跳续期
func (r *RunnerRegistry) StartHeartbeat() {
if r.rdb == nil {
return
}
ctx, cancel := context.WithCancel(context.Background())
r.cancel = cancel
go func() {
ticker := time.NewTicker(r.heartbeatInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
r.heartbeat()
}
}
}()
}
func (r *RunnerRegistry) heartbeat() {
ctx := context.Background()
pipe := r.rdb.Pipeline()
count := 0
r.localRunners.Range(func(key, _ any) bool {
runnerID := key.(string)
regKey := r.registryPrefix + runnerID
pipe.Expire(ctx, regKey, r.onlineTTL)
count++
return true
})
if count > 0 {
if _, err := pipe.Exec(ctx); err != nil {
r.logger.Error("Failed to heartbeat runner registry", zap.Error(err))
}
}
}
// Stop 停止心跳并清理本地 Runner 的 Redis 注册
func (r *RunnerRegistry) Stop() {
if r.cancel != nil {
r.cancel()
}
if r.rdb == nil {
return
}
r.localRunners.Range(func(key, value any) bool {
runnerID := key.(string)
capabilities, _ := value.(map[string]string)
r.Unregister(runnerID, capabilities)
return true
})
}
// parseTaskTypes 从 capabilities map 中提取支持的任务类型列表
func parseTaskTypes(capabilities map[string]string) []string {
var types []string
for capKey, capValue := range capabilities {
// key = "TASK_TYPE_GET_SCHEDULE", value = "true"
if capValue == "true" && isTaskType(capKey) {
types = append(types, capKey)
continue
}
// key = "schedule", value = "TASK_TYPE_GET_SCHEDULE"
if isTaskType(capValue) {
types = append(types, capValue)
}
}
return types
}
// isTaskType 检查字符串是否为有效的 TaskType 枚举值
func isTaskType(s string) bool {
switch s {
case "TASK_TYPE_LOGIN", "TASK_TYPE_GET_SCHEDULE", "TASK_TYPE_GET_GRADES",
"TASK_TYPE_GET_EXAMS", "TASK_TYPE_GET_USER_INFO":
return true
}
return false
}

View File

@@ -23,12 +23,11 @@ type Server struct {
grpcSrv *grpc.Server grpcSrv *grpc.Server
} }
// NewServer 创建新的 gRPC 服务器 // NewServer 创建新的 gRPC 服务器(内部构造 Hub + TaskManager
func NewServer(cfg *config.GRPCConfig, logger *zap.Logger) *Server { func NewServer(cfg *config.GRPCConfig, logger *zap.Logger) *Server {
// 创建 TaskManager 和 RunnerHub解决循环依赖
taskManager := NewTaskManager(nil, 60*time.Second, logger) taskManager := NewTaskManager(nil, 60*time.Second, logger)
hub := NewRunnerHub(taskManager, logger) hub := NewRunnerHub(taskManager, logger)
taskManager.SetHub(hub) taskManager.SetDispatcher(hub)
return &Server{ return &Server{
cfg: cfg, cfg: cfg,
@@ -37,6 +36,15 @@ func NewServer(cfg *config.GRPCConfig, logger *zap.Logger) *Server {
} }
} }
// NewServerWithDeps 使用外部依赖创建 gRPC 服务器Wire DI 使用)
func NewServerWithDeps(cfg *config.GRPCConfig, logger *zap.Logger, hub *RunnerHub) *Server {
return &Server{
cfg: cfg,
logger: logger,
hub: hub,
}
}
// Start 启动 gRPC 服务器 // Start 启动 gRPC 服务器
func (s *Server) Start() error { func (s *Server) Start() error {
if !s.cfg.Enabled { if !s.cfg.Enabled {

View File

@@ -16,6 +16,11 @@ import (
// TaskCallback 任务完成回调函数类型 // TaskCallback 任务完成回调函数类型
type TaskCallback func(result *runner.TaskResult) type TaskCallback func(result *runner.TaskResult)
// TaskDispatcher 任务分发接口
type TaskDispatcher interface {
DispatchTask(task *runner.Task) error
}
// PendingTask 表示一个等待处理的任务 // PendingTask 表示一个等待处理的任务
type PendingTask struct { type PendingTask struct {
Task *runner.Task Task *runner.Task
@@ -25,20 +30,20 @@ type PendingTask struct {
// TaskManager 管理任务的生命周期 // TaskManager 管理任务的生命周期
type TaskManager struct { type TaskManager struct {
mu sync.RWMutex mu sync.RWMutex
pending map[string]*PendingTask pending map[string]*PendingTask
hub *RunnerHub dispatcher TaskDispatcher
timeout time.Duration timeout time.Duration
logger *zap.Logger logger *zap.Logger
} }
// NewTaskManager 创建新的任务管理器 // NewTaskManager 创建新的任务管理器
func NewTaskManager(hub *RunnerHub, timeout time.Duration, logger *zap.Logger) *TaskManager { func NewTaskManager(dispatcher TaskDispatcher, timeout time.Duration, logger *zap.Logger) *TaskManager {
return &TaskManager{ return &TaskManager{
pending: make(map[string]*PendingTask), pending: make(map[string]*PendingTask),
hub: hub, dispatcher: dispatcher,
timeout: timeout, timeout: timeout,
logger: logger, logger: logger,
} }
} }
@@ -69,17 +74,10 @@ func (tm *TaskManager) SubmitTask(ctx context.Context, taskType runner.TaskType,
zap.String("task_id", task.TaskId), zap.String("task_id", task.TaskId),
zap.String("type", taskType.String())) zap.String("type", taskType.String()))
// 获取可用 Runner // 通过 dispatcher 分发任务
conn, err := tm.hub.GetAvailableRunner(taskType) if err := tm.dispatcher.DispatchTask(task); err != nil {
if err != nil {
tm.cleanupTask(task.TaskId) tm.cleanupTask(task.TaskId)
return nil, fmt.Errorf("no available runner: %w", err) return nil, fmt.Errorf("failed to dispatch task: %w", err)
}
// 发送任务
if err := tm.hub.SendTask(conn.ID, task); err != nil {
tm.cleanupTask(task.TaskId)
return nil, fmt.Errorf("failed to send task: %w", err)
} }
// 启动超时检查 // 启动超时检查
@@ -203,9 +201,9 @@ func (tm *TaskManager) GetPendingTaskCount() int {
return len(tm.pending) return len(tm.pending)
} }
// SetHub 设置 RunnerHub(用于解决循环依赖) // SetDispatcher 设置任务分发器(用于解决循环依赖)
func (tm *TaskManager) SetHub(hub *RunnerHub) { func (tm *TaskManager) SetDispatcher(dispatcher TaskDispatcher) {
tm.hub = hub tm.dispatcher = dispatcher
} }
// CleanupStaleTasks 清理超时的任务 // CleanupStaleTasks 清理超时的任务

View File

@@ -1764,6 +1764,50 @@ func (h *GroupHandler) HandleSetGroupAvatar(c *gin.Context) {
response.Success(c, dto.GroupToResponse(group)) response.Success(c, dto.GroupToResponse(group))
} }
// HandleSetGroupDescription 设置群描述
// PUT /api/v1/groups/:id/description
func (h *GroupHandler) HandleSetGroupDescription(c *gin.Context) {
userID := parseUserID(c)
if userID == "" {
response.Unauthorized(c, "")
return
}
groupID := parseGroupID(c)
if groupID == "" {
response.BadRequest(c, "group_id is required")
return
}
var params dto.SetGroupDescriptionParams
if err := c.ShouldBindJSON(&params); err != nil {
response.BadRequest(c, err.Error())
return
}
updates := map[string]any{
"description": params.Description,
}
err := h.groupService.UpdateGroup(userID, groupID, updates)
if err != nil {
if err == service.ErrNotGroupAdmin {
response.Forbidden(c, "没有权限修改群组信息")
return
}
if errors.Is(err, service.ErrGroupNotFound) {
response.NotFound(c, "群组不存在")
return
}
response.InternalServerError(c, err.Error())
return
}
// 获取更新后的群组信息
group, _ := h.groupService.GetGroupByID(groupID)
response.Success(c, dto.GroupToResponse(group))
}
// HandleSetGroupLeave 退出群组 // HandleSetGroupLeave 退出群组
// POST /api/v1/groups/:id/leave // POST /api/v1/groups/:id/leave
func (h *GroupHandler) HandleSetGroupLeave(c *gin.Context) { func (h *GroupHandler) HandleSetGroupLeave(c *gin.Context) {

View File

@@ -518,6 +518,7 @@ func (r *Router) setupRoutes() {
groups.PUT("/:id/join-type", r.groupHandler.HandleSetJoinType) // 加群方式 groups.PUT("/:id/join-type", r.groupHandler.HandleSetJoinType) // 加群方式
groups.PUT("/:id/name", r.groupHandler.HandleSetGroupName) // 群名称 groups.PUT("/:id/name", r.groupHandler.HandleSetGroupName) // 群名称
groups.PUT("/:id/avatar", r.groupHandler.HandleSetGroupAvatar) // 群头像 groups.PUT("/:id/avatar", r.groupHandler.HandleSetGroupAvatar) // 群头像
groups.PUT("/:id/description", r.groupHandler.HandleSetGroupDescription) // 群描述
// 群公告 // 群公告
groups.POST("/:id/announcements", r.groupHandler.HandleCreateAnnouncement) // 创建公告 groups.POST("/:id/announcements", r.groupHandler.HandleCreateAnnouncement) // 创建公告

View File

@@ -13,7 +13,7 @@ import (
"with_you/internal/config" "with_you/internal/config"
"with_you/internal/repository" "with_you/internal/repository"
"github.com/redis/go-redis/v9" redislib "github.com/redis/go-redis/v9"
"go.uber.org/zap" "go.uber.org/zap"
) )
@@ -23,29 +23,84 @@ type hotRankScored struct {
} }
// HotRankWorker 定时在候选池上重算热门分,写入 Redis ZSET仅 TopN及 top_idsRedis+本地缓存) // HotRankWorker 定时在候选池上重算热门分,写入 Redis ZSET仅 TopN及 top_idsRedis+本地缓存)
// 多实例部署时通过 Redis 分布式锁保证仅主实例执行定时刷新,避免重复计算。
type HotRankWorker struct { type HotRankWorker struct {
cfg *config.Config cfg *config.Config
postRepo repository.PostRepository postRepo repository.PostRepository
channelRepo repository.ChannelRepository channelRepo repository.ChannelRepository
c cache.Cache c cache.Cache
rdb *redislib.Client // 用于分布式锁(可为 nil降级为本地运行
mu sync.Mutex mu sync.Mutex
stopOnce sync.Once stopOnce sync.Once
stopCh chan struct{} stopCh chan struct{}
doneCh chan struct{} doneCh chan struct{}
} }
func NewHotRankWorker(cfg *config.Config, postRepo repository.PostRepository, channelRepo repository.ChannelRepository, c cache.Cache) *HotRankWorker { const (
// hotRankLockKey Redis 分布式锁键
hotRankLockKey = "hot_rank:leader_lock"
// hotRankLockTTL 锁持有时间(略大于刷新间隔,确保单实例持有)
hotRankLockTTL = 5 * time.Minute
// hotRankLockRetryInterval 竞争锁的重试间隔
hotRankLockRetryInterval = 30 * time.Second
)
func NewHotRankWorker(cfg *config.Config, postRepo repository.PostRepository, channelRepo repository.ChannelRepository, c cache.Cache, rdb *redislib.Client) *HotRankWorker {
return &HotRankWorker{ return &HotRankWorker{
cfg: cfg, cfg: cfg,
postRepo: postRepo, postRepo: postRepo,
channelRepo: channelRepo, channelRepo: channelRepo,
c: c, c: c,
rdb: rdb,
stopCh: make(chan struct{}), stopCh: make(chan struct{}),
doneCh: make(chan struct{}), doneCh: make(chan struct{}),
} }
} }
// Start 启动定时任务(启动时立即执行一次) // tryAcquireLock 尝试获取分布式锁,返回是否成功
func (w *HotRankWorker) tryAcquireLock(ctx context.Context) bool {
if w.rdb == nil {
return true // 无 Redis 时降级为本地运行
}
ok, err := w.rdb.SetNX(ctx, hotRankLockKey, "1", hotRankLockTTL).Result()
if err != nil {
zap.L().Warn("hot rank: failed to acquire leader lock, running locally", zap.Error(err))
return true // Redis 出错时降级为本地运行
}
return ok
}
// keepLockAlive 周期性续期分布式锁
func (w *HotRankWorker) keepLockAlive(ctx context.Context) {
if w.rdb == nil {
return
}
ticker := time.NewTicker(hotRankLockTTL / 2)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-w.stopCh:
return
case <-ticker.C:
w.rdb.Expire(ctx, hotRankLockKey, hotRankLockTTL)
}
}
}
// releaseLock 释放分布式锁
func (w *HotRankWorker) releaseLock() {
if w.rdb == nil {
return
}
w.rdb.Del(context.Background(), hotRankLockKey)
}
// Start 启动定时任务
// 多实例部署时通过 Redis 分布式锁竞争主实例角色:
// - 获得锁的实例负责执行定时刷新,并周期续期
// - 未获得锁的实例定期重试竞争(当主实例宕机时可接管)
func (w *HotRankWorker) Start(ctx context.Context) { func (w *HotRankWorker) Start(ctx context.Context) {
if w == nil || w.cfg == nil || !w.cfg.HotRank.Enabled || w.c == nil { if w == nil || w.cfg == nil || !w.cfg.HotRank.Enabled || w.c == nil {
close(w.doneCh) close(w.doneCh)
@@ -59,25 +114,63 @@ func (w *HotRankWorker) Start(ctx context.Context) {
go func() { go func() {
defer close(w.doneCh) defer close(w.doneCh)
if err := w.Refresh(ctx); err != nil { isLeader := false
if !errors.Is(err, cache.ErrKeyNotFound) {
zap.L().Warn("hot rank initial refresh failed", zap.Error(err))
}
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for { for {
select { select {
case <-w.stopCh: case <-w.stopCh:
if isLeader {
w.releaseLock()
}
return return
case <-ticker.C: default:
if err := w.Refresh(ctx); err != nil { }
if !errors.Is(err, cache.ErrKeyNotFound) {
zap.L().Warn("hot rank refresh failed", zap.Error(err)) acquired := w.tryAcquireLock(ctx)
if acquired {
if !isLeader {
zap.L().Info("hot rank: acquired leader lock, becoming primary")
isLeader = true
// 启动续期协程
lockCtx, lockCancel := context.WithCancel(ctx)
go w.keepLockAlive(lockCtx)
// 主实例立即执行一次刷新
if err := w.Refresh(ctx); err != nil {
if !errors.Is(err, cache.ErrKeyNotFound) {
zap.L().Warn("hot rank initial refresh failed", zap.Error(err))
}
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-w.stopCh:
lockCancel()
w.releaseLock()
return
case <-ticker.C:
if err := w.Refresh(ctx); err != nil {
if !errors.Is(err, cache.ErrKeyNotFound) {
zap.L().Warn("hot rank refresh failed", zap.Error(err))
}
}
}
} }
} }
// 不可能走到这里isLeader==true 的分支里有无限循环)
} else {
// 未获得锁,作为备用实例等待重试
zap.L().Debug("hot rank: another instance holds leader lock, retrying later")
}
select {
case <-w.stopCh:
return
case <-time.After(hotRankLockRetryInterval):
// 重试竞争锁
} }
} }
}() }()
@@ -92,6 +185,7 @@ func (w *HotRankWorker) Stop() {
close(w.stopCh) close(w.stopCh)
}) })
<-w.doneCh <-w.doneCh
w.releaseLock()
} }
// Refresh 立即重算热门分(线程安全) // Refresh 立即重算热门分(线程安全)
@@ -240,15 +334,15 @@ func (w *HotRankWorker) Refresh(ctx context.Context) error {
} }
const pinnedScoreBase = 1000.0 const pinnedScoreBase = 1000.0
zMembers := make([]redis.Z, 0, len(orderedIDs)) zMembers := make([]redislib.Z, 0, len(orderedIDs))
for i, id := range pinned { for i, id := range pinned {
zMembers = append(zMembers, redis.Z{ zMembers = append(zMembers, redislib.Z{
Score: pinnedScoreBase - float64(i)*1e-6, Score: pinnedScoreBase - float64(i)*1e-6,
Member: id, Member: id,
}) })
} }
for _, h := range hotPicks { for _, h := range hotPicks {
zMembers = append(zMembers, redis.Z{Score: h.norm * 0.99, Member: h.id}) zMembers = append(zMembers, redislib.Z{Score: h.norm * 0.99, Member: h.id})
} }
if err := w.c.ZReplaceSortedSet(ctx, zsetKey, zMembers); err != nil { if err := w.c.ZReplaceSortedSet(ctx, zsetKey, zMembers); err != nil {
@@ -448,15 +542,15 @@ func (w *HotRankWorker) refreshChannelRanks(
} }
const pinnedScoreBase = 1000.0 const pinnedScoreBase = 1000.0
chZMembers := make([]redis.Z, 0, len(chOrderedIDs)) chZMembers := make([]redislib.Z, 0, len(chOrderedIDs))
for i, id := range chPinned { for i, id := range chPinned {
chZMembers = append(chZMembers, redis.Z{ chZMembers = append(chZMembers, redislib.Z{
Score: pinnedScoreBase - float64(i)*1e-6, Score: pinnedScoreBase - float64(i)*1e-6,
Member: id, Member: id,
}) })
} }
for _, h := range hotPicks { for _, h := range hotPicks {
chZMembers = append(chZMembers, redis.Z{Score: h.norm * 0.99, Member: h.id}) chZMembers = append(chZMembers, redislib.Z{Score: h.norm * 0.99, Member: h.id})
} }
chZSetKey := cache.HotRankZSetChannelKey(ch.ID) chZSetKey := cache.HotRankZSetChannelKey(ch.ID)

View File

@@ -1,25 +1,108 @@
package wire package wire
import ( import (
"time"
"with_you/internal/config" "with_you/internal/config"
"with_you/internal/grpc/runner" "with_you/internal/grpc/runner"
"with_you/internal/pkg/redis"
"github.com/google/wire" "github.com/google/wire"
"github.com/google/uuid"
"go.uber.org/zap" "go.uber.org/zap"
) )
// GRPCSet gRPC 相关的 Provider Set // GRPCSet gRPC 相关的 Provider Set
var GRPCSet = wire.NewSet( var GRPCSet = wire.NewSet(
ProvideGRPCServer, ProvideGRPCServer,
ProvideRunnerHub,
ProvideTaskManager, ProvideTaskManager,
ProvideRunnerRegistry,
ProvideTaskDispatcher,
) )
// ProvideGRPCServer 提供 gRPC 服务器 // ProvideGRPCServer 提供 gRPC 服务器
func ProvideGRPCServer(cfg *config.Config, logger *zap.Logger) *runner.Server { func ProvideGRPCServer(cfg *config.Config, logger *zap.Logger, hub *runner.RunnerHub) *runner.Server {
return runner.NewServer(&cfg.GRPC, logger) return runner.NewServerWithDeps(&cfg.GRPC, logger, hub)
}
// ProvideRunnerHub 创建 RunnerHub
func ProvideRunnerHub(logger *zap.Logger) *runner.RunnerHub {
return runner.NewRunnerHub(nil, logger)
} }
// ProvideTaskManager 提供 TaskManager // ProvideTaskManager 提供 TaskManager
func ProvideTaskManager(server *runner.Server) *runner.TaskManager { func ProvideTaskManager(hub *runner.RunnerHub, logger *zap.Logger) *runner.TaskManager {
return server.GetTaskManager() tm := runner.NewTaskManager(nil, 60*time.Second, logger)
tm.SetDispatcher(hub)
return tm
}
// ProvideRunnerRegistry 提供 Runner 注册中心
// standalone 模式或 Redis 不可用时返回 nil
func ProvideRunnerRegistry(cfg *config.Config, redisClient *redis.Client) *runner.RunnerRegistry {
if cfg.Runner.Mode != "cluster" || redisClient == nil {
return nil
}
instanceID := cfg.Runner.Cluster.InstanceID
if instanceID == "" {
instanceID = uuid.New().String()
}
rdb := redisClient.GetClient()
return runner.NewRunnerRegistry(
rdb,
instanceID,
cfg.Runner.Cluster.RegistryPrefix,
cfg.Runner.Cluster.CapPrefix,
time.Duration(cfg.Runner.Cluster.RegistryTTL)*time.Second,
time.Duration(cfg.Runner.Cluster.HeartbeatInterval)*time.Second,
zap.L(),
)
}
// ProvideTaskDispatcher 提供任务分发器
// standalone 模式返回 *RunnerHubcluster 模式返回 *TaskBus
func ProvideTaskDispatcher(
hub *runner.RunnerHub,
registry *runner.RunnerRegistry,
cfg *config.Config,
redisClient *redis.Client,
logger *zap.Logger,
) runner.TaskDispatcher {
if cfg.Runner.Mode != "cluster" || redisClient == nil || registry == nil {
logger.Info("Runner running in standalone mode")
return hub
}
instanceID := cfg.Runner.Cluster.InstanceID
if instanceID == "" {
instanceID = uuid.New().String()
}
rdb := redisClient.GetClient()
bus := runner.NewTaskBus(
hub,
rdb,
registry,
instanceID,
cfg.Runner.Cluster.TaskChannel,
cfg.Runner.Cluster.ResultChannel,
logger,
)
if err := bus.Start(); err != nil {
logger.Error("Failed to start Runner Bus, falling back to standalone", zap.Error(err))
return hub
}
// 注入 Bus 和 Registry 到 Hub
hub.SetBus(bus)
hub.SetRegistry(registry)
registry.StartHeartbeat()
logger.Info("Runner running in cluster mode", zap.String("instance_id", bus.InstanceID()))
return bus
} }

View File

@@ -18,6 +18,7 @@ import (
"with_you/internal/service" "with_you/internal/service"
"github.com/google/wire" "github.com/google/wire"
redislib "github.com/redis/go-redis/v9"
"go.uber.org/zap" "go.uber.org/zap"
"gorm.io/gorm" "gorm.io/gorm"
) )
@@ -376,8 +377,12 @@ func ProvideLogService(
} }
// ProvideQRCodeLoginService 提供二维码登录服务 // ProvideQRCodeLoginService 提供二维码登录服务
func ProvideHotRankWorker(cfg *config.Config, postRepo repository.PostRepository, channelRepo repository.ChannelRepository, cacheBackend cache.Cache) *service.HotRankWorker { func ProvideHotRankWorker(cfg *config.Config, postRepo repository.PostRepository, channelRepo repository.ChannelRepository, cacheBackend cache.Cache, redisClient *redis.Client) *service.HotRankWorker {
return service.NewHotRankWorker(cfg, postRepo, channelRepo, cacheBackend) var rdb *redislib.Client
if redisClient != nil {
rdb = redisClient.GetClient()
}
return service.NewHotRankWorker(cfg, postRepo, channelRepo, cacheBackend, rdb)
} }
func ProvideQRCodeLoginService( func ProvideQRCodeLoginService(