feat(runner): implement cluster mode with Redis-based task dispatching and registry

Introduce a distributed task execution system for gRPC runners. This includes:
- A `TaskBus` for cluster-mode task dispatching via Redis Pub/Sub.
- A `RunnerRegistry` to track active runner instances in Redis.
- Support for both standalone and cluster modes via configuration.
- Refactored `RunnerHub` and `TaskManager` to use a `TaskDispatcher` interface, decoupling task submission from local execution.
- Added distributed locking to `HotRankWorker` to ensure only one instance performs periodic hot rank recalculations in a multi-instance deployment.
- Updated dependency injection (Wire) to support the new runner components and registry.

internal/grpc/runner/hub.go

@@ -26,22 +26,20 @@ type RunnerConnection struct {
 type RunnerHub struct {
 	runner.UnimplementedRunnerHubServer // 嵌入以实现 gRPC 接口
 
-	mu             sync.RWMutex
-	runners        map[string]*RunnerConnection
-	taskManager    *TaskManager
-	registerChan   chan *RunnerConnection
-	unregisterChan chan string
-	logger         *zap.Logger
+	mu          sync.RWMutex
+	runners     map[string]*RunnerConnection
+	taskManager *TaskManager
+	bus         *TaskBus
+	registry    *RunnerRegistry
+	logger      *zap.Logger
 }
 
 // NewRunnerHub 创建新的 RunnerHub
 func NewRunnerHub(taskManager *TaskManager, logger *zap.Logger) *RunnerHub {
 	return &RunnerHub{
-		runners:        make(map[string]*RunnerConnection),
-		taskManager:    taskManager,
-		registerChan:   make(chan *RunnerConnection, 100),
-		unregisterChan: make(chan string, 100),
-		logger:         logger,
+		runners:     make(map[string]*RunnerConnection),
+		taskManager: taskManager,
+		logger:      logger,
 	}
 }
 
@@ -137,6 +135,15 @@ func (h *RunnerHub) handleRegister(req *runner.RegisterRequest, stream runner.Ru
 	h.runners[req.RunnerId] = conn
 	h.mu.Unlock()
 
+	// 同步到 Redis 注册中心
+	if h.registry != nil {
+		if err := h.registry.Register(req.RunnerId, req.Version, req.Capabilities); err != nil {
+			h.logger.Error("Failed to register runner in registry",
+				zap.String("runner_id", req.RunnerId),
+				zap.Error(err))
+		}
+	}
+
 	// 发送注册响应
 	if err := conn.send(&runner.StreamMessage{
 		Message: &runner.StreamMessage_RegisterResponse{
@@ -161,6 +168,15 @@ func (h *RunnerHub) handleHeartbeat(conn *RunnerConnection, hb *runner.Heartbeat
 	}
 	conn.LastSeen = time.Now()
 
+	// 刷新 Redis 注册中心的 TTL
+	if h.registry != nil {
+		if err := h.registry.RefreshHeartbeat(conn.ID); err != nil {
+			h.logger.Debug("Failed to refresh runner heartbeat in registry",
+				zap.String("runner_id", conn.ID),
+				zap.Error(err))
+		}
+	}
+
 	if err := conn.send(&runner.StreamMessage{
 		Message: &runner.StreamMessage_HeartbeatAck{
 			HeartbeatAck: &runner.HeartbeatAck{
@@ -179,6 +195,12 @@ func (h *RunnerHub) handleTaskResult(result *runner.TaskResult) {
 		zap.String("task_id", result.TaskId),
 		zap.String("status", result.Status.String()))
 
+	// 如果有 Bus，委托给 Bus 处理（可能需要转发到远程实例）
+	if h.bus != nil {
+		h.bus.HandleResult(result)
+		return
+	}
+
 	if h.taskManager != nil {
 		h.taskManager.HandleResult(result)
 	}
@@ -187,12 +209,19 @@ func (h *RunnerHub) handleTaskResult(result *runner.TaskResult) {
 // unregisterRunner 注销 Runner
 func (h *RunnerHub) unregisterRunner(runnerID string) {
 	h.mu.Lock()
+	var capabilities map[string]string
 	if conn, ok := h.runners[runnerID]; ok {
+		capabilities = conn.Capabilities
 		close(conn.sendChan)
 		delete(h.runners, runnerID)
 	}
 	h.mu.Unlock()
 
+	// 从 Redis 注册中心移除
+	if h.registry != nil && capabilities != nil {
+		h.registry.Unregister(runnerID, capabilities)
+	}
+
 	h.logger.Info("Runner unregistered", zap.String("runner_id", runnerID))
 }
 
@@ -257,6 +286,37 @@ func (h *RunnerHub) SetTaskManager(tm *TaskManager) {
 	h.taskManager = tm
 }
 
+// SetBus 设置 TaskBus（集群模式）
+func (h *RunnerHub) SetBus(bus *TaskBus) {
+	h.bus = bus
+}
+
+// SetRegistry 设置 RunnerRegistry（集群模式）
+func (h *RunnerHub) SetRegistry(registry *RunnerRegistry) {
+	h.registry = registry
+}
+
+// DispatchTask 实现 TaskDispatcher 接口（standalone 模式）
+func (h *RunnerHub) DispatchTask(task *runner.Task) error {
+	conn, err := h.GetAvailableRunner(task.Type)
+	if err != nil {
+		return err
+	}
+	return h.SendTask(conn.ID, task)
+}
+
+// GetRunnerByID 根据 ID 获取 Runner 连接
+func (h *RunnerHub) GetRunnerByID(runnerID string) (*RunnerConnection, error) {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	conn, ok := h.runners[runnerID]
+	if !ok {
+		return nil, fmt.Errorf("runner not found: %s", runnerID)
+	}
+	return conn, nil
+}
+
 // send 向 RunnerConnection 发送消息
 func (c *RunnerConnection) send(msg *runner.StreamMessage) error {
 	select {
@@ -270,8 +330,11 @@ func (c *RunnerConnection) send(msg *runner.StreamMessage) error {
 // CheckStaleRunners 检查并清理超时的 Runner 连接
 func (h *RunnerHub) CheckStaleRunners(timeout time.Duration) {
 	h.mu.Lock()
-	defer h.mu.Unlock()
 
+	var toRemove []struct {
+		id           string
+		capabilities map[string]string
+	}
 	now := time.Now()
 	for id, conn := range h.runners {
 		if now.Sub(conn.LastSeen) > timeout {
@@ -280,6 +343,18 @@ func (h *RunnerHub) CheckStaleRunners(timeout time.Duration) {
 				zap.Time("last_seen", conn.LastSeen))
 			close(conn.sendChan)
 			delete(h.runners, id)
+			toRemove = append(toRemove, struct {
+				id           string
+				capabilities map[string]string
+			}{id: id, capabilities: conn.Capabilities})
+		}
+	}
+	h.mu.Unlock()
+
+	// 从 Redis 注册中心批量移除
+	if h.registry != nil {
+		for _, r := range toRemove {
+			h.registry.Unregister(r.id, r.capabilities)
 		}
 	}
 }