feat(runner): implement cluster mode with Redis-based task dispatching and registry
Introduce a distributed task execution system for gRPC runners. This includes: - A `TaskBus` for cluster-mode task dispatching via Redis Pub/Sub. - A `RunnerRegistry` to track active runner instances in Redis. - Support for both standalone and cluster modes via configuration. - Refactored `RunnerHub` and `TaskManager` to use a `TaskDispatcher` interface, decoupling task submission from local execution. - Added distributed locking to `HotRankWorker` to ensure only one instance performs periodic hot rank recalculations in a multi-instance deployment. - Updated dependency injection (Wire) to support the new runner components and registry.
This commit is contained in:
@@ -33,6 +33,7 @@ type Config struct {
|
||||
JPush JPushConfig `mapstructure:"jpush"`
|
||||
SetupSecret string `mapstructure:"setup_secret"`
|
||||
WebSocket WSConfig `mapstructure:"websocket"`
|
||||
Runner RunnerConfig `mapstructure:"runner"`
|
||||
}
|
||||
|
||||
// Load 加载配置文件
|
||||
@@ -180,6 +181,15 @@ func Load(configPath string) (*Config, error) {
|
||||
viper.SetDefault("websocket.cluster.msg_channel", "ws:msg")
|
||||
viper.SetDefault("websocket.cluster.online_ttl", 60)
|
||||
viper.SetDefault("websocket.cluster.heartbeat_interval", 20)
|
||||
// Runner 集群默认值(mode 和 instance_id 空则跟随 websocket)
|
||||
viper.SetDefault("runner.mode", "")
|
||||
viper.SetDefault("runner.cluster.instance_id", "")
|
||||
viper.SetDefault("runner.cluster.task_channel", "runner:task")
|
||||
viper.SetDefault("runner.cluster.result_channel", "runner:result")
|
||||
viper.SetDefault("runner.cluster.registry_prefix", "runner:reg:")
|
||||
viper.SetDefault("runner.cluster.cap_prefix", "runner:cap:")
|
||||
viper.SetDefault("runner.cluster.registry_ttl", 90)
|
||||
viper.SetDefault("runner.cluster.heartbeat_interval", 30)
|
||||
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
return nil, fmt.Errorf("failed to read config: %w", err)
|
||||
@@ -190,6 +200,9 @@ func Load(configPath string) (*Config, error) {
|
||||
return nil, fmt.Errorf("failed to unmarshal config: %w", err)
|
||||
}
|
||||
|
||||
// Runner mode 和 instance_id 空则跟随 WebSocket 配置
|
||||
cfg.Runner.ApplyWSDefaults(cfg.WebSocket)
|
||||
|
||||
// Convert seconds to duration
|
||||
cfg.JWT.AccessTokenExpire = time.Duration(viper.GetInt("jwt.access_token_expire")) * time.Second
|
||||
cfg.JWT.RefreshTokenExpire = time.Duration(viper.GetInt("jwt.refresh_token_expire")) * time.Second
|
||||
|
||||
Reference in New Issue
Block a user