lan
8aa85ca4b2
Introduce a distributed task execution system for gRPC runners. This includes: - A `TaskBus` for cluster-mode task dispatching via Redis Pub/Sub. - A `RunnerRegistry` to track active runner instances in Redis. - Support for both standalone and cluster modes via configuration. - Refactored `RunnerHub` and `TaskManager` to use a `TaskDispatcher` interface, decoupling task submission from local execution. - Added distributed locking to `HotRankWorker` to ensure only one instance performs periodic hot rank recalculations in a multi-instance deployment. - Updated dependency injection (Wire) to support the new runner components and registry.
226 lines
10 KiB
Go
226 lines
10 KiB
Go
package config
|
||
|
||
import (
|
||
"fmt"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/spf13/viper"
|
||
)
|
||
|
||
// Config 主配置结构体
|
||
type Config struct {
|
||
Server ServerConfig `mapstructure:"server"`
|
||
Database DatabaseConfig `mapstructure:"database"`
|
||
Redis RedisConfig `mapstructure:"redis"`
|
||
Cache CacheConfig `mapstructure:"cache"`
|
||
S3 S3Config `mapstructure:"s3"`
|
||
JWT JWTConfig `mapstructure:"jwt"`
|
||
Log LogConfig `mapstructure:"log"`
|
||
RateLimit RateLimitConfig `mapstructure:"rate_limit"`
|
||
Upload UploadConfig `mapstructure:"upload"`
|
||
OpenAI OpenAIConfig `mapstructure:"openai"`
|
||
TencentTMS TencentTMSConfig `mapstructure:"tencent_tms"`
|
||
Email EmailConfig `mapstructure:"email"`
|
||
ConversationCache ConversationCacheConfig `mapstructure:"conversation_cache"`
|
||
GRPC GRPCConfig `mapstructure:"grpc"`
|
||
Casbin CasbinConfig `mapstructure:"casbin"`
|
||
Encryption EncryptionConfig `mapstructure:"encryption"`
|
||
HotRank HotRankConfig `mapstructure:"hot_rank"`
|
||
WebRTC WebRTCConfig `mapstructure:"webrtc"`
|
||
Report ReportConfig `mapstructure:"report"`
|
||
Sensitive SensitiveConfig `mapstructure:"sensitive"`
|
||
JPush JPushConfig `mapstructure:"jpush"`
|
||
SetupSecret string `mapstructure:"setup_secret"`
|
||
WebSocket WSConfig `mapstructure:"websocket"`
|
||
Runner RunnerConfig `mapstructure:"runner"`
|
||
}
|
||
|
||
// Load 加载配置文件
|
||
func Load(configPath string) (*Config, error) {
|
||
viper.SetConfigFile(configPath)
|
||
viper.SetConfigType("yaml")
|
||
|
||
// 启用环境变量支持
|
||
viper.SetEnvPrefix("APP")
|
||
viper.AutomaticEnv()
|
||
// 允许环境变量使用下划线或连字符
|
||
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
|
||
|
||
// Set default values
|
||
viper.SetDefault("server.port", 8080)
|
||
viper.SetDefault("server.mode", "debug")
|
||
viper.SetDefault("server.host", "0.0.0.0")
|
||
viper.SetDefault("database.type", "sqlite")
|
||
viper.SetDefault("database.sqlite.path", "./data/with_you.db")
|
||
viper.SetDefault("database.max_idle_conns", 10)
|
||
viper.SetDefault("database.max_open_conns", 100)
|
||
viper.SetDefault("database.log_level", "warn")
|
||
viper.SetDefault("database.slow_threshold_ms", 200)
|
||
viper.SetDefault("database.ignore_record_not_found", true)
|
||
viper.SetDefault("database.parameterized_queries", true)
|
||
viper.SetDefault("redis.type", "miniredis")
|
||
viper.SetDefault("redis.redis.host", "localhost")
|
||
viper.SetDefault("redis.redis.port", 6379)
|
||
viper.SetDefault("redis.redis.password", "")
|
||
viper.SetDefault("redis.redis.db", 0)
|
||
viper.SetDefault("redis.miniredis.host", "localhost")
|
||
viper.SetDefault("redis.miniredis.port", 6379)
|
||
viper.SetDefault("redis.pool_size", 10)
|
||
viper.SetDefault("cache.enabled", true)
|
||
viper.SetDefault("cache.key_prefix", "")
|
||
viper.SetDefault("cache.default_ttl", 30)
|
||
viper.SetDefault("cache.null_ttl", 5)
|
||
viper.SetDefault("cache.jitter_ratio", 0.1)
|
||
viper.SetDefault("cache.disable_flushdb", true)
|
||
viper.SetDefault("cache.modules.post_list_ttl", 30)
|
||
viper.SetDefault("cache.modules.conversation_ttl", 60)
|
||
viper.SetDefault("cache.modules.unread_count_ttl", 30)
|
||
viper.SetDefault("cache.modules.group_members_ttl", 120)
|
||
viper.SetDefault("cache.local.enabled", true)
|
||
viper.SetDefault("cache.local.size", 10000)
|
||
viper.SetDefault("cache.local.buckets", 0)
|
||
viper.SetDefault("cache.local.default_ttl", 300)
|
||
viper.SetDefault("jwt.secret", "your-jwt-secret-key-change-in-production")
|
||
viper.SetDefault("jwt.access_token_expire", 86400)
|
||
viper.SetDefault("jwt.refresh_token_expire", 604800)
|
||
viper.SetDefault("log.level", "info")
|
||
viper.SetDefault("log.encoding", "json")
|
||
viper.SetDefault("log.output_paths", []string{"stdout", "./logs/app.log"})
|
||
viper.SetDefault("rate_limit.enabled", true)
|
||
viper.SetDefault("rate_limit.requests_per_minute", 60)
|
||
viper.SetDefault("upload.max_file_size", 10485760)
|
||
viper.SetDefault("upload.allowed_types", []string{"image/jpeg", "image/png", "image/gif", "image/webp"})
|
||
viper.SetDefault("s3.endpoint", "")
|
||
viper.SetDefault("s3.access_key", "")
|
||
viper.SetDefault("s3.secret_key", "")
|
||
viper.SetDefault("s3.bucket", "")
|
||
viper.SetDefault("s3.use_ssl", true)
|
||
viper.SetDefault("s3.region", "us-east-1")
|
||
viper.SetDefault("s3.domain", "")
|
||
viper.SetDefault("sensitive.enabled", true)
|
||
viper.SetDefault("sensitive.replace_str", "***")
|
||
viper.SetDefault("sensitive.min_match_len", 2)
|
||
viper.SetDefault("sensitive.load_from_db", true)
|
||
viper.SetDefault("sensitive.load_from_redis", false)
|
||
viper.SetDefault("sensitive.redis_key_prefix", "sensitive_words")
|
||
viper.SetDefault("audit.enabled", false)
|
||
viper.SetDefault("audit.provider", "local")
|
||
viper.SetDefault("openai.enabled", true)
|
||
viper.SetDefault("openai.base_url", "https://api.littlelan.cn/")
|
||
viper.SetDefault("openai.api_key", "")
|
||
viper.SetDefault("openai.moderation_model", "qwen3.5-122b")
|
||
viper.SetDefault("openai.moderation_max_images_per_request", 1)
|
||
viper.SetDefault("openai.request_timeout", 30)
|
||
viper.SetDefault("openai.strict_moderation", false)
|
||
viper.SetDefault("tencent_tms.enabled", false)
|
||
viper.SetDefault("tencent_tms.secret_id", "")
|
||
viper.SetDefault("tencent_tms.secret_key", "")
|
||
viper.SetDefault("tencent_tms.region", "ap-guangzhou")
|
||
viper.SetDefault("tencent_tms.biz_type", "")
|
||
viper.SetDefault("tencent_tms.timeout", 30)
|
||
viper.SetDefault("email.enabled", false)
|
||
viper.SetDefault("email.host", "")
|
||
viper.SetDefault("email.port", 587)
|
||
viper.SetDefault("email.username", "")
|
||
viper.SetDefault("email.password", "")
|
||
viper.SetDefault("email.from_address", "")
|
||
viper.SetDefault("email.from_name", "WithYou")
|
||
viper.SetDefault("email.use_tls", true)
|
||
viper.SetDefault("email.insecure_skip_verify", false)
|
||
viper.SetDefault("email.timeout", 15)
|
||
// ConversationCache 默认值
|
||
viper.SetDefault("conversation_cache.detail_ttl", "5m")
|
||
viper.SetDefault("conversation_cache.list_ttl", "60s")
|
||
viper.SetDefault("conversation_cache.participant_ttl", "5m")
|
||
viper.SetDefault("conversation_cache.unread_ttl", "30s")
|
||
viper.SetDefault("conversation_cache.message_detail_ttl", "30m")
|
||
viper.SetDefault("conversation_cache.message_list_ttl", "5m")
|
||
viper.SetDefault("conversation_cache.message_index_ttl", "30m")
|
||
viper.SetDefault("conversation_cache.message_count_ttl", "30m")
|
||
viper.SetDefault("conversation_cache.batch_interval", "5s")
|
||
viper.SetDefault("conversation_cache.batch_threshold", 100)
|
||
viper.SetDefault("conversation_cache.batch_max_size", 500)
|
||
viper.SetDefault("conversation_cache.buffer_max_size", 10000)
|
||
// GRPC 默认值
|
||
viper.SetDefault("grpc.enabled", true)
|
||
viper.SetDefault("grpc.port", 50051)
|
||
viper.SetDefault("grpc.tls_enabled", false)
|
||
viper.SetDefault("grpc.tls_cert_file", "")
|
||
viper.SetDefault("grpc.tls_key_file", "")
|
||
viper.SetDefault("casbin.model_path", "./configs/casbin/model.conf")
|
||
viper.SetDefault("casbin.auto_save", true)
|
||
viper.SetDefault("casbin.auto_load", true)
|
||
viper.SetDefault("casbin.enable_cache", true)
|
||
viper.SetDefault("casbin.cache_ttl", 300)
|
||
viper.SetDefault("casbin.skip_routes", []string{"/health", "/api/v1/auth/login", "/api/v1/auth/register"})
|
||
// Encryption 默认值
|
||
viper.SetDefault("encryption.enabled", false)
|
||
viper.SetDefault("encryption.key", "")
|
||
viper.SetDefault("encryption.key_version", 1)
|
||
viper.SetDefault("hot_rank.enabled", true)
|
||
viper.SetDefault("hot_rank.refresh_interval_seconds", 300)
|
||
viper.SetDefault("hot_rank.top_n", 100)
|
||
viper.SetDefault("hot_rank.recent_window_hours", 168)
|
||
viper.SetDefault("hot_rank.recent_fetch_limit", 500)
|
||
viper.SetDefault("hot_rank.candidate_cap", 800)
|
||
// WebRTC 默认值（Google 公共 STUN 服务器）
|
||
viper.SetDefault("webrtc.ice_servers", []map[string]any{
|
||
{"urls": []string{"stun:stun.l.google.com:19302"}},
|
||
})
|
||
// Report 默认值
|
||
viper.SetDefault("report.auto_hide_threshold", 3)
|
||
viper.SetDefault("jpush.enabled", false)
|
||
viper.SetDefault("jpush.app_key", "")
|
||
viper.SetDefault("jpush.master_secret", "")
|
||
viper.SetDefault("jpush.production", false)
|
||
viper.SetDefault("setup_secret", "")
|
||
// WebSocket 默认值
|
||
viper.SetDefault("websocket.mode", "standalone")
|
||
viper.SetDefault("websocket.cluster.instance_id", "")
|
||
viper.SetDefault("websocket.cluster.msg_channel", "ws:msg")
|
||
viper.SetDefault("websocket.cluster.online_ttl", 60)
|
||
viper.SetDefault("websocket.cluster.heartbeat_interval", 20)
|
||
// Runner 集群默认值（mode 和 instance_id 空则跟随 websocket）
|
||
viper.SetDefault("runner.mode", "")
|
||
viper.SetDefault("runner.cluster.instance_id", "")
|
||
viper.SetDefault("runner.cluster.task_channel", "runner:task")
|
||
viper.SetDefault("runner.cluster.result_channel", "runner:result")
|
||
viper.SetDefault("runner.cluster.registry_prefix", "runner:reg:")
|
||
viper.SetDefault("runner.cluster.cap_prefix", "runner:cap:")
|
||
viper.SetDefault("runner.cluster.registry_ttl", 90)
|
||
viper.SetDefault("runner.cluster.heartbeat_interval", 30)
|
||
|
||
if err := viper.ReadInConfig(); err != nil {
|
||
return nil, fmt.Errorf("failed to read config: %w", err)
|
||
}
|
||
|
||
var cfg Config
|
||
if err := viper.Unmarshal(&cfg); err != nil {
|
||
return nil, fmt.Errorf("failed to unmarshal config: %w", err)
|
||
}
|
||
|
||
// Runner mode 和 instance_id 空则跟随 WebSocket 配置
|
||
cfg.Runner.ApplyWSDefaults(cfg.WebSocket)
|
||
|
||
// Convert seconds to duration
|
||
cfg.JWT.AccessTokenExpire = time.Duration(viper.GetInt("jwt.access_token_expire")) * time.Second
|
||
cfg.JWT.RefreshTokenExpire = time.Duration(viper.GetInt("jwt.refresh_token_expire")) * time.Second
|
||
|
||
// Viper AutomaticEnv 已自动处理所有 APP_ 前缀的环境变量
|
||
|
||
// 安全检查：生产模式必须设置JWT密钥
|
||
if cfg.Server.Mode != "debug" {
|
||
if cfg.JWT.Secret == "" || cfg.JWT.Secret == "your-jwt-secret-key-change-in-production" {
|
||
return nil, fmt.Errorf("SECURITY ERROR: JWT secret must be set via APP_JWT_SECRET environment variable in production mode")
|
||
}
|
||
if len(cfg.JWT.Secret) < 32 {
|
||
return nil, fmt.Errorf("SECURITY ERROR: JWT secret must be at least 32 characters long in production mode")
|
||
}
|
||
}
|
||
|
||
return &cfg, nil
|
||
}
|
||
|
||
|