use anyhow::Result; use serde::{Deserialize, Deserializer, Serialize}; use std::env; use std::fs::File; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, OnceLock, RwLock}; use thiserror::Error; use tracing::{info, warn}; /// 默认配置文件内容 const DEFAULT_CONFIG_YAML: &str = include_str!("../config.yml"); /// 配置验证错误 #[derive(Error, Debug)] pub enum ConfigError { #[error("配置文件读取失败: {0}")] FileRead(String), #[error("配置解析失败: {0}")] Parse(String), #[error("配置验证失败: {field} - {message}")] Validation { field: String, message: String }, #[error("环境变量解析失败: {var} - {message}")] EnvVar { var: String, message: String }, #[error("路径无效: {path} - {message}")] InvalidPath { path: String, message: String }, } /// 文件大小单位 #[derive(Debug, Clone)] pub struct FileSize(pub u64); impl<'de> Deserialize<'de> for FileSize { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; parse_file_size(&s) .map(FileSize) .map_err(serde::de::Error::custom) } } impl FileSize { pub fn bytes(&self) -> u64 { self.0 } pub fn mb(&self) -> u64 { self.0 / (1024 * 1024) } pub fn from_mb(mb: u64) -> Self { Self(mb * 1024 * 1024) } } /// 全局文件大小配置 #[derive(Debug, Clone, Deserialize)] pub struct GlobalFileSizeConfig { /// 统一的最大文件大小限制 pub max_file_size: FileSize, /// 大文档阈值(用于流式处理) pub large_document_threshold: FileSize, } impl Default for GlobalFileSizeConfig { fn default() -> Self { Self { max_file_size: FileSize(100 * 1024 * 1024), // 100MB large_document_threshold: FileSize(1024 * 1024), // 1MB } } } impl GlobalFileSizeConfig { /// 创建新的全局文件大小配置实例 pub fn new() -> Self { get_global_file_size_config().clone() } /// 验证配置的有效性 pub fn validate(&self) -> Result<(), ConfigError> { let configs = [ ("max_file_size", self.max_file_size.bytes()), ( "large_document_threshold", self.large_document_threshold.bytes(), ), ]; for (name, size) in configs { if size == 0 { return Err(ConfigError::Validation { field: format!("file_size_config.{name}"), message: "文件大小不能为0".to_string(), }); } if size > 10 * 1024 * 1024 * 1024 { // 10GB return Err(ConfigError::Validation { field: format!("file_size_config.{name}"), message: "文件大小不能超过10GB".to_string(), }); } } Ok(()) } /// 获取指定用途的文件大小限制 pub fn get_max_size_for(&self, _purpose: FileSizePurpose) -> u64 { // 统一使用同一个文件大小限制 self.max_file_size.bytes() } /// 检查文件大小是否超过指定用途的限制 pub fn is_size_allowed(&self, file_size: u64, purpose: FileSizePurpose) -> bool { file_size <= self.get_max_size_for(purpose) } /// 获取大文档阈值 pub fn get_large_document_threshold(&self) -> u64 { self.large_document_threshold.bytes() } /// 检查是否为大文档 pub fn is_large_document(&self, file_size: u64) -> bool { file_size >= self.get_large_document_threshold() } /// 获取指定用途的文件大小限制(返回FileSize结构体) pub fn get_file_size_limit(&self, _purpose: &FileSizePurpose) -> &FileSize { // 统一使用同一个文件大小限制 &self.max_file_size } } /// 文件大小用途枚举 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FileSizePurpose { /// 默认用途 Default, /// 文档解析器 DocumentParser, /// MinerU解析器 MinerU, /// MarkItDown解析器 MarkItDown, /// 图片处理器 ImageProcessor, /// 文件上传 Upload, /// 格式检测器 FormatDetector, /// 内容验证 ContentValidation, /// 缓存 Cache, } /// 解析文件大小字符串 (例如: "100MB", "1GB", "500KB") pub fn parse_file_size(size_str: &str) -> Result { let size_str = size_str.trim().to_uppercase(); if let Some(pos) = size_str.find(|c: char| c.is_alphabetic()) { let (number_part, unit_part) = size_str.split_at(pos); let number: f64 = number_part .parse() .map_err(|_| format!("无效的数字: {number_part}"))?; let multiplier = match unit_part { "B" => 1, "KB" => 1024, "MB" => 1024 * 1024, "GB" => 1024 * 1024 * 1024, "TB" => 1024_u64.pow(4), _ => return Err(format!("不支持的单位: {unit_part}")), }; Ok((number * multiplier as f64) as u64) } else { // 如果没有单位,假设是字节 size_str .parse::() .map_err(|_| format!("无效的文件大小: {size_str}")) } } /// 配置构建器,用于测试和灵活配置创建 #[derive(Debug, Default)] pub struct ConfigBuilder { environment: Option, server: Option, log: Option, document_parser: Option, mineru: Option, markitdown: Option, storage: Option, external_integration: Option, file_size_config: Option, } impl ConfigBuilder { pub fn new() -> Self { Self::default() } pub fn environment(mut self, environment: String) -> Self { self.environment = Some(environment); self } pub fn server(mut self, server: ServerConfig) -> Self { self.server = Some(server); self } pub fn log(mut self, log: LogConfig) -> Self { self.log = Some(log); self } pub fn document_parser(mut self, document_parser: DocumentParserConfig) -> Self { self.document_parser = Some(document_parser); self } pub fn mineru(mut self, mineru: MinerUConfig) -> Self { self.mineru = Some(mineru); self } pub fn markitdown(mut self, markitdown: MarkItDownConfig) -> Self { self.markitdown = Some(markitdown); self } pub fn storage(mut self, storage: StorageConfig) -> Self { self.storage = Some(storage); self } pub fn external_integration(mut self, external_integration: ExternalIntegrationConfig) -> Self { self.external_integration = Some(external_integration); self } pub fn file_size_config(mut self, file_size_config: GlobalFileSizeConfig) -> Self { self.file_size_config = Some(file_size_config); self } pub fn build(self) -> Result { // 使用默认配置作为基础 let mut config: AppConfig = serde_yaml::from_str(DEFAULT_CONFIG_YAML) .map_err(|e| ConfigError::Parse(e.to_string()))?; // 应用构建器中的配置 if let Some(environment) = self.environment { config.environment = environment; } if let Some(server) = self.server { config.server = server; } if let Some(log) = self.log { config.log = log; } if let Some(document_parser) = self.document_parser { config.document_parser = document_parser; } if let Some(mineru) = self.mineru { config.mineru = mineru; } if let Some(markitdown) = self.markitdown { config.markitdown = markitdown; } if let Some(storage) = self.storage { config.storage = storage; } if let Some(external_integration) = self.external_integration { config.external_integration = external_integration; } if let Some(file_size_config) = self.file_size_config { config.file_size_config = file_size_config; } // 验证配置 config.validate()?; Ok(config) } } /// CUDA环境状态 #[derive(Debug, Clone, Default)] pub struct CudaStatus { pub available: bool, pub version: Option, pub device_count: usize, pub recommended_device: Option, } /// 应用配置 #[derive(Debug, Clone, Deserialize)] pub struct AppConfig { pub environment: String, pub server: ServerConfig, pub log: LogConfig, pub document_parser: DocumentParserConfig, pub mineru: MinerUConfig, pub markitdown: MarkItDownConfig, pub storage: StorageConfig, pub external_integration: ExternalIntegrationConfig, /// 全局文件大小配置 #[serde(default)] pub file_size_config: GlobalFileSizeConfig, } /// 服务器配置 #[derive(Debug, Clone, Deserialize)] pub struct ServerConfig { pub port: u16, pub host: String, } impl ServerConfig { pub fn validate(&self) -> Result<(), ConfigError> { if self.port == 0 { return Err(ConfigError::Validation { field: "server.port".to_string(), message: "端口号不能为0".to_string(), }); } if self.host.is_empty() { return Err(ConfigError::Validation { field: "server.host".to_string(), message: "主机地址不能为空".to_string(), }); } // 验证主机地址格式 if !self .host .chars() .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == ':' || c == '-') { return Err(ConfigError::Validation { field: "server.host".to_string(), message: "主机地址包含无效字符".to_string(), }); } Ok(()) } } /// 日志配置 #[derive(Debug, Clone, Deserialize)] pub struct LogConfig { pub level: String, pub path: String, /// The number of log files to retain (default: 20) #[serde(default = "default_retain_days")] pub retain_days: u32, } /// Default log files to retain fn default_retain_days() -> u32 { 20 } impl LogConfig { pub fn validate(&self) -> Result<(), ConfigError> { // 验证日志级别 let valid_levels = ["trace", "debug", "info", "warn", "error"]; if !valid_levels.contains(&self.level.to_lowercase().as_str()) { return Err(ConfigError::Validation { field: "log.level".to_string(), message: format!( "无效的日志级别: {},支持的级别: {:?}", self.level, valid_levels ), }); } if self.path.is_empty() { return Err(ConfigError::Validation { field: "log.path".to_string(), message: "日志路径不能为空".to_string(), }); } // 验证路径是否可以创建 let path = Path::new(&self.path); if let Some(parent) = path.parent() { if parent.exists() && !parent.is_dir() { return Err(ConfigError::InvalidPath { path: self.path.clone(), message: "父目录不是一个有效的目录".to_string(), }); } } Ok(()) } } /// 文档解析配置 #[derive(Debug, Clone, Deserialize)] pub struct DocumentParserConfig { pub max_concurrent: usize, pub queue_size: usize, pub download_timeout: u32, pub processing_timeout: u32, } impl DocumentParserConfig { pub fn validate(&self) -> Result<(), ConfigError> { if self.max_concurrent == 0 { return Err(ConfigError::Validation { field: "document_parser.max_concurrent".to_string(), message: "最大并发数不能为0".to_string(), }); } if self.max_concurrent > 100 { return Err(ConfigError::Validation { field: "document_parser.max_concurrent".to_string(), message: "最大并发数不能超过100".to_string(), }); } if self.queue_size == 0 { return Err(ConfigError::Validation { field: "document_parser.queue_size".to_string(), message: "队列大小不能为0".to_string(), }); } // 文件大小限制现在由全局配置管理 if self.download_timeout == 0 { return Err(ConfigError::Validation { field: "document_parser.download_timeout".to_string(), message: "下载超时时间不能为0".to_string(), }); } if self.processing_timeout == 0 { return Err(ConfigError::Validation { field: "document_parser.processing_timeout".to_string(), message: "处理超时时间不能为0".to_string(), }); } Ok(()) } } /// MinerU配置 #[derive(Debug, Clone, Deserialize)] pub struct MinerUConfig { #[serde(default = "default_backend")] pub backend: String, #[serde(default = "default_python_path")] pub python_path: String, pub max_concurrent: usize, pub queue_size: usize, #[serde(default)] pub timeout: u32, // 0表示使用统一的processing_timeout #[serde(default = "default_batch_size")] pub batch_size: usize, #[serde(default)] pub quality_level: QualityLevel, #[serde(default = "default_device")] pub device: String, // 推理设备:cpu/cuda/cuda:0/npu/mps等 #[serde(default = "default_vram")] pub vram: u32, // 单进程最大GPU显存占用(GB),仅对pipeline后端且支持CUDA时有效 } /// 质量级别 #[derive(Debug, Clone, PartialEq, Deserialize, Default)] pub enum QualityLevel { Fast, #[default] Balanced, HighQuality, } fn default_batch_size() -> usize { 1 } fn default_backend() -> String { "pipeline".to_string() } fn default_python_path() -> String { // Default to virtual environment python if available, otherwise system python if cfg!(windows) { "./venv/Scripts/python.exe".to_string() } else { "./venv/bin/python".to_string() } } fn default_device() -> String { "cpu".to_string() } fn default_vram() -> u32 { 8 // 默认8GB显存限制 } impl MinerUConfig { pub fn validate(&self) -> Result<(), ConfigError> { let valid_backends = [ "pipeline", "vlm-transformers", "vlm-sglang-engine", "vlm-sglang-client", ]; if !valid_backends.contains(&self.backend.as_str()) { return Err(ConfigError::Validation { field: "mineru.backend".to_string(), message: format!( "无效的后端类型: {},支持的类型: {:?}", self.backend, valid_backends ), }); } if self.python_path.is_empty() { return Err(ConfigError::Validation { field: "mineru.python_path".to_string(), message: "Python路径不能为空".to_string(), }); } if self.max_concurrent == 0 { return Err(ConfigError::Validation { field: "mineru.max_concurrent".to_string(), message: "最大并发数不能为0".to_string(), }); } if self.queue_size == 0 { return Err(ConfigError::Validation { field: "mineru.queue_size".to_string(), message: "队列大小不能为0".to_string(), }); } // timeout 为 0 表示使用统一的 processing_timeout,这是允许的 if self.batch_size == 0 { return Err(ConfigError::Validation { field: "mineru.batch_size".to_string(), message: "批处理大小不能为0".to_string(), }); } Ok(()) } /// Get the effective python path, auto-detecting virtual environment if needed pub fn get_effective_python_path(&self) -> String { // If the configured path is the default and a virtual environment exists, use it let default_path = default_python_path(); if self.python_path == default_path || self.python_path == "python3" || self.python_path == "python" { let venv_python = if cfg!(windows) { std::path::Path::new("./venv/Scripts/python.exe") } else { std::path::Path::new("./venv/bin/python") }; if venv_python.exists() { return venv_python.to_string_lossy().to_string(); } } self.python_path.clone() } } /// MarkItDown配置 #[derive(Debug, Clone, Deserialize)] pub struct MarkItDownConfig { #[serde(default = "default_python_path")] pub python_path: String, #[serde(default)] pub timeout: u32, // 0表示使用统一的processing_timeout pub enable_plugins: bool, pub features: MarkItDownFeatures, } impl MarkItDownConfig { pub fn validate(&self) -> Result<(), ConfigError> { if self.python_path.is_empty() { return Err(ConfigError::Validation { field: "markitdown.python_path".to_string(), message: "Python路径不能为空".to_string(), }); } // timeout 为 0 表示使用统一的 processing_timeout,这是允许的 Ok(()) } /// Get the effective python path, auto-detecting virtual environment if needed pub fn get_effective_python_path(&self) -> String { // If the configured path is the default and a virtual environment exists, use it let default_path = default_python_path(); if self.python_path == default_path || self.python_path == "python3" || self.python_path == "python" { let venv_python = if cfg!(windows) { std::path::Path::new("./venv/Scripts/python.exe") } else { std::path::Path::new("./venv/bin/python") }; if venv_python.exists() { return venv_python.to_string_lossy().to_string(); } } self.python_path.clone() } } /// MarkItDown功能配置 #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MarkItDownFeatures { pub ocr: bool, pub audio_transcription: bool, pub azure_doc_intel: bool, pub youtube_transcription: bool, } /// 存储配置 #[derive(Debug, Clone, Deserialize)] pub struct StorageConfig { pub sled: SledConfig, pub oss: OssConfig, } impl StorageConfig { pub fn validate(&self) -> Result<(), ConfigError> { self.sled.validate()?; self.oss.validate()?; Ok(()) } } /// Sled数据库配置 #[derive(Debug, Clone, Deserialize)] pub struct SledConfig { pub path: String, pub cache_capacity: usize, } impl SledConfig { pub fn validate(&self) -> Result<(), ConfigError> { if self.path.is_empty() { return Err(ConfigError::Validation { field: "storage.sled.path".to_string(), message: "数据库路径不能为空".to_string(), }); } if self.cache_capacity == 0 { return Err(ConfigError::Validation { field: "storage.sled.cache_capacity".to_string(), message: "缓存容量不能为0".to_string(), }); } Ok(()) } } /// OSS配置 #[derive(Debug, Clone, Deserialize)] pub struct OssConfig { pub endpoint: String, // pub bucket: String, /// 公有存储桶名称 (默认: nuwa-packages) pub public_bucket: String, /// 私有存储桶名称 (默认: edu-nuwa-packages) pub private_bucket: String, pub access_key_id: String, pub access_key_secret: String, /// 上传文件的统一子目录前缀 #[serde(default = "default_upload_directory")] pub upload_directory: String, /// 区域 (默认: oss-rg-china-mainland) pub region: String, } /// 默认上传目录 fn default_upload_directory() -> String { "edu".to_string() } impl OssConfig { pub fn validate(&self) -> Result<(), ConfigError> { if self.endpoint.is_empty() { return Err(ConfigError::Validation { field: "storage.oss.endpoint".to_string(), message: "OSS端点不能为空".to_string(), }); } if self.public_bucket.is_empty() { return Err(ConfigError::Validation { field: "storage.oss.public_bucket".to_string(), message: "OSS存储桶名称不能为空".to_string(), }); } if self.private_bucket.is_empty() { return Err(ConfigError::Validation { field: "storage.oss.private_bucket".to_string(), message: "OSS存储桶名称不能为空".to_string(), }); } // 注意:region 现在是可选的,可以为 None // 注意:access_key_id 和 access_key_secret 可以为空,因为它们可能通过环境变量设置 Ok(()) } /// 检查OSS配置是否完整(环境变量是否已设置) pub fn is_configured(&self) -> bool { !self.access_key_id.is_empty() && !self.access_key_secret.is_empty() } } /// 外部集成配置 #[derive(Debug, Clone, Deserialize)] pub struct ExternalIntegrationConfig { pub webhook_url: String, pub api_key: String, pub timeout: u32, } impl ExternalIntegrationConfig { pub fn validate(&self) -> Result<(), ConfigError> { if self.timeout == 0 { return Err(ConfigError::Validation { field: "external_integration.timeout".to_string(), message: "超时时间不能为0".to_string(), }); } // webhook_url 和 api_key 可以为空,因为外部集成是可选的 if !self.webhook_url.is_empty() { // 简单的URL格式验证 if !self.webhook_url.starts_with("http://") && !self.webhook_url.starts_with("https://") { return Err(ConfigError::Validation { field: "external_integration.webhook_url".to_string(), message: "Webhook URL必须以http://或https://开头".to_string(), }); } } Ok(()) } } impl AppConfig { /// 加载配置文件,支持多种配置源和环境变量覆盖 pub fn load_config() -> Result { // 1. 首先加载基础配置 let mut config = Self::load_base_config()?; // 2. 从环境变量覆盖配置 config.load_all_from_env()?; // 3. 验证最终配置 config.validate()?; // 4. 初始化必要的目录 config.initialize_directories()?; Ok(config) } /// 加载基础配置文件 pub fn load_base_config() -> Result { Self::load_base_config_with_path(None) } /// 加载基础配置文件,支持可选的配置文件路径 pub fn load_base_config_with_path(config_path: Option) -> Result { // 优先尝试从传入的路径加载 if let Some(path) = config_path { if Path::new(&path).exists() { return Self::load_from_file(&path); } } let config_paths = [ "/app/config.yml", "config.yml", "document-parser/config.yml", ]; // 尝试从环境变量指定的配置文件路径加载 if let Ok(env_config_path) = env::var("DOCUMENT_PARSER_CONFIG") { return Self::load_from_file(&env_config_path); } // 尝试从预定义路径加载 for path in &config_paths { if Path::new(path).exists() { return Self::load_from_file(path); } } // 如果都没有找到,尝试在当前目录创建默认配置文件 if let Err(e) = Self::create_default_config_in_current_dir() { warn!( "Unable to create default configuration file in current directory: {}", e ); } // 使用默认配置 serde_yaml::from_str::(DEFAULT_CONFIG_YAML) .map_err(|e| ConfigError::Parse(format!("解析默认配置失败: {e}"))) } /// 从指定文件加载配置 fn load_from_file(path: &str) -> Result { let file = File::open(path) .map_err(|e| ConfigError::FileRead(format!("无法打开配置文件 {path}: {e}")))?; serde_yaml::from_reader(file) .map_err(|e| ConfigError::Parse(format!("解析配置文件 {path} 失败: {e}"))) } /// 在当前目录创建默认配置文件 fn create_default_config_in_current_dir() -> Result<(), ConfigError> { let current_dir = std::env::current_dir() .map_err(|e| ConfigError::FileRead(format!("无法获取当前目录: {e}")))?; let config_path = current_dir.join("config.yml"); // 如果配置文件已存在,不覆盖 if config_path.exists() { return Ok(()); } // 创建配置文件 std::fs::write(&config_path, DEFAULT_CONFIG_YAML) .map_err(|e| ConfigError::FileRead(format!("无法创建默认配置文件: {e}")))?; info!( "The default configuration file has been created in the current directory: {}", config_path.display() ); Ok(()) } /// 验证整个配置的有效性 pub fn validate(&self) -> Result<(), ConfigError> { self.server.validate()?; self.log.validate()?; self.document_parser.validate()?; self.mineru.validate()?; self.markitdown.validate()?; self.storage.validate()?; self.external_integration.validate()?; self.file_size_config.validate()?; // 交叉验证 self.cross_validate()?; Ok(()) } /// 交叉验证不同配置项之间的一致性 fn cross_validate(&self) -> Result<(), ConfigError> { // 验证并发配置的一致性 if self.document_parser.max_concurrent < self.mineru.max_concurrent + 1 { return Err(ConfigError::Validation { field: "document_parser.max_concurrent".to_string(), message: "文档解析器的最大并发数应该大于MinerU的最大并发数".to_string(), }); } // 验证队列大小的合理性 if self.document_parser.queue_size < self.mineru.queue_size { return Err(ConfigError::Validation { field: "document_parser.queue_size".to_string(), message: "文档解析器的队列大小应该大于等于MinerU的队列大小".to_string(), }); } // 验证超时配置的合理性(0表示使用统一超时,跳过验证) if self.mineru.timeout > 0 && self.document_parser.processing_timeout < self.mineru.timeout { return Err(ConfigError::Validation { field: "document_parser.processing_timeout".to_string(), message: "文档解析器的处理超时时间应该大于等于MinerU的超时时间".to_string(), }); } if self.markitdown.timeout > 0 && self.document_parser.processing_timeout < self.markitdown.timeout { return Err(ConfigError::Validation { field: "document_parser.processing_timeout".to_string(), message: "文档解析器的处理超时时间应该大于等于MarkItDown的超时时间".to_string(), }); } Ok(()) } /// 初始化必要的目录 pub fn initialize_directories(&self) -> Result<(), ConfigError> { let directories = [&self.log.path, &self.storage.sled.path]; for dir_path in &directories { let path = Path::new(dir_path); // 如果是文件路径,获取父目录 let dir_to_create = if path.extension().is_some() { path.parent().unwrap_or(path) } else { path }; if !dir_to_create.exists() { std::fs::create_dir_all(dir_to_create).map_err(|e| ConfigError::InvalidPath { path: dir_to_create.to_string_lossy().to_string(), message: format!("无法创建目录: {e}"), })?; } } Ok(()) } /// 从环境变量加载所有配置,支持类型安全的解析和错误处理 pub fn load_all_from_env(&mut self) -> Result<(), ConfigError> { self.load_server_config_from_env()?; self.load_log_config_from_env()?; self.load_document_parser_config_from_env()?; self.load_oss_config_from_env()?; self.load_mineru_config_from_env()?; self.load_markitdown_config_from_env()?; self.load_external_integration_config_from_env()?; Ok(()) } /// 从环境变量加载服务器配置 fn load_server_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(port_str) = env::var("SERVER_PORT") { self.server.port = Self::parse_env_var("SERVER_PORT", &port_str)?; } if let Ok(host) = env::var("SERVER_HOST") { self.server.host = host; } Ok(()) } /// 从环境变量加载日志配置 fn load_log_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(level) = env::var("LOG_LEVEL") { self.log.level = level; } if let Ok(path) = env::var("LOG_PATH") { self.log.path = path; } Ok(()) } /// 从环境变量加载文档解析器配置 fn load_document_parser_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(max_concurrent_str) = env::var("DOCUMENT_PARSER_MAX_CONCURRENT") { self.document_parser.max_concurrent = Self::parse_env_var("DOCUMENT_PARSER_MAX_CONCURRENT", &max_concurrent_str)?; } if let Ok(queue_size_str) = env::var("DOCUMENT_PARSER_QUEUE_SIZE") { self.document_parser.queue_size = Self::parse_env_var("DOCUMENT_PARSER_QUEUE_SIZE", &queue_size_str)?; } // 文件大小限制现在由全局配置管理 if let Ok(download_timeout_str) = env::var("DOCUMENT_PARSER_DOWNLOAD_TIMEOUT") { self.document_parser.download_timeout = Self::parse_env_var("DOCUMENT_PARSER_DOWNLOAD_TIMEOUT", &download_timeout_str)?; } if let Ok(processing_timeout_str) = env::var("DOCUMENT_PARSER_PROCESSING_TIMEOUT") { self.document_parser.processing_timeout = Self::parse_env_var( "DOCUMENT_PARSER_PROCESSING_TIMEOUT", &processing_timeout_str, )?; } Ok(()) } /// 从环境变量加载OSS配置 fn load_oss_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(endpoint) = env::var("ALIYUN_OSS_ENDPOINT") { self.storage.oss.endpoint = endpoint; } if let Ok(public_bucket) = env::var("ALIYUN_OSS_PUBLIC_BUCKET") { self.storage.oss.public_bucket = public_bucket; } if let Ok(private_bucket) = env::var("ALIYUN_OSS_PRIVATE_BUCKET") { self.storage.oss.private_bucket = private_bucket; } if let Ok(access_key_id) = env::var("OSS_ACCESS_KEY_ID") { self.storage.oss.access_key_id = access_key_id; } if let Ok(access_key_secret) = env::var("OSS_ACCESS_KEY_SECRET") { self.storage.oss.access_key_secret = access_key_secret; } Ok(()) } /// 从环境变量加载MinerU配置 fn load_mineru_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(backend) = env::var("MINERU_BACKEND") { self.mineru.backend = backend; } if let Ok(python_path) = env::var("MINERU_PYTHON_PATH") { self.mineru.python_path = python_path; } if let Ok(max_concurrent_str) = env::var("MINERU_MAX_CONCURRENT") { self.mineru.max_concurrent = Self::parse_env_var("MINERU_MAX_CONCURRENT", &max_concurrent_str)?; } if let Ok(queue_size_str) = env::var("MINERU_QUEUE_SIZE") { self.mineru.queue_size = Self::parse_env_var("MINERU_QUEUE_SIZE", &queue_size_str)?; } if let Ok(timeout_str) = env::var("MINERU_TIMEOUT") { self.mineru.timeout = Self::parse_env_var("MINERU_TIMEOUT", &timeout_str)?; } if let Ok(batch_size_str) = env::var("MINERU_BATCH_SIZE") { self.mineru.batch_size = Self::parse_env_var("MINERU_BATCH_SIZE", &batch_size_str)?; } if let Ok(device) = env::var("MINERU_DEVICE") { self.mineru.device = device; } Ok(()) } /// 从环境变量加载MarkItDown配置 fn load_markitdown_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(python_path) = env::var("MARKITDOWN_PYTHON_PATH") { self.markitdown.python_path = python_path; } if let Ok(timeout_str) = env::var("MARKITDOWN_TIMEOUT") { self.markitdown.timeout = Self::parse_env_var("MARKITDOWN_TIMEOUT", &timeout_str)?; } if let Ok(enable_plugins_str) = env::var("MARKITDOWN_ENABLE_PLUGINS") { self.markitdown.enable_plugins = Self::parse_env_var("MARKITDOWN_ENABLE_PLUGINS", &enable_plugins_str)?; } if let Ok(enable_ocr_str) = env::var("MARKITDOWN_ENABLE_OCR") { self.markitdown.features.ocr = Self::parse_env_var("MARKITDOWN_ENABLE_OCR", &enable_ocr_str)?; } if let Ok(enable_audio_transcription_str) = env::var("MARKITDOWN_ENABLE_AUDIO_TRANSCRIPTION") { self.markitdown.features.audio_transcription = Self::parse_env_var( "MARKITDOWN_ENABLE_AUDIO_TRANSCRIPTION", &enable_audio_transcription_str, )?; } if let Ok(enable_azure_doc_intel_str) = env::var("MARKITDOWN_ENABLE_AZURE_DOC_INTEL") { self.markitdown.features.azure_doc_intel = Self::parse_env_var( "MARKITDOWN_ENABLE_AZURE_DOC_INTEL", &enable_azure_doc_intel_str, )?; } if let Ok(enable_youtube_transcription_str) = env::var("MARKITDOWN_ENABLE_YOUTUBE_TRANSCRIPTION") { self.markitdown.features.youtube_transcription = Self::parse_env_var( "MARKITDOWN_ENABLE_YOUTUBE_TRANSCRIPTION", &enable_youtube_transcription_str, )?; } Ok(()) } /// 从环境变量加载外部集成配置 fn load_external_integration_config_from_env(&mut self) -> Result<(), ConfigError> { if let Ok(webhook_url) = env::var("EXTERNAL_INTEGRATION_WEBHOOK_URL") { self.external_integration.webhook_url = webhook_url; } if let Ok(api_key) = env::var("EXTERNAL_INTEGRATION_API_KEY") { self.external_integration.api_key = api_key; } if let Ok(timeout_str) = env::var("EXTERNAL_INTEGRATION_TIMEOUT") { self.external_integration.timeout = Self::parse_env_var("EXTERNAL_INTEGRATION_TIMEOUT", &timeout_str)?; } Ok(()) } /// 类型安全的环境变量解析 fn parse_env_var(var_name: &str, value: &str) -> Result where T: FromStr, T::Err: std::fmt::Display, { value.parse::().map_err(|e| ConfigError::EnvVar { var: var_name.to_string(), message: format!("无法解析值 '{value}': {e}"), }) } /// 获取配置构建器,用于测试 pub fn builder() -> ConfigBuilder { ConfigBuilder::new() } /// 生成配置摘要,用于日志记录(隐藏敏感信息) pub fn summary(&self) -> String { format!( "AppConfig {{ server: {}:{}, log: {}, max_concurrent: {}, storage: sled={}, oss={}:{} }}", self.server.host, self.server.port, self.log.level, self.document_parser.max_concurrent, self.storage.sled.path, self.storage.oss.endpoint, self.storage.oss.public_bucket ) } } /// 全局配置实例 static GLOBAL_CONFIG: OnceLock = OnceLock::new(); /// 初始化全局配置 pub fn init_global_config(config: AppConfig) -> Result<(), ConfigError> { // 检查是否已经初始化 if GLOBAL_CONFIG.get().is_some() { return Ok(()); // 已经初始化过了,直接返回成功 } GLOBAL_CONFIG .set(config) .map_err(|_| ConfigError::Validation { field: "global_config".to_string(), message: "全局配置已经初始化过了".to_string(), })?; Ok(()) } /// 获取全局配置引用 pub fn get_global_config() -> &'static AppConfig { GLOBAL_CONFIG .get() .expect("全局配置尚未初始化,请先调用 init_global_config") } /// 获取全局文件大小配置 pub fn get_global_file_size_config() -> &'static GlobalFileSizeConfig { let max_file_sieze = &get_global_config().file_size_config; info!("Global file size configuration: {:?}", max_file_sieze); max_file_sieze } /// 便捷函数:获取指定用途的文件大小限制 pub fn get_file_size_limit(purpose: &FileSizePurpose) -> &'static FileSize { get_global_file_size_config().get_file_size_limit(purpose) } /// 便捷函数:检查文件大小是否允许 pub fn is_file_size_allowed(file_size: u64, purpose: FileSizePurpose) -> bool { get_global_file_size_config().is_size_allowed(file_size, purpose) } /// 便捷函数:检查是否为大文档 pub fn is_large_document(file_size: u64) -> bool { get_global_file_size_config().is_large_document(file_size) } /// 便捷函数:获取大文档阈值 pub fn get_large_document_threshold() -> u64 { get_global_file_size_config().get_large_document_threshold() } /// 便捷函数:检查CUDA是否可用 pub fn is_cuda_available() -> bool { get_global_cuda_status_clone().available } /// 便捷函数:获取推荐的CUDA设备 pub fn get_recommended_cuda_device() -> Option { get_global_cuda_status_clone().recommended_device } /// 全局CUDA状态管理,使用线程安全的Arc> static GLOBAL_CUDA_STATUS: OnceLock>> = OnceLock::new(); /// 初始化全局CUDA状态 pub fn init_global_cuda_status(cuda_status: CudaStatus) -> Result<(), ConfigError> { GLOBAL_CUDA_STATUS .set(Arc::new(RwLock::new(cuda_status))) .map_err(|_| ConfigError::Validation { field: "global_cuda_status".to_string(), message: "全局CUDA状态已经初始化过了".to_string(), })?; Ok(()) } /// 更新全局CUDA状态 pub fn update_global_cuda_status(cuda_status: CudaStatus) -> Result<(), ConfigError> { if let Some(global_status) = GLOBAL_CUDA_STATUS.get() { let mut status = global_status.write().map_err(|_| ConfigError::Validation { field: "global_cuda_status".to_string(), message: "无法获取CUDA状态写锁".to_string(), })?; *status = cuda_status; Ok(()) } else { Err(ConfigError::Validation { field: "global_cuda_status".to_string(), message: "全局CUDA状态尚未初始化".to_string(), }) } } /// 获取全局CUDA状态的克隆 pub fn get_global_cuda_status_clone() -> CudaStatus { if let Some(global_status) = GLOBAL_CUDA_STATUS.get() { if let Ok(status) = global_status.read() { return status.clone(); } } warn!("The global CUDA state has not been initialized and returns to the default value."); CudaStatus::default() } #[cfg(test)] mod tests { use super::*; use std::env; use tempfile::TempDir; #[test] fn test_file_size_parsing() { assert_eq!(parse_file_size("100B").unwrap(), 100); assert_eq!(parse_file_size("1KB").unwrap(), 1024); assert_eq!(parse_file_size("1MB").unwrap(), 1024 * 1024); assert_eq!(parse_file_size("1GB").unwrap(), 1024 * 1024 * 1024); assert_eq!( parse_file_size("2.5MB").unwrap(), (2.5 * 1024.0 * 1024.0) as u64 ); // 测试无效格式 assert!(parse_file_size("invalid").is_err()); assert!(parse_file_size("100XB").is_err()); assert!(parse_file_size("").is_err()); } #[test] fn test_default_config_loading() { let config = AppConfig::load_base_config().unwrap(); // 验证默认值 assert_eq!(config.server.port, 8087); assert_eq!(config.server.host, "0.0.0.0"); assert_eq!(config.log.level, "info"); assert_eq!(config.document_parser.max_concurrent, 5); // 配置文件中的实际值 } #[test] fn test_config_validation() { let mut config = AppConfig::load_base_config().unwrap(); // 测试有效配置 assert!(config.validate().is_ok()); // 测试无效端口 config.server.port = 0; assert!(config.validate().is_err()); // 恢复有效端口,测试无效日志级别 config.server.port = 8087; config.log.level = "invalid".to_string(); assert!(config.validate().is_err()); } #[test] fn test_config_builder() { let server_config = ServerConfig { port: 9000, host: "127.0.0.1".to_string(), }; let config = ConfigBuilder::new().server(server_config).build().unwrap(); assert_eq!(config.server.port, 9000); assert_eq!(config.server.host, "127.0.0.1"); } #[test] #[ignore = "Modifies global environment variables, causes race conditions with parallel tests"] fn test_environment_variable_override() { // 设置环境变量 unsafe { env::set_var("SERVER_PORT", "9999"); env::set_var("LOG_LEVEL", "debug"); } let mut config = AppConfig::load_base_config().unwrap(); config.load_all_from_env().unwrap(); assert_eq!(config.server.port, 9999); assert_eq!(config.log.level, "debug"); // 清理环境变量 unsafe { env::remove_var("SERVER_PORT"); env::remove_var("LOG_LEVEL"); } } #[test] #[ignore = "Modifies global environment variables, causes race conditions with parallel tests"] fn test_invalid_environment_variables() { // 设置无效的环境变量 unsafe { env::set_var("SERVER_PORT", "invalid_port"); } let mut config = AppConfig::load_base_config().unwrap(); let result = config.load_all_from_env(); assert!(result.is_err()); // 清理环境变量 unsafe { env::remove_var("SERVER_PORT"); } } #[test] fn test_cross_validation() { let mut config = AppConfig::load_base_config().unwrap(); // 设置不一致的并发配置 config.document_parser.max_concurrent = 1; config.mineru.max_concurrent = 5; assert!(config.validate().is_err()); } #[test] fn test_directory_initialization() { let temp_dir = TempDir::new().unwrap(); let temp_path = temp_dir.path().to_string_lossy().to_string(); let mut config = AppConfig::load_base_config().unwrap(); config.log.path = format!("{temp_path}/logs/app.log"); // temp_dir is now hardcoded, no need to set it config.storage.sled.path = format!("{temp_path}/sled"); assert!(config.initialize_directories().is_ok()); // 验证目录是否创建 assert!(Path::new(&format!("{temp_path}/logs")).exists()); // 注意:initialize_directories 方法不再创建 temp/mineru 和 temp/markitdown assert!(Path::new(&format!("{temp_path}/sled")).exists()); } #[test] fn test_server_config_validation() { let mut config = ServerConfig { port: 8080, host: "localhost".to_string(), }; assert!(config.validate().is_ok()); // 测试无效端口 config.port = 0; assert!(config.validate().is_err()); // 测试空主机 config.port = 8080; config.host = "".to_string(); assert!(config.validate().is_err()); } #[test] fn test_log_config_validation() { let mut config = LogConfig { level: "info".to_string(), path: "/tmp/test.log".to_string(), retain_days: 20, }; assert!(config.validate().is_ok()); // 测试无效日志级别 config.level = "invalid".to_string(); assert!(config.validate().is_err()); // 测试空路径 config.level = "info".to_string(); config.path = "".to_string(); assert!(config.validate().is_err()); } #[test] fn test_document_parser_config_validation() { let config = DocumentParserConfig { max_concurrent: 3, queue_size: 100, download_timeout: 3600, processing_timeout: 1800, }; assert!(config.validate().is_ok()); // 测试零并发 let mut invalid_config = config.clone(); invalid_config.max_concurrent = 0; assert!(invalid_config.validate().is_err()); // 测试过大的并发数 invalid_config.max_concurrent = 200; assert!(invalid_config.validate().is_err()); } #[test] fn test_mineru_config_validation() { let config = MinerUConfig { backend: "pipeline".to_string(), python_path: "/usr/bin/python3".to_string(), max_concurrent: 3, queue_size: 100, timeout: 0, // 使用统一超时配置 batch_size: 10, quality_level: QualityLevel::Balanced, device: "cpu".to_string(), vram: 8, // 默认显存限制 }; assert!(config.validate().is_ok()); // 测试无效后端 let mut invalid_config = config.clone(); invalid_config.backend = "invalid".to_string(); assert!(invalid_config.validate().is_err()); } #[test] fn test_external_integration_config_validation() { let config = ExternalIntegrationConfig { webhook_url: "https://example.com/webhook".to_string(), api_key: "test-key".to_string(), timeout: 30, }; assert!(config.validate().is_ok()); // 测试无效URL let mut invalid_config = config.clone(); invalid_config.webhook_url = "invalid-url".to_string(); assert!(invalid_config.validate().is_err()); // 测试零超时 invalid_config.webhook_url = "https://example.com/webhook".to_string(); invalid_config.timeout = 0; assert!(invalid_config.validate().is_err()); } #[test] fn test_config_summary() { let config = AppConfig::load_base_config().unwrap(); let summary = config.summary(); assert!(summary.contains("AppConfig")); assert!(summary.contains("0.0.0.0:8087")); assert!(summary.contains("info")); assert!(!summary.contains("access_key")); // 确保敏感信息不在摘要中 } }