byy/qiming
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交qiming-mcp-proxy

Browse Source
This commit is contained in:
Codex
2026-06-01 13:03:20 +08:00
parent 9e9486b7c2
commit afb3d9f4e6
394 changed files with 124494 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

439
qiming-mcp-proxy/document-parser/src/performance/mod.rs Normal file
View File

@@ -0,0 +1,439 @@
//! 性能优化模块
//!
//! 包含内存使用优化、并发处理优化和缓存策略
pub mod cache_manager;
pub mod concurrency_optimizer;
pub mod memory_optimizer;
pub mod metrics_collector;
// pub mod resource_monitor; // 模块不存在，暂时注释
use crate::config::AppConfig;
use crate::error::AppError;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
/// 性能优化器主结构
#[derive(Clone)]
pub struct PerformanceOptimizer {
memory_optimizer: Arc<memory_optimizer::MemoryOptimizer>,
concurrency_optimizer: Arc<concurrency_optimizer::ConcurrencyOptimizer>,
cache_manager: Arc<cache_manager::CacheManager>,
metrics_collector: Arc<metrics_collector::MetricsCollector>,
// resource_monitor: Arc<resource_monitor::ResourceMonitor>, // 模块不存在，暂时注释
_config: PerformanceConfig,
}
impl PerformanceOptimizer {
/// 创建新的性能优化器
pub async fn new(config: &AppConfig) -> Result<Self, AppError> {
let performance_config = PerformanceConfig::default();
let memory_optimizer = Arc::new(memory_optimizer::MemoryOptimizer::new(config).await?);
let concurrency_optimizer =
Arc::new(concurrency_optimizer::ConcurrencyOptimizer::new(config).await?);
let cache_manager = Arc::new(cache_manager::CacheManager::new(config).await?);
let metrics_collector = Arc::new(metrics_collector::MetricsCollector::new(config).await?);
// let resource_monitor = Arc::new(resource_monitor::ResourceMonitor::new(config).await?); // 模块不存在，暂时注释
Ok(Self {
memory_optimizer,
concurrency_optimizer,
cache_manager,
metrics_collector,
// resource_monitor, // 模块不存在，暂时注释
_config: performance_config,
})
}
/// 启动性能监控
pub async fn start_monitoring(&self) -> Result<(), AppError> {
// self.resource_monitor.start_monitoring().await?; // 模块不存在，暂时注释
// self.metrics_collector.start_monitoring().await?; // 方法不存在，暂时注释
Ok(())
}
/// 停止性能监控
pub async fn stop_monitoring(&self) -> Result<(), AppError> {
// self.resource_monitor.stop_monitoring().await?; // 模块不存在，暂时注释
// self.metrics_collector.stop_monitoring().await?; // 方法不存在，暂时注释
Ok(())
}
/// 获取内存优化器
pub fn memory_optimizer(&self) -> &Arc<memory_optimizer::MemoryOptimizer> {
&self.memory_optimizer
}
/// 获取并发优化器
pub fn concurrency_optimizer(&self) -> &Arc<concurrency_optimizer::ConcurrencyOptimizer> {
&self.concurrency_optimizer
}
/// 获取缓存管理器
pub fn cache_manager(&self) -> &Arc<cache_manager::CacheManager> {
&self.cache_manager
}
/// 获取指标收集器
pub fn metrics_collector(&self) -> &Arc<metrics_collector::MetricsCollector> {
&self.metrics_collector
}
// /// 获取资源监控器
// pub fn resource_monitor(&self) -> &Arc<resource_monitor::ResourceMonitor> {
// &self.resource_monitor
// } // 模块不存在，暂时注释
// /// 启动资源监控
// pub async fn start_resource_monitoring(&self) -> Result<(), DocumentParserError> {
// self.resource_monitor.start_monitoring().await
// } // 模块不存在，暂时注释
// /// 停止资源监控
// pub async fn stop_resource_monitoring(&self) -> Result<(), DocumentParserError> {
// self.resource_monitor.stop_monitoring().await
// } // 模块不存在，暂时注释
// /// 获取资源统计
// pub async fn get_resource_stats(&self) -> Result<resource_monitor::ResourceStats, DocumentParserError> {
// self.resource_monitor.get_stats().await
// } // 模块不存在，暂时注释
/// 优化资源使用
pub async fn optimize_resources(&self) -> Result<(), AppError> {
// self.resource_monitor.optimize().await // 模块不存在，暂时注释
Ok(())
}
/// 执行性能优化
pub async fn optimize(&self) -> Result<(), AppError> {
// 执行内存优化
self.memory_optimizer.optimize().await?;
// 执行并发优化
self.concurrency_optimizer.optimize().await?;
// 执行缓存优化
self.cache_manager.optimize().await?;
Ok(())
}
/// 获取性能报告
pub async fn get_performance_report(&self) -> Result<PerformanceReport, AppError> {
// let system_resources = self.resource_monitor.get_system_resources().await?; // 模块不存在，暂时注释
// let app_resources = self.resource_monitor.get_application_resources().await?; // 模块不存在，暂时注释
let metrics = self.metrics_collector.get_stats().await?;
let cache_stats = self.cache_manager.get_stats().await?;
Ok(PerformanceReport {
system_resources: Default::default(),
application_resources: Default::default(),
metrics,
cache_stats,
generated_at: SystemTime::now(),
})
}
}
/// 性能报告
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceReport {
pub system_resources: serde_json::Value, // resource_monitor::SystemResourceStatus, // 模块不存在，暂时使用通用类型
pub application_resources: serde_json::Value, // resource_monitor::ApplicationResourceStatus, // 模块不存在，暂时使用通用类型
pub metrics: serde_json::Value,
pub cache_stats: serde_json::Value,
pub generated_at: SystemTime,
}
/// 详细性能报告
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct DetailedPerformanceReport {
pub memory_stats: memory_optimizer::MemoryStats,
pub concurrency_stats: concurrency_optimizer::ConcurrencyStats,
pub cache_stats: cache_manager::CacheStats,
pub metrics: metrics_collector::MetricsSnapshot,
// pub resource_stats: resource_monitor::ResourceStats, // 模块不存在，暂时注释
pub timestamp: chrono::DateTime<chrono::Utc>,
}
/// 性能配置
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct PerformanceConfig {
/// 内存优化配置
pub memory: MemoryConfig,
/// 并发优化配置
pub concurrency: ConcurrencyConfig,
/// 缓存配置
pub cache: CacheConfig,
/// 资源配置
pub resource: ResourceConfig,
/// 监控配置
pub monitoring: MonitoringConfig,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ResourceConfig {
pub max_cpu_usage: f64,
pub max_memory_usage: u64,
pub max_disk_usage: u64,
pub max_network_bandwidth: u64,
pub max_connections: usize,
pub max_file_descriptors: usize,
pub min_instances: usize,
pub max_instances: usize,
pub monitoring_interval: Duration,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct MemoryConfig {
/// 最大内存使用量（字节）
pub max_memory_usage: u64,
/// 内存清理阈值（百分比）
pub cleanup_threshold: f64,
/// 内存池大小
pub pool_size: usize,
/// 启用内存压缩
pub enable_compression: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ConcurrencyConfig {
/// 最大并发任务数
pub max_concurrent_tasks: usize,
/// 任务队列大小
pub task_queue_size: usize,
/// 工作线程数
pub worker_threads: usize,
/// 任务超时时间
pub task_timeout: Duration,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CacheConfig {
/// 缓存大小（字节）
pub cache_size: u64,
/// 文档缓存大小
pub document_cache_size: usize,
/// 结果缓存大小
pub result_cache_size: usize,
/// 元数据缓存大小
pub metadata_cache_size: usize,
/// 缓存TTL
pub ttl: Duration,
/// 文档缓存TTL
pub document_ttl: Duration,
/// 结果缓存TTL
pub result_ttl: Duration,
/// 元数据缓存TTL
pub metadata_ttl: Duration,
/// 清理间隔
pub cleanup_interval: Duration,
/// 启用LRU淘汰
pub enable_lru: bool,
/// 缓存压缩
pub enable_compression: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct MonitoringConfig {
/// 监控间隔
pub interval: Duration,
/// 启用详细监控
pub enable_detailed: bool,
/// 保留历史数据时间
pub retention_period: Duration,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct MetricsConfig {
/// 指标收集间隔
pub collection_interval: Duration,
/// 启用系统指标收集
pub enable_system_metrics: bool,
/// 启用应用指标收集
pub enable_application_metrics: bool,
/// 启用自定义指标
pub enable_custom_metrics: bool,
/// 指标保留时间
pub retention_period: Duration,
/// 聚合窗口大小
pub aggregation_window: Duration,
/// 聚合间隔
pub aggregation_interval: Duration,
/// 启用指标导出
pub enable_export: bool,
/// 导出格式
pub export_format: String,
/// 报告生成间隔
pub report_interval: Duration,
/// 报告间隔
pub reporting_interval: Duration,
}
impl Default for PerformanceConfig {
fn default() -> Self {
Self {
memory: MemoryConfig {
max_memory_usage: 2 * 1024 * 1024 * 1024, // 2GB
cleanup_threshold: 0.8, // 80%
pool_size: 100,
enable_compression: true,
},
concurrency: ConcurrencyConfig {
max_concurrent_tasks: 10,
task_queue_size: 100,
worker_threads: num_cpus::get(),
task_timeout: Duration::from_secs(1800), // 30分钟
},
cache: CacheConfig {
cache_size: 512 * 1024 * 1024, // 512MB
document_cache_size: 1000,
result_cache_size: 500,
metadata_cache_size: 200,
ttl: Duration::from_secs(3600), // 1小时
document_ttl: Duration::from_secs(3600), // 1小时
result_ttl: Duration::from_secs(1800), // 30分钟
metadata_ttl: Duration::from_secs(7200), // 2小时
cleanup_interval: Duration::from_secs(300), // 5分钟
enable_lru: true,
enable_compression: true,
},
resource: ResourceConfig {
max_cpu_usage: 80.0,
max_memory_usage: 8 * 1024 * 1024 * 1024, // 8GB
max_disk_usage: 100 * 1024 * 1024 * 1024, // 100GB
max_network_bandwidth: 1024 * 1024 * 1024, // 1GB/s
max_connections: 1000,
max_file_descriptors: 10000,
min_instances: 1,
max_instances: 10,
monitoring_interval: Duration::from_secs(30),
},
monitoring: MonitoringConfig {
interval: Duration::from_secs(30),
enable_detailed: false,
retention_period: Duration::from_secs(24 * 3600), // 24小时
},
}
}
}
impl Default for MemoryConfig {
fn default() -> Self {
Self {
max_memory_usage: 1024 * 1024 * 1024, // 1GB
cleanup_threshold: 0.8,
pool_size: 100,
enable_compression: true,
}
}
}
impl Default for ConcurrencyConfig {
fn default() -> Self {
Self {
max_concurrent_tasks: 10,
task_queue_size: 1000,
worker_threads: 4,
task_timeout: Duration::from_secs(300),
}
}
}
impl Default for CacheConfig {
fn default() -> Self {
Self {
cache_size: 100 * 1024 * 1024, // 100MB
document_cache_size: 1000,
result_cache_size: 500,
metadata_cache_size: 2000,
ttl: Duration::from_secs(3600), // 1 hour
document_ttl: Duration::from_secs(3600), // 1 hour
result_ttl: Duration::from_secs(1800), // 30 minutes
metadata_ttl: Duration::from_secs(7200), // 2 hours
cleanup_interval: Duration::from_secs(300), // 5 minutes
enable_lru: true,
enable_compression: false,
}
}
}
impl Default for MonitoringConfig {
fn default() -> Self {
Self {
interval: Duration::from_secs(30),
enable_detailed: false,
retention_period: Duration::from_secs(86400), // 24 hours
}
}
}
impl Default for ResourceConfig {
fn default() -> Self {
Self {
max_cpu_usage: 80.0,
max_memory_usage: 1024 * 1024 * 1024, // 1GB
max_disk_usage: 10 * 1024 * 1024 * 1024, // 10GB
max_network_bandwidth: 100 * 1024 * 1024, // 100MB/s
max_connections: 1000,
max_file_descriptors: 1024,
min_instances: 1,
max_instances: 10,
monitoring_interval: Duration::from_secs(60),
}
}
}
impl Default for MetricsConfig {
fn default() -> Self {
Self {
collection_interval: Duration::from_secs(30),
enable_system_metrics: true,
enable_application_metrics: true,
enable_custom_metrics: false,
retention_period: Duration::from_secs(3600 * 24), // 24小时
aggregation_window: Duration::from_secs(300), // 5分钟
aggregation_interval: Duration::from_secs(60), // 1分钟
enable_export: false,
export_format: "json".to_string(),
report_interval: Duration::from_secs(300), // 5分钟
reporting_interval: Duration::from_secs(300), // 5分钟
}
}
}
/// 性能优化特征
#[async_trait::async_trait]
pub trait PerformanceOptimizable {
/// 执行性能优化
async fn optimize(&self) -> Result<(), AppError>;
/// 获取性能统计
async fn get_stats(&self) -> Result<serde_json::Value, AppError>;
/// 重置性能统计
async fn reset_stats(&self) -> Result<(), AppError>;
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_performance_optimizer_creation() {
// 使用默认配置进行测试
let config = crate::config::AppConfig::load_base_config().unwrap();
let optimizer = PerformanceOptimizer::new(&config).await;
assert!(optimizer.is_ok());
}
#[tokio::test]
async fn test_performance_config_default() {
let config = PerformanceConfig::default();
assert!(config.memory.max_memory_usage > 0);
assert!(config.concurrency.max_concurrent_tasks > 0);
assert!(config.cache.cache_size > 0);
}
}