提交qiming-mcp-proxy

qiming-mcp-proxy/document-parser/src/services/document_task_processor.rs

@@ -0,0 +1,70 @@
+use std::sync::Arc;
+
+use log::info;
+
+use crate::error::AppError;
+use crate::models::SourceType;
+
+use super::{DocumentService, TaskProcessor, TaskService};
+
+/// 文档任务处理器
+///
+/// 基于任务ID从任务服务获取任务详情，根据 `source_type` 分派到对应的解析逻辑。
+pub struct DocumentTaskProcessor {
+    document_service: Arc<DocumentService>,
+    task_service: Arc<TaskService>,
+}
+
+impl DocumentTaskProcessor {
+    /// 创建新的文档任务处理器
+    pub fn new(document_service: Arc<DocumentService>, task_service: Arc<TaskService>) -> Self {
+        Self {
+            document_service,
+            task_service,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TaskProcessor for DocumentTaskProcessor {
+    async fn process_task(&self, task_id: &str) -> Result<(), AppError> {
+        // 获取任务
+        let task = self
+            .task_service
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        match task.source_type {
+            SourceType::Upload => {
+                let file_path = task
+                    .source_path
+                    .ok_or_else(|| AppError::Task("上传任务缺少文件路径".to_string()))?;
+
+                // 使用文件路径解析
+                // 由 DocumentService 负责更新状态、进度、结果等
+                let _ = self
+                    .document_service
+                    .parse_document(&task.id, &file_path)
+                    .await
+                    .map_err(|e| AppError::Processing(e.to_string()))?;
+                Ok(())
+            }
+            SourceType::Url => {
+                // 优先使用新的 source_url 字段，兼容旧数据回退到 source_path
+                let url = task
+                    .source_url
+                    .or(task.source_path.clone())
+                    .ok_or_else(|| AppError::Task("URL 任务缺少下载地址".to_string()))?;
+
+                info!("Start parsing URL task: {url}");
+                let _ = self
+                    .document_service
+                    .parse_document_from_url(&task.id, &url)
+                    .await
+                    .map_err(|e| AppError::Processing(e.to_string()))?;
+                Ok(())
+            }
+        }
+    }
+}

qiming-mcp-proxy/document-parser/src/services/image_processor.rs

@@ -0,0 +1,409 @@
+use crate::error::AppError;
+use crate::services::OssService;
+use anyhow::{Context, Result};
+use regex::Regex;
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::Arc;
+use tokio::fs;
+use tokio::sync::Mutex;
+use tracing::{debug, info, instrument, warn};
+
+/// 图片处理服务配置
+#[derive(Debug, Clone)]
+pub struct ImageProcessorConfig {
+    /// 是否启用图片上传
+    pub enable_upload: bool,
+    /// 支持的图片格式
+    pub supported_formats: Vec<String>,
+    /// 最大图片大小（字节）
+    pub max_image_size: usize,
+    /// 图片存储路径前缀
+    pub image_path_prefix: String,
+}
+
+impl Default for ImageProcessorConfig {
+    fn default() -> Self {
+        Self {
+            enable_upload: true,
+            supported_formats: vec![
+                "jpg".to_string(),
+                "jpeg".to_string(),
+                "png".to_string(),
+                "gif".to_string(),
+                "webp".to_string(),
+                "bmp".to_string(),
+            ],
+            max_image_size: 10 * 1024 * 1024, // 10MB
+            image_path_prefix: "images/".to_string(),
+        }
+    }
+}
+
+/// 图片上传结果
+#[derive(Debug, Clone)]
+pub struct ImageUploadResult {
+    /// 原始图片路径
+    pub original_path: String,
+    /// OSS图片URL
+    pub oss_url: String,
+    /// 图片文件名
+    pub filename: String,
+    /// 上传状态
+    pub success: bool,
+    /// 错误信息（如果有）
+    pub error_message: Option<String>,
+}
+
+/// 图片处理服务
+pub struct ImageProcessor {
+    config: ImageProcessorConfig,
+    oss_service: Option<Arc<OssService>>,
+    upload_cache: Arc<Mutex<HashMap<String, String>>>, // 本地路径 -> OSS URL 映射
+}
+
+impl ImageProcessor {
+    /// 创建新的图片处理服务
+    pub fn new(config: ImageProcessorConfig, oss_service: Option<Arc<OssService>>) -> Self {
+        Self {
+            config,
+            oss_service,
+            upload_cache: Arc::new(Mutex::new(HashMap::new())),
+        }
+    }
+
+    /// 批量上传图片到OSS
+    #[instrument(skip(self, image_paths))]
+    pub async fn batch_upload_images(
+        &self,
+        image_paths: Vec<String>,
+    ) -> Result<Vec<ImageUploadResult>> {
+        let mut results = Vec::new();
+
+        for image_path in image_paths {
+            let result = self.upload_single_image(&image_path).await;
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// 上传单个图片到OSS
+    #[instrument(skip(self))]
+    async fn upload_single_image(&self, image_path: &str) -> ImageUploadResult {
+        // 检查缓存
+        if let Some(cached_url) = self.upload_cache.lock().await.get(image_path) {
+            return ImageUploadResult {
+                original_path: image_path.to_string(),
+                oss_url: cached_url.clone(),
+                filename: self.extract_filename(image_path),
+                success: true,
+                error_message: None,
+            };
+        }
+
+        // 检查文件是否存在
+        if !Path::new(image_path).exists() {
+            return ImageUploadResult {
+                original_path: image_path.to_string(),
+                oss_url: String::new(),
+                filename: self.extract_filename(image_path),
+                success: false,
+                error_message: Some("Image file not found".to_string()),
+            };
+        }
+
+        // 检查文件大小
+        if let Ok(metadata) = fs::metadata(image_path).await {
+            if metadata.len() > self.config.max_image_size as u64 {
+                return ImageUploadResult {
+                    original_path: image_path.to_string(),
+                    oss_url: String::new(),
+                    filename: self.extract_filename(image_path),
+                    success: false,
+                    error_message: Some("Image file too large".to_string()),
+                };
+            }
+        }
+
+        // 检查文件格式
+        if let Some(extension) = Path::new(image_path).extension() {
+            let ext = extension.to_string_lossy().to_lowercase();
+            if !self.config.supported_formats.contains(&ext) {
+                return ImageUploadResult {
+                    original_path: image_path.to_string(),
+                    oss_url: String::new(),
+                    filename: self.extract_filename(image_path),
+                    success: false,
+                    error_message: Some(format!("Unsupported image format: {ext}")),
+                };
+            }
+        }
+
+        // 上传到OSS
+        match self.upload_to_oss(image_path).await {
+            Ok(oss_url) => {
+                // 缓存结果
+                self.upload_cache
+                    .lock()
+                    .await
+                    .insert(image_path.to_string(), oss_url.clone());
+
+                ImageUploadResult {
+                    original_path: image_path.to_string(),
+                    oss_url,
+                    filename: self.extract_filename(image_path),
+                    success: true,
+                    error_message: None,
+                }
+            }
+            Err(e) => ImageUploadResult {
+                original_path: image_path.to_string(),
+                oss_url: String::new(),
+                filename: self.extract_filename(image_path),
+                success: false,
+                error_message: Some(e.to_string()),
+            },
+        }
+    }
+
+    /// 上传图片到OSS（复用现有OSS服务）
+    #[instrument(skip(self))]
+    async fn upload_to_oss(&self, image_path: &str) -> Result<String> {
+        let oss_service = self
+            .oss_service
+            .as_ref()
+            .ok_or_else(|| AppError::Oss("OSS service not initialized".to_string()))?;
+
+        // 使用现有的OSS服务上传图片
+        let image_info = oss_service
+            .upload_image(image_path)
+            .await
+            .with_context(|| format!("Failed to upload image to OSS: {image_path}"))?;
+
+        info!(
+            "Successfully uploaded image to OSS: {} -> {}",
+            image_path, image_info.oss_url
+        );
+        Ok(image_info.oss_url)
+    }
+
+    /// 提取文件名
+    fn extract_filename(&self, path: &str) -> String {
+        Path::new(path)
+            .file_name()
+            .and_then(|name| name.to_str())
+            .unwrap_or("unknown")
+            .to_string()
+    }
+
+    /// 替换Markdown中的图片路径
+    #[instrument(skip(self, markdown_content))]
+    pub async fn replace_markdown_images(&self, markdown_content: &str) -> Result<String> {
+        // 正则表达式匹配Markdown图片语法
+        let image_regex = Regex::new(r"!\[([^\]]*)\]\(([^)]+)\)").unwrap();
+        let mut result = markdown_content.to_string();
+        let mut replacements = Vec::new();
+
+        // 收集所有需要替换的图片
+        for cap in image_regex.captures_iter(markdown_content) {
+            let _alt_text = &cap[1];
+            let image_path = &cap[2];
+
+            // 跳过已经是URL的图片
+            if image_path.starts_with("http://") || image_path.starts_with("https://") {
+                continue;
+            }
+
+            // 检查缓存
+            if let Some(oss_url) = self.upload_cache.lock().await.get(image_path) {
+                replacements.push((image_path.to_string(), oss_url.clone()));
+            } else {
+                // 尝试上传图片
+                let upload_result = self.upload_single_image(image_path).await;
+                if upload_result.success {
+                    replacements.push((image_path.to_string(), upload_result.oss_url));
+                } else {
+                    warn!(
+                        "Failed to upload image: {} - {}",
+                        image_path,
+                        upload_result.error_message.unwrap_or_default()
+                    );
+                }
+            }
+        }
+
+        // 执行替换
+        for (old_path, new_url) in replacements {
+            result = result.replace(&old_path, &new_url);
+        }
+
+        Ok(result)
+    }
+
+    /// 获取上传缓存统计
+    pub async fn get_cache_stats(&self) -> (usize, usize) {
+        let cache = self.upload_cache.lock().await;
+        let total = cache.len();
+        let successful = cache.values().filter(|url| !url.is_empty()).count();
+        (total, successful)
+    }
+
+    /// 清空上传缓存
+    pub async fn clear_cache(&self) {
+        self.upload_cache.lock().await.clear();
+    }
+
+    /// 从Markdown内容中提取所有图片路径
+    pub fn extract_image_paths(markdown_content: &str) -> Vec<String> {
+        let image_regex = Regex::new(r"!\[([^\]]*)\]\(([^)]+)\)").unwrap();
+        let mut image_paths = Vec::new();
+
+        for cap in image_regex.captures_iter(markdown_content) {
+            let image_path = &cap[2];
+
+            // 跳过已经是URL的图片
+            if !image_path.starts_with("http://") && !image_path.starts_with("https://") {
+                image_paths.push(image_path.to_string());
+            }
+        }
+
+        debug!("Extracted image path: {:?}", image_paths);
+
+        image_paths
+    }
+
+    /// 验证图片文件
+    pub async fn validate_image_file(&self, image_path: &str) -> Result<bool> {
+        let path = Path::new(image_path);
+
+        // 检查文件是否存在
+        if !path.exists() {
+            return Ok(false);
+        }
+
+        // 检查文件大小
+        if let Ok(metadata) = fs::metadata(image_path).await {
+            if metadata.len() > self.config.max_image_size as u64 {
+                return Ok(false);
+            }
+        }
+
+        // 检查文件格式
+        if let Some(extension) = path.extension() {
+            let ext = extension.to_string_lossy().to_lowercase();
+            if !self.config.supported_formats.contains(&ext) {
+                return Ok(false);
+            }
+        }
+
+        Ok(true)
+    }
+
+    /// 批量处理Markdown文件中的图片
+    #[instrument(skip(self, markdown_files))]
+    pub async fn process_markdown_files(
+        &self,
+        markdown_files: Vec<(String, String)>,
+    ) -> Result<Vec<(String, String)>> {
+        let mut results = Vec::new();
+
+        for (file_path, content) in markdown_files {
+            match self.replace_markdown_images(&content).await {
+                Ok(processed_content) => {
+                    results.push((file_path, processed_content));
+                }
+                Err(e) => {
+                    warn!("Failed to process markdown file {}: {}", file_path, e);
+                    // 如果处理失败，保留原内容
+                    results.push((file_path, content));
+                }
+            }
+        }
+
+        Ok(results)
+    }
+
+    /// 从目录中提取所有图片路径
+    pub async fn extract_images_from_directory(&self, directory_path: &str) -> Result<Vec<String>> {
+        let mut image_paths = Vec::new();
+
+        let mut entries = fs::read_dir(directory_path)
+            .await
+            .with_context(|| format!("Failed to read directory: {directory_path}"))?;
+
+        while let Some(entry) = entries.next_entry().await? {
+            let path = entry.path();
+
+            if path.is_file() {
+                if let Some(extension) = path.extension() {
+                    let ext = extension.to_string_lossy().to_lowercase();
+                    if self.config.supported_formats.contains(&ext) {
+                        if let Some(path_str) = path.to_str() {
+                            image_paths.push(path_str.to_string());
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(image_paths)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::tempdir;
+
+    #[tokio::test]
+    async fn test_extract_image_paths() {
+        let markdown = r#"
+        # Test Document
+
+        ![Image 1](images/test1.jpg)
+        ![Image 2](images/test2.png)
+        ![External Image](https://example.com/image.jpg)
+        ![Local Image](./local/image.gif)
+        "#;
+
+        let paths = ImageProcessor::extract_image_paths(markdown);
+        assert_eq!(paths.len(), 3);
+        assert!(paths.contains(&"images/test1.jpg".to_string()));
+        assert!(paths.contains(&"images/test2.png".to_string()));
+        assert!(paths.contains(&"./local/image.gif".to_string()));
+        assert!(!paths.contains(&"https://example.com/image.jpg".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_extract_filename() {
+        let config = ImageProcessorConfig::default();
+        let processor = ImageProcessor::new(config, None);
+
+        assert_eq!(processor.extract_filename("path/to/image.jpg"), "image.jpg");
+        assert_eq!(processor.extract_filename("image.png"), "image.png");
+        assert_eq!(processor.extract_filename(""), "unknown");
+    }
+
+    #[tokio::test]
+    async fn test_validate_image_file() {
+        let config = ImageProcessorConfig::default();
+        let processor = ImageProcessor::new(config, None);
+
+        // 创建临时目录和测试文件
+        let temp_dir = tempdir().unwrap();
+        let test_file_path = temp_dir.path().join("test.txt");
+        fs::write(&test_file_path, "not an image").unwrap();
+
+        // 测试无效文件
+        let result = processor
+            .validate_image_file(test_file_path.to_str().unwrap())
+            .await
+            .unwrap();
+        assert!(!result);
+
+        // 清理
+        temp_dir.close().unwrap();
+    }
+}

qiming-mcp-proxy/document-parser/src/services/mod.rs

@@ -0,0 +1,17 @@
+// 服务模块
+// TODO: 实现具体的服务
+pub mod document_service;
+pub mod document_task_processor;
+pub mod image_processor;
+pub mod oss_service;
+pub mod storage_service;
+pub mod task_queue_service;
+pub mod task_service;
+
+pub use document_service::{DocumentService, DocumentServiceConfig};
+pub use document_task_processor::DocumentTaskProcessor;
+pub use image_processor::{ImageProcessor, ImageProcessorConfig};
+pub use oss_service::OssService;
+pub use storage_service::*;
+pub use task_queue_service::*;
+pub use task_service::{TaskService, TaskStats};

qiming-mcp-proxy/document-parser/src/services/oss_service.rs

@@ -0,0 +1,948 @@
+use crate::config::OssConfig;
+use crate::error::AppError;
+use crate::models::ImageInfo;
+use aliyun_oss_rust_sdk::oss::OSS;
+use aliyun_oss_rust_sdk::request::RequestBuilder;
+use aliyun_oss_rust_sdk::url::UrlApi;
+use futures::stream::{self, StreamExt};
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Semaphore;
+use tokio::time::sleep;
+use tracing::{error, info, instrument, warn};
+
+/// 批量操作进度回调
+pub type ProgressCallback = Arc<dyn Fn(usize, usize) + Send + Sync>;
+
+/// 批量上传结果
+#[derive(Debug, Clone)]
+pub struct BatchUploadResult {
+    pub successful: Vec<BatchUploadItem>,
+    pub failed: Vec<BatchUploadError>,
+    pub total_processed: usize,
+    pub total_bytes: u64,
+    pub duration: Duration,
+}
+
+/// 批量上传成功项
+#[derive(Debug, Clone)]
+pub struct BatchUploadItem {
+    pub local_path: String,
+    pub object_key: String,
+    pub url: String,
+    pub size: u64,
+    pub content_type: String,
+}
+
+/// 批量上传失败项
+#[derive(Debug, Clone)]
+pub struct BatchUploadError {
+    pub local_path: String,
+    pub object_key: String,
+    pub error: String,
+    pub retry_count: u32,
+}
+
+/// OSS服务配置
+#[derive(Debug, Clone)]
+pub struct OssServiceConfig {
+    pub max_concurrent_uploads: usize,
+    pub retry_attempts: u32,
+    pub retry_delay_ms: u64,
+    pub upload_timeout_secs: u64,
+    pub chunk_size: usize,
+}
+
+impl Default for OssServiceConfig {
+    fn default() -> Self {
+        Self {
+            max_concurrent_uploads: 10,
+            retry_attempts: 3,
+            retry_delay_ms: 1000,
+            upload_timeout_secs: 300,
+            chunk_size: 8 * 1024 * 1024, // 8MB
+        }
+    }
+}
+
+/// OSS服务
+#[derive(Debug)]
+pub struct OssService {
+    client: OSS,
+    bucket: String,
+    endpoint: String,
+    base_url: String,
+    config: OssServiceConfig,
+    semaphore: Arc<Semaphore>,
+}
+
+impl OssService {
+    /// 创建新的OSS服务实例
+    #[instrument(skip(oss_config), fields(public_bucket = %oss_config.public_bucket, endpoint = %oss_config.endpoint))]
+    pub async fn new(oss_config: &OssConfig) -> Result<Self, AppError> {
+        Self::new_with_config(oss_config, OssServiceConfig::default()).await
+    }
+
+    /// 使用自定义配置创建OSS服务实例
+    #[instrument(skip(oss_config, service_config), fields(public_bucket = %oss_config.public_bucket, endpoint = %oss_config.endpoint))]
+    pub async fn new_with_config(
+        oss_config: &OssConfig,
+        service_config: OssServiceConfig,
+    ) -> Result<Self, AppError> {
+        info!("Initialize OSS service");
+
+        // 检查OSS配置是否完整（环境变量是否已设置）
+        if oss_config.access_key_id.is_empty() || oss_config.access_key_secret.is_empty() {
+            warn!(
+                "OSS environment variables are not configured and OSS service initialization is skipped."
+            );
+            return Err(AppError::Config(
+                "OSS环境变量未配置，请设置OSS_ACCESS_KEY_ID和OSS_ACCESS_KEY_SECRET".to_string(),
+            ));
+        }
+
+        //这里使用公网的公有 bucket
+        let bucket = oss_config.public_bucket.clone();
+
+        // 创建OSS客户端 - 配置文件中endpoint已不包含协议前缀
+        let client = OSS::new(
+            &oss_config.access_key_id,
+            &oss_config.access_key_secret,
+            &oss_config.endpoint,
+            &bucket,
+        );
+
+        // 构建base_url
+        let base_url = format!("https://{}.{}", bucket, oss_config.endpoint);
+
+        let service = Self {
+            client,
+            bucket,
+            endpoint: oss_config.endpoint.clone(),
+            base_url,
+            config: service_config.clone(),
+            semaphore: Arc::new(Semaphore::new(service_config.max_concurrent_uploads)),
+        };
+
+        // 验证连接
+        service.validate_connection().await?;
+
+        info!("OSS service initialization successful");
+        Ok(service)
+    }
+
+    /// 验证OSS连接
+    #[instrument(skip(self))]
+    async fn validate_connection(&self) -> Result<(), AppError> {
+        info!("Verify OSS connection");
+
+        // 使用上传临时文件的方式验证连接
+        let test_key = format!("health-check-{}", chrono::Utc::now().timestamp_millis());
+        let test_content = b"OSS connection test";
+
+        match self
+            .upload_content(test_content, &test_key, Some("text/plain"))
+            .await
+        {
+            Ok(_) => {
+                info!("OSS connection verification successful");
+                // 尝试删除测试文件（忽略删除失败）
+                let _ = self.delete_object(&test_key).await;
+                Ok(())
+            }
+            Err(e) => {
+                error!("OSS connection verification failed: {}", e);
+                Err(AppError::Oss(format!(
+                    "无法连接到OSS存储桶 {}: {}",
+                    self.bucket, e
+                )))
+            }
+        }
+    }
+
+    /// 上传文件到OSS
+    #[instrument(skip(self), fields(file_path, object_key))]
+    pub async fn upload_file(&self, file_path: &str, object_key: &str) -> Result<String, AppError> {
+        info!(
+            "Start uploading files to OSS: {} -> {}",
+            file_path, object_key
+        );
+
+        let _permit = self
+            .semaphore
+            .acquire()
+            .await
+            .map_err(|e| AppError::Oss(format!("获取上传许可失败: {e}")))?;
+
+        let content_type = self.detect_mime_type(file_path)?;
+
+        // 检查文件大小决定上传方式
+        let metadata = std::fs::metadata(file_path)
+            .map_err(|e| AppError::Oss(format!("读取文件元数据失败: {e}")))?;
+
+        if metadata.len() > self.config.chunk_size as u64 {
+            self.upload_large_file(file_path, object_key, &content_type)
+                .await
+        } else {
+            self.upload_small_file(file_path, object_key, &content_type)
+                .await
+        }
+    }
+
+    /// 上传小文件
+    #[instrument(skip(self))]
+    async fn upload_small_file(
+        &self,
+        file_path: &str,
+        object_key: &str,
+        content_type: &str,
+    ) -> Result<String, AppError> {
+        let builder = RequestBuilder::new().with_content_type(content_type);
+
+        match self
+            .client
+            .put_object_from_file(object_key, file_path, builder)
+            .await
+        {
+            Ok(_) => Ok(format!("{}/{}", self.base_url, object_key)),
+            Err(e) => Err(AppError::Oss(format!("上传文件失败: {e}"))),
+        }
+    }
+
+    /// 上传大文件（分片上传）
+    #[instrument(skip(self))]
+    async fn upload_large_file(
+        &self,
+        file_path: &str,
+        object_key: &str,
+        content_type: &str,
+    ) -> Result<String, AppError> {
+        // 对于大文件，我们仍然使用简单上传，因为aliyun-oss-rust-sdk的分片上传API可能不同
+        // 如果需要分片上传，需要查看具体的API文档
+        warn!("Large file upload, currently using the simple upload method");
+
+        let builder = RequestBuilder::new().with_content_type(content_type);
+
+        match self
+            .client
+            .put_object_from_file(object_key, file_path, builder)
+            .await
+        {
+            Ok(_) => Ok(format!("{}/{}", self.base_url, object_key)),
+            Err(e) => Err(AppError::Oss(format!("上传大文件失败: {e}"))),
+        }
+    }
+
+    /// 上传内容到OSS
+    #[instrument(skip(self, content), fields(object_key, content_size = content.len()))]
+    pub async fn upload_content(
+        &self,
+        content: &[u8],
+        object_key: &str,
+        content_type: Option<&str>,
+    ) -> Result<String, AppError> {
+        self.upload_content_with_retry(content, object_key, content_type)
+            .await
+    }
+
+    /// 上传markdown内容到OSS，返回(URL, object_key)
+    #[instrument(skip(self, content), fields(task_id, content_size = content.len()))]
+    pub async fn upload_markdown(
+        &self,
+        task_id: &str,
+        content: &[u8],
+        original_filename: Option<&str>,
+    ) -> Result<(String, String), AppError> {
+        // 生成唯一的object key
+        let object_key = self.generate_markdown_object_key(task_id, original_filename);
+
+        // 上传内容
+        let url = self
+            .upload_content(content, &object_key, Some("text/markdown; charset=utf-8"))
+            .await?;
+
+        Ok((url, object_key))
+    }
+
+    /// 上传用户文件到OSS，返回(URL, object_key, download_url)
+    #[instrument(skip(self), fields(file_path))]
+    pub async fn upload_user_file(
+        &self,
+        file_path: &str,
+        original_filename: Option<&str>,
+    ) -> Result<(String, String, String), AppError> {
+        // 生成唯一的object key
+        let object_key = self.generate_user_file_object_key(original_filename);
+
+        // 上传文件
+        let url = self.upload_file(file_path, &object_key).await?;
+
+        // 生成4小时有效期的下载链接
+        let download_url = self
+            .generate_download_url(&object_key, Some(Duration::from_secs(4 * 3600)))
+            .await?;
+
+        Ok((url, object_key, download_url))
+    }
+
+    /// 根据object_key生成下载链接（如果文件存在）
+    #[instrument(skip(self), fields(object_key))]
+    pub async fn get_download_url_for_file(&self, object_key: &str) -> Result<String, AppError> {
+        // 先检查文件是否存在
+        if !self.file_exists(object_key).await? {
+            return Err(AppError::Oss(format!("文件不存在: {object_key}")));
+        }
+
+        // 生成4小时有效期的下载链接
+        self.generate_download_url(object_key, Some(Duration::from_secs(4 * 3600)))
+            .await
+    }
+
+    /// 根据object_key和指定bucket生成下载链接（如果文件存在）
+    /// 注意：这个方法仅适用于同一OSS账户下的不同bucket
+    #[instrument(skip(self), fields(object_key, bucket))]
+    pub async fn get_download_url_for_file_with_bucket(
+        &self,
+        object_key: &str,
+        bucket: &str,
+    ) -> Result<String, AppError> {
+        // 如果指定的bucket与当前bucket相同，直接使用现有方法
+        if bucket == self.bucket {
+            return self.get_download_url_for_file(object_key).await;
+        }
+
+        // 对于不同的bucket，我们构建一个临时的下载URL
+        // 注意：这假设所有bucket都在同一endpoint下，且访问权限相同
+
+        // 构建临时URL (这是一个简化版本，实际环境中可能需要更复杂的签名逻辑)
+        let temp_url = format!(
+            "https://{}.{}/{}",
+            bucket,
+            self.endpoint.trim_start_matches("https://"),
+            object_key
+        );
+
+        // 由于我们无法直接验证不同bucket中的文件存在性，
+        // 这里返回URL但在实际使用时可能需要额外的验证
+        warn!(
+            "Use different buckets to generate download URLs, and the file existence cannot be verified in advance: bucket={}, object_key={}",
+            bucket, object_key
+        );
+
+        Ok(temp_url)
+    }
+
+    /// 生成markdown文件的OSS object key
+    fn generate_markdown_object_key(
+        &self,
+        task_id: &str,
+        original_filename: Option<&str>,
+    ) -> String {
+        let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S").to_string();
+        let uid = uuid::Uuid::new_v4().to_string()[..8].to_string(); // 取前8位作为短UID
+
+        // 如果有原始文件名，使用原始文件名和后缀
+        let filename = if let Some(original) = original_filename {
+            let clean_name = self.sanitize_filename(original);
+
+            // 分离文件名和扩展名
+            if let Some(dot_pos) = clean_name.rfind('.') {
+                let name_part = &clean_name[..dot_pos];
+                let ext_part = &clean_name[dot_pos..]; // 包含点号
+                format!("{name_part}_{timestamp}_{uid}{ext_part}")
+            } else {
+                // 没有扩展名，默认加上.md
+                format!("{clean_name}_{timestamp}_{uid}. md")
+            }
+        } else {
+            // 没有原始文件名，生成默认名称
+            format!("document_{timestamp}_{uid}.md")
+        };
+
+        format!("markdown/{task_id}/{filename}")
+    }
+
+    /// 生成用户文件的OSS object key
+    fn generate_user_file_object_key(&self, original_filename: Option<&str>) -> String {
+        let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S").to_string();
+        let uid = uuid::Uuid::new_v4().to_string()[..8].to_string(); // 取前8位作为短UID
+
+        // 如果有原始文件名，使用原始文件名和后缀
+        let filename = if let Some(original) = original_filename {
+            let clean_name = self.sanitize_filename(original);
+
+            // 分离文件名和扩展名
+            if let Some(dot_pos) = clean_name.rfind('.') {
+                let name_part = &clean_name[..dot_pos];
+                let ext_part = &clean_name[dot_pos..]; // 包含点号
+                format!("{name_part}_{timestamp}_{uid}{ext_part}")
+            } else {
+                // 没有扩展名，保持原名
+                format!("{clean_name}_{timestamp}_{uid}.")
+            }
+        } else {
+            // 没有原始文件名，生成默认名称
+            format!("file_{timestamp}_{uid}")
+        };
+
+        format!("uploads/{filename}")
+    }
+
+    /// 清理文件名，移除特殊字符
+    fn sanitize_filename(&self, filename: &str) -> String {
+        filename
+            .chars()
+            .map(|c| {
+                if c.is_alphanumeric() || c == '.' || c == '-' || c == '_' {
+                    c
+                } else {
+                    '_'
+                }
+            })
+            .collect()
+    }
+
+    /// 带重试的内容上传
+    #[instrument(skip(self, content))]
+    async fn upload_content_with_retry(
+        &self,
+        content: &[u8],
+        object_key: &str,
+        content_type: Option<&str>,
+    ) -> Result<String, AppError> {
+        let mut last_error = None;
+
+        for attempt in 1..=self.config.retry_attempts {
+            match self.do_upload(content, object_key, content_type).await {
+                Ok(url) => {
+                    if attempt > 1 {
+                        info!("The {}th retry upload was successful.", attempt);
+                    }
+                    return Ok(url);
+                }
+                Err(e) => {
+                    last_error = Some(e);
+                    if attempt < self.config.retry_attempts {
+                        warn!(
+                            "The {} upload failed, try again after {}ms",
+                            attempt, self.config.retry_delay_ms
+                        );
+                        sleep(Duration::from_millis(self.config.retry_delay_ms)).await;
+                    } else {
+                        error!("Upload failed, maximum number of retries reached");
+                    }
+                }
+            }
+        }
+
+        Err(last_error.unwrap_or_else(|| AppError::Oss("上传失败，未知错误".to_string())))
+    }
+
+    /// 执行上传
+    #[instrument(skip(self, content))]
+    async fn do_upload(
+        &self,
+        content: &[u8],
+        object_key: &str,
+        content_type: Option<&str>,
+    ) -> Result<String, AppError> {
+        let _permit = self
+            .semaphore
+            .acquire()
+            .await
+            .map_err(|e| AppError::Oss(format!("获取上传许可失败: {e}")))?;
+
+        let mut builder = RequestBuilder::new();
+        if let Some(ct) = content_type {
+            builder = builder.with_content_type(ct);
+        }
+
+        // 将内容写入临时文件，然后使用put_object_from_file上传
+        let temp_file = tempfile::NamedTempFile::new()
+            .map_err(|e| AppError::Oss(format!("创建临时文件失败: {e}")))?;
+
+        std::fs::write(temp_file.path(), content)
+            .map_err(|e| AppError::Oss(format!("写入临时文件失败: {e}")))?;
+
+        match self
+            .client
+            .put_object_from_file(object_key, temp_file.path().to_str().unwrap(), builder)
+            .await
+        {
+            Ok(_) => Ok(format!("{}/{}", self.base_url, object_key)),
+            Err(e) => Err(AppError::Oss(format!("上传内容失败: {e}"))),
+        }
+    }
+
+    /// 上传图片
+    #[instrument(skip(self), fields(image_path))]
+    pub async fn upload_image(&self, image_path: &str) -> Result<ImageInfo, AppError> {
+        let path = Path::new(image_path);
+        let file_name = path
+            .file_name()
+            .and_then(|n| n.to_str())
+            .ok_or_else(|| AppError::Oss("无效的文件名".to_string()))?;
+
+        let object_key = format!("images/{file_name}");
+        let url = self.upload_file(image_path, &object_key).await?;
+
+        let metadata = std::fs::metadata(image_path)
+            .map_err(|e| AppError::Oss(format!("读取图片元数据失败: {e}")))?;
+
+        Ok(ImageInfo::with_full_info(
+            image_path.to_string(),
+            file_name.to_string(),
+            object_key,
+            url,
+            metadata.len(),
+            self.detect_mime_type(image_path)?,
+        ))
+    }
+
+    /// 批量上传图片
+    #[instrument(skip(self, image_paths), fields(count = image_paths.len()))]
+    pub async fn upload_images(&self, image_paths: &[String]) -> Result<Vec<ImageInfo>, AppError> {
+        let result = self.upload_images_with_progress(image_paths, None).await?;
+        Ok(result
+            .successful
+            .into_iter()
+            .map(|item| {
+                let original_filename = Path::new(&item.local_path)
+                    .file_name()
+                    .and_then(|n| n.to_str())
+                    .unwrap_or("unknown")
+                    .to_string();
+
+                ImageInfo::with_full_info(
+                    item.local_path,
+                    original_filename,
+                    item.object_key,
+                    item.url,
+                    item.size,
+                    item.content_type,
+                )
+            })
+            .collect())
+    }
+
+    /// 带进度的批量上传图片
+    #[instrument(skip(self, image_paths, progress_callback), fields(count = image_paths.len()))]
+    pub async fn upload_images_with_progress(
+        &self,
+        image_paths: &[String],
+        progress_callback: Option<ProgressCallback>,
+    ) -> Result<BatchUploadResult, AppError> {
+        let total_count = image_paths.len();
+
+        info!("Start batch uploading {} pictures", total_count);
+
+        let files: Vec<(String, String)> = image_paths
+            .iter()
+            .map(|path| {
+                let file_name = Path::new(path)
+                    .file_name()
+                    .and_then(|n| n.to_str())
+                    .unwrap_or("unknown");
+                let object_key = format!("images/{file_name}");
+                (path.clone(), object_key)
+            })
+            .collect();
+
+        self.upload_batch(files, progress_callback).await
+    }
+
+    /// 批量上传文件
+    #[instrument(skip(self, files, progress_callback), fields(count = files.len()))]
+    pub async fn upload_batch<P: AsRef<Path> + Send + Sync>(
+        &self,
+        files: Vec<(P, String)>, // (local_path, object_key)
+        progress_callback: Option<ProgressCallback>,
+    ) -> Result<BatchUploadResult, AppError> {
+        let start_time = std::time::Instant::now();
+        let total_count = files.len();
+        let mut successful = Vec::new();
+        let mut failed = Vec::new();
+        let mut total_bytes = 0u64;
+
+        info!("Start batch uploading {} files", total_count);
+
+        // 使用流处理来控制并发
+        let mut stream = stream::iter(files.into_iter().enumerate())
+            .map(|(index, (local_path, object_key))| {
+                let local_path_str = local_path.as_ref().to_string_lossy().to_string();
+                async move {
+                    let result = self.upload_file(&local_path_str, &object_key).await;
+                    (index, local_path_str, object_key, result)
+                }
+            })
+            .buffer_unordered(self.config.max_concurrent_uploads);
+
+        let mut processed = 0;
+
+        while let Some((_index, local_path, object_key, result)) = stream.next().await {
+            processed += 1;
+
+            match result {
+                Ok(url) => {
+                    let metadata = std::fs::metadata(&local_path).unwrap_or_else(|_| {
+                        std::fs::metadata("/dev/null").unwrap_or_else(|_| {
+                            // 创建一个默认的元数据结构
+                            std::fs::metadata(std::env::current_dir().unwrap()).unwrap()
+                        })
+                    });
+                    let size = metadata.len();
+                    let content_type = self
+                        .detect_mime_type(&local_path)
+                        .unwrap_or_else(|_| "application/octet-stream".to_string());
+
+                    total_bytes += size;
+                    successful.push(BatchUploadItem {
+                        local_path,
+                        object_key,
+                        url,
+                        size,
+                        content_type,
+                    });
+                }
+                Err(e) => {
+                    failed.push(BatchUploadError {
+                        local_path,
+                        object_key,
+                        error: e.to_string(),
+                        retry_count: 0,
+                    });
+                }
+            }
+
+            // 调用进度回调
+            if let Some(ref callback) = progress_callback {
+                callback(processed, total_count);
+            }
+
+            if processed % 10 == 0 {
+                info!("{}/{} files processed", processed, total_count);
+            }
+        }
+
+        let duration = start_time.elapsed();
+
+        info!(
+            "Batch upload completed: success {}, failure {}, total size {} bytes, time consumption {:?}",
+            successful.len(),
+            failed.len(),
+            total_bytes,
+            duration
+        );
+
+        Ok(BatchUploadResult {
+            successful,
+            failed,
+            total_processed: processed,
+            total_bytes,
+            duration,
+        })
+    }
+
+    /// 从URL提取object key
+    pub fn extract_object_key(&self, url: &str) -> String {
+        url.trim_start_matches(&format!("{}/", self.base_url))
+            .trim_start_matches('/')
+            .to_string()
+    }
+
+    /// 下载文件到临时目录
+    #[instrument(skip(self), fields(object_key))]
+    pub async fn download_to_temp(&self, object_key: &str) -> Result<String, AppError> {
+        let temp_dir = std::env::temp_dir();
+        let file_name = Path::new(object_key)
+            .file_name()
+            .and_then(|n| n.to_str())
+            .unwrap_or("download");
+
+        let temp_path = temp_dir.join(format!("oss_download_{file_name}"));
+
+        self.download_to_path(object_key, &temp_path).await?;
+
+        Ok(temp_path.to_string_lossy().to_string())
+    }
+
+    /// 下载文件到指定路径
+    #[instrument(skip(self), fields(object_key, target_path = %target_path.as_ref().display()))]
+    pub async fn download_to_path<P: AsRef<Path>>(
+        &self,
+        object_key: &str,
+        target_path: P,
+    ) -> Result<(), AppError> {
+        let target_path = target_path.as_ref();
+
+        // 创建目标目录
+        if let Some(parent) = target_path.parent() {
+            std::fs::create_dir_all(parent)
+                .map_err(|e| AppError::Oss(format!("创建目标目录失败: {e}")))?;
+        }
+
+        self.do_download(object_key, target_path).await
+    }
+
+    /// 执行下载
+    #[instrument(skip(self))]
+    async fn do_download(&self, object_key: &str, target_path: &Path) -> Result<(), AppError> {
+        let builder = RequestBuilder::new();
+
+        let content = match self.client.get_object(object_key, builder).await {
+            Ok(content) => content,
+            Err(e) => return Err(AppError::Oss(format!("下载文件失败: {e}"))),
+        };
+
+        std::fs::write(target_path, content)
+            .map_err(|e| AppError::Oss(format!("写入文件失败: {e}")))?;
+
+        info!("File download successful: {}", target_path.display());
+        Ok(())
+    }
+
+    /// 获取对象内容（直接返回字节数组）
+    #[instrument(skip(self), fields(object_key))]
+    pub async fn get_object_content(&self, object_key: &str) -> Result<Vec<u8>, AppError> {
+        let builder = RequestBuilder::new();
+
+        match self.client.get_object(object_key, builder).await {
+            Ok(content) => {
+                info!(
+                    "Successfully obtained the object content: object_key={}, size={} bytes",
+                    object_key,
+                    content.len()
+                );
+                Ok(content)
+            }
+            Err(e) => {
+                let error_msg = e.to_string();
+                if error_msg.contains("404") || error_msg.contains("NoSuchKey") {
+                    Err(AppError::Oss(format!("OSS文件不存在: {object_key}")))
+                } else {
+                    Err(AppError::Oss(format!("获取文件内容失败: {e}")))
+                }
+            }
+        }
+    }
+
+    /// 生成下载URL
+    #[instrument(skip(self), fields(object_key, expires_in = ?expires_in))]
+    pub async fn generate_download_url(
+        &self,
+        object_key: &str,
+        expires_in: Option<Duration>,
+    ) -> Result<String, AppError> {
+        let expire_seconds = expires_in.unwrap_or(Duration::from_secs(3600)).as_secs() as i64;
+
+        let builder = RequestBuilder::new().with_expire(expire_seconds);
+
+        let url = self.client.sign_download_url(object_key, &builder);
+
+        Ok(url)
+    }
+
+    /// 生成上传签名URL
+    #[instrument(skip(self), fields(object_key, expires_in = ?expires_in))]
+    pub async fn generate_upload_url(
+        &self,
+        object_key: &str,
+        expires_in: Option<Duration>,
+    ) -> Result<String, AppError> {
+        let expire_seconds = expires_in.unwrap_or(Duration::from_secs(3600)).as_secs() as i64;
+
+        let builder = RequestBuilder::new()
+            .with_expire(expire_seconds)
+            .with_content_type("application/octet-stream"); // 默认内容类型
+
+        let url = self.client.sign_upload_url(object_key, &builder);
+
+        Ok(url)
+    }
+
+    /// 生成带自定义内容类型的上传签名URL
+    #[instrument(skip(self), fields(object_key, content_type, expires_in = ?expires_in))]
+    pub async fn generate_upload_url_with_content_type(
+        &self,
+        object_key: &str,
+        content_type: &str,
+        expires_in: Option<Duration>,
+    ) -> Result<String, AppError> {
+        let expire_seconds = expires_in.unwrap_or(Duration::from_secs(3600)).as_secs() as i64;
+
+        let builder = RequestBuilder::new()
+            .with_expire(expire_seconds)
+            .with_content_type(content_type);
+
+        let url = self.client.sign_upload_url(object_key, &builder);
+
+        Ok(url)
+    }
+
+    /// 删除对象
+    #[instrument(skip(self), fields(object_key))]
+    pub async fn delete_object(&self, object_key: &str) -> Result<(), AppError> {
+        let builder = RequestBuilder::new();
+
+        match self.client.delete_object(object_key, builder).await {
+            Ok(_) => {
+                info!("Object deleted successfully: {}", object_key);
+                Ok(())
+            }
+            Err(e) => Err(AppError::Oss(format!("删除对象失败: {e}"))),
+        }
+    }
+
+    /// 批量删除对象
+    #[instrument(skip(self, object_keys), fields(count = object_keys.len()))]
+    pub async fn delete_objects(&self, object_keys: &[String]) -> Result<Vec<String>, AppError> {
+        let mut deleted = Vec::new();
+
+        for object_key in object_keys {
+            match self.delete_object(object_key).await {
+                Ok(_) => deleted.push(object_key.clone()),
+                Err(e) => {
+                    warn!("Failed to delete object {}: {}", object_key, e);
+                }
+            }
+        }
+
+        Ok(deleted)
+    }
+
+    /// 检查文件是否存在
+    #[instrument(skip(self), fields(object_key))]
+    pub async fn file_exists(&self, object_key: &str) -> Result<bool, AppError> {
+        let builder = RequestBuilder::new();
+
+        // 尝试获取对象来检查是否存在
+        match self.client.get_object(object_key, builder).await {
+            Ok(_) => Ok(true),
+            Err(_) => Ok(false),
+        }
+    }
+
+    /// 获取对象元数据
+    #[instrument(skip(self), fields(object_key))]
+    pub async fn get_object_metadata(
+        &self,
+        _object_key: &str,
+    ) -> Result<HashMap<String, String>, AppError> {
+        // 暂时返回空的元数据，因为 get_object_metadata 方法可能不存在
+        // 如果需要元数据，可能需要使用其他方法或者升级SDK版本
+        warn!(
+            "The get_object_metadata method has not been implemented yet and returns empty metadata."
+        );
+        Ok(HashMap::new())
+    }
+
+    /// 检测MIME类型
+    fn detect_mime_type(&self, file_path: &str) -> Result<String, AppError> {
+        let path = Path::new(file_path);
+        let extension = path.extension().and_then(|ext| ext.to_str()).unwrap_or("");
+
+        let mime_type = match extension.to_lowercase().as_str() {
+            "jpg" | "jpeg" => "image/jpeg",
+            "png" => "image/png",
+            "gif" => "image/gif",
+            "webp" => "image/webp",
+            "svg" => "image/svg+xml",
+            "pdf" => "application/pdf",
+            "doc" => "application/msword",
+            "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "xls" => "application/vnd.ms-excel",
+            "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            "ppt" => "application/vnd.ms-powerpoint",
+            "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            "txt" => "text/plain",
+            "html" | "htm" => "text/html",
+            "css" => "text/css",
+            "js" => "application/javascript",
+            "json" => "application/json",
+            "xml" => "application/xml",
+            "zip" => "application/zip",
+            "rar" => "application/x-rar-compressed",
+            "7z" => "application/x-7z-compressed",
+            "mp3" => "audio/mpeg",
+            "wav" => "audio/wav",
+            "mp4" => "video/mp4",
+            "avi" => "video/x-msvideo",
+            "mov" => "video/quicktime",
+            _ => "application/octet-stream",
+        };
+
+        Ok(mime_type.to_string())
+    }
+
+    /// 获取存储桶名称
+    pub fn get_bucket_name(&self) -> &str {
+        &self.bucket
+    }
+
+    /// 获取基础URL
+    pub fn get_base_url(&self) -> &str {
+        &self.base_url
+    }
+
+    /// 获取配置
+    pub fn get_config(&self) -> &OssServiceConfig {
+        &self.config
+    }
+
+    /// 列出对象（简化版本）
+    #[instrument(skip(self), fields(prefix, max_keys))]
+    pub async fn list_objects(
+        &self,
+        _prefix: Option<&str>,
+        _max_keys: Option<i32>,
+    ) -> Result<Vec<String>, AppError> {
+        // 注意：aliyun-oss-rust-sdk可能没有直接的list_objects API
+        // 这里返回空列表，实际使用时需要根据SDK的API来实现
+        warn!(
+            "The list_objects function needs to be implemented according to the API of aliyun-oss-rust-sdk"
+        );
+        Ok(Vec::new())
+    }
+
+    /// 获取存储统计信息（简化版本）
+    #[instrument(skip(self))]
+    pub async fn get_storage_stats(&self, prefix: Option<&str>) -> Result<StorageStats, AppError> {
+        // 注意：这个功能需要根据aliyun-oss-rust-sdk的API来实现
+        warn!(
+            "The get_storage_stats function needs to be implemented according to the API of aliyun-oss-rust-sdk"
+        );
+        Ok(StorageStats {
+            total_objects: 0,
+            total_size: 0,
+            file_count: 0,
+            last_modified: None,
+        })
+    }
+}
+
+/// 存储统计信息
+#[derive(Debug, Clone)]
+pub struct StorageStats {
+    pub total_objects: usize,
+    pub total_size: u64,
+    pub file_count: usize,
+    pub last_modified: Option<String>,
+}
+
+impl StorageStats {
+    /// 格式化大小显示
+    pub fn formatted_size(&self) -> String {
+        let size = self.total_size as f64;
+        if size < 1024.0 {
+            format!("{size} B")
+        } else if size < 1024.0 * 1024.0 {
+            format!("{:.2} KB", size / 1024.0)
+        } else if size < 1024.0 * 1024.0 * 1024.0 {
+            format!("{:.2} MB", size / (1024.0 * 1024.0))
+        } else {
+            format!("{:.2} GB", size / (1024.0 * 1024.0 * 1024.0))
+        }
+    }
+}

qiming-mcp-proxy/document-parser/src/services/task_queue_service.rs

@@ -0,0 +1,814 @@
+use std::collections::{HashMap, VecDeque};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use tokio::sync::{Mutex, RwLock, mpsc, watch};
+use tokio::time::{Duration, Instant, interval, sleep};
+use tracing::{debug, error, info, warn};
+
+use crate::error::AppError;
+use crate::models::{ProcessingStage, TaskStatus};
+use crate::services::TaskService;
+
+/// 任务队列项
+#[derive(Debug, Clone)]
+pub struct QueueItem {
+    pub task_id: String,
+    pub priority: u8,
+    pub created_at: Instant,
+    pub retry_count: u32,
+    pub max_retries: u32,
+}
+
+impl QueueItem {
+    pub fn new(task_id: String, priority: u8) -> Self {
+        Self {
+            task_id,
+            priority,
+            created_at: Instant::now(),
+            retry_count: 0,
+            max_retries: 3,
+        }
+    }
+
+    pub fn can_retry(&self) -> bool {
+        self.retry_count < self.max_retries
+    }
+
+    pub fn increment_retry(&mut self) {
+        self.retry_count += 1;
+    }
+}
+
+/// 队列统计信息
+#[derive(Debug, Clone)]
+pub struct QueueStats {
+    pub pending_count: usize,
+    pub processing_count: usize,
+    pub completed_count: u64,
+    pub failed_count: u64,
+    pub retry_count: u64,
+    pub average_processing_time: Duration,
+    pub queue_throughput: f64, // 任务/秒
+    pub backpressure_events: u64,
+    pub queue_overflow_events: u64,
+    pub worker_utilization: f64, // 0.0 - 1.0
+    pub memory_usage_bytes: u64,
+    pub last_updated: Instant,
+}
+
+impl Default for QueueStats {
+    fn default() -> Self {
+        Self {
+            pending_count: 0,
+            processing_count: 0,
+            completed_count: 0,
+            failed_count: 0,
+            retry_count: 0,
+            average_processing_time: Duration::from_secs(0),
+            queue_throughput: 0.0,
+            backpressure_events: 0,
+            queue_overflow_events: 0,
+            worker_utilization: 0.0,
+            memory_usage_bytes: 0,
+            last_updated: Instant::now(),
+        }
+    }
+}
+
+/// 任务处理器trait
+#[async_trait::async_trait]
+pub trait TaskProcessor: Send + Sync {
+    async fn process_task(&self, task_id: &str) -> Result<(), AppError>;
+}
+
+/// 队列配置
+#[derive(Debug, Clone)]
+pub struct QueueConfig {
+    pub max_concurrent_tasks: usize,
+    pub max_queue_size: usize,
+    pub task_timeout: Duration,
+    pub backpressure_threshold: f64, // 0.0 - 1.0
+    pub retry_base_delay: Duration,
+    pub retry_max_delay: Duration,
+    pub metrics_update_interval: Duration,
+    pub health_check_interval: Duration,
+}
+
+impl Default for QueueConfig {
+    fn default() -> Self {
+        Self {
+            max_concurrent_tasks: 10,
+            max_queue_size: 1000,
+            task_timeout: Duration::from_secs(300),
+            backpressure_threshold: 0.8,
+            retry_base_delay: Duration::from_secs(1),
+            retry_max_delay: Duration::from_secs(60),
+            metrics_update_interval: Duration::from_secs(5),
+            health_check_interval: Duration::from_secs(30),
+        }
+    }
+}
+
+/// 任务队列服务
+pub struct TaskQueueService {
+    // 队列通道 - 使用有界通道实现背压
+    task_sender: Option<mpsc::Sender<QueueItem>>,
+
+    // 正在处理的任务
+    processing_tasks: Arc<RwLock<HashMap<String, TaskExecutionContext>>>,
+
+    // 统计信息
+    stats: Arc<RwLock<QueueStats>>,
+    completed_count: Arc<AtomicU64>,
+    failed_count: Arc<AtomicU64>,
+    retry_count: Arc<AtomicU64>,
+    // SPMC 队列中的待处理任务计数
+    queued_count: Arc<AtomicU64>,
+    backpressure_events: Arc<AtomicU64>,
+    overflow_events: Arc<AtomicU64>,
+
+    // 配置
+    config: QueueConfig,
+
+    // 服务
+    task_service: Arc<TaskService>,
+
+    // 控制通道
+    shutdown_sender: watch::Sender<bool>,
+    shutdown_receiver: watch::Receiver<bool>,
+
+    // 健康状态
+    is_healthy: Arc<AtomicUsize>, // 0 = unhealthy, 1 = healthy
+}
+
+/// 任务执行上下文
+#[derive(Debug, Clone)]
+struct TaskExecutionContext {
+    _task_id: String,
+    started_at: Instant,
+    _worker_id: usize,
+    _retry_count: u32,
+}
+
+impl TaskQueueService {
+    /// 创建新的任务队列服务
+    pub fn new(task_service: Arc<TaskService>) -> Self {
+        Self::with_config(task_service, QueueConfig::default())
+    }
+
+    /// 使用自定义配置创建任务队列服务
+    pub fn with_config(task_service: Arc<TaskService>, config: QueueConfig) -> Self {
+        let (shutdown_sender, shutdown_receiver) = watch::channel(false);
+
+        Self {
+            task_sender: None, // 初始时不创建channel
+            processing_tasks: Arc::new(RwLock::new(HashMap::new())),
+            stats: Arc::new(RwLock::new(QueueStats {
+                last_updated: Instant::now(),
+                ..Default::default()
+            })),
+            completed_count: Arc::new(AtomicU64::new(0)),
+            failed_count: Arc::new(AtomicU64::new(0)),
+            retry_count: Arc::new(AtomicU64::new(0)),
+            queued_count: Arc::new(AtomicU64::new(0)),
+            backpressure_events: Arc::new(AtomicU64::new(0)),
+            overflow_events: Arc::new(AtomicU64::new(0)),
+            config,
+            task_service,
+            shutdown_sender,
+            shutdown_receiver,
+            is_healthy: Arc::new(AtomicUsize::new(1)),
+        }
+    }
+
+    /// 启动队列处理器
+    pub async fn start<P>(&mut self, processor: Arc<P>) -> Result<(), AppError>
+    where
+        P: TaskProcessor + 'static,
+    {
+        info!(
+            "Start the task queue service, maximum concurrency: {}, queue size: {}",
+            self.config.max_concurrent_tasks, self.config.max_queue_size
+        );
+
+        // 创建channel并保存sender
+        let (task_sender, task_receiver) = mpsc::channel(self.config.max_queue_size);
+        self.task_sender = Some(task_sender);
+
+        // 启动多个工作协程
+        self.spawn_workers(processor, task_receiver).await?;
+
+        // 启动监控协程
+        self.spawn_monitors().await?;
+
+        // 恢复数据库中的待执行和进行中任务
+        self.restore_pending_tasks().await?;
+
+        Ok(())
+    }
+
+    /// 从数据库恢复待执行和进行中的任务 - 统一重置状态，让 worker 执行时重新设置
+    async fn restore_pending_tasks(&self) -> Result<(), AppError> {
+        info!("Start restoring pending and ongoing tasks in the database...");
+
+        let stats = self.task_service.get_task_stats().await?;
+
+        let all_need_process_tasks = stats
+            .pending_ids
+            .iter()
+            .chain(stats.processing_ids.iter())
+            .collect::<Vec<_>>();
+
+        info!(
+            "Number of tasks to be restored: {}",
+            all_need_process_tasks.len()
+        );
+        info!(
+            "Tasks that need to be restored: {:?}",
+            all_need_process_tasks
+        );
+
+        // 将所有进行中任务统一改为 pending 状态，然后重新入队
+        // 这样 worker 真正执行时会重新设置为 processing 状态
+        for task_id in all_need_process_tasks.clone() {
+            // 使用强制更新，跳过状态转换验证（仅在服务重启恢复时使用）
+            if let Err(e) = self
+                .task_service
+                .update_task_status(task_id, TaskStatus::new_pending())
+                .await
+            {
+                warn!("Failed to re-mark ongoing task status {}: {}", task_id, e);
+            } else {
+                // 重新入队
+                if let Err(e) = self.enqueue_task(task_id.clone(), 1).await {
+                    warn!("Requeue task failed {}: {}", task_id, e);
+                } else {
+                    info!(
+                        "The ongoing task has been remarked as pending and added to the queue: {}",
+                        task_id
+                    );
+                }
+            }
+        }
+        // 恢复所有待执行任务
+        for task_id in all_need_process_tasks.clone() {
+            if let Err(e) = self.enqueue_task(task_id.clone(), 1).await {
+                warn!("Failed to restore pending tasks {}: {}", task_id, e);
+            } else {
+                debug!("Restored pending tasks: {}", task_id);
+            }
+        }
+
+        info!(
+            "Task recovery is completed, with a total of {} tasks recovered. The status of all tasks has been reset to pending, and will be reset to processing when the worker executes",
+            all_need_process_tasks.len()
+        );
+        Ok(())
+    }
+
+    /// 启动工作协程 - 采用 SPMC 模式，worker 直接从 channel 消费
+    async fn spawn_workers<P>(
+        &self,
+        processor: Arc<P>,
+        task_receiver: mpsc::Receiver<QueueItem>,
+    ) -> Result<(), AppError>
+    where
+        P: TaskProcessor + 'static,
+    {
+        // 将 task_receiver 包装为 Arc<Mutex<>>，让多个 worker 共享
+        let shared_receiver = Arc::new(Mutex::new(task_receiver));
+
+        // 启动 N 个 worker，每个 worker 直接从 channel 消费任务
+        for worker_id in 0..self.config.max_concurrent_tasks {
+            self.spawn_simple_worker(
+                worker_id,
+                Arc::clone(&processor),
+                Arc::clone(&shared_receiver),
+            )
+            .await;
+        }
+
+        info!(
+            "{} worker coroutines have been started, using SPMC mode to directly consume tasks",
+            self.config.max_concurrent_tasks
+        );
+        Ok(())
+    }
+
+    /// 启动简化的 worker - 直接从共享 channel 消费任务
+    async fn spawn_simple_worker<P>(
+        &self,
+        worker_id: usize,
+        processor: Arc<P>,
+        shared_receiver: Arc<Mutex<mpsc::Receiver<QueueItem>>>,
+    ) where
+        P: TaskProcessor + 'static,
+    {
+        let task_service = Arc::clone(&self.task_service);
+        let processing_tasks = Arc::clone(&self.processing_tasks);
+        let completed_count = Arc::clone(&self.completed_count);
+        let failed_count = Arc::clone(&self.failed_count);
+        let queued_count = Arc::clone(&self.queued_count);
+        let config = self.config.clone();
+        let mut shutdown_rx = self.shutdown_receiver.clone();
+
+        tokio::spawn(async move {
+            debug!(
+                "Simplified Worker {} has been started and consumes tasks directly from the channel",
+                worker_id
+            );
+
+            loop {
+                tokio::select! {
+                    // 检查关闭信号
+                    _ = shutdown_rx.changed() => {
+                        if *shutdown_rx.borrow() {
+                            info!("Worker {} received a shutdown signal", worker_id);
+                            break;
+                        }
+                    }
+
+                    // 从共享 channel 接收任务
+                    task_result = async {
+                        let mut receiver = shared_receiver.lock().await;
+                        receiver.recv().await
+                    } => {
+                        match task_result {
+                            Some(queue_item) => {
+                                let task_id = queue_item.task_id.clone();
+                                let start_time = Instant::now();
+
+                                debug!("Worker {} starts processing task: {}", worker_id, task_id);
+
+                                // 更新任务状态为处理中
+                                if let Err(e) = task_service.update_task_status(
+                                    &task_id,
+                                    TaskStatus::new_processing(ProcessingStage::FormatDetection)
+                                ).await {
+                                    error!("Worker {} failed to update task status: {}", worker_id, e);
+                                }
+                                // 出队计数减一
+                                queued_count.fetch_sub(1, Ordering::Relaxed);
+
+                                // 记录到 processing_tasks，便于统计与健康检查
+                                {
+                                    let mut tasks = processing_tasks.write().await;
+                                    tasks.insert(
+                                        task_id.clone(),
+                                        TaskExecutionContext {
+                                            _task_id: task_id.clone(),
+                                            started_at: start_time,
+                                            _worker_id: worker_id,
+                                            _retry_count: queue_item.retry_count,
+                                        },
+                                    );
+                                }
+
+                                // 执行任务处理
+                                let result = tokio::time::timeout(
+                                    config.task_timeout,
+                                    processor.process_task(&task_id)
+                                ).await;
+
+                                let processing_time = start_time.elapsed();
+
+                                // 处理结果
+                                match result {
+                                    Ok(Ok(())) => {
+                                        completed_count.fetch_add(1, Ordering::Relaxed);
+
+                                        if let Err(e) = task_service.update_task_status(
+                                            &task_id,
+                                            TaskStatus::new_completed(processing_time)
+                                        ).await {
+                                            error!("Worker {} failed to update task completion status: {}", worker_id, e);
+                                        }
+
+                                        info!("Worker {} completed the task: {} (time taken: {:?})",
+                                                  worker_id, task_id, processing_time);
+                                        // 从 processing 列表移除
+                                        {
+                                            let mut tasks = processing_tasks.write().await;
+                                            tasks.remove(&task_id);
+                                        }
+                                    }
+                                    Ok(Err(e)) => {
+                                        failed_count.fetch_add(1, Ordering::Relaxed);
+
+                                        if let Err(err) = task_service.set_task_error(&task_id, e.to_string()).await {
+                                            error!("Worker {} failed to set task error: {}", worker_id, err);
+                                        }
+
+                                        error!("Worker {} task failed: {} - {}", worker_id, task_id, e);
+                                        // 从 processing 列表移除
+                                        {
+                                            let mut tasks = processing_tasks.write().await;
+                                            tasks.remove(&task_id);
+                                        }
+                                    }
+                                    Err(_) => {
+                                        failed_count.fetch_add(1, Ordering::Relaxed);
+
+                                        if let Err(e) = task_service.set_task_error(&task_id, "任务处理超时".to_string()).await {
+                                            error!("Worker {} failed to set task timeout error: {}", worker_id, e);
+                                        }
+
+                                        error!("Worker {} task timeout: {} (timeout time: {:?})",worker_id, task_id, config.task_timeout);
+                                        // 从 processing 列表移除
+                                        {
+                                            let mut tasks = processing_tasks.write().await;
+                                            tasks.remove(&task_id);
+                                        }
+                                    }
+                                }
+                            }
+                            None => {
+                                // Channel 已关闭
+                                info!("Worker {} detected that the channel was closed and stopped working", worker_id);
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+
+            debug!("Worker {} has stopped", worker_id);
+        });
+    }
+
+    /// 启动监控协程
+    async fn spawn_monitors(&self) -> Result<(), AppError> {
+        // 启动统计更新协程
+        self.spawn_stats_updater().await;
+
+        // 启动健康检查协程
+        self.spawn_health_checker().await;
+
+        Ok(())
+    }
+
+    /// 启动统计更新协程
+    async fn spawn_stats_updater(&self) {
+        let stats = Arc::clone(&self.stats);
+        let processing_tasks = Arc::clone(&self.processing_tasks);
+        let completed_count = Arc::clone(&self.completed_count);
+        let failed_count = Arc::clone(&self.failed_count);
+        let retry_count = Arc::clone(&self.retry_count);
+        let backpressure_events = Arc::clone(&self.backpressure_events);
+        let overflow_events = Arc::clone(&self.overflow_events);
+        let queued_count = Arc::clone(&self.queued_count);
+        let config = self.config.clone();
+        let mut shutdown_rx = self.shutdown_receiver.clone();
+
+        tokio::spawn(async move {
+            let mut interval = interval(config.metrics_update_interval);
+            let mut processing_times = VecDeque::with_capacity(100);
+
+            loop {
+                tokio::select! {
+                    _ = shutdown_rx.changed() => {
+                        if *shutdown_rx.borrow() {
+                            break;
+                        }
+                    }
+
+                    _ = interval.tick() => {
+                        let now = Instant::now();
+
+                        // 收集当前统计信息
+                        // 队列中的等待任务使用原子计数器近似
+                        let pending_count = queued_count.load(Ordering::Relaxed) as usize;
+
+                        let processing_count = {
+                            let tasks = processing_tasks.read().await;
+                            tasks.len()
+                        };
+
+                        let completed = completed_count.load(Ordering::Relaxed);
+                        let failed = failed_count.load(Ordering::Relaxed);
+                        let retries = retry_count.load(Ordering::Relaxed);
+                        let backpressure = backpressure_events.load(Ordering::Relaxed);
+                        let overflow = overflow_events.load(Ordering::Relaxed);
+
+                        // 计算工作协程利用率
+                        // 利用率直接使用 processing_count / max_concurrent_tasks
+                        let worker_utilization = {
+                            let tasks = processing_tasks.read().await;
+                            if config.max_concurrent_tasks > 0 {
+                                (tasks.len() as f64 / config.max_concurrent_tasks as f64).min(1.0)
+                            } else { 0.0 }
+                        };
+
+                        // 计算平均处理时间
+                        let average_processing_time = if processing_times.is_empty() {
+                            Duration::from_secs(0)
+                        } else {
+                            let total_ms: u64 = processing_times.iter().map(|d: &Duration| d.as_millis() as u64).sum();
+                            Duration::from_millis(total_ms / processing_times.len() as u64)
+                        };
+
+                        // 计算吞吐量
+                        let queue_throughput = if completed > 0 && !average_processing_time.is_zero() {
+                            1000.0 / average_processing_time.as_millis() as f64
+                        } else {
+                            0.0
+                        };
+
+                        // 估算内存使用
+                        let memory_usage_bytes = (pending_count + processing_count) * 1024; // 简化估算
+
+                        // 更新统计信息
+                        {
+                            let mut stats_guard = stats.write().await;
+                            stats_guard.pending_count = pending_count;
+                            stats_guard.processing_count = processing_count;
+                            stats_guard.completed_count = completed;
+                            stats_guard.failed_count = failed;
+                            stats_guard.retry_count = retries;
+                            stats_guard.backpressure_events = backpressure;
+                            stats_guard.queue_overflow_events = overflow;
+                            stats_guard.worker_utilization = worker_utilization;
+                            stats_guard.average_processing_time = average_processing_time;
+                            stats_guard.queue_throughput = queue_throughput;
+                            stats_guard.memory_usage_bytes = memory_usage_bytes as u64;
+                            stats_guard.last_updated = now;
+                        }
+
+                        // 记录处理时间样本
+                        {
+                            let tasks = processing_tasks.read().await;
+                            for (_, context) in tasks.iter() {
+                                let elapsed = now.duration_since(context.started_at);
+                                processing_times.push_back(elapsed);
+
+                                // 保持队列大小
+                                if processing_times.len() > 100 {
+                                    processing_times.pop_front();
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    /// 启动健康检查协程
+    async fn spawn_health_checker(&self) {
+        let is_healthy = Arc::clone(&self.is_healthy);
+        let processing_tasks = Arc::clone(&self.processing_tasks);
+        let config = self.config.clone();
+        let mut shutdown_rx = self.shutdown_receiver.clone();
+
+        tokio::spawn(async move {
+            let mut interval = interval(config.health_check_interval);
+
+            loop {
+                tokio::select! {
+                    _ = shutdown_rx.changed() => {
+                        if *shutdown_rx.borrow() {
+                            break;
+                        }
+                    }
+
+                    _ = interval.tick() => {
+                        let now = Instant::now();
+                        let mut unhealthy_tasks = 0;
+
+                        // 检查是否有任务超时
+                        {
+                            let tasks = processing_tasks.read().await;
+                            for (task_id, context) in tasks.iter() {
+                                let elapsed = now.duration_since(context.started_at);
+                                if elapsed > config.task_timeout * 2 {
+                                    warn!("Possibly stuck task detected: {} (running time: {:?})", task_id, elapsed);
+                                    unhealthy_tasks += 1;
+                                }
+                            }
+                        }
+
+                        // 更新健康状态
+                        let healthy = unhealthy_tasks == 0;
+                        is_healthy.store(if healthy { 1 } else { 0 }, Ordering::Relaxed);
+
+                        if !healthy {
+                            warn!("Queue service health check failed: {} tasks may be stuck", unhealthy_tasks);
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    /// 添加任务到队列
+    pub async fn enqueue_task(&self, task_id: String, priority: u8) -> Result<(), AppError> {
+        // 检查队列是否已启动
+        let task_sender = self.task_sender.as_ref().ok_or_else(|| {
+            AppError::Queue("队列服务尚未启动，请先调用 start() 方法".to_string())
+        })?;
+
+        let item = QueueItem::new(task_id.clone(), priority);
+
+        // 尝试发送任务，如果队列满了则触发背压
+        match task_sender.try_send(item) {
+            Ok(()) => {
+                self.queued_count.fetch_add(1, Ordering::Relaxed);
+                debug!(
+                    "Task has been added to the queue: {} (Priority: {})",
+                    task_id, priority
+                );
+                Ok(())
+            }
+            Err(mpsc::error::TrySendError::Full(_)) => {
+                self.overflow_events.fetch_add(1, Ordering::Relaxed);
+                warn!(
+                    "The queue is full, triggering back pressure control: {}",
+                    task_id
+                );
+                Err(AppError::Queue("队列已满，请稍后重试".to_string()))
+            }
+            Err(mpsc::error::TrySendError::Closed(_)) => {
+                Err(AppError::Queue("队列已关闭".to_string()))
+            }
+        }
+    }
+
+    /// 获取队列统计信息
+    pub async fn get_stats(&self) -> QueueStats {
+        let stats = self.stats.read().await;
+        stats.clone()
+    }
+
+    /// 获取正在处理的任务列表
+    pub async fn get_processing_tasks(&self) -> Vec<(String, Duration)> {
+        let tasks = self.processing_tasks.read().await;
+        let now = Instant::now();
+
+        tasks
+            .iter()
+            .map(|(task_id, context)| (task_id.clone(), now.duration_since(context.started_at)))
+            .collect()
+    }
+
+    /// 检查队列是否健康
+    pub fn is_healthy(&self) -> bool {
+        self.is_healthy.load(Ordering::Relaxed) == 1
+    }
+
+    /// 检查队列是否已启动
+    pub fn is_started(&self) -> bool {
+        self.task_sender.is_some()
+    }
+
+    /// 优雅关闭
+    pub async fn shutdown(&self) -> Result<(), AppError> {
+        info!("Starting to shut down the task queue service...");
+
+        // 发送关闭信号
+        if let Err(e) = self.shutdown_sender.send(true) {
+            error!("Failed to send shutdown signal: {}", e);
+        }
+
+        // 等待所有正在处理的任务完成
+        let mut wait_count = 0;
+        while wait_count < 30 {
+            // 最多等待30秒
+            let processing_count = {
+                let tasks = self.processing_tasks.read().await;
+                tasks.len()
+            };
+
+            if processing_count == 0 {
+                break;
+            }
+
+            info!("Waiting for {} tasks to complete...", processing_count);
+            sleep(Duration::from_secs(1)).await;
+            wait_count += 1;
+        }
+
+        info!("Task queue service is down");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use tempfile::TempDir;
+
+    struct TestProcessor {
+        processed_count: AtomicUsize,
+        should_fail: bool,
+    }
+
+    impl TestProcessor {
+        fn new() -> Self {
+            Self {
+                processed_count: AtomicUsize::new(0),
+                should_fail: false,
+            }
+        }
+
+        fn with_failure() -> Self {
+            Self {
+                processed_count: AtomicUsize::new(0),
+                should_fail: true,
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TaskProcessor for TestProcessor {
+        async fn process_task(&self, task_id: &str) -> Result<(), AppError> {
+            // 模拟处理时间
+            sleep(Duration::from_millis(50)).await;
+
+            if self.should_fail && task_id.contains("fail") {
+                return Err(AppError::Parse("模拟处理失败".to_string()));
+            }
+
+            self.processed_count.fetch_add(1, Ordering::SeqCst);
+            info!("Processing task: {}", task_id);
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_backpressure_control() {
+        let temp_dir = TempDir::new().unwrap();
+        let db = Arc::new(sled::open(temp_dir.path()).unwrap());
+        let task_service = Arc::new(TaskService::new(db).unwrap());
+
+        let config = QueueConfig {
+            max_concurrent_tasks: 1,
+            max_queue_size: 2,
+            backpressure_threshold: 0.5,
+            ..Default::default()
+        };
+
+        let mut queue_service = TaskQueueService::with_config(task_service, config);
+        let processor = Arc::new(TestProcessor::new());
+
+        queue_service.start(processor).await.unwrap();
+
+        // 添加任务直到触发背压
+        let mut success_count = 0;
+        let mut backpressure_triggered = false;
+
+        for i in 0..5 {
+            match queue_service.enqueue_task(format!("task{i}"), 1).await {
+                Ok(()) => success_count += 1,
+                Err(_) => {
+                    backpressure_triggered = true;
+                    break;
+                }
+            }
+        }
+
+        assert!(backpressure_triggered, "背压控制应该被触发");
+        assert!(success_count < 5, "不应该所有任务都成功入队");
+
+        queue_service.shutdown().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_metrics_collection() {
+        let app_config = crate::tests::test_helpers::create_real_environment_test_config();
+        crate::config::init_global_config(app_config).unwrap();
+        let temp_dir = TempDir::new().unwrap();
+        let db = Arc::new(sled::open(temp_dir.path()).unwrap());
+        let task_service = Arc::new(TaskService::new(db).unwrap());
+
+        let config = QueueConfig {
+            metrics_update_interval: Duration::from_millis(100),
+            ..Default::default()
+        };
+
+        let mut queue_service = TaskQueueService::with_config(task_service, config);
+        let processor = Arc::new(TestProcessor::new());
+
+        queue_service.start(processor).await.expect("start ok");
+
+        // 添加任务
+        for i in 0..3 {
+            queue_service
+                .enqueue_task(format!("task{i}"), 1)
+                .await
+                .expect("enqueue ok");
+        }
+
+        // 等待处理和统计更新
+        sleep(Duration::from_millis(400)).await;
+
+        let stats = queue_service.get_stats().await;
+        assert!(stats.last_updated.elapsed() < Duration::from_secs(1));
+        assert!(stats.worker_utilization >= 0.0 && stats.worker_utilization <= 1.0);
+        // Memory usage might be 0 if tasks are processed quickly, so we just check it's non-negative
+        assert!(stats.memory_usage_bytes >= 0);
+
+        queue_service.shutdown().await.expect("shutdown ok");
+    }
+}

qiming-mcp-proxy/document-parser/src/services/task_service.rs

@@ -0,0 +1,634 @@
+use crate::error::AppError;
+use crate::models::{
+    DocumentFormat, DocumentTask, ParserEngine, ProcessingStage, SourceType, TaskStatus,
+};
+use sled::Db;
+use std::sync::Arc;
+use tracing::{debug, error, info, warn};
+use uuid::{NoContext, Timestamp, Uuid};
+
+/// 任务服务
+pub struct TaskService {
+    tasks_tree: sled::Tree,
+}
+
+impl TaskService {
+    /// 创建新的任务服务
+    pub fn new(db: Arc<Db>) -> Result<Self, AppError> {
+        let tasks_tree = db
+            .open_tree("tasks")
+            .map_err(|e| AppError::Database(format!("打开任务树失败: {e}")))?;
+
+        Ok(Self { tasks_tree })
+    }
+
+    /// 创建新任务
+    pub async fn create_task(
+        &self,
+        source_type: SourceType,
+        source_path: Option<String>,
+        original_filename: Option<String>,
+        format: Option<DocumentFormat>,
+    ) -> Result<DocumentTask, AppError> {
+        let task_id = Uuid::new_v7(Timestamp::now(NoContext)).to_string();
+
+        info!(
+            "Create new task: {} ({:?} -> {:?})",
+            task_id, source_type, format
+        );
+
+        let task = DocumentTask::new(
+            task_id.clone(),
+            source_type.clone(),
+            source_path,
+            original_filename,
+            format,
+            Some("pipeline".to_string()),
+            Some(24),
+            Some(3),
+        );
+
+        // 保存到数据库
+        self.save_task(&task).await?;
+
+        Ok(task)
+    }
+
+    /// 获取任务
+    pub async fn get_task(&self, task_id: &str) -> Result<Option<DocumentTask>, AppError> {
+        debug!("Query task: {}", task_id);
+
+        match self.tasks_tree.get(task_id) {
+            Ok(Some(data)) => {
+                let task: DocumentTask = serde_json::from_slice(&data)
+                    .map_err(|e| AppError::Database(format!("反序列化任务失败: {e}")))?;
+                Ok(Some(task))
+            }
+            Ok(None) => Ok(None),
+            Err(e) => Err(AppError::Database(format!("查询任务失败: {e}"))),
+        }
+    }
+
+    /// 保存任务
+    pub async fn save_task(&self, task: &DocumentTask) -> Result<(), AppError> {
+        let data = serde_json::to_vec(task)
+            .map_err(|e| AppError::Database(format!("序列化任务失败: {e}")))?;
+
+        self.tasks_tree
+            .insert(&task.id, data)
+            .map_err(|e| AppError::Database(format!("保存任务失败: {e}")))?;
+
+        self.tasks_tree
+            .flush()
+            .map_err(|e| AppError::Database(format!("刷新数据库失败: {e}")))?;
+
+        debug!("Task saved: {}", task.id);
+        Ok(())
+    }
+
+    /// 更新任务基本信息
+    pub async fn update_task(
+        &self,
+        task_id: &str,
+        source_path: Option<String>,
+        original_filename: Option<String>,
+        document_format: DocumentFormat,
+    ) -> Result<(), AppError> {
+        debug!("Update basic task information: {}", task_id);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        // 更新任务信息
+        if let Some(path) = source_path {
+            task.source_path = Some(path);
+        }
+        if let Some(filename) = original_filename {
+            task.original_filename = Some(filename);
+        }
+        // 根据文档格式更新解析引擎
+        task.parser_engine = Some(if document_format == DocumentFormat::PDF {
+            ParserEngine::MinerU
+        } else {
+            ParserEngine::MarkItDown
+        });
+
+        task.document_format = Some(document_format);
+
+        // 更新时间戳
+        task.updated_at = chrono::Utc::now();
+
+        self.save_task(&task).await?;
+        Ok(())
+    }
+
+    /// 更新任务状态
+    pub async fn update_task_status(
+        &self,
+        task_id: &str,
+        status: TaskStatus,
+    ) -> Result<(), AppError> {
+        info!("Update task status: {} -> {:?}", task_id, status);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        task.update_status(status)?;
+        self.save_task(&task).await?;
+
+        Ok(())
+    }
+
+    /// 更新任务处理阶段
+    pub async fn update_task_stage(
+        &self,
+        task_id: &str,
+        stage: ProcessingStage,
+    ) -> Result<(), AppError> {
+        info!("Update task stage: {} -> {:?}", task_id, stage);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        let _ = task.update_status(TaskStatus::new_processing(stage));
+        self.save_task(&task).await?;
+
+        Ok(())
+    }
+
+    /// 更新任务进度
+    pub async fn update_task_progress(&self, task_id: &str, progress: u32) -> Result<(), AppError> {
+        debug!("Update task progress: {} -> {}%", task_id, progress);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        task.update_progress(progress)?;
+        self.save_task(&task).await?;
+
+        Ok(())
+    }
+
+    /// 设置任务错误
+    pub async fn set_task_error(
+        &self,
+        task_id: &str,
+        error_message: String,
+    ) -> Result<(), AppError> {
+        error!("Task error: {} -> {}", task_id, error_message);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        task.set_error(error_message)?;
+        self.save_task(&task).await?;
+
+        Ok(())
+    }
+
+    /// 设置任务解析引擎
+    pub async fn set_task_parser_engine(
+        &self,
+        task_id: &str,
+        engine: ParserEngine,
+    ) -> Result<(), AppError> {
+        info!("Set task parsing engine: {} -> {:?}", task_id, engine);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        task.parser_engine = Some(engine);
+        self.save_task(&task).await?;
+
+        Ok(())
+    }
+
+    /// 设置任务文件信息
+    pub async fn set_task_file_info(
+        &self,
+        task_id: &str,
+        file_size: Option<u64>,
+        mime_type: Option<String>,
+    ) -> Result<(), AppError> {
+        debug!(
+            "Set task file information: {} (size: {:?}, type: {:?})",
+            task_id, file_size, mime_type
+        );
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        if let (Some(size), Some(mime)) = (file_size, mime_type.clone()) {
+            task.set_file_info(size, mime)?;
+        } else {
+            if let Some(size) = file_size {
+                task.file_size = Some(size);
+            }
+            if let Some(mime) = mime_type {
+                task.mime_type = Some(mime);
+            }
+        }
+
+        self.save_task(&task).await?;
+        Ok(())
+    }
+
+    /// 更新任务的来源信息（本地路径、URL、原始文件名）
+    pub async fn update_task_source_info(
+        &self,
+        task_id: &str,
+        source_path: Option<String>,
+        source_url: Option<String>,
+        original_filename: Option<String>,
+    ) -> Result<(), AppError> {
+        debug!(
+            "Update task source information: task_id={}, path={:?}, url={:?}, filename={:?}",
+            task_id, source_path, source_url, original_filename
+        );
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        if let Some(path) = source_path {
+            task.source_path = Some(path);
+        }
+        if let Some(url) = source_url {
+            task.source_url = Some(url);
+        }
+        if let Some(name) = original_filename {
+            task.original_filename = Some(name);
+        }
+
+        task.updated_at = chrono::Utc::now();
+
+        self.save_task(&task).await?;
+        Ok(())
+    }
+
+    /// 设置任务的 OSS 子目录（bucket_dir）
+    pub async fn set_task_bucket_dir(
+        &self,
+        task_id: &str,
+        bucket_dir: Option<String>,
+    ) -> Result<(), AppError> {
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        task.bucket_dir = bucket_dir;
+        task.updated_at = chrono::Utc::now();
+
+        self.save_task(&task).await?;
+        Ok(())
+    }
+
+    /// 列出所有任务
+    pub async fn list_tasks(&self, limit: Option<usize>) -> Result<Vec<DocumentTask>, AppError> {
+        let mut tasks = Vec::new();
+        let mut count = 0;
+
+        for result in self.tasks_tree.iter() {
+            if let Some(max_count) = limit {
+                if count >= max_count {
+                    break;
+                }
+            }
+
+            match result {
+                Ok((_, data)) => match serde_json::from_slice::<DocumentTask>(&data) {
+                    Ok(task) => {
+                        tasks.push(task);
+                        count += 1;
+                    }
+                    Err(e) => {
+                        warn!("Deserialization task failed: {}", e);
+                    }
+                },
+                Err(e) => {
+                    warn!("Failed to read task data: {}", e);
+                }
+            }
+        }
+
+        // 按创建时间倒序排列
+        tasks.sort_by(|a, b| b.created_at.cmp(&a.created_at));
+
+        Ok(tasks)
+    }
+
+    /// 取消任务
+    pub async fn cancel_task(
+        &self,
+        task_id: &str,
+        reason: Option<String>,
+    ) -> Result<DocumentTask, AppError> {
+        info!("Cancel task: {} (reason: {:?})", task_id, reason);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        // 使用任务模型的 cancel 方法
+        task.cancel()?;
+
+        // 如果提供了原因，更新取消状态
+        if let Some(cancel_reason) = reason {
+            task.status = TaskStatus::new_cancelled(Some(cancel_reason));
+        }
+
+        self.save_task(&task).await?;
+
+        Ok(task)
+    }
+
+    /// 删除任务
+    pub async fn delete_task(&self, task_id: &str) -> Result<bool, AppError> {
+        info!("Delete task: {}", task_id);
+
+        // 获取任务信息以便清理相关文件
+        let task = self.get_task(task_id).await?;
+
+        match self.tasks_tree.remove(task_id) {
+            Ok(Some(_)) => {
+                self.tasks_tree
+                    .flush()
+                    .map_err(|e| AppError::Database(format!("刷新数据库失败: {e}")))?;
+
+                // 清理任务相关的临时文件
+                if let Some(task) = task {
+                    self.cleanup_task_files(&task).await;
+                }
+
+                Ok(true)
+            }
+            Ok(None) => Ok(false),
+            Err(e) => Err(AppError::Database(format!("删除任务失败: {e}"))),
+        }
+    }
+
+    /// 重试任务
+    pub async fn retry_task(&self, task_id: &str) -> Result<DocumentTask, AppError> {
+        info!("Retry task: {}", task_id);
+
+        let mut task = self
+            .get_task(task_id)
+            .await?
+            .ok_or_else(|| AppError::Task(format!("任务不存在: {task_id}")))?;
+
+        // 使用任务模型的 reset 方法
+        task.reset()?;
+
+        self.save_task(&task).await?;
+
+        Ok(task)
+    }
+
+    /// 清理过期任务
+    pub async fn cleanup_expired_tasks(&self) -> Result<usize, AppError> {
+        let mut cleaned_count = 0;
+        let mut to_remove = Vec::new();
+
+        for result in self.tasks_tree.iter() {
+            match result {
+                Ok((key, data)) => {
+                    match serde_json::from_slice::<DocumentTask>(&data) {
+                        Ok(task) => {
+                            if task.is_expired() {
+                                to_remove.push(key);
+                            }
+                        }
+                        Err(e) => {
+                            warn!("Deserialization task failed: {}", e);
+                            // 损坏的数据也删除
+                            to_remove.push(key);
+                        }
+                    }
+                }
+                Err(e) => {
+                    warn!("Failed to read task data: {}", e);
+                }
+            }
+        }
+
+        // 删除过期任务并清理相关文件
+        for key in to_remove {
+            // 获取任务信息以便清理文件
+            if let Ok(data) = self.tasks_tree.get(&key) {
+                if let Some(data) = data {
+                    if let Ok(task) = serde_json::from_slice::<DocumentTask>(&data) {
+                        // 清理任务相关的临时文件
+                        self.cleanup_task_files(&task).await;
+                    }
+                }
+            }
+
+            if let Err(e) = self.tasks_tree.remove(&key) {
+                warn!("Failed to delete expired tasks: {}", e);
+            } else {
+                cleaned_count += 1;
+            }
+        }
+
+        if cleaned_count > 0 {
+            self.tasks_tree
+                .flush()
+                .map_err(|e| AppError::Database(format!("刷新数据库失败: {e}")))?;
+
+            info!("Cleaned up {} expired tasks", cleaned_count);
+        }
+
+        Ok(cleaned_count)
+    }
+
+    /// 清理任务相关的临时文件
+    async fn cleanup_task_files(&self, task: &DocumentTask) {
+        // 清理基于 taskId 的临时文件
+        if let Some(source_path) = &task.source_path {
+            // 如果是基于 taskId 的文件路径，进行清理
+            if source_path.contains(&task.id) {
+                if let Err(e) = tokio::fs::remove_file(source_path).await {
+                    warn!(
+                        "Cleanup task {}'s temporary files failed: {} - {}",
+                        task.id, source_path, e
+                    );
+                } else {
+                    info!(
+                        "Cleaned temporary files of task {}: {}",
+                        task.id, source_path
+                    );
+                }
+            }
+        }
+
+        // 清理可能的工作目录
+        let temp_dir = std::env::temp_dir();
+        let task_work_dir = temp_dir.join(format!("document_parser_{}", task.id));
+        if task_work_dir.exists() {
+            if let Err(e) = tokio::fs::remove_dir_all(&task_work_dir).await {
+                warn!(
+                    "Cleanup task {}'s working directory failed: {} - {}",
+                    task.id,
+                    task_work_dir.display(),
+                    e
+                );
+            } else {
+                info!(
+                    "Cleaned working directory of task {}: {}",
+                    task.id,
+                    task_work_dir.display()
+                );
+            }
+        }
+    }
+
+    /// 获取任务统计信息
+    pub async fn get_task_stats(&self) -> Result<TaskStats, AppError> {
+        let mut stats = TaskStats::default();
+
+        for result in self.tasks_tree.iter() {
+            match result {
+                Ok((_, data)) => {
+                    match serde_json::from_slice::<DocumentTask>(&data) {
+                        Ok(task) => {
+                            stats.total_count += 1;
+                            let id = task.id.clone();
+
+                            match task.status {
+                                TaskStatus::Pending { .. } => {
+                                    stats.pending_count += 1;
+                                    stats.pending_ids.push(id);
+                                }
+                                TaskStatus::Processing { .. } => {
+                                    stats.processing_count += 1;
+                                    stats.processing_ids.push(id);
+                                }
+                                TaskStatus::Completed {
+                                    processing_time, ..
+                                } => {
+                                    stats.completed_count += 1;
+                                    stats.completed_ids.push(id.clone());
+
+                                    // 记录执行时间信息
+                                    let processing_time_ms = processing_time.as_millis() as u64;
+                                    stats.completed_task_times.push(CompletedTaskTime {
+                                        task_id: id,
+                                        processing_time_ms,
+                                    });
+                                }
+                                TaskStatus::Failed { error, .. } => {
+                                    stats.failed_count += 1;
+                                    stats.failed_ids.push(id.clone());
+                                    stats.failed_details.push(FailedTaskSummary {
+                                        task_id: id,
+                                        error_code: error.error_code,
+                                        error_message: error.error_message,
+                                        stage: error.stage,
+                                    });
+                                }
+                                TaskStatus::Cancelled { .. } => {
+                                    stats.cancelled_count += 1;
+                                    stats.cancelled_ids.push(id);
+                                }
+                            }
+
+                            if let Some(engine) = task.parser_engine {
+                                match engine {
+                                    ParserEngine::MinerU => stats.mineru_count += 1,
+                                    ParserEngine::MarkItDown => stats.markitdown_count += 1,
+                                }
+                            }
+                        }
+                        Err(e) => {
+                            warn!("Deserialization task failed: {}", e);
+                        }
+                    }
+                }
+                Err(e) => {
+                    warn!("Failed to read task data: {}", e);
+                }
+            }
+        }
+
+        // 计算已完成任务的平均执行时间
+        if !stats.completed_task_times.is_empty() {
+            let total_time_ms: u64 = stats
+                .completed_task_times
+                .iter()
+                .map(|task_time| task_time.processing_time_ms)
+                .sum();
+            stats.average_processing_time_ms =
+                Some(total_time_ms / stats.completed_task_times.len() as u64);
+        }
+
+        Ok(stats)
+    }
+}
+
+/// 任务统计信息
+#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+pub struct TaskStats {
+    pub total_count: usize,
+    pub pending_count: usize,
+    pub processing_count: usize,
+    pub completed_count: usize,
+    pub failed_count: usize,
+    pub cancelled_count: usize,
+    pub mineru_count: usize,
+    pub markitdown_count: usize,
+    /// 待处理任务ID列表
+    pub pending_ids: Vec<String>,
+    /// 处理中任务ID列表
+    pub processing_ids: Vec<String>,
+    /// 已完成任务ID列表
+    pub completed_ids: Vec<String>,
+    /// 已取消任务ID列表
+    pub cancelled_ids: Vec<String>,
+    /// 失败任务ID列表
+    pub failed_ids: Vec<String>,
+    /// 失败任务详情列表（包含错误码、错误信息与阶段）
+    pub failed_details: Vec<FailedTaskSummary>,
+    /// 已完成任务的执行时间详情列表
+    pub completed_task_times: Vec<CompletedTaskTime>,
+    /// 已完成任务的平均执行时间（毫秒）
+    pub average_processing_time_ms: Option<u64>,
+}
+
+/// 失败任务简要信息
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+pub struct FailedTaskSummary {
+    /// 任务ID
+    pub task_id: String,
+    /// 错误码（如 E009、E003）
+    pub error_code: String,
+    /// 错误信息
+    pub error_message: String,
+    /// 发生错误时的处理阶段
+    pub stage: Option<ProcessingStage>,
+}
+
+/// 已完成任务执行时间信息
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+pub struct CompletedTaskTime {
+    /// 任务ID
+    pub task_id: String,
+    /// 执行耗时（毫秒）
+    pub processing_time_ms: u64,
+}