添加qiming-rcoder模块
This commit is contained in:
Codex
@@ -0,0 +1,7 @@
|
||||
//! Agent 模块
|
||||
|
||||
pub mod scanner;
|
||||
pub mod status_checker;
|
||||
|
||||
pub use scanner::AgentScanner;
|
||||
pub use status_checker::AgentStatusChecker;
|
||||
202
qiming-rcoder/crates/rcoder/src/cleanup_task/agent/scanner.rs
Normal file
202
qiming-rcoder/crates/rcoder/src/cleanup_task/agent/scanner.rs
Normal file
@@ -0,0 +1,202 @@
|
||||
//! Agent 扫描器
|
||||
//!
|
||||
//! 扫描并识别需要清理的闲置 agent
|
||||
|
||||
use crate::AgentStatus;
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Agent 扫描器
|
||||
pub struct AgentScanner {
|
||||
pub state: Arc<crate::router::AppState>,
|
||||
pub config: crate::cleanup_task::config::CleanupConfig,
|
||||
pub status_checker: super::AgentStatusChecker,
|
||||
}
|
||||
|
||||
impl AgentScanner {
|
||||
pub fn new(
|
||||
state: Arc<crate::router::AppState>,
|
||||
config: crate::cleanup_task::config::CleanupConfig,
|
||||
) -> Self {
|
||||
use crate::cleanup_task::agent::AgentStatusChecker;
|
||||
let status_checker = AgentStatusChecker::new(state.grpc_pool.clone());
|
||||
Self {
|
||||
state,
|
||||
config,
|
||||
status_checker,
|
||||
}
|
||||
}
|
||||
|
||||
/// 扫描需要清理的 agent
|
||||
pub async fn scan_idle_agents(&self) -> Result<Vec<String>> {
|
||||
let mut idle_agents = Vec::new();
|
||||
let current_time = Utc::now();
|
||||
|
||||
info!("[scanner] Starting agent scan");
|
||||
|
||||
// 收集所有项目 ID
|
||||
let project_ids: Vec<String> = self.state.projects.iter().map(|(id, _)| id).collect();
|
||||
|
||||
for project_id in project_ids {
|
||||
if let Some(agent) = self.state.get_project(&project_id) {
|
||||
if self.should_cleanup_agent(&agent, current_time).await {
|
||||
idle_agents.push(project_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"🎯 [scanner] Scan completed: found {} idle agents",
|
||||
idle_agents.len()
|
||||
);
|
||||
Ok(idle_agents)
|
||||
}
|
||||
|
||||
async fn should_cleanup_agent(
|
||||
&self,
|
||||
agent: &shared_types::ProjectAndContainerInfo,
|
||||
current_time: chrono::DateTime<Utc>,
|
||||
) -> bool {
|
||||
// 状态检查
|
||||
let status = agent.status();
|
||||
match status {
|
||||
Some(AgentStatus::Idle) => {
|
||||
debug!("[scanner] status=Idle: {}", agent.project_id());
|
||||
}
|
||||
Some(AgentStatus::Pending) | Some(AgentStatus::Active) => {
|
||||
// 🔧 修复：即使是 Active/Pending 状态，也要检查是否真的活跃
|
||||
// 如果状态卡住（比如 gRPC 服务异常），仍需要清理
|
||||
debug!("⏸️ [scanner] status={:?}, checking", status);
|
||||
// 继续检查，不要直接返回 false
|
||||
}
|
||||
None => {
|
||||
// 状态为 None，检查保护期
|
||||
let age = current_time - agent.created_at();
|
||||
if age.num_seconds() < self.config.container_protection_duration.as_secs() as i64 {
|
||||
debug!("⏸️ [scanner] status=None, in protection period");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Some(AgentStatus::Terminating) => {
|
||||
// 🔧 修复：Terminating 状态不应该持续很久（最多 30 秒）
|
||||
// 如果长时间停留在 Terminating，说明操作卡住了，应该清理
|
||||
let terminating_duration = current_time - agent.last_activity();
|
||||
let terminating_stuck_secs = terminating_duration.num_seconds();
|
||||
let max_terminating_secs = 30; // docker_stop_timeout 默认 30 秒
|
||||
|
||||
if terminating_stuck_secs > max_terminating_secs {
|
||||
warn!(
|
||||
"⚠️ [scanner] Terminating status stuck for more than {} seconds, forcing cleanup: project_id={}, stuck_duration={}s",
|
||||
max_terminating_secs,
|
||||
agent.project_id(),
|
||||
terminating_stuck_secs
|
||||
);
|
||||
// 继续检查，不要返回 false
|
||||
} else {
|
||||
debug!("⏸️ [scanner] status=Terminating, waiting...");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 超时检查
|
||||
let idle_duration = current_time - agent.last_activity();
|
||||
let is_timeout = idle_duration
|
||||
> chrono::Duration::from_std(self.config.idle_timeout).unwrap_or_default();
|
||||
|
||||
if !is_timeout {
|
||||
// 未超时，但如果状态是 Active/Pending，仍需要通过 gRPC 确认
|
||||
if matches!(
|
||||
status,
|
||||
Some(AgentStatus::Active) | Some(AgentStatus::Pending)
|
||||
) {
|
||||
debug!(
|
||||
"⏸️ [scanner] Not timeout, status active, skip: {:?}",
|
||||
status
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// 保护期检查
|
||||
if self.should_skip_cleanup_due_to_protection(agent.created_at(), &agent.project_id()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 🆕 gRPC 二次确认：查询容器内 agent 的真实状态
|
||||
if let Some(container) = agent.container() {
|
||||
// 从 service_url 提取 gRPC 地址
|
||||
let grpc_addr = match crate::handler::utils::extract_grpc_addr_with_port(
|
||||
&container.service_url,
|
||||
shared_types::GRPC_DEFAULT_PORT,
|
||||
) {
|
||||
Ok(addr) => addr,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"⚠️ [scanner] Failed to parse gRPC address: project_id={}, error={}",
|
||||
agent.project_id(),
|
||||
e
|
||||
);
|
||||
return true; // 解析失败，允许清理
|
||||
}
|
||||
};
|
||||
|
||||
let project_id = agent.project_id();
|
||||
// 根据 service_type 获取正确的容器标识符
|
||||
// - RCoder: 使用 project_id
|
||||
// - ComputerAgentRunner: 使用 user_id（如果存在），否则使用 project_id
|
||||
let user_id = agent.user_id().unwrap_or(project_id);
|
||||
|
||||
match self
|
||||
.status_checker
|
||||
.is_container_active(&grpc_addr, user_id, project_id)
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
info!(
|
||||
"🔄 [scanner] gRPC secondary confirmation: agent in container is still active, skipping cleanup: project_id={}, user_id={}",
|
||||
project_id, user_id
|
||||
);
|
||||
return false;
|
||||
}
|
||||
Ok(false) => {
|
||||
debug!(
|
||||
"💤 [scanner] gRPC secondary confirmation: agent in container is idle, can cleanup: project_id={}, user_id={}",
|
||||
project_id, user_id
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"⚠️ [scanner] gRPC secondary confirmation failed, allowing cleanup: project_id={}, user_id={}, error={}",
|
||||
project_id, user_id, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn should_skip_cleanup_due_to_protection(
|
||||
&self,
|
||||
created_at: chrono::DateTime<Utc>,
|
||||
project_id: &str,
|
||||
) -> bool {
|
||||
let current_time = Utc::now();
|
||||
let age = current_time.signed_duration_since(created_at);
|
||||
|
||||
if age.num_seconds() < self.config.container_protection_duration.as_secs() as i64 {
|
||||
info!(
|
||||
"🛡️ [scanner] Container in protection period, skipping cleanup: project_id={}, age={}s",
|
||||
project_id,
|
||||
age.num_seconds()
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
//! Agent 状态检查器
|
||||
//!
|
||||
//! 通过 gRPC 查询容器内 agent 的真实状态
|
||||
|
||||
use anyhow::Result;
|
||||
use shared_types::grpc::GetContainerStatusRequest;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tracing::debug;
|
||||
|
||||
/// Agent 状态检查器
|
||||
pub struct AgentStatusChecker {
|
||||
pub grpc_pool: Arc<crate::grpc::GrpcChannelPool>,
|
||||
}
|
||||
|
||||
impl AgentStatusChecker {
|
||||
pub fn new(grpc_pool: Arc<crate::grpc::GrpcChannelPool>) -> Self {
|
||||
Self { grpc_pool }
|
||||
}
|
||||
|
||||
/// 查询容器内 agent 是否正在执行任务
|
||||
///
|
||||
/// 返回 true 表示活跃（不应清理），false 表示空闲（可以清理）
|
||||
pub async fn is_container_active(
|
||||
&self,
|
||||
grpc_addr: &str,
|
||||
user_id: &str,
|
||||
project_id: &str,
|
||||
) -> Result<bool> {
|
||||
let timeout_duration = Duration::from_secs(3);
|
||||
|
||||
match timeout(
|
||||
timeout_duration,
|
||||
self.query_container_status(grpc_addr, user_id, project_id),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(is_active)) => Ok(is_active),
|
||||
Ok(Err(e)) => {
|
||||
debug!("⚠️ [status_checker] gRPC Query failed: {}", e);
|
||||
Ok(false) // Query failed，允许清理
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("⏰ [status_checker] gRPC timeout");
|
||||
Ok(false) // 超时，允许清理
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn query_container_status(
|
||||
&self,
|
||||
grpc_addr: &str,
|
||||
user_id: &str,
|
||||
project_id: &str,
|
||||
) -> Result<bool> {
|
||||
let mut client = self.grpc_pool.get_client(grpc_addr).await?;
|
||||
|
||||
let request = tonic::Request::new(GetContainerStatusRequest {
|
||||
user_id: user_id.to_string(),
|
||||
project_id: project_id.to_string(),
|
||||
});
|
||||
|
||||
let response = client.get_container_status(request).await?;
|
||||
let status = response.into_inner();
|
||||
|
||||
debug!(
|
||||
"📊 [status_checker] Container status: is_active={}, active_tasks={}",
|
||||
status.is_active, status.active_tasks
|
||||
);
|
||||
|
||||
Ok(status.is_active || status.active_tasks > 0)
|
||||
}
|
||||
}
|
||||
250
qiming-rcoder/crates/rcoder/src/cleanup_task/cleaner.rs
Normal file
250
qiming-rcoder/crates/rcoder/src/cleanup_task/cleaner.rs
Normal file
@@ -0,0 +1,250 @@
|
||||
//! 主清理器
|
||||
//!
|
||||
//! 协调各模块完成清理任务
|
||||
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
use shared_types::ServiceType;
|
||||
use std::sync::Arc;
|
||||
use tokio::time::interval;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// 主清理器（协调各模块）
|
||||
pub struct AgentCleaner {
|
||||
config: super::config::CleanupConfig,
|
||||
stats: super::config::CleanupStats,
|
||||
state: Arc<crate::router::AppState>,
|
||||
|
||||
// 策略
|
||||
rcoder_strategy: super::strategies::rcoder::RCoderStrategy,
|
||||
computer_runner_strategy: super::strategies::computer_runner::ComputerRunnerStrategy,
|
||||
|
||||
// 组件
|
||||
container_destroyer: super::container::ContainerDestroyer,
|
||||
agent_scanner: super::agent::AgentScanner,
|
||||
log_cleaner: super::logs::LogCleaner,
|
||||
}
|
||||
|
||||
impl AgentCleaner {
|
||||
pub fn new(
|
||||
config: super::config::CleanupConfig,
|
||||
state: Arc<crate::router::AppState>,
|
||||
docker_manager: Arc<docker_manager::DockerManager>,
|
||||
pingora_service: Option<Arc<rcoder_proxy::PingoraProxyService>>,
|
||||
) -> Self {
|
||||
let config_clone = config.clone();
|
||||
let state_clone = state.clone();
|
||||
let grpc_pool = state.grpc_pool.clone();
|
||||
|
||||
// 创建日志清理器（使用配置）
|
||||
let log_cleaner = super::logs::LogCleaner::new(
|
||||
config.log_dir.clone(),
|
||||
config.log_retention_duration.as_secs() / 24 / 60 / 60,
|
||||
);
|
||||
|
||||
Self {
|
||||
config,
|
||||
stats: super::config::CleanupStats::default(),
|
||||
state: state_clone,
|
||||
rcoder_strategy: super::strategies::rcoder::RCoderStrategy,
|
||||
computer_runner_strategy: super::strategies::computer_runner::ComputerRunnerStrategy,
|
||||
container_destroyer: super::container::ContainerDestroyer::new(
|
||||
docker_manager.clone(),
|
||||
grpc_pool,
|
||||
pingora_service,
|
||||
),
|
||||
agent_scanner: {
|
||||
use crate::cleanup_task::agent::AgentScanner;
|
||||
AgentScanner::new(state.clone(), config_clone)
|
||||
},
|
||||
log_cleaner,
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一次清理
|
||||
pub async fn cleanup_once(&mut self) -> Result<super::config::CleanupStats> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
// 重置本次清理的统计
|
||||
let mut current_stats = super::config::CleanupStats::default();
|
||||
|
||||
// 1. 清理过期日志文件
|
||||
match self.log_cleaner.cleanup_once().await {
|
||||
Ok(log_stats) => {
|
||||
if log_stats.files_deleted > 0 || log_stats.failed_deletions > 0 {
|
||||
info!(
|
||||
"🗑️ [cleaner] Cleanup completed: {}",
|
||||
log_stats.summary()
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("[cleaner] Cleanup failed: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 扫描需要清理的 agent
|
||||
let idle_agents = self.agent_scanner.scan_idle_agents().await?;
|
||||
info!("[cleaner] Found {} idle agents to clean", idle_agents.len());
|
||||
|
||||
// 3. 清理每个 agent
|
||||
// 记录已销毁的容器名称，避免共享容器被重复销毁（ComputerAgentRunner 场景）
|
||||
let mut destroyed_containers: std::collections::HashSet<String> =
|
||||
std::collections::HashSet::new();
|
||||
|
||||
for project_id in idle_agents {
|
||||
// 检查该项目的容器是否已被本轮销毁（共享容器场景）
|
||||
let container_name: Option<String> = self
|
||||
.state
|
||||
.get_project(&project_id)
|
||||
.and_then(|info| info.container().map(|c| c.container_name.clone()));
|
||||
|
||||
if let Some(ref name) = container_name {
|
||||
if destroyed_containers.contains(name) {
|
||||
// 容器已被销毁，只需删除项目记录
|
||||
self.state.remove_project(&project_id);
|
||||
current_stats.total_cleaned += 1;
|
||||
current_stats.success_cleaned += 1;
|
||||
info!(
|
||||
"[cleaner] Container already destroyed, removed project record only: project_id={}",
|
||||
project_id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
current_stats.total_cleaned += 1;
|
||||
|
||||
match self.cleanup_agent(&project_id).await {
|
||||
Ok(destroyed) => {
|
||||
current_stats.success_cleaned += 1;
|
||||
if destroyed {
|
||||
current_stats.containers_destroyed += 1;
|
||||
// 记录已销毁的容器名称，跳过后续同容器的销毁
|
||||
if let Some(name) = container_name {
|
||||
destroyed_containers.insert(name);
|
||||
}
|
||||
}
|
||||
info!("[cleaner] Agent cleanupsucceeded: {}", project_id);
|
||||
}
|
||||
Err(e) => {
|
||||
current_stats.failed_cleaned += 1;
|
||||
warn!("[cleaner] Agent cleanupfailed: {} - {}", project_id, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 更新累计统计
|
||||
current_stats.last_cleanup = Some(Utc::now());
|
||||
self.stats.total_cleaned += current_stats.total_cleaned;
|
||||
self.stats.success_cleaned += current_stats.success_cleaned;
|
||||
self.stats.failed_cleaned += current_stats.failed_cleaned;
|
||||
self.stats.containers_destroyed += current_stats.containers_destroyed;
|
||||
self.stats.last_cleanup = current_stats.last_cleanup;
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
info!(
|
||||
"✅ [cleaner] Cleanup completed, duration: {:.2}s, this run: {}",
|
||||
duration.as_secs_f64(),
|
||||
current_stats.summary()
|
||||
);
|
||||
info!("[cleaner] stats: {}", self.stats.summary());
|
||||
|
||||
Ok(current_stats)
|
||||
}
|
||||
|
||||
/// 清理单个 agent
|
||||
/// 返回 Ok(true) 表示销毁了容器，Ok(false) 表示只删除了记录
|
||||
async fn cleanup_agent(&self, project_id: &str) -> Result<bool> {
|
||||
info!("[cleaner] startingcleanup agent: {}", project_id);
|
||||
|
||||
// 1. 获取项目信息
|
||||
let agent_info = self
|
||||
.state
|
||||
.get_project(project_id)
|
||||
.ok_or_else(|| anyhow::anyhow!("Agent does not exist: {}", project_id))?;
|
||||
|
||||
let service_type = agent_info.service_type().unwrap_or(ServiceType::RCoder);
|
||||
|
||||
// 2. 选择策略
|
||||
let strategy: &dyn super::strategies::CleanupStrategy = match service_type {
|
||||
ServiceType::RCoder => &self.rcoder_strategy,
|
||||
ServiceType::ComputerAgentRunner => &self.computer_runner_strategy,
|
||||
};
|
||||
|
||||
// 3. 检查是否需要销毁容器，并获取销毁原因
|
||||
let context = super::strategies::CleanupContext {
|
||||
state: self.state.clone(),
|
||||
config: self.config.clone(),
|
||||
};
|
||||
|
||||
let destroy_reason = strategy
|
||||
.should_destroy_container(project_id, &context)
|
||||
.await?;
|
||||
|
||||
// 4. 如果需要销毁容器
|
||||
let mut container_destroyed = false;
|
||||
if let Some(reason) = destroy_reason {
|
||||
if let Some(container_info) = agent_info.container() {
|
||||
let project_info = super::strategies::ProjectInfo {
|
||||
project_id: agent_info.project_id().to_string(),
|
||||
user_id: agent_info.user_id().map(|s| s.to_string()),
|
||||
pod_id: agent_info.pod_id().map(|s| s.to_string()),
|
||||
last_activity: agent_info.last_activity(),
|
||||
};
|
||||
|
||||
let container_identifier = strategy.get_container_identifier(&project_info)?;
|
||||
|
||||
// 🔧 使用容器名称而不是 container_id 来销毁容器
|
||||
// 容器名称更稳定，不会因为容器重启而改变
|
||||
// Docker API 的 remove_container 既接受 ID 也接受名称
|
||||
self.container_destroyer
|
||||
.destroy_with_reason(
|
||||
&container_info.container_name,
|
||||
&service_type,
|
||||
&container_identifier,
|
||||
&reason,
|
||||
)
|
||||
.await?;
|
||||
|
||||
container_destroyed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 从存储中移除项目记录（始终执行）
|
||||
self.state.remove_project(project_id);
|
||||
info!(
|
||||
"[cleaner] already removed project: project_id={}",
|
||||
project_id
|
||||
);
|
||||
|
||||
Ok(container_destroyed)
|
||||
}
|
||||
|
||||
/// 运行清理任务（定时）
|
||||
pub async fn run(&mut self) {
|
||||
info!("[cleaner] cleanup task already started");
|
||||
|
||||
let mut interval = interval(self.config.cleanup_interval);
|
||||
let mut memory_log_counter = 0u32;
|
||||
const MEMORY_LOG_INTERVAL: u32 = 12; // 每 12 次清理（大约 1 小时）输出一次内存日志
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
match self.cleanup_once().await {
|
||||
Ok(_) => debug!("[cleaner] Cleanup completed"),
|
||||
Err(e) => warn!("[cleaner] Cleanup failed: {}", e),
|
||||
}
|
||||
|
||||
// 定期输出 DuckDB 内存使用统计
|
||||
memory_log_counter += 1;
|
||||
if memory_log_counter >= MEMORY_LOG_INTERVAL {
|
||||
memory_log_counter = 0;
|
||||
if let Ok(stats) = self.state.projects.get_memory_stats() {
|
||||
debug!("[cleaner] DuckDB Memory Stats:\n{}", stats);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
76
qiming-rcoder/crates/rcoder/src/cleanup_task/config.rs
Normal file
76
qiming-rcoder/crates/rcoder/src/cleanup_task/config.rs
Normal file
@@ -0,0 +1,76 @@
|
||||
//! 清理任务配置和统计
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use std::time::Duration;
|
||||
|
||||
/// 清理配置
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CleanupConfig {
|
||||
/// 闲置超时时间（默认30分钟）
|
||||
pub idle_timeout: Duration,
|
||||
/// 清理检查间隔（默认5分钟）
|
||||
pub cleanup_interval: Duration,
|
||||
/// Docker容器停止超时时间（默认30秒）
|
||||
#[allow(dead_code)] // 保留用于未来的超时配置
|
||||
pub docker_stop_timeout: Duration,
|
||||
/// 容器最小保护时间（默认5分钟）
|
||||
pub container_protection_duration: Duration,
|
||||
/// Agent 活跃判断时间窗口（默认5分钟）
|
||||
pub active_window: Duration,
|
||||
/// 日志目录路径
|
||||
pub log_dir: String,
|
||||
/// 日志保留时长
|
||||
pub log_retention_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for CleanupConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
idle_timeout: Duration::from_secs(30 * 60),
|
||||
cleanup_interval: Duration::from_secs(5 * 60),
|
||||
docker_stop_timeout: Duration::from_secs(30),
|
||||
container_protection_duration: Duration::from_secs(5 * 60),
|
||||
active_window: Duration::from_secs(5 * 60),
|
||||
log_dir: "/app/logs/container".to_string(),
|
||||
log_retention_duration: Duration::from_secs(7 * 24 * 60 * 60),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 清理任务统计信息
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CleanupStats {
|
||||
/// 总共清理的agent数量
|
||||
pub total_cleaned: u64,
|
||||
/// 成功清理的agent数量
|
||||
pub success_cleaned: u64,
|
||||
/// 清理失败的agent数量
|
||||
pub failed_cleaned: u64,
|
||||
/// 销毁的容器数量
|
||||
pub containers_destroyed: u64,
|
||||
/// 最后清理时间
|
||||
pub last_cleanup: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl CleanupStats {
|
||||
/// 获取清理成功率
|
||||
pub fn success_rate(&self) -> f64 {
|
||||
if self.total_cleaned == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(self.success_cleaned as f64 / self.total_cleaned as f64) * 100.0
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取格式化的统计摘要
|
||||
pub fn summary(&self) -> String {
|
||||
format!(
|
||||
"Total cleanup: {}, success: {}, failed: {}, containers destroyed: {}, success rate: {:.1}%",
|
||||
self.total_cleaned,
|
||||
self.success_cleaned,
|
||||
self.failed_cleaned,
|
||||
self.containers_destroyed,
|
||||
self.success_rate()
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//! 容器销毁器
|
||||
//!
|
||||
//! 销毁容器并清理相关资源（gRPC 连接池、Pingora VNC 后端）
|
||||
|
||||
use anyhow::Result;
|
||||
use shared_types::ServiceType;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info};
|
||||
|
||||
use crate::cleanup_task::strategies::DestroyReason;
|
||||
|
||||
/// 容器销毁器
|
||||
pub struct ContainerDestroyer {
|
||||
pub docker_manager: Arc<docker_manager::DockerManager>,
|
||||
pub grpc_pool: Arc<crate::grpc::GrpcChannelPool>,
|
||||
pub pingora_service: Option<Arc<rcoder_proxy::PingoraProxyService>>,
|
||||
}
|
||||
|
||||
impl ContainerDestroyer {
|
||||
pub fn new(
|
||||
docker_manager: Arc<docker_manager::DockerManager>,
|
||||
grpc_pool: Arc<crate::grpc::GrpcChannelPool>,
|
||||
pingora_service: Option<Arc<rcoder_proxy::PingoraProxyService>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
docker_manager,
|
||||
grpc_pool,
|
||||
pingora_service,
|
||||
}
|
||||
}
|
||||
|
||||
/// 销毁容器并清理相关资源（带原因）
|
||||
///
|
||||
/// # 参数
|
||||
/// * `container_name` - 容器名称（稳定不变，优先使用）
|
||||
/// * `service_type` - 服务类型（用于决定是否清理 VNC 后端）
|
||||
/// * `container_identifier` - 容器标识符（project_id 或 user_id）
|
||||
/// * `reason` - 销毁原因
|
||||
pub async fn destroy_with_reason(
|
||||
&self,
|
||||
container_name: &str,
|
||||
service_type: &ServiceType,
|
||||
container_identifier: &str,
|
||||
reason: &DestroyReason,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
"🔥 [destroyer] Starting container destruction: container_name={}, service_type={:?}, identifier={}, reason={}",
|
||||
container_name,
|
||||
service_type,
|
||||
container_identifier,
|
||||
reason.as_str()
|
||||
);
|
||||
|
||||
// 输出详细原因
|
||||
debug!("📋 [destroyer] destroy reason: {}", reason.description());
|
||||
|
||||
// 1. 🔍 通过容器名称实时查询最新的容器信息
|
||||
// 这样可以获取最新的 container_id，避免使用缓存中过期的 ID
|
||||
let (actual_container_id, container_ip) = match self
|
||||
.docker_manager
|
||||
.find_container_realtime(container_name)
|
||||
.await
|
||||
{
|
||||
Ok(Some(result)) => {
|
||||
debug!(
|
||||
"✅ [destroyer] Found container: name={}, id={}, ip={}",
|
||||
container_name, result.container_id, result.container_ip
|
||||
);
|
||||
(result.container_id, result.container_ip)
|
||||
}
|
||||
Ok(None) => {
|
||||
// 容器不存在，可能已经被删除了，这不是错误
|
||||
info!(
|
||||
"⚠️ [destroyer] Container does not exist, may have been deleted: name={}",
|
||||
container_name
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
// 查询出错，返回错误
|
||||
return Err(anyhow::anyhow!(
|
||||
"Failed to query container info: name={}, error={}",
|
||||
container_name,
|
||||
e
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// 2. 执行物理销毁（使用最新的 container_id）
|
||||
docker_manager::container_stop::runtime_cleanup_container(
|
||||
&self.docker_manager,
|
||||
&actual_container_id,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to stop container: {}", e))?;
|
||||
|
||||
// 3. 清理 DockerManager 内存缓存（防止缓存残留导致孤立容器无法被清理）
|
||||
let _: Option<_> = self
|
||||
.docker_manager
|
||||
.remove_container_cache(container_identifier)
|
||||
.await;
|
||||
debug!(
|
||||
"🧹 [destroyer] DockerManager memory cache cleaned: identifier={}",
|
||||
container_identifier
|
||||
);
|
||||
|
||||
// 4. 清理关联资源
|
||||
// 清理 gRPC 连接池中的旧连接（避免复用已失效的 TCP 连接）
|
||||
if !container_ip.is_empty() {
|
||||
let old_grpc_addr = format!(
|
||||
"{}:{}",
|
||||
container_ip,
|
||||
shared_types::GRPC_DEFAULT_PORT
|
||||
);
|
||||
self.grpc_pool.remove(&old_grpc_addr);
|
||||
}
|
||||
|
||||
if *service_type == ServiceType::ComputerAgentRunner {
|
||||
// 清理 Pingora VNC 后端
|
||||
if let Some(ref pingora_service) = self.pingora_service {
|
||||
let _: Option<String> = pingora_service.remove_vnc_backend(container_identifier);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"✅ [destroyer] Container destruction completed: container_name={}, actual_id={}, reason={}",
|
||||
container_name,
|
||||
actual_container_id,
|
||||
reason.as_str()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 销毁容器并清理相关资源（兼容旧接口）
|
||||
///
|
||||
/// # 参数
|
||||
/// * `container_name` - 容器名称
|
||||
/// * `service_type` - 服务类型（用于决定是否清理 VNC 后端）
|
||||
/// * `container_identifier` - 容器标识符（project_id 或 user_id）
|
||||
pub async fn destroy(
|
||||
&self,
|
||||
container_name: &str,
|
||||
service_type: &ServiceType,
|
||||
container_identifier: &str,
|
||||
) -> Result<()> {
|
||||
// 使用默认原因
|
||||
let reason = DestroyReason::ManualStop {
|
||||
source: "unknown".to_string(),
|
||||
};
|
||||
self.destroy_with_reason(container_name, service_type, container_identifier, &reason)
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! 容器操作模块
|
||||
|
||||
pub mod destroyer;
|
||||
|
||||
pub use destroyer::ContainerDestroyer;
|
||||
@@ -0,0 +1,413 @@
|
||||
//! 清理任务集成测试
|
||||
//!
|
||||
//! 测试核心业务逻辑：
|
||||
//! 1. ComputerAgentRunner 引用计数检查（核心）
|
||||
//! 2. 活跃窗口边界条件
|
||||
//! 3. 容器标识符获取策略
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::*;
|
||||
use crate::cleanup_task::strategies::CleanupStrategy;
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use duckdb_manager::ProjectRecord;
|
||||
use shared_types::ServiceType;
|
||||
use std::time::Duration;
|
||||
|
||||
/// 创建测试用的 ProjectRecord
|
||||
fn create_test_project(
|
||||
project_id: &str,
|
||||
user_id: &str,
|
||||
service_type: ServiceType,
|
||||
last_activity_seconds_ago: i64,
|
||||
) -> ProjectRecord {
|
||||
ProjectRecord {
|
||||
project_id: project_id.to_string(),
|
||||
session_id: None,
|
||||
service_type,
|
||||
container_id: format!("container_{}", project_id),
|
||||
user_id: Some(user_id.to_string()),
|
||||
pod_id: None,
|
||||
agent_status_code: None,
|
||||
agent_status_name: None,
|
||||
request_id: None,
|
||||
model_provider_json: None,
|
||||
created_at: Utc::now() - ChronoDuration::hours(2),
|
||||
last_activity: Utc::now() - ChronoDuration::seconds(last_activity_seconds_ago),
|
||||
session_created_at: None,
|
||||
session_last_activity: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建带 pod_id 的测试用 ProjectRecord（RCoder 共享容器模式）
|
||||
fn create_test_project_with_pod(
|
||||
project_id: &str,
|
||||
user_id: &str,
|
||||
pod_id: &str,
|
||||
service_type: ServiceType,
|
||||
last_activity_seconds_ago: i64,
|
||||
) -> ProjectRecord {
|
||||
ProjectRecord {
|
||||
project_id: project_id.to_string(),
|
||||
session_id: None,
|
||||
service_type,
|
||||
container_id: format!("container_{}", pod_id),
|
||||
user_id: Some(user_id.to_string()),
|
||||
pod_id: Some(pod_id.to_string()),
|
||||
agent_status_code: None,
|
||||
agent_status_name: None,
|
||||
request_id: None,
|
||||
model_provider_json: None,
|
||||
created_at: Utc::now() - ChronoDuration::hours(2),
|
||||
last_activity: Utc::now() - ChronoDuration::seconds(last_activity_seconds_ago),
|
||||
session_created_at: None,
|
||||
session_last_activity: None,
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// 核心测试：ComputerAgentRunner 引用计数逻辑
|
||||
// ========================================================================
|
||||
|
||||
/// 测试场景：有活跃项目时，不应该销毁容器
|
||||
#[test]
|
||||
fn test_computer_runner_ref_count_with_active_projects() {
|
||||
// 场景：user_1 有 3 个项目
|
||||
// - proj_A: 闲置30分钟（准备清理）
|
||||
// - proj_B: 活跃2分钟（仍在使用）
|
||||
// - proj_C: 闲置30分钟
|
||||
//
|
||||
// 预期：清理 proj_A 时，因为 proj_B 仍活跃，容器应该被保留
|
||||
|
||||
let proj_a =
|
||||
create_test_project("proj_A", "user_1", ServiceType::ComputerAgentRunner, 1800);
|
||||
let proj_b = create_test_project("proj_B", "user_1", ServiceType::ComputerAgentRunner, 120);
|
||||
let proj_c =
|
||||
create_test_project("proj_C", "user_1", ServiceType::ComputerAgentRunner, 1800);
|
||||
|
||||
let config = CleanupConfig {
|
||||
active_window: Duration::from_secs(300), // 5分钟窗口
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// proj_B 应该是活跃的
|
||||
assert!(
|
||||
strategies::computer_runner::is_project_active(&proj_b, &config),
|
||||
"proj_B (2分钟前活动) 应该被认为是活跃的"
|
||||
);
|
||||
|
||||
// proj_A 和 proj_C 应该是闲置的
|
||||
assert!(
|
||||
!strategies::computer_runner::is_project_active(&proj_a, &config),
|
||||
"proj_A (30分钟前活动) 应该被认为是闲置的"
|
||||
);
|
||||
|
||||
// 验证引用计数逻辑：存在活跃引用，不应该销毁容器
|
||||
let related_projects = vec![proj_a.clone(), proj_b.clone(), proj_c.clone()];
|
||||
let has_active_refs = related_projects.iter().any(|p| {
|
||||
p.project_id != "proj_A" && strategies::computer_runner::is_project_active(p, &config)
|
||||
});
|
||||
|
||||
assert!(
|
||||
has_active_refs,
|
||||
"应该存在活跃的引用项目 (proj_B)，因此不应该销毁容器"
|
||||
);
|
||||
}
|
||||
|
||||
/// 测试场景：所有项目都闲置时，应该销毁容器
|
||||
#[test]
|
||||
fn test_computer_runner_ref_count_all_idle() {
|
||||
// 场景：user_2 有 3 个项目，全部闲置
|
||||
// - proj_D: 闲置1小时
|
||||
// - proj_E: 闲置2小时
|
||||
// - proj_F: 闲置30分钟
|
||||
//
|
||||
// 预期：清理 proj_D 时，因为所有项目都闲置，应该销毁容器
|
||||
|
||||
let proj_d =
|
||||
create_test_project("proj_D", "user_2", ServiceType::ComputerAgentRunner, 3600);
|
||||
let proj_e =
|
||||
create_test_project("proj_E", "user_2", ServiceType::ComputerAgentRunner, 7200);
|
||||
let proj_f =
|
||||
create_test_project("proj_F", "user_2", ServiceType::ComputerAgentRunner, 1800);
|
||||
|
||||
let config = CleanupConfig {
|
||||
active_window: Duration::from_secs(300),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let related_projects = vec![proj_d.clone(), proj_e.clone(), proj_f.clone()];
|
||||
|
||||
// 所有项目都应该是闲置的
|
||||
for proj in &related_projects {
|
||||
assert!(
|
||||
!strategies::computer_runner::is_project_active(proj, &config),
|
||||
"{} 应该被认为是闲置的",
|
||||
proj.project_id
|
||||
);
|
||||
}
|
||||
|
||||
// 没有活跃引用，应该销毁容器
|
||||
let has_active_refs = related_projects.iter().any(|p| {
|
||||
p.project_id != "proj_D" && strategies::computer_runner::is_project_active(p, &config)
|
||||
});
|
||||
|
||||
assert!(!has_active_refs, "不存在活跃的引用项目，因此应该销毁容器");
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// 测试：活跃窗口边界条件
|
||||
// ========================================================================
|
||||
|
||||
#[test]
|
||||
fn test_active_window_boundary_conditions() {
|
||||
// is_project_active 使用 idle_timeout 作为判断标准（与 scanner 一致）
|
||||
let config = CleanupConfig {
|
||||
idle_timeout: Duration::from_secs(600), // 10分钟
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 测试边界：(描述, 距离上次活动秒数, 是否应该活跃)
|
||||
let test_cases = vec![
|
||||
("刚刚活动", 0, true),
|
||||
("1分钟前", 60, true),
|
||||
("9分59秒前", 599, true),
|
||||
("恰好在边界", 600, false),
|
||||
("10分1秒前", 601, false),
|
||||
("30分钟前", 1800, false),
|
||||
];
|
||||
|
||||
for (desc, seconds_ago, expected_active) in test_cases {
|
||||
let project = create_test_project(
|
||||
&format!("project_{}", desc.replace(' ', "_")),
|
||||
"test_user",
|
||||
ServiceType::ComputerAgentRunner,
|
||||
seconds_ago,
|
||||
);
|
||||
|
||||
let is_active = strategies::computer_runner::is_project_active(&project, &config);
|
||||
|
||||
assert_eq!(
|
||||
is_active, expected_active,
|
||||
"{}: {}秒前活动, 预期={}, 实际={}",
|
||||
desc, seconds_ago, expected_active, is_active
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// 测试：容器标识符获取策略
|
||||
// ========================================================================
|
||||
|
||||
#[test]
|
||||
fn test_container_identifier_extraction() {
|
||||
let rcoder_strategy = strategies::rcoder::RCoderStrategy;
|
||||
let computer_runner_strategy = strategies::computer_runner::ComputerRunnerStrategy;
|
||||
|
||||
// RCoder 无 pod_id: 使用 project_id
|
||||
let rcoder_info = strategies::ProjectInfo {
|
||||
project_id: "project_abc".to_string(),
|
||||
user_id: Some("user_xyz".to_string()),
|
||||
pod_id: None,
|
||||
last_activity: Utc::now(),
|
||||
};
|
||||
|
||||
let rcoder_id = rcoder_strategy
|
||||
.get_container_identifier(&rcoder_info)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
rcoder_id, "project_abc",
|
||||
"RCoder 无 pod_id 时应该使用 project_id 作为容器标识符"
|
||||
);
|
||||
|
||||
// RCoder 有 pod_id: 使用 pod_id（共享容器模式）
|
||||
let rcoder_pod_info = strategies::ProjectInfo {
|
||||
project_id: "project_jkl".to_string(),
|
||||
user_id: Some("user_xyz".to_string()),
|
||||
pod_id: Some("pod_123".to_string()),
|
||||
last_activity: Utc::now(),
|
||||
};
|
||||
|
||||
let rcoder_pod_id = rcoder_strategy
|
||||
.get_container_identifier(&rcoder_pod_info)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
rcoder_pod_id, "pod_123",
|
||||
"RCoder 有 pod_id 时应该使用 pod_id 作为容器标识符"
|
||||
);
|
||||
|
||||
// ComputerAgentRunner: 使用 user_id
|
||||
let runner_info = strategies::ProjectInfo {
|
||||
project_id: "project_def".to_string(),
|
||||
user_id: Some("user_123".to_string()),
|
||||
pod_id: None,
|
||||
last_activity: Utc::now(),
|
||||
};
|
||||
|
||||
let runner_id = computer_runner_strategy
|
||||
.get_container_identifier(&runner_info)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
runner_id, "user_123",
|
||||
"ComputerAgentRunner 应该使用 user_id 作为容器标识符"
|
||||
);
|
||||
|
||||
// ComputerAgentRunner 缺少 user_id 应该返回错误
|
||||
let runner_info_missing_user = strategies::ProjectInfo {
|
||||
project_id: "project_ghi".to_string(),
|
||||
user_id: None,
|
||||
pod_id: None,
|
||||
last_activity: Utc::now(),
|
||||
};
|
||||
|
||||
let result = computer_runner_strategy.get_container_identifier(&runner_info_missing_user);
|
||||
assert!(result.is_err(), "缺少 user_id 时应该返回错误");
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// 测试：RCoder pod_id 共享容器引用计数逻辑
|
||||
// ========================================================================
|
||||
|
||||
/// 测试场景：RCoder 有 pod_id 时，有活跃项目不应该销毁容器
|
||||
#[test]
|
||||
fn test_rcoder_pod_id_ref_count_with_active_projects() {
|
||||
// 场景：pod_1 下有 3 个 RCoder 项目
|
||||
// - proj_A: 闲置30分钟（准备清理）
|
||||
// - proj_B: 活跃2分钟（仍在使用）
|
||||
// - proj_C: 闲置30分钟
|
||||
//
|
||||
// 预期：清理 proj_A 时，因为 proj_B 仍活跃，容器应该被保留
|
||||
|
||||
let proj_a = create_test_project_with_pod(
|
||||
"proj_A",
|
||||
"user_1",
|
||||
"pod_1",
|
||||
ServiceType::RCoder,
|
||||
1800,
|
||||
);
|
||||
let proj_b = create_test_project_with_pod(
|
||||
"proj_B",
|
||||
"user_1",
|
||||
"pod_1",
|
||||
ServiceType::RCoder,
|
||||
120,
|
||||
);
|
||||
let proj_c = create_test_project_with_pod(
|
||||
"proj_C",
|
||||
"user_1",
|
||||
"pod_1",
|
||||
ServiceType::RCoder,
|
||||
1800,
|
||||
);
|
||||
|
||||
let config = CleanupConfig {
|
||||
idle_timeout: Duration::from_secs(600), // 10分钟
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// proj_B 应该是活跃的
|
||||
assert!(
|
||||
strategies::computer_runner::is_project_active(&proj_b, &config),
|
||||
"proj_B (2分钟前活动) 应该被认为是活跃的"
|
||||
);
|
||||
|
||||
// proj_A 和 proj_C 应该是闲置的
|
||||
assert!(
|
||||
!strategies::computer_runner::is_project_active(&proj_a, &config),
|
||||
"proj_A (30分钟前活动) 应该被认为是闲置的"
|
||||
);
|
||||
|
||||
// 验证引用计数逻辑：存在活跃引用，不应该销毁容器
|
||||
let related_projects = vec![proj_a.clone(), proj_b.clone(), proj_c.clone()];
|
||||
let has_active_refs = related_projects.iter().any(|p| {
|
||||
p.project_id != "proj_A" && strategies::computer_runner::is_project_active(p, &config)
|
||||
});
|
||||
|
||||
assert!(
|
||||
has_active_refs,
|
||||
"应该存在活跃的引用项目 (proj_B)，因此不应该销毁容器"
|
||||
);
|
||||
}
|
||||
|
||||
/// 测试场景：RCoder 有 pod_id 时，所有项目都闲置应该销毁容器
|
||||
#[test]
|
||||
fn test_rcoder_pod_id_ref_count_all_idle() {
|
||||
// 场景：pod_2 下有 3 个 RCoder 项目，全部闲置
|
||||
// - proj_D: 闲置1小时
|
||||
// - proj_E: 闲置2小时
|
||||
// - proj_F: 闲置30分钟
|
||||
//
|
||||
// 预期：清理 proj_D 时，因为所有项目都闲置，应该销毁容器
|
||||
|
||||
let proj_d = create_test_project_with_pod(
|
||||
"proj_D",
|
||||
"user_2",
|
||||
"pod_2",
|
||||
ServiceType::RCoder,
|
||||
3600,
|
||||
);
|
||||
let proj_e = create_test_project_with_pod(
|
||||
"proj_E",
|
||||
"user_2",
|
||||
"pod_2",
|
||||
ServiceType::RCoder,
|
||||
7200,
|
||||
);
|
||||
let proj_f = create_test_project_with_pod(
|
||||
"proj_F",
|
||||
"user_2",
|
||||
"pod_2",
|
||||
ServiceType::RCoder,
|
||||
1800,
|
||||
);
|
||||
|
||||
let config = CleanupConfig {
|
||||
idle_timeout: Duration::from_secs(600),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let related_projects = vec![proj_d.clone(), proj_e.clone(), proj_f.clone()];
|
||||
|
||||
// 所有项目都应该是闲置的
|
||||
for proj in &related_projects {
|
||||
assert!(
|
||||
!strategies::computer_runner::is_project_active(proj, &config),
|
||||
"{} 应该被认为是闲置的",
|
||||
proj.project_id
|
||||
);
|
||||
}
|
||||
|
||||
// 没有活跃引用，应该销毁容器
|
||||
let has_active_refs = related_projects.iter().any(|p| {
|
||||
p.project_id != "proj_D" && strategies::computer_runner::is_project_active(p, &config)
|
||||
});
|
||||
|
||||
assert!(!has_active_refs, "不存在活跃的引用项目，因此应该销毁容器");
|
||||
}
|
||||
|
||||
/// 测试场景：RCoder 无 pod_id 时，始终应该销毁容器（1:1 模式）
|
||||
#[test]
|
||||
fn test_rcoder_no_pod_id_always_destroy() {
|
||||
// 场景：无 pod_id 的 RCoder 项目，1容器=1项目
|
||||
// 预期：无论其他项目如何，清理时始终销毁容器
|
||||
|
||||
let proj = create_test_project("proj_solo", "user_3", ServiceType::RCoder, 1800);
|
||||
|
||||
let config = CleanupConfig {
|
||||
idle_timeout: Duration::from_secs(600),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 确认没有 pod_id
|
||||
assert!(
|
||||
proj.pod_id.is_none(),
|
||||
"无 pod_id 的项目应该直接销毁容器"
|
||||
);
|
||||
|
||||
// 项目闲置
|
||||
assert!(
|
||||
!strategies::computer_runner::is_project_active(&proj, &config),
|
||||
"proj_solo (30分钟前活动) 应该被认为是闲置的"
|
||||
);
|
||||
}
|
||||
}
|
||||
274
qiming-rcoder/crates/rcoder/src/cleanup_task/logs/mod.rs
Normal file
274
qiming-rcoder/crates/rcoder/src/cleanup_task/logs/mod.rs
Normal file
@@ -0,0 +1,274 @@
|
||||
//! 日志清理模块
|
||||
//!
|
||||
//! 负责清理 /app/logs/container 目录下的过期日志文件
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use std::vec::Vec;
|
||||
use tokio::fs;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// 日志清理器
|
||||
pub struct LogCleaner {
|
||||
/// 日志目录路径
|
||||
log_dir: String,
|
||||
/// 日志保留时长（默认 7 天）
|
||||
retention_duration: Duration,
|
||||
}
|
||||
|
||||
impl LogCleaner {
|
||||
/// 创建新的日志清理器
|
||||
pub fn new(log_dir: impl Into<String>, retention_days: u64) -> Self {
|
||||
Self {
|
||||
log_dir: log_dir.into(),
|
||||
retention_duration: Duration::from_secs(retention_days * 24 * 60 * 60),
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一次日志清理
|
||||
///
|
||||
/// # 返回
|
||||
/// 返回清理的文件数量和释放的字节数
|
||||
pub async fn cleanup_once(&self) -> Result<LogCleanupStats, std::io::Error> {
|
||||
let log_path = Path::new(&self.log_dir);
|
||||
|
||||
// 检查目录是否存在
|
||||
if !log_path.exists() {
|
||||
debug!(
|
||||
"📋 [log_cleaner] Log directory does not exist, skipping cleanup: {}",
|
||||
self.log_dir
|
||||
);
|
||||
return Ok(LogCleanupStats::default());
|
||||
}
|
||||
|
||||
// 检查是否是目录
|
||||
if !log_path.is_dir() {
|
||||
warn!(
|
||||
"📋 [log_cleaner] Path is not a directory, skip cleanup: {}",
|
||||
self.log_dir
|
||||
);
|
||||
return Ok(LogCleanupStats::default());
|
||||
}
|
||||
|
||||
info!(
|
||||
"🧹 [log_cleaner] Starting log directory cleanup: {}, retention: {} days",
|
||||
self.log_dir,
|
||||
self.retention_duration.as_secs() / 86400
|
||||
);
|
||||
|
||||
let mut stats = LogCleanupStats::default();
|
||||
let cutoff_time = match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) {
|
||||
Ok(duration) => duration.as_secs(),
|
||||
Err(e) => {
|
||||
warn!("📋 [log_cleaner] Invalid timestamp, skip cleanup: {}", e);
|
||||
return Ok(LogCleanupStats::default());
|
||||
}
|
||||
};
|
||||
let cutoff_time = cutoff_time.saturating_sub(self.retention_duration.as_secs());
|
||||
|
||||
// 读取目录内容
|
||||
let mut entries = match fs::read_dir(log_path).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
warn!("📋 [log_cleaner] Failed to read directory: {}", e);
|
||||
return Ok(LogCleanupStats::default());
|
||||
}
|
||||
};
|
||||
|
||||
// 遍历目录中的文件和子目录
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let metadata = match entry.metadata().await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"📋 [log_cleaner] get file failed: {:?} - {}",
|
||||
path, e
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// 获取修改时间（更可靠，所有文件系统都支持）
|
||||
let modified = match metadata.modified() {
|
||||
Ok(time) => time,
|
||||
Err(e) => {
|
||||
debug!("📋 [log_cleaner] get modified time failed: {:?} - {}", path, e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// 转换为 Unix 时间戳
|
||||
let modified_secs = match modified.duration_since(std::time::UNIX_EPOCH) {
|
||||
Ok(duration) => duration.as_secs(),
|
||||
Err(_) => {
|
||||
debug!("📋 [log_cleaner] skip: {:?}", path);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// 判断是否过期（基于修改时间）
|
||||
if modified_secs < cutoff_time {
|
||||
if metadata.is_file() {
|
||||
// 删除过期文件
|
||||
let file_size = metadata.len();
|
||||
|
||||
match fs::remove_file(&path).await {
|
||||
Ok(_) => {
|
||||
stats.files_deleted += 1;
|
||||
stats.bytes_freed += file_size;
|
||||
debug!(
|
||||
"🗑️ [log_cleaner] Deleting expired file: {:?} ({:.2} MB)",
|
||||
path,
|
||||
file_size as f64 / 1024.0 / 1024.0
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
stats.failed_deletions += 1;
|
||||
warn!("📋 [log_cleaner] Failed to delete file: {:?} - {}", path, e);
|
||||
}
|
||||
}
|
||||
} else if metadata.is_dir() {
|
||||
// 删除过期目录（递归删除整个目录）
|
||||
match fs::remove_dir_all(&path).await {
|
||||
Ok(_) => {
|
||||
stats.dirs_deleted += 1;
|
||||
debug!("🗑️ [log_cleaner] Deleted directory: {:?}", path);
|
||||
}
|
||||
Err(e) => {
|
||||
stats.failed_deletions += 1;
|
||||
warn!(
|
||||
"📋 [log_cleaner] Failed to delete directory: {:?} - {}",
|
||||
path, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if stats.files_deleted > 0 || stats.dirs_deleted > 0 {
|
||||
info!(
|
||||
"✅ [log_cleaner] Log cleanup completed: deleted {} files, {} dirs, freed {:.2} MB",
|
||||
stats.files_deleted,
|
||||
stats.dirs_deleted,
|
||||
stats.bytes_freed as f64 / 1024.0 / 1024.0
|
||||
);
|
||||
} else {
|
||||
info!("[log_cleaner] Cleanup completed");
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// 获取日志目录路径
|
||||
pub fn log_dir(&self) -> &str {
|
||||
&self.log_dir
|
||||
}
|
||||
|
||||
/// 获取保留时长（秒）
|
||||
pub fn retention_duration(&self) -> Duration {
|
||||
self.retention_duration
|
||||
}
|
||||
}
|
||||
|
||||
/// 日志清理统计
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct LogCleanupStats {
|
||||
/// 删除的文件数量
|
||||
pub files_deleted: u64,
|
||||
/// 删除的目录数量
|
||||
pub dirs_deleted: u64,
|
||||
/// 释放的字节数
|
||||
pub bytes_freed: u64,
|
||||
/// 删除失败的文件/目录数量
|
||||
pub failed_deletions: u64,
|
||||
}
|
||||
|
||||
impl LogCleanupStats {
|
||||
/// 获取格式化的统计摘要
|
||||
pub fn summary(&self) -> String {
|
||||
if self.files_deleted == 0 && self.dirs_deleted == 0 && self.failed_deletions == 0 {
|
||||
"No expired logs".to_string()
|
||||
} else {
|
||||
let mut parts = Vec::new();
|
||||
if self.files_deleted > 0 {
|
||||
parts.push(format!(
|
||||
"Deleted: {} files, freed: {:.2} MB",
|
||||
self.files_deleted,
|
||||
self.bytes_freed as f64 / 1024.0 / 1024.0
|
||||
));
|
||||
}
|
||||
if self.dirs_deleted > 0 {
|
||||
parts.push(format!("deleted: {} dirs", self.dirs_deleted));
|
||||
}
|
||||
if self.failed_deletions > 0 {
|
||||
parts.push(format!("failed: {}", self.failed_deletions));
|
||||
}
|
||||
parts.join(", ")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_log_cleanup_basic() {
|
||||
// 创建临时测试目录
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let log_dir = temp_dir.path().join("logs");
|
||||
|
||||
// 创建日志目录
|
||||
fs::create_dir_all(&log_dir).await.unwrap();
|
||||
|
||||
// 创建一些测试日志文件
|
||||
let test_files = vec![
|
||||
("old_log_1.txt", "old content 1"),
|
||||
("old_log_2.txt", "old content 2"),
|
||||
("recent_log.txt", "recent content"),
|
||||
];
|
||||
|
||||
for (filename, content) in &test_files {
|
||||
let file_path = log_dir.join(filename);
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(content.as_bytes()).unwrap();
|
||||
}
|
||||
|
||||
// 设置旧文件的修改时间为 15 天前
|
||||
let old_time =
|
||||
std::time::SystemTime::now() - std::time::Duration::from_secs(15 * 24 * 60 * 60);
|
||||
for (filename, _) in &test_files[..2] {
|
||||
let file_path = log_dir.join(filename);
|
||||
if let Err(e) = filetime::set_file_mtime(&file_path, old_time.into()) {
|
||||
// 系统不支持设置修改时间，跳过此测试
|
||||
println!(" set mtime not supported, skip: {}", e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建日志清理器，保留 10 天
|
||||
let cleaner = LogCleaner::new(log_dir.to_str().unwrap(), 10);
|
||||
|
||||
// 执行清理
|
||||
let stats = cleaner.cleanup_once().await.unwrap();
|
||||
|
||||
// 验证结果
|
||||
assert_eq!(stats.files_deleted, 2);
|
||||
assert!(log_dir.join("recent_log.txt").exists());
|
||||
assert!(!log_dir.join("old_log_1.txt").exists());
|
||||
assert!(!log_dir.join("old_log_2.txt").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_nonexistent_dir() {
|
||||
let cleaner = LogCleaner::new("/nonexistent/path", 10);
|
||||
let stats = cleaner.cleanup_once().await.unwrap();
|
||||
assert_eq!(stats.files_deleted, 0);
|
||||
assert_eq!(stats.failed_deletions, 0);
|
||||
}
|
||||
}
|
||||
132
qiming-rcoder/crates/rcoder/src/cleanup_task/mod.rs
Normal file
132
qiming-rcoder/crates/rcoder/src/cleanup_task/mod.rs
Normal file
@@ -0,0 +1,132 @@
|
||||
//! 清理任务模块
|
||||
//!
|
||||
//! 重构后的清理任务，修复 ComputerAgentRunner 引用计数问题并模块化拆分
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
pub mod agent;
|
||||
pub mod cleaner;
|
||||
pub mod config;
|
||||
pub mod container;
|
||||
pub mod logs;
|
||||
pub mod storage;
|
||||
pub mod strategies;
|
||||
|
||||
// 集成测试
|
||||
#[cfg(test)]
|
||||
mod integration_tests;
|
||||
|
||||
pub use cleaner::AgentCleaner;
|
||||
pub use config::CleanupConfig;
|
||||
#[allow(unused_imports)] // CleanupStats 用于类型导出
|
||||
pub use config::CleanupStats;
|
||||
|
||||
/// 启动清理任务
|
||||
///
|
||||
/// # Errors
|
||||
/// 如果Failed to get DockerManager，返回错误而不是静默失败
|
||||
pub async fn start_cleanup_task(
|
||||
config: CleanupConfig,
|
||||
state: Arc<crate::router::AppState>,
|
||||
) -> anyhow::Result<tokio::task::JoinHandle<()>> {
|
||||
let docker_manager = match docker_manager::global::get_global_docker_manager().await {
|
||||
Ok(dm) => Some(dm),
|
||||
Err(e) => {
|
||||
if matches!(
|
||||
docker_manager::runtime::RuntimeManager::runtime_type(),
|
||||
docker_manager::runtime_selection::RuntimeType::Kubernetes
|
||||
) {
|
||||
tracing::warn!(
|
||||
"⚠️ [CLEANUP_TASK] DockerManager unavailable in Kubernetes mode, starting lightweight cleanup task: {}",
|
||||
e
|
||||
);
|
||||
None
|
||||
} else {
|
||||
tracing::error!(
|
||||
"🚨 [CLEANUP_TASK] Failed to get DockerManager: {}, cleanup task cannot start",
|
||||
e
|
||||
);
|
||||
return Err(anyhow::anyhow!("Failed to get DockerManager: {}", e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if docker_manager.is_none() {
|
||||
let state_for_k8s = state.clone();
|
||||
return Ok(tokio::task::spawn(async move {
|
||||
let mut interval = tokio::time::interval(config.cleanup_interval);
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let runtime = match docker_manager::runtime::RuntimeManager::get().await {
|
||||
Ok(rt) => rt,
|
||||
Err(e) => {
|
||||
tracing::warn!("[CLEANUP_TASK] failed to get runtime: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let idle_threshold = match chrono::Duration::from_std(config.idle_timeout) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!("[CLEANUP_TASK] invalid idle_timeout config: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let now = chrono::Utc::now();
|
||||
let projects: Vec<(String, Arc<shared_types::ProjectAndContainerInfo>)> =
|
||||
state_for_k8s.projects.iter().collect();
|
||||
|
||||
for (project_id, project_info) in projects {
|
||||
let idle = now.signed_duration_since(project_info.last_activity());
|
||||
if idle < idle_threshold {
|
||||
continue;
|
||||
}
|
||||
|
||||
let service_type = project_info
|
||||
.service_type()
|
||||
.unwrap_or(shared_types::ServiceType::RCoder);
|
||||
let identifier = match service_type {
|
||||
shared_types::ServiceType::ComputerAgentRunner => project_info
|
||||
.user_id()
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| project_id.clone()),
|
||||
shared_types::ServiceType::RCoder => project_id.clone(),
|
||||
};
|
||||
|
||||
if let Err(e) = runtime
|
||||
.stop_container_by_identifier(&identifier, &service_type)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
"[CLEANUP_TASK] failed to stop runtime container: identifier={}, service_type={:?}, error={}",
|
||||
identifier,
|
||||
service_type,
|
||||
e
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
state_for_k8s.remove_project(&project_id);
|
||||
tracing::info!(
|
||||
"[CLEANUP_TASK] cleaned idle runtime container: project_id={}, identifier={}, service_type={:?}",
|
||||
project_id,
|
||||
identifier,
|
||||
service_type
|
||||
);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
let pingora_service = state.pingora_service.clone();
|
||||
|
||||
let mut cleaner = AgentCleaner::new(
|
||||
config,
|
||||
state,
|
||||
docker_manager.expect("docker_manager checked above"),
|
||||
pingora_service,
|
||||
);
|
||||
|
||||
Ok(tokio::task::spawn(async move {
|
||||
cleaner.run().await;
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
//! 存储操作辅助模块
|
||||
@@ -0,0 +1,100 @@
|
||||
//! ComputerAgentRunner 清理策略
|
||||
//!
|
||||
//! ComputerAgentRunner 模式: 1容器 = N项目，需要引用计数检查
|
||||
//!
|
||||
//! 核心修复：只有当容器的所有项目都闲置时才销毁容器
|
||||
|
||||
use super::{CleanupContext, CleanupStrategy, DestroyReason, ProjectInfo};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use duckdb_manager::ProjectRecord;
|
||||
use tracing::info;
|
||||
|
||||
/// ComputerAgentRunner 清理策略
|
||||
///
|
||||
/// 在 ComputerAgentRunner 模式中，一个用户对应一个容器
|
||||
/// 该容器可能被多个 project_id 共享
|
||||
///
|
||||
/// 清理时需要检查：
|
||||
/// 1. 获取 user_id
|
||||
/// 2. 查询该用户的所有项目
|
||||
/// 3. 检查是否还有其他活跃项目
|
||||
/// 4. 只有所有项目都闲置时才销毁容器
|
||||
pub struct ComputerRunnerStrategy;
|
||||
|
||||
#[async_trait]
|
||||
impl CleanupStrategy for ComputerRunnerStrategy {
|
||||
async fn should_destroy_container(
|
||||
&self,
|
||||
project_id: &str,
|
||||
context: &CleanupContext,
|
||||
) -> Result<Option<DestroyReason>> {
|
||||
// 获取 user_id
|
||||
let user_id = context
|
||||
.state
|
||||
.get_project(project_id)
|
||||
.and_then(|p| p.user_id().map(|s| s.to_string()))
|
||||
.ok_or_else(|| anyhow::anyhow!("Failed to get user_id: {}", project_id))?;
|
||||
|
||||
// 查询该用户的所有项目
|
||||
let related_projects = context.state.projects.find_projects_by_user_id(&user_id);
|
||||
|
||||
// 检查是否还有其他活跃项目（排除当前项目）
|
||||
let has_active_refs = related_projects
|
||||
.iter()
|
||||
.any(|p| p.project_id != project_id && is_project_active(p, &context.config));
|
||||
|
||||
if has_active_refs {
|
||||
// 还有其他活跃项目，不销毁容器
|
||||
info!(
|
||||
"🛡️ [cleanup] 容器还被其他项目使用，只删除项目记录: project_id={}, user_id={}",
|
||||
project_id, user_id
|
||||
);
|
||||
Ok(None)
|
||||
} else {
|
||||
// 所有项目都闲置，计算最大闲置时间
|
||||
let now = Utc::now();
|
||||
let max_idle_duration = related_projects
|
||||
.iter()
|
||||
.map(|p| (now - p.last_activity).num_seconds())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
|
||||
let timeout_secs = context.config.idle_timeout.as_secs();
|
||||
|
||||
info!(
|
||||
"🔥 [cleanup] 容器所有项目都已闲置，可以销毁: project_id={}, user_id={}",
|
||||
project_id, user_id
|
||||
);
|
||||
|
||||
Ok(Some(DestroyReason::IdleTimeout {
|
||||
idle_duration_secs: max_idle_duration,
|
||||
timeout_secs,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn get_container_identifier(&self, project_info: &ProjectInfo) -> Result<String> {
|
||||
// ComputerAgentRunner: 容器标识符是 user_id
|
||||
project_info
|
||||
.user_id
|
||||
.clone()
|
||||
.ok_or_else(|| anyhow::anyhow!("user_id is missing"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 判断项目是否活跃
|
||||
///
|
||||
/// 使用 idle_timeout 作为判断标准：如果项目的闲置时间小于 idle_timeout，
|
||||
/// 则认为项目仍然活跃，不应销毁其关联的容器。
|
||||
/// 这与 scanner 的 idle 判断标准一致，避免出现 scanner 认为项目未超时
|
||||
/// 但策略却认为项目不活跃的矛盾情况。
|
||||
pub fn is_project_active(
|
||||
project: &ProjectRecord,
|
||||
config: &crate::cleanup_task::config::CleanupConfig,
|
||||
) -> bool {
|
||||
let now = Utc::now();
|
||||
let idle_duration = now - project.last_activity;
|
||||
idle_duration.num_seconds() < config.idle_timeout.as_secs() as i64
|
||||
}
|
||||
123
qiming-rcoder/crates/rcoder/src/cleanup_task/strategies/mod.rs
Normal file
123
qiming-rcoder/crates/rcoder/src/cleanup_task/strategies/mod.rs
Normal file
@@ -0,0 +1,123 @@
|
||||
//! 清理策略模块
|
||||
//!
|
||||
//! 定义不同服务类型的清理策略 trait
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// 容器销毁原因
|
||||
///
|
||||
/// 记录容器被销毁的具体原因，便于日志追踪和问题排查
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum DestroyReason {
|
||||
/// 闲置超时 - 所有项目都超过闲置时间限制
|
||||
/// - RCoder: 项目闲置超时
|
||||
/// - ComputerAgentRunner: 容器下所有项目都闲置
|
||||
IdleTimeout {
|
||||
/// 闲置时长（秒）
|
||||
idle_duration_secs: i64,
|
||||
/// 超时阈值（秒）
|
||||
timeout_secs: u64,
|
||||
},
|
||||
|
||||
/// 孤立容器 - DuckDB 中没有对应记录
|
||||
/// - 容器存在但状态管理系统中没有记录
|
||||
/// - 可能是由于系统重启、异常退出等原因导致
|
||||
Orphaned {
|
||||
/// 容器创建时间
|
||||
created_at: DateTime<Utc>,
|
||||
/// 是否在保护期内
|
||||
was_protected: bool,
|
||||
},
|
||||
|
||||
/// 手动停止 - 用户主动停止或重启
|
||||
/// - 通过 API 调用停止
|
||||
/// - 通过 Agent 生命周期管理停止
|
||||
ManualStop {
|
||||
/// 触发来源
|
||||
source: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl DestroyReason {
|
||||
/// 获取销毁原因的简短描述（用于日志）
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
DestroyReason::IdleTimeout { .. } => "Idle timeout",
|
||||
DestroyReason::Orphaned { .. } => "Orphaned container",
|
||||
DestroyReason::ManualStop { .. } => "Manual stop",
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取销毁原因的详细描述
|
||||
pub fn description(&self) -> String {
|
||||
match self {
|
||||
DestroyReason::IdleTimeout {
|
||||
idle_duration_secs,
|
||||
timeout_secs,
|
||||
} => {
|
||||
format!(
|
||||
"Idle timeout (idle {}s / timeout {}s)",
|
||||
idle_duration_secs, timeout_secs
|
||||
)
|
||||
}
|
||||
DestroyReason::Orphaned {
|
||||
created_at,
|
||||
was_protected,
|
||||
} => {
|
||||
format!(
|
||||
"Orphaned container (created at {}, protected:{})",
|
||||
created_at.format("%Y-%m-%d %H:%M:%S"),
|
||||
was_protected
|
||||
)
|
||||
}
|
||||
DestroyReason::ManualStop { source } => {
|
||||
format!("manual stop (source:{})", source)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 清理策略 trait
|
||||
///
|
||||
/// 不同服务类型有不同的容器管理策略：
|
||||
/// - RCoder: 1容器 = 1项目，直接销毁
|
||||
/// - ComputerAgentRunner: 1容器 = N项目，需要引用计数检查
|
||||
#[async_trait]
|
||||
pub trait CleanupStrategy: Send + Sync {
|
||||
/// 检查容器是否应该被销毁
|
||||
///
|
||||
/// 返回 `Some(DestroyReason)` 表示应该销毁容器，并附带原因
|
||||
/// 返回 `None` 表示不应该销毁容器
|
||||
async fn should_destroy_container(
|
||||
&self,
|
||||
project_id: &str,
|
||||
context: &CleanupContext,
|
||||
) -> Result<Option<DestroyReason>>;
|
||||
|
||||
/// 获取容器的唯一标识符（用于查找容器）
|
||||
///
|
||||
/// RCoder 返回 project_id
|
||||
/// ComputerAgentRunner 返回 user_id
|
||||
fn get_container_identifier(&self, project_info: &ProjectInfo) -> Result<String>;
|
||||
}
|
||||
|
||||
/// 清理上下文
|
||||
pub struct CleanupContext {
|
||||
pub state: Arc<crate::router::AppState>,
|
||||
pub config: super::config::CleanupConfig,
|
||||
}
|
||||
|
||||
/// 项目信息摘要
|
||||
pub struct ProjectInfo {
|
||||
pub project_id: String,
|
||||
pub user_id: Option<String>,
|
||||
pub pod_id: Option<String>,
|
||||
pub last_activity: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub mod computer_runner;
|
||||
pub mod rcoder;
|
||||
@@ -0,0 +1,91 @@
|
||||
//! RCoder 清理策略
|
||||
//!
|
||||
//! RCoder 模式支持两种容器关系：
|
||||
//! - 无 pod_id: 1容器 = 1项目，直接销毁
|
||||
//! - 有 pod_id: N容器 = 1项目（共享容器），需要引用计数检查
|
||||
|
||||
use super::{CleanupContext, CleanupStrategy, DestroyReason, ProjectInfo};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use duckdb_manager::ProjectRecord;
|
||||
use tracing::info;
|
||||
|
||||
/// RCoder 清理策略
|
||||
///
|
||||
/// 在 RCoder 模式中：
|
||||
/// - 当 pod_id 为空时，每个项目对应一个独立的容器，清理时直接销毁
|
||||
/// - 当 pod_id 有值时，多个项目共享同一个容器，需要检查其他项目是否仍然活跃
|
||||
pub struct RCoderStrategy;
|
||||
|
||||
#[async_trait]
|
||||
impl CleanupStrategy for RCoderStrategy {
|
||||
async fn should_destroy_container(
|
||||
&self,
|
||||
project_id: &str,
|
||||
context: &CleanupContext,
|
||||
) -> Result<Option<DestroyReason>> {
|
||||
let project = context
|
||||
.state
|
||||
.get_project(project_id)
|
||||
.ok_or_else(|| anyhow::anyhow!("Project does not exist: {}", project_id))?;
|
||||
|
||||
let now = Utc::now();
|
||||
let idle_duration = (now - project.last_activity()).num_seconds();
|
||||
let timeout_secs = context.config.idle_timeout.as_secs();
|
||||
|
||||
// 检查是否有 pod_id（共享容器模式）
|
||||
let effective_idle_duration = if let Some(pod_id) = project.pod_id() {
|
||||
// 共享容器模式：检查同 pod_id 下是否有其他活跃项目
|
||||
let related_projects = context.state.projects.find_projects_by_pod_id(pod_id);
|
||||
|
||||
let has_active_refs = related_projects.iter().any(|p| {
|
||||
p.project_id != project_id
|
||||
&& super::computer_runner::is_project_active(p, &context.config)
|
||||
});
|
||||
|
||||
if has_active_refs {
|
||||
info!(
|
||||
"🛡️ [cleanup] RCoder 容器还被其他项目使用，只删除项目记录: project_id={}, pod_id={}",
|
||||
project_id, pod_id
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// 计算所有相关项目的最大闲置时间（与 ComputerRunnerStrategy 一致）
|
||||
let max_idle = compute_max_idle_duration(&related_projects);
|
||||
|
||||
info!(
|
||||
"🔥 [cleanup] RCoder 共享容器所有项目都已闲置，可以销毁: project_id={}, pod_id={}, max_idle={}s",
|
||||
project_id, pod_id, max_idle
|
||||
);
|
||||
|
||||
max_idle
|
||||
} else {
|
||||
idle_duration
|
||||
};
|
||||
|
||||
Ok(Some(DestroyReason::IdleTimeout {
|
||||
idle_duration_secs: effective_idle_duration,
|
||||
timeout_secs,
|
||||
}))
|
||||
}
|
||||
|
||||
fn get_container_identifier(&self, project_info: &ProjectInfo) -> Result<String> {
|
||||
// RCoder: 有 pod_id 时使用 pod_id（共享容器），否则使用 project_id
|
||||
Ok(project_info
|
||||
.pod_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| project_info.project_id.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
/// 计算一组项目的最大闲置时间（秒）
|
||||
fn compute_max_idle_duration(projects: &[ProjectRecord]) -> i64 {
|
||||
let now = Utc::now();
|
||||
projects
|
||||
.iter()
|
||||
.map(|p| (now - p.last_activity).num_seconds())
|
||||
.max()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
Reference in New Issue
Block a user