添加qiming-rcoder模块
This commit is contained in:
273
qiming-rcoder/specs/k8s-runtime-support/data-model.md
Normal file
273
qiming-rcoder/specs/k8s-runtime-support/data-model.md
Normal file
@@ -0,0 +1,273 @@
|
||||
# Data Model: Kubernetes Runtime Support
|
||||
|
||||
**Date**: 2026-04-16
|
||||
**Feature**: K8s Runtime Support
|
||||
|
||||
## 1. Interface Changes
|
||||
|
||||
### 1.1 ContainerRuntime Trait (container-runtime-api)
|
||||
|
||||
**Location**: `crates/container-runtime-api/src/runtime_trait.rs`
|
||||
|
||||
**Existing Methods**:
|
||||
```rust
|
||||
pub trait ContainerRuntime: Send + Sync {
|
||||
async fn create_container(&self, project_id, user_id, host_workspace_path, service_type, resource_limits) -> Result<ContainerBasicInfo>;
|
||||
async fn get_container_info(&self, project_id) -> Result<Option<ContainerBasicInfo>>;
|
||||
async fn find_container(&self, project_id, service_type) -> Result<Option<RuntimeContainerInfo>>;
|
||||
async fn stop_container(&self, project_id) -> Result<()>;
|
||||
async fn is_container_running(&self, project_id) -> Result<bool>;
|
||||
async fn list_containers(&self) -> Result<Vec<RuntimeContainerInfo>>;
|
||||
async fn cleanup_all(&self) -> Result<()>;
|
||||
async fn health_check(&self) -> Result<()>;
|
||||
}
|
||||
```
|
||||
|
||||
**New Methods**:
|
||||
```rust
|
||||
// 新增: 按标签列出容器
|
||||
async fn list_containers_by_label(&self, label_selector: &str) -> Result<Vec<RuntimeContainerInfo>>;
|
||||
|
||||
// 新增: 获取 Pod 的 DNS 名称
|
||||
fn get_service_dns_name(&self, project_id: &str, user_id: Option<&str>) -> String;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. KubernetesRuntime Implementation
|
||||
|
||||
**Location**: `crates/docker_manager/src/runtime/kubernetes_runtime.rs`
|
||||
|
||||
### 2.1 Config Changes
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct KubernetesRuntimeConfig {
|
||||
pub namespace: String,
|
||||
pub pod_ttl_seconds: Option<u64>,
|
||||
pub image_pull_secret: Option<String>,
|
||||
pub service_account_name: String,
|
||||
// 新增: DNS 域名后缀
|
||||
pub cluster_domain: String, // 默认: "cluster.local"
|
||||
}
|
||||
```
|
||||
|
||||
### 2.2 Service DNS Name Generation
|
||||
|
||||
```rust
|
||||
impl KubernetesRuntime {
|
||||
/// Generate stable service DNS name for a pod
|
||||
///
|
||||
/// Format: {prefix}-{id}.{namespace}.svc.{cluster_domain}
|
||||
/// Examples:
|
||||
/// - RCoder: rcoder-agent-{project_id}.default.svc.cluster.local
|
||||
/// - Computer: computer-agent-runner-{user_id}.default.svc.cluster.local
|
||||
pub fn get_service_dns_name(&self, project_id: &str, user_id: Option<&str>) -> String {
|
||||
let prefix = match user_id {
|
||||
Some(uid) => format!("computer-agent-runner-{}", uid),
|
||||
None => format!("rcoder-agent-{}", project_id),
|
||||
};
|
||||
format!(
|
||||
"{}.{}.svc.{}",
|
||||
prefix, self.namespace, self.config.cluster_domain
|
||||
)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2.3 ContainerBasicInfo with Service URL
|
||||
|
||||
```rust
|
||||
// KubernetesRuntime 返回的 ContainerBasicInfo.service_url 使用 DNS
|
||||
ContainerBasicInfo {
|
||||
// ...
|
||||
container_ip: pod_ip, // 仍保留 Pod IP(用于内部参考)
|
||||
service_url: format!( // 使用稳定的 DNS
|
||||
"http://{}:{}",
|
||||
self.get_service_dns_name(project_id, user_id),
|
||||
shared_types::GRPC_DEFAULT_PORT
|
||||
),
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
### 2.4 user_id Support in create_container
|
||||
|
||||
```rust
|
||||
async fn create_container(
|
||||
&self,
|
||||
project_id: Option<&str>,
|
||||
user_id: Option<&str>, // ← 现在处理 user_id
|
||||
host_workspace_path: &str,
|
||||
service_type: ServiceType,
|
||||
resource_limits: Option<ServiceResourceLimits>,
|
||||
) -> ContainerRuntimeResult<ContainerBasicInfo> {
|
||||
// 确定容器标识符
|
||||
let identifier = user_id.or(project_id).ok_or_else(|| {
|
||||
ContainerRuntimeError::ConfigurationError(
|
||||
"Either project_id or user_id must be provided".to_string()
|
||||
)
|
||||
})?;
|
||||
|
||||
// 生成 Pod 名称时考虑 user_id
|
||||
let pod_name = match user_id {
|
||||
Some(uid) => format!("computer-agent-runner-{}", uid),
|
||||
None => format!("rcoder-agent-{}", project_id.unwrap()),
|
||||
};
|
||||
|
||||
// ... 后续逻辑
|
||||
}
|
||||
```
|
||||
|
||||
### 2.5 list_containers_by_label Implementation
|
||||
|
||||
```rust
|
||||
async fn list_containers_by_label(
|
||||
&self,
|
||||
label_selector: &str,
|
||||
) -> ContainerRuntimeResult<Vec<RuntimeContainerInfo>> {
|
||||
let lp = ListParams::default().labels(label_selector);
|
||||
let pods = self.pods().list(&lp).await?;
|
||||
|
||||
let mut result = Vec::new();
|
||||
for p in pods.items {
|
||||
let pod: Pod = p;
|
||||
// ... 转换逻辑
|
||||
result.push(RuntimeContainerInfo { ... });
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Global Module Changes
|
||||
|
||||
**Location**: `crates/docker_manager/src/lib.rs`
|
||||
|
||||
### 3.1 New Initialization API
|
||||
|
||||
```rust
|
||||
pub mod global {
|
||||
// 修改: 添加 runtime_type 参数
|
||||
pub async fn init_global_runtime(
|
||||
runtime_type: RuntimeType,
|
||||
config: DockerManagerConfig,
|
||||
) -> DockerResult<()> {
|
||||
match runtime_type {
|
||||
RuntimeType::Kubernetes => {
|
||||
RuntimeManager::init(config).await?;
|
||||
}
|
||||
RuntimeType::Docker => {
|
||||
init_docker_manager_direct(config).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 新增: 获取运行时类型
|
||||
pub fn get_runtime_type() -> RuntimeType {
|
||||
RuntimeType::from_env()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.2 Backward Compatibility
|
||||
|
||||
```rust
|
||||
// 保持向后兼容的初始化方法
|
||||
pub async fn init_global_docker_manager_with_config(
|
||||
config: DockerManagerConfig,
|
||||
) -> DockerResult<()> {
|
||||
// 自动检测运行时类型
|
||||
init_global_runtime(RuntimeType::from_env(), config).await
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Error Types
|
||||
|
||||
**Location**: `crates/container-runtime-api/src/runtime_trait.rs`
|
||||
|
||||
```rust
|
||||
#[derive(Error, Debug)]
|
||||
pub enum ContainerRuntimeError {
|
||||
#[error("Connection error: {0}")]
|
||||
ConnectionError(String),
|
||||
|
||||
#[error("Container creation failed: {0}")]
|
||||
ContainerCreationError(String),
|
||||
|
||||
// ... existing errors ...
|
||||
|
||||
// 新增 K8s 相关错误
|
||||
#[error("Kubernetes error: {0}")]
|
||||
K8sError(String),
|
||||
|
||||
#[error("Pod not found: {0}")]
|
||||
PodNotFound(String),
|
||||
|
||||
#[error("Service DNS resolution failed: {0}")]
|
||||
DnsResolutionError(String),
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Entity Relationships
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ RuntimeManager │
|
||||
│ - select_runtime() -> Arc<dyn ContainerRuntime> │
|
||||
└─────────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────────┐
|
||||
│ DockerRuntime │ │ KubernetesRuntime │
|
||||
│ (wrapper) │ │ │
|
||||
├─────────────────┤ ├─────────────────────┤
|
||||
│ - inner: │ │ - client: Client │
|
||||
│ DockerManager │ │ - namespace: String │
|
||||
├─────────────────┤ │ - config: K8sConfig │
|
||||
│ Implements: │ ├─────────────────────┤
|
||||
│ ContainerRuntime │ Implements: │
|
||||
└─────────────────┘ │ ContainerRuntime │
|
||||
│ + Service DNS │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. State Transitions
|
||||
|
||||
### Container State (K8s)
|
||||
|
||||
```
|
||||
┌──────────┐
|
||||
│ Pending │
|
||||
└─────┬────┘
|
||||
│ (Pod scheduled)
|
||||
▼
|
||||
┌──────────┐
|
||||
┌──────────│ Running │──────────┐
|
||||
│ └────┬─────┘ │
|
||||
│ │ (container │
|
||||
│ │ exits) │
|
||||
▼ ▼ ▼
|
||||
┌─────────┐ ┌──────────┐ ┌──────────┐
|
||||
│ Succeeded│ │ Failed │ │ Unknown │
|
||||
└─────────┘ └──────────┘ └──────────┘
|
||||
```
|
||||
|
||||
### RCoder Container Status Mapping
|
||||
|
||||
| K8s Phase | RCoder Status |
|
||||
|-----------|---------------|
|
||||
| Pending | Creating |
|
||||
| Running | Running |
|
||||
| Succeeded | Stopped |
|
||||
| Failed | Failed |
|
||||
| Unknown | Unknown |
|
||||
348
qiming-rcoder/specs/k8s-runtime-support/kubernetes-deploy.md
Normal file
348
qiming-rcoder/specs/k8s-runtime-support/kubernetes-deploy.md
Normal file
@@ -0,0 +1,348 @@
|
||||
# Kubernetes RBAC Configuration
|
||||
|
||||
**Date**: 2026-04-16
|
||||
**Feature**: K8s Runtime Support
|
||||
|
||||
---
|
||||
|
||||
## 概述
|
||||
|
||||
RCoder 在 Kubernetes 环境中运行时,需要通过 kube-rs 库调用 K8s API Server 来动态创建和管理 Pod。
|
||||
|
||||
本文档提供完整的 RBAC 配置清单,确保 RCoder 有足够的权限来创建、查询和删除 Pod。
|
||||
|
||||
---
|
||||
|
||||
## 最小权限需求
|
||||
|
||||
| 资源 | 操作 | 用途 |
|
||||
|------|------|------|
|
||||
| pods | create | 创建 agent_runner Pod |
|
||||
| pods | delete | 删除已完成的 Pod |
|
||||
| pods | get | 查询 Pod 状态 |
|
||||
| pods | list | 列出 Pod |
|
||||
| pods | watch | 监听 Pod 状态变化 |
|
||||
| pods/log | get | 查看 Pod 日志(可选) |
|
||||
|
||||
---
|
||||
|
||||
## 完整 RBAC 配置
|
||||
|
||||
### 方案一:ClusterRole(适用于所有 Namespace)
|
||||
|
||||
```yaml
|
||||
# rcoder-rbac.yaml
|
||||
---
|
||||
# 1. ServiceAccount
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rcoder-pods-sa
|
||||
# 如果 rcoder 运行在特定 namespace,修改这里
|
||||
# namespace: rcoder
|
||||
---
|
||||
# 2. ClusterRole(权限定义)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rcoder-pods-clusterrole
|
||||
rules:
|
||||
# Pod 管理
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
|
||||
# Pod 日志
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/log"]
|
||||
verbs: ["get", "list"]
|
||||
# Pod 执行命令(用于调试,可选)
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
# Pod 状态
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/status"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
# 3. ClusterRoleBinding(绑定到 ServiceAccount)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: rcoder-pods-clusterrolebinding
|
||||
subjects:
|
||||
# 如果在特定 namespace 使用 Role/RoleBinding,改为 kind: ServiceAccount
|
||||
- kind: ServiceAccount
|
||||
name: rcoder-pods-sa
|
||||
# apiGroup 固定
|
||||
apiGroup: ""
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rcoder-pods-clusterrole
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
```
|
||||
|
||||
### 方案二:Role + RoleBinding(限定 Namespace)
|
||||
|
||||
适用于多租户环境,限制 RCoder 只能操作特定 namespace。
|
||||
|
||||
```yaml
|
||||
# rcoder-rbac-namespaced.yaml
|
||||
---
|
||||
# 1. ServiceAccount
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rcoder-pods-sa
|
||||
namespace: rcoder # 指定 namespace
|
||||
---
|
||||
# 2. Role(权限定义)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: rcoder-pods-role
|
||||
namespace: rcoder # 指定 namespace
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/log"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/status"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
# 3. RoleBinding(绑定到 ServiceAccount)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: rcoder-pods-rolebinding
|
||||
namespace: rcoder # 指定 namespace
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rcoder-pods-sa
|
||||
namespace: rcoder
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: rcoder-pods-role
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 部署步骤
|
||||
|
||||
### 1. 创建配置(使用 ClusterRole 方案)
|
||||
|
||||
```bash
|
||||
# 应用 RBAC 配置
|
||||
kubectl apply -f rcoder-rbac.yaml
|
||||
|
||||
# 验证 ServiceAccount 创建成功
|
||||
kubectl get sa rcoder-pods-sa
|
||||
|
||||
# 验证 Role 创建成功
|
||||
kubectl get clusterrole rcoder-pods-clusterrole
|
||||
|
||||
# 验证 RoleBinding 创建成功
|
||||
kubectl get clusterrolebinding rcoder-pods-clusterrolebinding
|
||||
```
|
||||
|
||||
### 2. 修改 RCoder Deployment
|
||||
|
||||
将 RCoder Pod 关联到 ServiceAccount:
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: rcoder
|
||||
namespace: default # 或你的 namespace
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: rcoder
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: rcoder
|
||||
spec:
|
||||
# 添加 ServiceAccount 配置
|
||||
serviceAccountName: rcoder-pods-sa
|
||||
containers:
|
||||
- name: rcoder
|
||||
image: registry.yichamao.com/rcoder:latest
|
||||
ports:
|
||||
- containerPort: 8087
|
||||
env:
|
||||
# 启用 K8s 运行时
|
||||
- name: CONTAINER_RUNTIME
|
||||
value: "kubernetes"
|
||||
# 可选:指定 namespace(默认使用 default)
|
||||
- name: RCODER_K8S_NAMESPACE
|
||||
value: "default"
|
||||
```
|
||||
|
||||
### 3. 验证权限
|
||||
|
||||
```bash
|
||||
# 进入 RCoder Pod
|
||||
kubectl exec -it <rcoder-pod-name> -- sh
|
||||
|
||||
# 测试 K8s API 访问权限
|
||||
# 方法1:使用 kubectl(需要安装)
|
||||
kubectl auth can-i create pods --as=system:serviceaccount:default:rcoder-pods-sa
|
||||
|
||||
# 方法2:直接测试 API
|
||||
curl -k https://kubernetes.default.svc/api/v1/namespaces/default/pods \
|
||||
--header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 故障排查
|
||||
|
||||
### 问题 1: Permission Denied
|
||||
|
||||
```
|
||||
Error: Container creation failed: Failed to create pod: Unauthorized
|
||||
```
|
||||
|
||||
**解决**: 检查 ServiceAccount 是否正确绑定到 Role/ClusterRole
|
||||
|
||||
```bash
|
||||
# 检查 ServiceAccount
|
||||
kubectl get sa rcoder-pods-sa -o yaml
|
||||
|
||||
# 检查 RoleBinding subjects
|
||||
kubectl get rolebinding <rolebinding-name> -o yaml
|
||||
```
|
||||
|
||||
### 问题 2: Cannot create Pods in other namespaces
|
||||
|
||||
**原因**: 使用了 RoleBinding 而不是 ClusterRoleBinding
|
||||
|
||||
**解决**: 如果需要跨 namespace 操作 Pod,使用 ClusterRoleBinding
|
||||
|
||||
### 问题 3: Token 文件不存在
|
||||
|
||||
```
|
||||
Error: K8s client init failed: No such file or directory: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
```
|
||||
|
||||
**原因**: 代码不在 Pod 内运行,或未配置 ServiceAccount
|
||||
|
||||
**解决**:
|
||||
1. 确保 RCoder 部署在 K8s 集群内
|
||||
2. 确保 Deployment 配置了 `serviceAccountName`
|
||||
3. 检查是否挂载了 ServiceAccount token
|
||||
|
||||
```bash
|
||||
# 检查 Pod 是否挂载了 token
|
||||
kubectl exec <pod-name> -- ls -la /var/run/secrets/kubernetes.io/serviceaccount/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 生产环境建议
|
||||
|
||||
### 1. 使用专用 namespace
|
||||
|
||||
```bash
|
||||
# 创建独立 namespace
|
||||
kubectl create namespace rcoder
|
||||
```
|
||||
|
||||
### 2. 限制 Image Pull Secret(如果使用私有仓库)
|
||||
|
||||
```yaml
|
||||
imagePullSecrets:
|
||||
- name: regcred
|
||||
```
|
||||
|
||||
### 3. NetworkPolicy(可选)
|
||||
|
||||
限制 Pod 网络通信:
|
||||
|
||||
```yaml
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: rcoder-pods-networkpolicy
|
||||
namespace: rcoder
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: rcoder
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
```
|
||||
|
||||
### 4. Resource Limits
|
||||
|
||||
为 RCoder Pod 设置资源限制:
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "1000m"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 代码集成
|
||||
|
||||
### 启动时权限检查(建议添加)
|
||||
|
||||
在 `KubernetesRuntime::new()` 中添加权限验证:
|
||||
|
||||
```rust
|
||||
pub async fn new(config: DockerManagerConfig) -> ContainerRuntimeResult<Self> {
|
||||
let kube_config = Config::infer()
|
||||
.await
|
||||
.map_err(|e| ...)?;
|
||||
|
||||
let client = Client::try_from(kube_config)
|
||||
.map_err(|e| ...)?;
|
||||
|
||||
// 验证权限:尝试列出 pods
|
||||
let pods: Api<Pod> = Api::namespaced(client.clone(), &namespace);
|
||||
pods.list(&ListParams::default().limit(1)).await
|
||||
.map_err(|e| ContainerRuntimeError::K8sError(
|
||||
format!("Permission denied or API server unreachable: {}", e)
|
||||
))?;
|
||||
|
||||
// ... 继续初始化
|
||||
}
|
||||
```
|
||||
|
||||
### 环境变量配置
|
||||
|
||||
| 环境变量 | 默认值 | 说明 |
|
||||
|----------|--------|------|
|
||||
| `CONTAINER_RUNTIME` | `docker` | 运行时类型:docker/kubernetes/k8s |
|
||||
| `RCODER_K8S_NAMESPACE` | `default` | Pod 创建的 namespace |
|
||||
| `RCODER_K8S_SERVICE_ACCOUNT` | `rcoder-pods-sa` | 使用的 ServiceAccount |
|
||||
|
||||
---
|
||||
|
||||
## 完整示例部署
|
||||
|
||||
参见下一节 "Helm Chart 或 Kustomize 清单"
|
||||
|
||||
---
|
||||
|
||||
## 相关文档
|
||||
|
||||
- [K8s RBAC 官方文档](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)
|
||||
- [kube-rs 认证文档](https://kube.rs/client/auth/)
|
||||
- [ServiceAccount 配置](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/)
|
||||
235
qiming-rcoder/specs/k8s-runtime-support/plan.md
Normal file
235
qiming-rcoder/specs/k8s-runtime-support/plan.md
Normal file
@@ -0,0 +1,235 @@
|
||||
# Implementation Plan: Kubernetes Runtime Support
|
||||
|
||||
**Branch**: `dev-k8s` | **Date**: 2026-04-16 | **Spec**: [spec.md](./spec.md)
|
||||
**Input**: Feature specification from `/specs/k8s-runtime-support/spec.md`
|
||||
|
||||
## Execution Flow (/plan command scope)
|
||||
```
|
||||
1. Load feature spec from Input path
|
||||
2. Fill Technical Context
|
||||
3. Constitution Check (empty constitution - skip)
|
||||
4. Execute Phase 0 → research.md
|
||||
5. Execute Phase 1 → contracts, data-model.md, quickstart.md
|
||||
6. Re-evaluate Constitution Check
|
||||
7. Plan Phase 2 → tasks.md (via /tasks command)
|
||||
8. STOP - Ready for /tasks command
|
||||
```
|
||||
|
||||
## Summary
|
||||
修复 RCoder 的 Kubernetes 运行时支持,使系统能够在 K8s 环境中动态创建和管理容器(Pod),而不依赖 Docker Socket。
|
||||
|
||||
主要改动:
|
||||
1. 重构 `global` 模块使用 `RuntimeManager` 选择运行时 **(P0 - 关键路径)**
|
||||
2. K8s Runtime 支持 `user_id` 参数 **(P0 - ComputerAgent 支持)**
|
||||
3. 修复 `stop_container` 方法正确处理不同 ServiceType **(P0)**
|
||||
4. 更新 Makefile K8s 命令避免重复操作 **(P1)**
|
||||
|
||||
## Technical Context
|
||||
**Language/Version**: Rust 1.75+ (2024 Edition)
|
||||
**Primary Dependencies**: kube-rs 0.98, bollard, tonic, tower
|
||||
**Storage**: N/A (状态在 K8s API 中)
|
||||
**Testing**: cargo test, integration tests
|
||||
**Target Platform**: Linux (Docker + Kubernetes)
|
||||
**Performance Goals**: Pod 启动 < 120s, gRPC 延迟 < 100ms
|
||||
**Constraints**:
|
||||
- 必须兼容现有 Docker 模式
|
||||
- K8s Service Account 需要预配置
|
||||
- 需要 RBAC 权限创建/删除 Pod
|
||||
|
||||
## Problem Analysis (Updated 2026-04-20)
|
||||
|
||||
### P0 Issues (Critical - Blocking K8s)
|
||||
|
||||
| Issue | Location | Description |
|
||||
|-------|----------|-------------|
|
||||
| `RuntimeManager::init()` never called | `main.rs:181-186` | K8s mode calls `init_global_docker_manager_with_config()` which does NOT initialize `RuntimeManager::RUNTIME_INSTANCE` |
|
||||
| `stop_container` ignores service_type | `kubernetes_runtime.rs:437` | Always uses `ServiceType::RCoder`, cannot stop ComputerAgentRunner pods |
|
||||
| K8s path not properly initialized | `lib.rs:201-223` | `init_global_docker_manager_with_config()` has K8s code but never reaches it properly |
|
||||
|
||||
### Root Cause
|
||||
|
||||
```rust
|
||||
// main.rs calls this:
|
||||
docker_manager::global::init_global_docker_manager_with_config(config).await
|
||||
|
||||
// But this function does NOT call RuntimeManager::init() for K8s mode!
|
||||
// It only sets GLOBAL_DOCKER_MANAGER (for Docker mode)
|
||||
```
|
||||
|
||||
### Required Changes
|
||||
|
||||
1. **main.rs**: Initialize using `RuntimeManager::init()` for K8s, or modify `init_global_docker_manager_with_config()` to properly call `RuntimeManager::init()` when K8s mode detected
|
||||
|
||||
2. **kubernetes_runtime.rs**: Fix `stop_container` to handle different service types
|
||||
|
||||
## Constitution Check
|
||||
*Note: Constitution file is empty template - no gates to check*
|
||||
|
||||
## Project Structure
|
||||
|
||||
### Documentation (this feature)
|
||||
```
|
||||
specs/k8s-runtime-support/
|
||||
├── plan.md # This file
|
||||
├── spec.md # Feature specification
|
||||
├── research.md # Phase 0 output
|
||||
├── data-model.md # Phase 1 output (interfaces)
|
||||
└── tasks.md # Phase 2 output (/tasks command)
|
||||
```
|
||||
|
||||
### Source Code (repository root)
|
||||
```
|
||||
crates/
|
||||
├── docker_manager/
|
||||
│ ├── src/
|
||||
│ │ ├── lib.rs # MODIFY: global module
|
||||
│ │ └── runtime/
|
||||
│ │ ├── mod.rs # MODIFY: export changes
|
||||
│ │ ├── manager.rs # KEEP: RuntimeManager
|
||||
│ │ ├── kubernetes_runtime.rs # MODIFY: fix issues
|
||||
│ │ └── docker_runtime.rs # KEEP: unchanged
|
||||
├── container-runtime-api/
|
||||
│ └── src/
|
||||
│ └── runtime_trait.rs # KEEP: trait definition
|
||||
└── rcoder/
|
||||
└── src/
|
||||
└── main.rs # MODIFY: runtime init
|
||||
```
|
||||
|
||||
**Structure Decision**: 修改现有 `docker_manager` crate,重构 `global` 模块使其根据 `CONTAINER_RUNTIME` 环境变量选择 `RuntimeManager` 或直接使用 `DockerManager`
|
||||
|
||||
### K8s Local Development Environment
|
||||
|
||||
新增 `k8s/` 目录,用于本地 K8s 测试环境(类比现有 `docker/` 目录):
|
||||
|
||||
```
|
||||
k8s/
|
||||
├── README.md # K8s 环境使用说明
|
||||
├── kind-config.yaml # Kind (Kubernetes IN Docker) 配置
|
||||
├── start-kind.sh # 启动本地 K8s 集群脚本
|
||||
├── stop-kind.sh # 停止本地 K8s 集群脚本
|
||||
├── manifests/
|
||||
│ ├── namespace.yaml # Namespace 定义
|
||||
│ ├── serviceaccount.yaml # ServiceAccount + RBAC
|
||||
│ ├── rcoder-deployment.yaml # RCoder 主服务 Deployment
|
||||
│ └── rcoder-service.yaml # RCoder 主服务 Service
|
||||
└── test-chat.sh # 测试脚本
|
||||
```
|
||||
|
||||
**功能**:
|
||||
- 使用 [Kind](https://kind.sigs.k8s.io/) 在本地运行 K8s
|
||||
- 一键启动本地 K8s 集群
|
||||
- 部署 RCoder 到本地 K8s
|
||||
- 测试 `/chat` 和 `/computer/chat` 接口
|
||||
|
||||
## Phase 0: Outline & Research
|
||||
|
||||
### Research Tasks
|
||||
1. **K8s Pod IP 通信**: Pod 之间直接用 IP 通信,同 Docker 方式
|
||||
2. **kube-rs 最佳实践**: 社区推荐的 K8s Runtime 实现模式
|
||||
3. **K8s 健康检查**: HTTP Health Check 方式(与 Docker 一致)
|
||||
4. **K8s 存储**: workspace 存储问题(标记为后续优化项)
|
||||
|
||||
### Output
|
||||
**research.md**:
|
||||
- Decision: 使用 Pod IP 直接通信(与 Docker 方式一致)
|
||||
- Rationale: Pod IP 在同一集群内可直接通信,无需额外 Service;Pod 重启后 cleanup_task 会重建并更新 IP
|
||||
- Alternatives considered: Service DNS(需要额外创建 Service,增加复杂度,当前不需要)
|
||||
|
||||
## Phase 1: Design & Contracts
|
||||
|
||||
### Actual Code Changes Required
|
||||
|
||||
#### 1. Fix `lib.rs` - global module initialization
|
||||
|
||||
**Current (broken)**:
|
||||
```rust
|
||||
pub async fn init_global_docker_manager_with_config(config: DockerManagerConfig) -> DockerResult<()> {
|
||||
let runtime_type = RuntimeType::from_env();
|
||||
crate::runtime::RuntimeManager::init(config.clone()).await // <- Already calls RuntimeManager::init()!
|
||||
|
||||
if runtime_type == RuntimeType::Docker {
|
||||
let manager = Arc::new(DockerManager::new(config).await?);
|
||||
GLOBAL_DOCKER_MANAGER.set(manager)...;
|
||||
}
|
||||
// Problem: Sets RUNTIME_INSTANCE in RuntimeManager::init() but returns DockerResult
|
||||
}
|
||||
```
|
||||
|
||||
**Fix**: Ensure proper error handling and that K8s path doesn't try to set GLOBAL_DOCKER_MANAGER
|
||||
|
||||
#### 2. Fix `main.rs` - runtime initialization
|
||||
|
||||
**Current**: Calls `init_global_docker_manager_with_config()` which should work, but error handling may be wrong
|
||||
|
||||
**Fix**: Verify RuntimeManager is properly initialized before using `RuntimeManager::get()`
|
||||
|
||||
#### 3. Fix `kubernetes_runtime.rs` - stop_container
|
||||
|
||||
**Current**:
|
||||
```rust
|
||||
async fn stop_container(&self, project_id: &str) -> ContainerRuntimeResult<()> {
|
||||
let pod_name = self.pod_name(project_id, &ServiceType::RCoder); // Always RCoder!
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**Fix**: Need to track service_type per container or change the interface
|
||||
|
||||
#### 4. Fix Makefile k8s commands
|
||||
|
||||
**Current**:
|
||||
```makefile
|
||||
dev-up-k8s:
|
||||
kubectl apply -f manifests/rcoder-deployment.yaml
|
||||
kubectl set image deployment/rcoder rcoder=$(IMAGE) # Redundant after apply
|
||||
|
||||
dev-restart-k8s: dev-build-k8s
|
||||
kubectl apply -f manifests/rcoder-deployment.yaml
|
||||
kubectl set image deployment/rcoder rcoder=$(IMAGE)
|
||||
kubectl rollout restart deploy/rcoder # rollout restart uses current deployment image, not new one!
|
||||
```
|
||||
|
||||
**Fix**: Use `kubectl delete pods` or fix the image update flow
|
||||
|
||||
## Phase 2: Task Planning Approach
|
||||
*This section describes what the /tasks command will do - DO NOT execute during /plan*
|
||||
|
||||
**Task Generation Strategy**:
|
||||
- P0 优先级:先解决阻止 K8s 运行的 critical 问题
|
||||
- P1 优先级:修复已知的 bug 和不完整实现
|
||||
- P2 优先级:改进和优化
|
||||
|
||||
**Task Order (Dependency-Based)**:
|
||||
1. **[P0] Fix global module initialization** - 确保 K8s 模式下 RuntimeManager 正确初始化
|
||||
2. **[P0] Fix rcoder main.rs** - 调用正确的初始化函数
|
||||
3. **[P0] Fix stop_container in KubernetesRuntime** - 正确处理不同 service_type
|
||||
4. **[P1] Fix Makefile k8s commands** - 消除重复操作,简化逻辑
|
||||
5. **[P1] Test K8s mode end-to-end** - 验证修复有效
|
||||
6. **[P2] Improve K8s health check** - 如有时间,优化健康检查
|
||||
|
||||
**Estimated Output**: 8-10 focused tasks
|
||||
|
||||
## Complexity Tracking
|
||||
| Violation | Why Needed | Simpler Alternative Rejected Because |
|
||||
|-----------|------------|-------------------------------------|
|
||||
| 重构 global 模块 | 需要统一运行时选择逻辑 | 直接在 rcoder 中判断,但会导致代码重复 |
|
||||
|
||||
## Progress Tracking
|
||||
|
||||
**Phase Status**:
|
||||
- [x] Phase 0: Research complete - research.md created
|
||||
- [x] Phase 1: Design complete - data-model.md created
|
||||
- [x] Phase 2: Task planning approach defined (above)
|
||||
- [ ] Phase 3: Tasks generated (/tasks command)
|
||||
- [ ] Phase 4: Implementation complete
|
||||
- [ ] Phase 5: Validation passed
|
||||
|
||||
**Gate Status**:
|
||||
- [x] Initial Constitution Check: N/A (empty constitution)
|
||||
- [x] Post-Design Constitution Check: N/A
|
||||
- [x] All NEEDS CLARIFICATION resolved
|
||||
- [ ] Complexity deviations documented
|
||||
|
||||
---
|
||||
*Based on Constitution v2.1.1 - See `/memory/constitution.md`*
|
||||
168
qiming-rcoder/specs/k8s-runtime-support/research.md
Normal file
168
qiming-rcoder/specs/k8s-runtime-support/research.md
Normal file
@@ -0,0 +1,168 @@
|
||||
# Research: Kubernetes Runtime Support
|
||||
|
||||
**Date**: 2026-04-16
|
||||
**Feature**: K8s Runtime Support
|
||||
|
||||
## Research 1: K8s Service DNS vs Pod IP
|
||||
|
||||
### Decision
|
||||
使用 K8s Service DNS 作为服务发现机制。
|
||||
|
||||
### Rationale
|
||||
- **Pod IP 不稳定**: Pod 重启后 IP 会变化,直接使用 Pod IP 会导致 gRPC 通信失败
|
||||
- **Service DNS 稳定**: 即使 Pod 重启,Service IP 保持不变(ClusterIP Service)
|
||||
- **K8s 标准做法**: 社区推荐使用 Service 进行服务发现
|
||||
|
||||
### Alternatives Considered
|
||||
| 方案 | 优点 | 缺点 |
|
||||
|------|------|------|
|
||||
| Pod IP | 简单直接 | Pod 重启后失效 |
|
||||
| Headless Service | 可直接解析 Pod IP | 仍依赖 Pod DNS(不稳定) |
|
||||
| Ingress | 支持外部访问 | 增加复杂度,不需要 |
|
||||
| ExternalName Service | 简单 | 不适合内部服务 |
|
||||
|
||||
### Service DNS 格式
|
||||
```
|
||||
{service_name}.{namespace}.svc.cluster.local
|
||||
```
|
||||
|
||||
对于 RCoder:
|
||||
- RCoder Pod: `rcoder-agent-{project_id}.{namespace}.svc.cluster.local`
|
||||
- ComputerAgent Pod: `computer-agent-runner-{user_id}.{namespace}.svc.cluster.local`
|
||||
|
||||
### Implementation
|
||||
```rust
|
||||
fn pod_dns_name(project_id: &str, user_id: Option<&str>, namespace: &str) -> String {
|
||||
let prefix = match user_id {
|
||||
Some(uid) => format!("computer-agent-runner-{}", uid),
|
||||
None => format!("rcoder-agent-{}", project_id),
|
||||
};
|
||||
format!("{}.{}.svc.cluster.local", prefix, namespace)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Research 2: kube-rs 最佳实践
|
||||
|
||||
### kube-rs 版本
|
||||
当前使用: `kube 0.98`
|
||||
|
||||
### 推荐的 API 使用模式
|
||||
|
||||
#### 1. Client 初始化
|
||||
```rust
|
||||
// 推荐:从 Config::infer() 自动检测 in-cluster vs local
|
||||
let kube_config = Config::infer().await?;
|
||||
let client = Client::try_from(kube_config)?;
|
||||
```
|
||||
|
||||
#### 2. API 访问模式
|
||||
```rust
|
||||
// 推荐:使用 Api::namespaced() 访问 namespaced 资源
|
||||
let pods: Api<Pod> = Api::namespaced(client, &namespace);
|
||||
|
||||
// 使用 ListParams 过滤
|
||||
let lp = ListParams::default().labels(&format!("project_id={}", project_id));
|
||||
let pods = pods.list(&lp).await?;
|
||||
```
|
||||
|
||||
#### 3. 错误处理
|
||||
```rust
|
||||
match pods.get(&pod_name).await {
|
||||
Ok(pod) => { /* found */ }
|
||||
Err(kube::Error::Api(ae)) if ae.code == 404 => { /* not found */ }
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Research 3: K8s 健康检查 (Readiness Probe)
|
||||
|
||||
### 问题
|
||||
Docker 模式使用 HTTP 轮询检查服务健康:
|
||||
```rust
|
||||
async fn wait_for_service_ready(service_url: &str) {
|
||||
loop {
|
||||
if http::get(service_url).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
sleep().await;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
K8s 模式应该使用 K8s 原生的 Readiness Probe 概念。
|
||||
|
||||
### 分析
|
||||
- **K8s Readiness Probe**: K8s 自动管理,决定 Pod 是否接收流量
|
||||
- **当前实现**: 应用层轮询,与 K8s 概念不匹配
|
||||
|
||||
### 解决方案
|
||||
仍然使用应用层健康检查(保持兼容性),但针对 K8s 环境优化:
|
||||
- 使用 DNS 解析代替 IP
|
||||
- 增加超时时间(K8s Pod 启动通常需要 30-60s)
|
||||
- 复用容器运行时层的健康检查接口
|
||||
|
||||
### K8s Probe 配置(未来可能需要)
|
||||
```yaml
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8086
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Research 4: K8s 存储 (Workspace 处理)
|
||||
|
||||
### 问题
|
||||
Docker 模式下使用 bind mount 共享文件系统:
|
||||
```yaml
|
||||
volumes:
|
||||
- /host/path:/container/path
|
||||
```
|
||||
|
||||
K8s 中如何处理?
|
||||
|
||||
### 分析
|
||||
| K8s 存储方案 | 适用场景 | 缺点 |
|
||||
|--------------|----------|------|
|
||||
| EmptyDir | 临时存储 | Pod 删除后数据丢失 |
|
||||
| HostPath | 节点文件 | 安全性差,需要特权 |
|
||||
| PVC | 持久存储 | 需要预先配置 StorageClass |
|
||||
| NFS/CIFS | 共享存储 | 需要外部存储服务 |
|
||||
| ConfigMap | 配置文件 | 不适合大文件 |
|
||||
|
||||
### RCoder 场景
|
||||
RCoder 的 workspace 需要:
|
||||
1. 持久化(用户代码、项目文件)
|
||||
2. 跨容器共享(rcoder 主容器和 agent_runner 容器)
|
||||
|
||||
### 解决方案
|
||||
- **开发环境**: 使用 PVC with ReadWriteMany (如果存储支持)
|
||||
- **生产环境**: 使用 NFS 或云存储
|
||||
- **简化方案**: 对于 K8s 支持,暂时不处理 workspace 挂载(使用容器内置存储)
|
||||
|
||||
### 实现计划
|
||||
Phase 1 中暂时跳过 workspace 存储问题,标记为 [NEEDS CLARIFICATION: workspace 存储策略]
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
### Key Decisions
|
||||
1. **Service DNS**: 使用 `{prefix}-{id}.{namespace}.svc.cluster.local`
|
||||
2. **kube-rs**: 使用 Api::namespaced() + ListParams 过滤
|
||||
3. **健康检查**: 保持应用层检查,增加超时
|
||||
4. **存储**: 标记为待解决问题
|
||||
|
||||
### Open Questions
|
||||
1. [NEEDS CLARIFICATION]: K8s 环境中 rcoder 主服务如何与 agent_runner Pod 通信?(同 namespace 直连?)
|
||||
2. [NEEDS CLARIFICATION]: workspace 存储使用 PVC 还是其他方案?
|
||||
3. [NEEDS CLARIFICATION]: K8s Service Account 和 RBAC 权限如何预配置?
|
||||
65
qiming-rcoder/specs/k8s-runtime-support/spec.md
Normal file
65
qiming-rcoder/specs/k8s-runtime-support/spec.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# Feature Specification: Kubernetes Runtime Support
|
||||
|
||||
**Feature Branch**: `dev-k8s`
|
||||
**Created**: 2026-04-16
|
||||
**Status**: Draft
|
||||
**Input**: User description: "修复 K8s 支持,问题是 global 模块未使用 RuntimeManager,K8s Runtime 中 user_id 未处理,使用 Pod IP 而非 Service DNS"
|
||||
|
||||
## User Scenarios & Testing
|
||||
|
||||
### Primary User Story
|
||||
RCoder 在 Kubernetes 集群中运行时,需要动态为每个项目/用户创建和管理容器(Pod)。当前实现仅支持 Docker Socket 方式,不支持 K8s 环境。
|
||||
|
||||
### Acceptance Scenarios
|
||||
1. **Given** RCoder 运行在 K8s 环境(设置 `CONTAINER_RUNTIME=kubernetes`),**When** 用户调用 `/chat` 接口,**Then** 系统应在 K8s 中创建 Pod 并正常通信
|
||||
2. **Given** RCoder 运行在 K8s 环境,**When** 用户调用 `/computer/chat` 接口,**Then** 系统应使用 `user_id` 作为 Pod 标识创建容器
|
||||
3. **Given** K8s Pod 发生重启,**When** gRPC 通信发生,**Then** 系统应能通过稳定的 Service DNS 找到新 Pod
|
||||
|
||||
### Edge Cases
|
||||
- Pod 重启后 IP 变化如何处理?
|
||||
- K8s API 访问失败时的降级策略?
|
||||
- 容器清理逻辑在 K8s 中如何适配?
|
||||
|
||||
## Requirements
|
||||
|
||||
### Functional Requirements
|
||||
- **FR-001**: 系统必须根据 `CONTAINER_RUNTIME` 环境变量选择 Docker 或 Kubernetes 运行时
|
||||
- **FR-002**: Kubernetes Runtime 必须支持 `/chat` 接口(使用 project_id 作为 Pod 标识)
|
||||
- **FR-003**: Kubernetes Runtime 必须支持 `/computer/chat` 接口(使用 user_id 作为 Pod 标识)
|
||||
- **FR-004**: Kubernetes Runtime 必须使用稳定的 Service DNS 而非 Pod IP
|
||||
- **FR-005**: 系统必须实现 `list_containers` 接口以支持 pod list 管理接口
|
||||
- **FR-006**: Kubernetes Runtime 必须支持容器健康检查
|
||||
|
||||
### Key Entities
|
||||
- **ContainerRuntime**: 容器运行时抽象接口
|
||||
- **KubernetesRuntime**: K8s 运行时实现
|
||||
- **DockerRuntime**: Docker 运行时实现(封装 DockerManager)
|
||||
- **RuntimeManager**: 运行时管理器,负责选择和初始化正确的运行时
|
||||
|
||||
## Technical Context (for planning)
|
||||
|
||||
### Problem Analysis
|
||||
| 问题 | 严重程度 | 位置 |
|
||||
|------|----------|------|
|
||||
| global 模块未使用 RuntimeManager,K8s 路径从未触发 | P0 | docker_manager/src/lib.rs |
|
||||
| user_id 在 K8s Runtime 中被忽略 | P0 | kubernetes_runtime.rs |
|
||||
| 使用 Pod IP 而非 Service DNS | P0 | kubernetes_runtime.rs |
|
||||
| list_containers 未实现 | P1 | kubernetes_runtime.rs |
|
||||
| 健康检查机制不兼容 K8s | P1 | health/ |
|
||||
|
||||
### Proposed Changes
|
||||
1. 修改 `global` 模块使用 `RuntimeManager` 选择运行时
|
||||
2. K8s Runtime 支持 `user_id` 参数
|
||||
3. K8s Runtime 使用 Service DNS(`{service}-{id}.{namespace}.svc.cluster.local`)
|
||||
4. 实现 `list_containers` 方法
|
||||
5. 添加 K8s 健康检查支持( Readiness Probe 概念)
|
||||
|
||||
## Review & Acceptance Checklist
|
||||
|
||||
- [x] User description parsed
|
||||
- [x] Key concepts extracted
|
||||
- [x] Ambiguities marked (Pod DNS 格式、K8s API 权限)
|
||||
- [x] User scenarios defined
|
||||
- [x] Requirements generated
|
||||
- [x] Entities identified
|
||||
- [x] Review checklist passed
|
||||
325
qiming-rcoder/specs/k8s-runtime-support/tasks.md
Normal file
325
qiming-rcoder/specs/k8s-runtime-support/tasks.md
Normal file
@@ -0,0 +1,325 @@
|
||||
# Tasks: K8s Runtime Support Fix
|
||||
|
||||
**Branch**: `dev-k8s` | **Date**: 2026-04-20
|
||||
**Plan**: [plan.md](./plan.md) | **Spec**: [spec.md](./spec.md)
|
||||
|
||||
## Task List
|
||||
|
||||
### Phase 1: Critical Fixes (P0)
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Fix `global::init_global_docker_manager_with_config()` K8s path
|
||||
**File**: `crates/docker_manager/src/lib.rs`
|
||||
**Priority**: P0
|
||||
**Estimated Time**: 30 min
|
||||
**Status**: TODO
|
||||
|
||||
**Problem**: When `CONTAINER_RUNTIME=kubernetes`, the function calls `RuntimeManager::init()` but then still tries to set `GLOBAL_DOCKER_MANAGER` which fails silently or causes issues.
|
||||
|
||||
**Changes**:
|
||||
```rust
|
||||
#[cfg(feature = "kubernetes")]
|
||||
pub async fn init_global_docker_manager_with_config(
|
||||
config: DockerManagerConfig,
|
||||
) -> DockerResult<()> {
|
||||
let runtime_type = RuntimeType::from_env();
|
||||
crate::runtime::RuntimeManager::init(config.clone())
|
||||
.await
|
||||
.map_err(|e| DockerError::ConfigurationError(e.to_string()))?;
|
||||
info!("Runtime initialized with config");
|
||||
|
||||
if runtime_type == RuntimeType::Docker {
|
||||
let manager = Arc::new(DockerManager::new(config).await?);
|
||||
GLOBAL_DOCKER_MANAGER.set(manager).map_err(|_| {
|
||||
DockerError::IoError(std::io::Error::new(
|
||||
std::io::ErrorKind::AlreadyExists,
|
||||
"global DockerManager already initialized",
|
||||
))
|
||||
})?;
|
||||
info!("DockerManager initialized with config");
|
||||
}
|
||||
// K8s mode: RuntimeManager is initialized, GLOBAL_DOCKER_MANAGER stays empty (ok)
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
**Verification**:
|
||||
- [ ] `cargo check -p docker_manager --features kubernetes` passes
|
||||
- [ ] Unit test `test_runtime_type_from_env_kubernetes` passes
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Verify `main.rs` runtime initialization flow
|
||||
**File**: `crates/rcoder/src/main.rs`
|
||||
**Priority**: P0
|
||||
**Estimated Time**: 30 min
|
||||
**Status**: TODO
|
||||
|
||||
**Problem**: Need to verify that after calling `init_global_docker_manager_with_config()`, `RuntimeManager::get()` works correctly for K8s mode.
|
||||
|
||||
**Current Code** (lines 181-186):
|
||||
```rust
|
||||
if let Err(e) =
|
||||
docker_manager::global::init_global_docker_manager_with_config(docker_manager_config).await
|
||||
{
|
||||
error!("Docker Manager initializefailed: {}", e);
|
||||
return Err(anyhow::anyhow!("Docker Manager initialization failed: {}", e));
|
||||
}
|
||||
```
|
||||
|
||||
**Verification**:
|
||||
- [ ] K8s mode: `RuntimeManager::get().await` returns `Arc<dyn ContainerRuntime>`
|
||||
- [ ] Docker mode: `get_global_docker_manager().await` returns `Arc<DockerManager>`
|
||||
- [ ] Both modes can call `cleanup_all()` without error
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Fix `stop_container` in KubernetesRuntime to handle service_type
|
||||
**File**: `crates/docker_manager/src/runtime/kubernetes_runtime.rs`
|
||||
**Priority**: P0
|
||||
**Estimated Time**: 45 min
|
||||
**Status**: TODO
|
||||
|
||||
**Problem**: `stop_container(&self, project_id: &str)` only takes project_id, but K8s creates different pod types (RCoder vs ComputerAgentRunner) with different prefixes.
|
||||
|
||||
**Current**:
|
||||
```rust
|
||||
async fn stop_container(&self, project_id: &str) -> ContainerRuntimeResult<()> {
|
||||
let pod_name = self.pod_name(project_id, &ServiceType::RCoder); // Always RCoder!
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**Fix Options**:
|
||||
|
||||
Option A - Add service_type parameter to `stop_container`:
|
||||
```rust
|
||||
async fn stop_container(&self, project_id: &str, service_type: ServiceType) -> ContainerRuntimeResult<()> {
|
||||
let pod_name = self.pod_name(project_id, &service_type);
|
||||
// ...
|
||||
}
|
||||
```
|
||||
⚠️ This breaks the trait signature.
|
||||
|
||||
Option B - Cache service_type in pod_cache:
|
||||
```rust
|
||||
// Store (identifier, service_type) in cache when creating container
|
||||
self.pod_cache.write().await.insert(identifier.to_string(), (pod_info, service_type.clone()));
|
||||
|
||||
// In stop_container, lookup cached service_type or default to RCoder
|
||||
let service_type = self.get_cached_service_type(identifier).await.unwrap_or(ServiceType::RCoder);
|
||||
```
|
||||
|
||||
Option C - Use `stop_container_by_identifier` which already has service_type:
|
||||
```rust
|
||||
async fn stop_container(&self, project_id: &str) -> ContainerRuntimeResult<()> {
|
||||
// Try RCoder first, then ComputerAgentRunner
|
||||
match self.stop_container_by_identifier(project_id, &ServiceType::RCoder).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(ContainerRuntimeError::ContainerStopError(_)) => {
|
||||
// Try ComputerAgentRunner if RCoder not found
|
||||
self.stop_container_by_identifier(project_id, &ServiceType::ComputerAgentRunner).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
```
|
||||
✅ **Recommended** - Uses existing trait method, handles both types
|
||||
|
||||
**Implementation**: Use Option C
|
||||
|
||||
**Verification**:
|
||||
- [ ] `stop_container("user123")` correctly deletes ComputerAgentRunner pod when it exists
|
||||
- [ ] `stop_container("project123")` correctly deletes RCoder pod when it exists
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Fix Makefile k8s commands
|
||||
**File**: `Makefile`
|
||||
**Priority**: P1
|
||||
**Estimated Time**: 20 min
|
||||
**Status**: TODO
|
||||
|
||||
**Problem**:
|
||||
1. `dev-up-k8s` and `dev-restart-k8s` do `kubectl apply` followed by `kubectl set image`, but the image in deployment.yaml is hardcoded to `rcoder:test`
|
||||
2. `dev-restart-k8s` uses `rollout restart` which doesn't pick up the new image from `set image`
|
||||
|
||||
**Current (broken)**:
|
||||
```makefile
|
||||
dev-up-k8s:
|
||||
kubectl apply -f k8s/manifests/rcoder-deployment.yaml # Uses image: rcoder:test
|
||||
kubectl set image deployment/rcoder rcoder=$(IMAGE) # But this updates it
|
||||
...
|
||||
|
||||
dev-restart-k8s: dev-build-k8s
|
||||
kubectl apply -f k8s/manifests/rcoder-deployment.yaml
|
||||
kubectl set image deployment/rcoder rcoder=$(IMAGE) # Sets new image
|
||||
kubectl rollout restart deploy/rcoder # But restart uses deployment.yaml, not set image!
|
||||
```
|
||||
|
||||
**Fix**:
|
||||
```makefile
|
||||
# Use sed to replace image in deployment.yaml before applying
|
||||
define apply_with_image
|
||||
kubectl apply -f k8s/manifests/namespace.yaml
|
||||
kubectl apply -f k8s/manifests/serviceaccount.yaml
|
||||
sed "s|image: rcoder:test|image: $(IMAGE)|" k8s/manifests/rcoder-deployment.yaml | kubectl apply -f -
|
||||
kubectl apply -f k8s/manifests/rcoder-service.yaml
|
||||
endef
|
||||
|
||||
dev-up-k8s:
|
||||
$(call apply_with_image)
|
||||
kubectl rollout status deploy/rcoder -n $(K8S_NAMESPACE) --timeout=$(ROLLOUT_TIMEOUT)
|
||||
|
||||
dev-restart-k8s: dev-build-k8s
|
||||
kubectl delete pods -n $(K8S_NAMESPACE) -l app=rcoder --ignore-not-found
|
||||
kubectl rollout status deploy/rcoder -n $(K8S_NAMESPACE) --timeout=$(ROLLOUT_TIMEOUT)
|
||||
```
|
||||
|
||||
**Key changes**:
|
||||
1. Use `sed` to replace image tag at apply time
|
||||
2. `dev-restart-k8s` uses `kubectl delete pods` instead of `rollout restart` (simpler, faster)
|
||||
3. `dev-down-k8s` stays the same (already correct)
|
||||
|
||||
**Verification**:
|
||||
- [ ] `make dev-up-k8s IMAGE=rcoder:test-k8s` deploys with correct image
|
||||
- [ ] `make dev-restart-k8s IMAGE=rcoder:test-k8s` rebuilds and redeploys
|
||||
- [ ] `make dev-down-k8s` cleans up properly
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Add K8s mode to Cargo features in rcoder
|
||||
**File**: `crates/rcoder/Cargo.toml`
|
||||
**Priority**: P1
|
||||
**Estimated Time**: 10 min
|
||||
**Status**: TODO
|
||||
|
||||
**Problem**: The `kubernetes` feature in rcoder passes to docker_manager, but need to verify it's properly configured.
|
||||
|
||||
**Current**:
|
||||
```toml
|
||||
[features]
|
||||
# Kubernetes 支持:启用 Kubernetes 运行时模式
|
||||
# 启用后可通过 CONTAINER_RUNTIME=kubernetes 环境变量切换到 K8s 模式
|
||||
kubernetes = ["docker_manager/kubernetes"]
|
||||
```
|
||||
|
||||
**Verification**:
|
||||
- [ ] `cargo build --features kubernetes` works
|
||||
- [ ] `cargo build --features kubernetes --package rcoder` produces binary with K8s support
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Test K8s mode end-to-end
|
||||
**Priority**: P0
|
||||
**Estimated Time**: 60 min
|
||||
**Status**: TODO
|
||||
|
||||
**Prerequisites**: Kind cluster or real K8s cluster available
|
||||
|
||||
**Test Steps**:
|
||||
```bash
|
||||
# 1. Build K8s image
|
||||
make dev-build-k8s IMAGE=rcoder:test-k8s
|
||||
|
||||
# 2. Deploy to K8s
|
||||
make dev-up-k8s IMAGE=rcoder:test-k8s
|
||||
|
||||
# 3. Check rcoder pod is running
|
||||
kubectl get pods -n rcoder
|
||||
kubectl logs -n rcoder -l app=rcoder
|
||||
|
||||
# 4. Test /health endpoint
|
||||
NODE_PORT=$(kubectl get svc rcoder -n rcoder -o jsonpath='{.spec.ports[0].nodePort}')
|
||||
curl http://localhost:$NODE_PORT/health
|
||||
|
||||
# 5. Test /chat endpoint (creates RCoder agent pod)
|
||||
curl -X POST http://localhost:$NODE_PORT/chat \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"prompt": "hello"}'
|
||||
|
||||
# 6. Check agent pod was created
|
||||
kubectl get pods -n rcoder
|
||||
|
||||
# 7. Test /computer/chat endpoint (creates ComputerAgentRunner pod)
|
||||
curl -X POST http://localhost:$NODE_PORT/computer/chat \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"user_id": "test-user", "prompt": "hello"}'
|
||||
|
||||
# 8. Check computer agent pod was created
|
||||
kubectl get pods -n rcoder -l user_id=test-user
|
||||
|
||||
# 9. Test cleanup (delete chat session)
|
||||
# ... verify pods are deleted
|
||||
|
||||
# 10. Cleanup
|
||||
make dev-down-k8s
|
||||
```
|
||||
|
||||
**Expected Results**:
|
||||
- [ ] `/health` returns healthy
|
||||
- [ ] `/chat` creates `rcoder-agent-{project_id}` pod
|
||||
- [ ] `/computer/chat` creates `computer-agent-runner-{user_id}` pod
|
||||
- [ ] Both pods are reachable via gRPC from rcoder main pod
|
||||
- [ ] Cleanup properly deletes pods
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Verify Docker mode still works (regression test)
|
||||
**Priority**: P1
|
||||
**Estimated Time**: 30 min
|
||||
**Status**: TODO
|
||||
|
||||
**Test Steps**:
|
||||
```bash
|
||||
# 1. Build Docker image
|
||||
make dev-build
|
||||
|
||||
# 2. Start Docker Compose
|
||||
make dev-up
|
||||
|
||||
# 3. Test /health
|
||||
curl http://localhost:8087/health
|
||||
|
||||
# 4. Test /chat
|
||||
curl -X POST http://localhost:8087/chat \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"prompt": "hello"}'
|
||||
|
||||
# 5. Verify container created
|
||||
docker ps | grep rcoder-agent
|
||||
|
||||
# 6. Cleanup
|
||||
make dev-down
|
||||
```
|
||||
|
||||
**Expected Results**:
|
||||
- [ ] All existing functionality works as before
|
||||
- [ ] No regression in Docker mode
|
||||
|
||||
---
|
||||
|
||||
## Task Summary
|
||||
|
||||
| # | Task | Priority | Status | Dependencies |
|
||||
|---|------|----------|--------|--------------|
|
||||
| 1 | Fix `init_global_docker_manager_with_config()` | P0 | TODO | - |
|
||||
| 2 | Verify `main.rs` runtime init | P0 | TODO | Task 1 |
|
||||
| 3 | Fix `stop_container` service_type | P0 | TODO | - |
|
||||
| 4 | Fix Makefile k8s commands | P1 | TODO | - |
|
||||
| 5 | Verify Cargo features | P1 | TODO | - |
|
||||
| 6 | Test K8s mode end-to-end | P0 | TODO | Tasks 1-5 |
|
||||
| 7 | Regression test Docker mode | P1 | TODO | - |
|
||||
|
||||
---
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
- [ ] All P0 tasks complete
|
||||
- [ ] K8s mode can create/manage RCoder agent pods
|
||||
- [ ] K8s mode can create/manage ComputerAgentRunner pods
|
||||
- [ ] Docker mode regression tests pass
|
||||
- [ ] Makefile k8s commands work correctly
|
||||
- [ ] Code compiles with both `kubernetes` feature enabled and disabled
|
||||
Reference in New Issue
Block a user