添加qiming-rcoder模块

2026-06-01 13:54:52 +08:00
parent 8092c4b1f8
commit 4b1a580132
539 changed files with 151650 additions and 0 deletions
--- a/qiming-rcoder/k8s/scripts/deploy-juicefs.sh
+++ b/qiming-rcoder/k8s/scripts/deploy-juicefs.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+# ============================================================
+# JuiceFS + MinIO + PostgreSQL 部署脚本
+# 使用 Kustomize overlay (dev / prod) 部署，避免集群级资源冲突
+# 用法 (可在任意目录运行):
+#   ENV=dev  k8s/scripts/deploy-juicefs.sh   # 部署到 nuwax-rcoder-dev（默认）
+#   ENV=prod k8s/scripts/deploy-juicefs.sh   # 部署到 nuwax-rcoder-prod
+# ============================================================
+
+set -e
+
+ENV="${ENV:-dev}"
+
+case "$ENV" in
+    dev|prod) ;;
+    *)
+        echo "Error: ENV 必须是 dev 或 prod，当前值: $ENV"
+        exit 1
+        ;;
+esac
+
+# 路径解析：脚本位于 k8s/scripts/，manifests 位于 k8s/manifests/
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+K8S_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+NAMESPACE="${NAMESPACE:-nuwax-rcoder-${ENV}}"
+MANIFESTS_DIR="${K8S_ROOT}/manifests"
+OVERLAY_DIR="${MANIFESTS_DIR}/overlays/${ENV}"
+
+echo "=========================================="
+echo "  RCoder - JuiceFS 存储部署 (${ENV})"
+echo "  namespace: ${NAMESPACE}"
+echo "  overlay:   ${OVERLAY_DIR}"
+echo "=========================================="
+
+# 检查 kubectl
+if ! command -v kubectl &> /dev/null; then
+    echo "Error: kubectl not found"
+    exit 1
+fi
+
+# 检查 kubectl kustomize 插件
+if ! kubectl kustomize --help &> /dev/null; then
+    echo "Error: kubectl kustomize not found (需要 kubectl 1.14+)"
+    exit 1
+fi
+
+# 部署 JuiceFS CSI Driver
+echo ""
+echo "[1/6] 检查 JuiceFS CSI Driver..."
+if ! kubectl get daemonset juicefs-csi-driver-node -n kube-system &> /dev/null; then
+    echo "JuiceFS CSI Driver 未部署，是否部署? (y/n)"
+    read -r response
+    if [[ "$response" =~ ^[Yy]$ ]]; then
+        helm repo add juicefs https://juicefs.github.io/charts || true
+        helm repo update
+        helm install juicefs-csi-driver juicefs/juicefs-csi-driver \
+          --namespace kube-system \
+          --set webhook.enabled=false
+        echo "等待 JuiceFS CSI Driver 就绪..."
+        kubectl rollout status daemonset/juicefs-csi-driver-node -n kube-system --timeout=120s
+    fi
+fi
+
+# 使用 Kustomize overlay 部署
+echo ""
+echo "[2/6] 使用 Kustomize overlay 部署..."
+kubectl apply -k "${OVERLAY_DIR}"
+
+# 等待存储层就绪
+echo ""
+echo "[3/6] 等待 PostgreSQL 就绪..."
+kubectl wait --for=condition=ready pod -l app=postgresql -n ${NAMESPACE} --timeout=180s 2>/dev/null || echo "⚠️  PostgreSQL 等待超时"
+
+echo ""
+echo "[4/6] 等待 MinIO 就绪..."
+kubectl wait --for=condition=ready pod -l app=minio -n ${NAMESPACE} --timeout=180s 2>/dev/null || echo "⚠️  MinIO 等待超时"
+
+echo ""
+echo "[5/6] 等待 MinIO Bucket 初始化..."
+kubectl wait --for=condition=complete job/minio-init -n ${NAMESPACE} --timeout=120s 2>/dev/null || echo "⚠️  MinIO 初始化等待超时"
+
+# 验证部署
+echo ""
+echo "[6/6] 验证部署状态..."
+echo ""
+echo "StorageClass:"
+kubectl get sc | grep juicefs || echo "  (未找到)"
+
+echo ""
+echo "PVC:"
+kubectl get pvc -n ${NAMESPACE}
+
+echo ""
+echo "Pods:"
+kubectl get pods -n ${NAMESPACE}
+
+echo ""
+echo "=========================================="
+echo "  部署完成！"
+echo "=========================================="
--- a/qiming-rcoder/k8s/scripts/install-k3s-registry-mirrors-cn.sh
+++ b/qiming-rcoder/k8s/scripts/install-k3s-registry-mirrors-cn.sh
@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+# 为 K3s 配置中国大陆常用的 containerd 镜像加速（docker.io / registry.k8s.io / ghcr.io / quay.io）
+# 用法（在 K3s 节点上）:
+#   sudo k8s/scripts/install-k3s-registry-mirrors-cn.sh
+#
+# 可选: 本地内网 registry 作首选 mirror (命中时走千兆内网)
+#   sudo REGISTRY_HOST=192.168.32.228:5000 k8s/scripts/install-k3s-registry-mirrors-cn.sh
+#   # 多个 registry 用逗号隔开, 按顺序作为 endpoint:
+#   sudo REGISTRY_HOST=192.168.32.228:5000,192.168.32.228:5002 k8s/scripts/install-k3s-registry-mirrors-cn.sh
+#
+# 可选：禁止回退到 Docker Hub / GHCR 官方地址（对已配置 mirrors 的仓库不再走官方 endpoint）
+#   sudo env INSTALL_DISABLE_REGISTRY_FALLBACK=1 k8s/scripts/install-k3s-registry-mirrors-cn.sh
+#
+# 完成后会重启 k3s，短暂影响调度中的 Pod。
+
+set -euo pipefail
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+if [[ "${EUID:-}" -ne 0 ]]; then
+  echo -e "${RED}请使用 root 运行: sudo $0${NC}"
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SRC="${SCRIPT_DIR}/k3s-registries-cn.yaml"
+DEST="/etc/rancher/k3s/registries.yaml"
+
+if [[ ! -f "$SRC" ]]; then
+  echo -e "${RED}找不到模板: $SRC${NC}"
+  exit 1
+fi
+
+install -d /etc/rancher/k3s
+
+# 备份现有配置 (如果有)
+if [[ -f "$DEST" ]]; then
+  BACKUP="${DEST}.bak.$(date +%s)"
+  cp -a "$DEST" "$BACKUP"
+  echo -e "${YELLOW}已备份现有配置:${NC} $BACKUP"
+fi
+
+# ---- 可选: 生成带本地 registry 的配置 ----
+if [[ -n "${REGISTRY_HOST:-}" ]]; then
+  echo -e "${GREEN}使用本地 registry 前缀:${NC} $REGISTRY_HOST"
+
+  IFS=',' read -ra LOCAL_REGS <<< "$REGISTRY_HOST"
+
+  # 纯粹用 printf 累加真实换行 (不依赖命令替换保留 \n)
+  _emit_endpoints() {
+    for r in "${LOCAL_REGS[@]}"; do
+      printf '      - "http://%s"\n' "$r"
+    done
+  }
+  _emit_self_mirrors() {
+    for r in "${LOCAL_REGS[@]}"; do
+      printf '  "%s":\n    endpoint:\n      - "http://%s"\n' "$r" "$r"
+    done
+  }
+  _emit_self_configs() {
+    for r in "${LOCAL_REGS[@]}"; do
+      printf '  "%s":\n    tls:\n      insecure_skip_verify: true\n' "$r"
+    done
+  }
+
+  {
+    cat <<EOF
+# 由 install-k3s-registry-mirrors-cn.sh 生成 (REGISTRY_HOST=${REGISTRY_HOST})
+# 文档: https://docs.k3s.io/installation/private-registry
+mirrors:
+  "docker.io":
+    endpoint:
+EOF
+    _emit_endpoints
+    cat <<'EOF'
+      - "https://docker.m.daocloud.io"
+      - "https://docker.1ms.run"
+
+  "registry.k8s.io":
+    endpoint:
+EOF
+    _emit_endpoints
+    cat <<'EOF'
+      - "https://registry.aliyuncs.com/google_containers"
+      - "https://k8s.m.daocloud.io"
+
+  "ghcr.io":
+    endpoint:
+EOF
+    _emit_endpoints
+    cat <<'EOF'
+      - "https://ghcr.m.daocloud.io"
+
+  "quay.io":
+    endpoint:
+EOF
+    _emit_endpoints
+    cat <<'EOF'
+      - "https://quay.m.daocloud.io"
+
+  "nuwax-docker-images-registry.cn-hangzhou.cr.aliyuncs.com":
+    endpoint:
+EOF
+    _emit_endpoints
+    cat <<'EOF'
+      - "https://nuwax-docker-images-registry.cn-hangzhou.cr.aliyuncs.com"
+
+  # 自指: 允许 "crictl pull <host>/foo" 走 plain HTTP
+EOF
+    _emit_self_mirrors
+    cat <<'EOF'
+
+configs:
+EOF
+    _emit_self_configs
+  } > "$DEST"
+
+else
+  cp -a "$SRC" "$DEST"
+fi
+chmod 0644 "$DEST"
+
+echo -e "${GREEN}已写入:${NC} $DEST"
+echo -e "${YELLOW}内容预览:${NC}"
+cat "$DEST"
+
+# 可选：对已在 registries.yaml 中声明的仓库，不再回退到官方默认 endpoint（见 K3s 文档 Default Endpoint Fallback）
+if [[ "${INSTALL_DISABLE_REGISTRY_FALLBACK:-0}" == "1" ]]; then
+  install -d /etc/rancher/k3s/config.yaml.d
+  DROPIN="/etc/rancher/k3s/config.yaml.d/99-rcoder-disable-default-registry-endpoint.yaml"
+  cat >"$DROPIN" <<'EOF'
+disable-default-registry-endpoint: true
+EOF
+  chmod 0644 "$DROPIN"
+  echo ""
+  echo -e "${GREEN}已写入 K3s 配置片段:${NC} $DROPIN"
+  echo -e "${YELLOW}（仅对 registries.yaml 里配置过 mirrors 的仓库生效；避免回退到被墙地址导致长时间卡住）${NC}"
+fi
+
+if systemctl is-active --quiet k3s 2>/dev/null; then
+  echo ""
+  echo -e "${YELLOW}正在重启 k3s 使镜像配置生效...${NC}"
+  systemctl restart k3s
+  echo -e "${GREEN}k3s 已重启。${NC}"
+elif systemctl is-active --quiet k3s-agent 2>/dev/null; then
+  echo ""
+  echo -e "${YELLOW}正在重启 k3s-agent...${NC}"
+  systemctl restart k3s-agent
+  echo -e "${GREEN}k3s-agent 已重启。${NC}"
+else
+  echo -e "${YELLOW}未检测到 k3s / k3s-agent 服务处于 active，请手动重启:${NC}"
+  echo "  systemctl restart k3s"
+  echo "  # 或 agent 节点:"
+  echo "  systemctl restart k3s-agent"
+fi
+
+echo ""
+echo -e "${YELLOW}说明:${NC} ${GREEN}k3s ctr images pull${NC} 往往 ${RED}不会${NC} 使用 /etc/rancher/k3s/registries.yaml，"
+echo "  仍会直连 registry-1.docker.io，因此在国内容易误报「镜像加速没生效」。"
+echo -e "  与 kubelet/CRI 一致的正确验证方式是用 ${GREEN}crictl pull${NC}："
+
+RUNTIME_SOCK=""
+for s in /run/k3s/containerd/containerd.sock /var/run/k3s/containerd/containerd.sock; do
+  if [[ -S "$s" ]]; then
+    RUNTIME_SOCK="$s"
+    break
+  fi
+done
+
+if [[ -n "$RUNTIME_SOCK" ]]; then
+  echo ""
+  echo -e "${GREEN}验证拉取（推荐）:${NC}"
+  echo "  sudo crictl --runtime-endpoint unix://${RUNTIME_SOCK} pull docker.io/rancher/mirrored-pause:3.6"
+  if command -v crictl &>/dev/null; then
+    echo ""
+    echo -e "${YELLOW}正在尝试自动拉取 pause 镜像（失败则忽略，你可手动执行上一行）...${NC}"
+    if crictl --runtime-endpoint "unix://${RUNTIME_SOCK}" pull docker.io/rancher/mirrored-pause:3.6; then
+      echo -e "${GREEN}crictl 拉取成功：镜像加速已对 CRI/kubelet 生效。${NC}"
+    else
+      echo -e "${RED}crictl 拉取失败，请把完整终端输出保存后排查（勿用 k3s ctr 判断）。${NC}"
+    fi
+  else
+    echo -e "${YELLOW}未安装 crictl，可安装后再验证。${NC}"
+  fi
+else
+  echo ""
+  echo -e "${GREEN}验证拉取（推荐）:${NC}"
+  echo "  sudo crictl --runtime-endpoint unix:///run/k3s/containerd/containerd.sock pull docker.io/rancher/mirrored-pause:3.6"
+fi
--- a/qiming-rcoder/k8s/scripts/k3s-registries-cn.yaml
+++ b/qiming-rcoder/k8s/scripts/k3s-registries-cn.yaml
@@ -0,0 +1,63 @@
+# 复制到节点: /etc/rancher/k3s/registries.yaml 后执行: sudo systemctl restart k3s
+#
+# 注意: 「k3s ctr images pull」通常不走本文件；验证请用 crictl（与 kubelet 一致），例如:
+#   sudo crictl --runtime-endpoint unix:///run/k3s/containerd/containerd.sock pull docker.io/rancher/mirrored-pause:3.6
+#
+# 文档: https://docs.k3s.io/installation/private-registry
+#
+# ============================================================================
+# 可选: 内网本地 registry 加速
+# ----------------------------------------------------------------------------
+# 如果你在局域网起了 registry:2 (见 k8s/register2/docker-compose.yml), 建议把它
+# 作为首选 mirror —— 命中时走内网 (~千兆), 未命中自动回退到公网 cn mirror。
+#
+# 使用 install-k3s-registry-mirrors-cn.sh 时, 设置 REGISTRY_HOST=<ip>:<port> 会
+# 自动生成包含本地 registry 的完整配置; 或者手动把下方注释掉的 `- "http://..."`
+# 行取消注释并改成你的 registry 地址。
+# ============================================================================
+
+mirrors:
+  "docker.io":
+    endpoint:
+      # - "http://192.168.32.228:5000"       # 本地私有 (按需 push 加速专用镜像)
+      # - "http://192.168.32.228:5002"       # 本地 pull-through 缓存
+      - "https://docker.m.daocloud.io"
+      - "https://docker.1ms.run"
+
+  "registry.k8s.io":
+    endpoint:
+      # - "http://192.168.32.228:5000"
+      - "https://registry.aliyuncs.com/google_containers"
+      - "https://k8s.m.daocloud.io"
+
+  "ghcr.io":
+    endpoint:
+      # - "http://192.168.32.228:5000"
+      - "https://ghcr.m.daocloud.io"
+
+  "quay.io":
+    endpoint:
+      # - "http://192.168.32.228:5000"
+      - "https://quay.m.daocloud.io"
+
+  # RCoder 自有阿里云镜像 —— 本地 registry 命中时走内网
+  # "nuwax-docker-images-registry.cn-hangzhou.cr.aliyuncs.com":
+  #   endpoint:
+  #     - "http://192.168.32.228:5000"
+  #     - "https://nuwax-docker-images-registry.cn-hangzhou.cr.aliyuncs.com"
+
+  # 自指 mirror: 允许 "crictl pull 192.168.32.228:5000/foo" 直接走 plain HTTP
+  # "192.168.32.228:5000":
+  #   endpoint:
+  #     - "http://192.168.32.228:5000"
+  # "192.168.32.228:5002":
+  #   endpoint:
+  #     - "http://192.168.32.228:5002"
+
+# configs:
+#   "192.168.32.228:5000":
+#     tls:
+#       insecure_skip_verify: true
+#   "192.168.32.228:5002":
+#     tls:
+#       insecure_skip_verify: true
--- a/qiming-rcoder/k8s/scripts/test-chat.sh
+++ b/qiming-rcoder/k8s/scripts/test-chat.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+set -e
+
+NAMESPACE="${NAMESPACE:-nuwax-rcoder-dev}"
+
+echo "=== Testing RCoder in K8s ==="
+
+# 获取 NodePort
+NODE_PORT=$(kubectl get svc rcoder -n "${NAMESPACE}" -o jsonpath='{.spec.ports[0].nodePort}')
+echo "NodePort: $NODE_PORT"
+
+# 获取节点 IP（优先使用第一个 IPv4，避免 dual-stack 返回多个地址导致 URL 拼接失败）
+NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' | awk '{print $1}')
+echo "Node IP: $NODE_IP"
+
+BASE_URL="http://${NODE_IP}:${NODE_PORT}"
+
+echo ""
+echo "Testing /health endpoint..."
+curl -s "${BASE_URL}/health" | jq .
+
+echo ""
+echo "Testing /chat endpoint..."
+CHAT_RESP=$(curl -s -X POST "${BASE_URL}/chat" \
+  -H "Content-Type: application/json" \
+  -d '{"prompt": "hello"}')
+echo "${CHAT_RESP}" | jq .
+PROJECT_ID=$(echo "${CHAT_RESP}" | jq -r '.data.project_id // empty')
+
+echo ""
+echo "Testing /computer/chat endpoint..."
+USER_ID="k8s-test-user"
+COMPUTER_RESP=$(curl -s -X POST "${BASE_URL}/computer/chat" \
+  -H "Content-Type: application/json" \
+  -d "{\"user_id\":\"${USER_ID}\",\"prompt\":\"hello from k8s\"}")
+echo "${COMPUTER_RESP}" | jq .
+COMPUTER_PROJECT_ID=$(echo "${COMPUTER_RESP}" | jq -r '.data.project_id // empty')
+
+echo ""
+echo "Verifying created pods by labels..."
+if [ -n "${PROJECT_ID}" ]; then
+  kubectl get pods -n "${NAMESPACE}" -l "project_id=${PROJECT_ID}" -o wide
+fi
+kubectl get pods -n "${NAMESPACE}" -l "user_id=${USER_ID}" -o wide
+
+echo ""
+echo "Verifying pod status API..."
+curl -s "${BASE_URL}/computer/pod/status?user_id=${USER_ID}" | jq .
+if [ -n "${PROJECT_ID}" ]; then
+  curl -s "${BASE_URL}/computer/pod/status?project_id=${PROJECT_ID}" | jq .
+fi
+if [ -n "${COMPUTER_PROJECT_ID}" ]; then
+  curl -s "${BASE_URL}/computer/pod/status?project_id=${COMPUTER_PROJECT_ID}" | jq .
+fi
+
+echo ""
+echo "=== Test complete ==="