From 913c6da8f396635419bba755b73dd4eeda32e047 Mon Sep 17 00:00:00 2001 From: baiyanyun Date: Mon, 8 Jun 2026 18:44:45 +0800 Subject: [PATCH] feat(client): project local GUI and file tools --- .../src/main/ipc/digitalEmployeeHandlers.ts | 5 +- .../skillCatalogService.test.ts | 98 ++++++++ .../digitalEmployee/skillCatalogService.ts | 233 +++++++++++++++++- ...ital-employee-frontend-integration-plan.md | 16 +- 4 files changed, 348 insertions(+), 4 deletions(-) diff --git a/qimingclaw/crates/agent-electron-client/src/main/ipc/digitalEmployeeHandlers.ts b/qimingclaw/crates/agent-electron-client/src/main/ipc/digitalEmployeeHandlers.ts index 17392f6b..7bd05583 100644 --- a/qimingclaw/crates/agent-electron-client/src/main/ipc/digitalEmployeeHandlers.ts +++ b/qimingclaw/crates/agent-electron-client/src/main/ipc/digitalEmployeeHandlers.ts @@ -53,7 +53,10 @@ export function registerDigitalEmployeeHandlers(ctx: HandlerContext): void { }); ipcMain.handle("digitalEmployee:getSkillCapabilities", async () => { - return listDigitalEmployeeSkillCapabilities(); + return listDigitalEmployeeSkillCapabilities({ + fileServerStatus: normalizeServiceStatus(ctx.fileServer.status()), + guiServerStatus: normalizeServiceStatus(getGuiStatus()), + }); }); ipcMain.handle("digitalEmployee:getPlanTemplates", async () => { diff --git a/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.test.ts b/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.test.ts index 26568c06..e695877f 100644 --- a/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.test.ts +++ b/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.test.ts @@ -63,6 +63,32 @@ const mockRuntime = vi.hoisted(() => ({ }, occurredAt: "2026-06-07T08:02:00.000Z", }, + { + id: "event-4", + kind: "computer_tool_call", + message: "工具调用开始：gui_click", + payload: { + source: "gui_agent", + toolName: "gui_click", + projectId: "project-gui", + sessionId: "session-gui", + status: "completed", + }, + occurredAt: "2026-06-07T08:03:00.000Z", + }, + { + id: "event-5", + kind: "file_server.workspace.created", + message: "create-workspace 完成", + payload: { + source: "qiming-file-server", + toolName: "create_workspace", + projectId: "project-file", + sessionId: "session-file", + status: "completed", + }, + occurredAt: "2026-06-07T08:04:00.000Z", + }, ], }, })); @@ -95,6 +121,22 @@ vi.mock("../engines/unifiedAgent", () => ({ }, })); +vi.mock("../packages/guiAgentServer", () => ({ + getGuiAgentServerStatus: () => ({ running: true, port: 60008 }), +})); + +vi.mock("../packages/windowsMcp", () => ({ + getWindowsMcpStatus: () => ({ running: false }), +})); + +vi.mock("../system/shellEnv", () => ({ + isWindows: () => false, +})); + +vi.mock("../startupPorts", () => ({ + getConfiguredPorts: () => ({ fileServer: 60005 }), +})); + describe("digital employee skill catalog service", () => { beforeEach(() => { vi.resetModules(); @@ -165,4 +207,60 @@ describe("digital employee skill catalog service", () => { }), ])); }); + + it("projects GUI agent tools as local digital skills", async () => { + const { listDigitalEmployeeSkillCapabilities } = await import("./skillCatalogService"); + + const result = listDigitalEmployeeSkillCapabilities({ + guiServerStatus: { running: true, port: 60008 }, + }); + + expect(result.skills).toEqual(expect.arrayContaining([ + expect.objectContaining({ + skill_id: "qimingclaw-gui-agent", + source: "qimingclaw-gui", + server_id: "qimingclaw-gui-agent", + transport: "local", + running: true, + description: expect.stringContaining("MCP 端口：60008"), + tools: expect.arrayContaining(["gui_execute_task", "gui_click", "gui_analyze_screen"]), + tool_count: 15, + recent_calls: [expect.objectContaining({ event_id: "event-4", tool_name: "gui_click" })], + project_ids: ["project-gui"], + }), + expect.objectContaining({ + skill_id: "qimingclaw-gui-agent-tool-gui-click", + tools: ["gui_click"], + recent_calls: [expect.objectContaining({ event_id: "event-4" })], + }), + ])); + }); + + it("projects file server tools as local digital skills", async () => { + const { listDigitalEmployeeSkillCapabilities } = await import("./skillCatalogService"); + + const result = listDigitalEmployeeSkillCapabilities({ + fileServerStatus: { running: true, port: 60005 }, + }); + + expect(result.skills).toEqual(expect.arrayContaining([ + expect.objectContaining({ + skill_id: "qimingclaw-file-server", + source: "qimingclaw-file-server", + server_id: "qimingclaw-file-server", + transport: "local", + running: true, + description: expect.stringContaining("HTTP 端口：60005"), + tools: expect.arrayContaining(["create_workspace", "upload_file", "download_all_files"]), + tool_count: 18, + recent_calls: [expect.objectContaining({ event_id: "event-5", tool_name: "create_workspace" })], + project_ids: ["project-file"], + }), + expect.objectContaining({ + skill_id: "qimingclaw-file-server-tool-create-workspace", + tools: ["create_workspace"], + recent_calls: [expect.objectContaining({ event_id: "event-5" })], + }), + ])); + }); }); diff --git a/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.ts b/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.ts index f0f2f395..45a012be 100644 --- a/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.ts +++ b/qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.ts @@ -1,6 +1,10 @@ import { mcpProxyManager } from "../packages/mcp"; import type { McpServerEntry } from "../packages/mcp"; import { agentService } from "../engines/unifiedAgent"; +import { getGuiAgentServerStatus } from "../packages/guiAgentServer"; +import { getWindowsMcpStatus } from "../packages/windowsMcp"; +import { isWindows } from "../system/shellEnv"; +import { getConfiguredPorts } from "../startupPorts"; import { readDigitalEmployeeRuntimeRecords } from "./stateService"; export interface DigitalEmployeeSkillRecentCall { @@ -31,7 +35,7 @@ export interface DigitalEmployeeSkillCapability { description: string; category: string; capability_level: "core" | "standard" | "specialized" | "experimental"; - source: "qimingclaw-mcp" | "qimingclaw-acp"; + source: "qimingclaw-mcp" | "qimingclaw-acp" | "qimingclaw-gui" | "qimingclaw-file-server"; server_id: string; transport: "stdio" | "remote" | "local"; running: boolean; @@ -69,6 +73,17 @@ interface DigitalEmployeeRuntimeEventLike { occurredAt: string; } +interface DigitalEmployeeLocalServiceStatus { + running?: boolean; + port?: number; + error?: string | null; +} + +export interface DigitalEmployeeSkillCatalogOptions { + fileServerStatus?: DigitalEmployeeLocalServiceStatus | null; + guiServerStatus?: DigitalEmployeeLocalServiceStatus | null; +} + const ACP_SKILLS: AcpSkillDefinition[] = [ { skill_id: "qimingclaw-acp-session", @@ -108,7 +123,48 @@ const ACP_SKILLS: AcpSkillDefinition[] = [ }, ]; -export function listDigitalEmployeeSkillCapabilities(): { +const GUI_AGENT_TOOLS = [ + "gui_execute_task", + "gui_analyze_screen", + "gui_screenshot", + "gui_click", + "gui_double_click", + "gui_move_mouse", + "gui_drag", + "gui_scroll", + "gui_type", + "gui_press_key", + "gui_hotkey", + "gui_cursor_position", + "gui_list_displays", + "gui_find_image", + "gui_wait_for_image", +]; + +const FILE_SERVER_TOOLS = [ + "create_workspace", + "create_workspace_v2", + "push_skills_to_workspace", + "push_skills_to_workspace_v2", + "get_file_list", + "files_update", + "upload_file", + "upload_files", + "download_all_files", + "create_project", + "upload_project", + "get_project_content", + "get_project_content_by_version", + "backup_current_version", + "export_project", + "delete_project", + "upload_attachment_file", + "copy_project", +]; + +export function listDigitalEmployeeSkillCapabilities( + options: DigitalEmployeeSkillCatalogOptions = {}, +): { generatedAt: string; skills: DigitalEmployeeSkillCapability[]; } { @@ -198,6 +254,8 @@ export function listDigitalEmployeeSkillCapabilities(): { } skills.push(...buildAcpSkills(runtime.events)); + skills.push(...buildGuiAgentSkills(runtime.events, options.guiServerStatus)); + skills.push(...buildFileServerSkills(runtime.events, options.fileServerStatus)); return { generatedAt: new Date().toISOString(), @@ -205,6 +263,120 @@ export function listDigitalEmployeeSkillCapabilities(): { }; } +function buildGuiAgentSkills( + events: DigitalEmployeeRuntimeEventLike[], + inputStatus?: DigitalEmployeeLocalServiceStatus | null, +): DigitalEmployeeSkillCapability[] { + const status = normalizeLocalStatus(inputStatus ?? safeGuiServerStatus()); + return buildLocalServiceSkills({ + source: "qimingclaw-gui", + serverId: "qimingclaw-gui-agent", + aggregateSkillId: "qimingclaw-gui-agent", + aggregateName: "GUI 桌面自动化", + aggregateDescription: [ + "通过 qimingclaw GUI Agent 执行截图分析、鼠标键盘动作和自然语言桌面任务。", + status.port ? `MCP 端口：${status.port}。` : null, + status.error ? `当前状态：${status.error}。` : null, + ].filter(Boolean).join(" "), + category: "GUI", + capabilityLevel: "specialized", + tools: GUI_AGENT_TOOLS, + status, + matchNeedles: ["gui_agent", "gui-agent", "gui_", "desktop", "screenshot", "mouse", "keyboard", "display"], + toolNamePrefix: "GUI 工具", + }, events); +} + +function buildFileServerSkills( + events: DigitalEmployeeRuntimeEventLike[], + inputStatus?: DigitalEmployeeLocalServiceStatus | null, +): DigitalEmployeeSkillCapability[] { + const status = normalizeLocalStatus(inputStatus ?? safeFileServerStatus()); + return buildLocalServiceSkills({ + source: "qimingclaw-file-server", + serverId: "qimingclaw-file-server", + aggregateSkillId: "qimingclaw-file-server", + aggregateName: "文件服务与工作区交付", + aggregateDescription: [ + "通过 qiming-file-server 创建工作区、同步技能包、上传下载文件和导出项目产物。", + status.port ? `HTTP 端口：${status.port}。` : null, + status.error ? `当前状态：${status.error}。` : null, + ].filter(Boolean).join(" "), + category: "文件服务", + capabilityLevel: "standard", + tools: FILE_SERVER_TOOLS, + status, + matchNeedles: ["file_server", "file-server", "qiming-file-server", "create-workspace", "workspace", "upload", "download", "artifact"], + toolNamePrefix: "文件工具", + }, events); +} + +function buildLocalServiceSkills( + definition: { + source: Extract; + serverId: string; + aggregateSkillId: string; + aggregateName: string; + aggregateDescription: string; + category: string; + capabilityLevel: DigitalEmployeeSkillCapability["capability_level"]; + tools: string[]; + status: Required> & DigitalEmployeeLocalServiceStatus; + matchNeedles: string[]; + toolNamePrefix: string; + }, + events: DigitalEmployeeRuntimeEventLike[], +): DigitalEmployeeSkillCapability[] { + const governance = buildGovernance(true, [], [], [], []); + const serviceCalls = localRecentCalls(events, definition.tools, definition.matchNeedles, definition.tools[0]); + const skills: DigitalEmployeeSkillCapability[] = [{ + skill_id: definition.aggregateSkillId, + name: definition.aggregateName, + version: "1.0.0", + enabled: true, + description: definition.aggregateDescription, + category: definition.category, + capability_level: definition.capabilityLevel, + source: definition.source, + server_id: definition.serverId, + transport: "local", + running: definition.status.running, + governance, + recent_calls: serviceCalls.slice(0, 5), + call_count: serviceCalls.length, + last_called_at: serviceCalls[0]?.occurred_at ?? null, + tools: definition.tools, + tool_count: definition.tools.length, + project_ids: projectIds(serviceCalls), + }]; + + for (const tool of definition.tools) { + const toolCalls = localRecentCalls(events, [tool], toolNeedles(tool), tool); + skills.push({ + skill_id: `${definition.aggregateSkillId}-tool-${slugifySkillPart(tool)}`, + name: `${definition.toolNamePrefix}：${tool}`, + version: "1.0.0", + enabled: true, + description: `由 ${definition.aggregateName} 提供，可被 qimingclaw Agent 编排为本地数字员工动作。`, + category: `${definition.category}工具`, + capability_level: "standard", + source: definition.source, + server_id: definition.serverId, + transport: "local", + running: definition.status.running, + governance, + recent_calls: toolCalls.slice(0, 5), + call_count: toolCalls.length, + last_called_at: toolCalls[0]?.occurred_at ?? null, + tools: [tool], + tool_count: 1, + project_ids: projectIds(toolCalls), + }); + } + + return skills; +} + function buildAcpSkills(events: DigitalEmployeeRuntimeEventLike[]): DigitalEmployeeSkillCapability[] { const sessions = safeAgentSessions(); const running = safeAgentReady(); @@ -278,6 +450,63 @@ function includesAny(event: DigitalEmployeeRuntimeEventLike, needles: string[]): return needles.some((needle) => text.includes(needle.toLowerCase())); } +function localRecentCalls( + events: DigitalEmployeeRuntimeEventLike[], + tools: string[], + needles: string[], + fallbackToolName: string, +): DigitalEmployeeSkillRecentCall[] { + const toolSet = new Set(tools.map((tool) => tool.toLowerCase())); + return events + .filter((event) => { + const payload = objectRecord(event.payload); + const toolName = stringValue(payload?.toolName ?? payload?.tool_name) || toolNameFromMessage(event.message); + if (toolName && toolSet.has(normalizeToolName(toolName))) return true; + return includesAny(event, needles); + }) + .map((event) => toAcpRecentCall(event, fallbackToolName)) + .sort((left, right) => right.occurred_at.localeCompare(left.occurred_at)); +} + +function toolNeedles(tool: string): string[] { + return [ + tool, + tool.replace(/_/g, "-"), + tool.replace(/_/g, " "), + ]; +} + +function normalizeToolName(value: string): string { + return value.trim().toLowerCase().replace(/-/g, "_"); +} + +function normalizeLocalStatus( + value: DigitalEmployeeLocalServiceStatus | null | undefined, +): Required> & DigitalEmployeeLocalServiceStatus { + return { + running: value?.running === true, + port: typeof value?.port === "number" ? value.port : undefined, + error: typeof value?.error === "string" ? value.error : null, + }; +} + +function safeGuiServerStatus(): DigitalEmployeeLocalServiceStatus { + try { + return isWindows() ? getWindowsMcpStatus() : getGuiAgentServerStatus(); + } catch (error) { + return { running: false, error: error instanceof Error ? error.message : String(error) }; + } +} + +function safeFileServerStatus(): DigitalEmployeeLocalServiceStatus { + try { + const { fileServer } = getConfiguredPorts(); + return { running: false, port: fileServer }; + } catch (error) { + return { running: false, error: error instanceof Error ? error.message : String(error) }; + } +} + function safeAgentReady(): boolean { try { return agentService.isReady; diff --git a/qimingclaw/docs/digital-employee-frontend-integration-plan.md b/qimingclaw/docs/digital-employee-frontend-integration-plan.md index 406fd813..9bf186cc 100644 --- a/qimingclaw/docs/digital-employee-frontend-integration-plan.md +++ b/qimingclaw/docs/digital-employee-frontend-integration-plan.md @@ -872,7 +872,8 @@ file server tools 当前客户端实现由 `digitalEmployee:getSkillCapabilities` IPC 输出本地 Skill Catalog， embedded digital 页面通过 `window.QimingClawBridge.digital.getSkillCapabilities()` 合并到 `/api/skill`。目录已覆盖 MCP server / tool，并把 qimingclaw ACP 内建能力 -投射为 Skill；后续再继续扩展 GUI agent tool 和 file server tool。 +投射为 Skill；GUI agent tool 和 file server tool 也已作为 qimingclaw 本地技能源 +进入目录。 MCP 技能字段已包含： @@ -899,6 +900,18 @@ qimingclaw-artifact-reporting 这些技能会展示 Agent 引擎、运行状态、活跃会话数、内建 command/tool 列表， 并从本地 digital runtime events 归集最近调用、项目 ID 和会话 ID。 +GUI / File Server 技能目录已包含： + +```text +qimingclaw-gui-agent +qimingclaw-file-server +gui_execute_task / gui_analyze_screen / gui_click / gui_type / ... +create_workspace / upload_file / download_all_files / export_project / ... +``` + +这些本地技能会展示对应服务运行状态、端口、工具列表，并从 runtime event 的 +`toolName`、`source`、workspace/upload/download/artifact 等线索归集最近调用。 + 用户可以： - 查看技能 @@ -907,6 +920,7 @@ qimingclaw-artifact-reporting - 查看 MCP allow / deny 策略 - 查看 MCP proxy 运行状态 - 查看 ACP 内建能力与最近调用 +- 查看 GUI agent / file server 本地工具和最近调用 - 查看输入输出说明 - 查看最近调用记录