byy/qiming
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(client): project local GUI and file tools

Browse Source
This commit is contained in:
baiyanyun
2026-06-08 18:44:45 +08:00
parent 38e5bdbc04
commit 913c6da8f3
4 changed files with 348 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qimingclaw/crates/agent-electron-client/src/main/ipc/digitalEmployeeHandlers.ts
View File

@@ -53,7 +53,10 @@ export function registerDigitalEmployeeHandlers(ctx: HandlerContext): void {
});
ipcMain.handle("digitalEmployee:getSkillCapabilities", async () => {
return listDigitalEmployeeSkillCapabilities();
return listDigitalEmployeeSkillCapabilities({
fileServerStatus: normalizeServiceStatus(ctx.fileServer.status()),
guiServerStatus: normalizeServiceStatus(getGuiStatus()),
});
});
ipcMain.handle("digitalEmployee:getPlanTemplates", async () => {

98
qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.test.ts
View File

@@ -63,6 +63,32 @@ const mockRuntime = vi.hoisted(() => ({
},
occurredAt: "2026-06-07T08:02:00.000Z",
},
{
id: "event-4",
kind: "computer_tool_call",
message: "工具调用开始：gui_click",
payload: {
source: "gui_agent",
toolName: "gui_click",
projectId: "project-gui",
sessionId: "session-gui",
status: "completed",
},
occurredAt: "2026-06-07T08:03:00.000Z",
},
{
id: "event-5",
kind: "file_server.workspace.created",
message: "create-workspace 完成",
payload: {
source: "qiming-file-server",
toolName: "create_workspace",
projectId: "project-file",
sessionId: "session-file",
status: "completed",
},
occurredAt: "2026-06-07T08:04:00.000Z",
},
],
},
}));
@@ -95,6 +121,22 @@ vi.mock("../engines/unifiedAgent", () => ({
},
}));
vi.mock("../packages/guiAgentServer", () => ({
getGuiAgentServerStatus: () => ({ running: true, port: 60008 }),
}));
vi.mock("../packages/windowsMcp", () => ({
getWindowsMcpStatus: () => ({ running: false }),
}));
vi.mock("../system/shellEnv", () => ({
isWindows: () => false,
}));
vi.mock("../startupPorts", () => ({
getConfiguredPorts: () => ({ fileServer: 60005 }),
}));
describe("digital employee skill catalog service", () => {
beforeEach(() => {
vi.resetModules();
@@ -165,4 +207,60 @@ describe("digital employee skill catalog service", () => {
}),
]));
});
it("projects GUI agent tools as local digital skills", async () => {
const { listDigitalEmployeeSkillCapabilities } = await import("./skillCatalogService");
const result = listDigitalEmployeeSkillCapabilities({
guiServerStatus: { running: true, port: 60008 },
});
expect(result.skills).toEqual(expect.arrayContaining([
expect.objectContaining({
skill_id: "qimingclaw-gui-agent",
source: "qimingclaw-gui",
server_id: "qimingclaw-gui-agent",
transport: "local",
running: true,
description: expect.stringContaining("MCP 端口：60008"),
tools: expect.arrayContaining(["gui_execute_task", "gui_click", "gui_analyze_screen"]),
tool_count: 15,
recent_calls: [expect.objectContaining({ event_id: "event-4", tool_name: "gui_click" })],
project_ids: ["project-gui"],
}),
expect.objectContaining({
skill_id: "qimingclaw-gui-agent-tool-gui-click",
tools: ["gui_click"],
recent_calls: [expect.objectContaining({ event_id: "event-4" })],
}),
]));
});
it("projects file server tools as local digital skills", async () => {
const { listDigitalEmployeeSkillCapabilities } = await import("./skillCatalogService");
const result = listDigitalEmployeeSkillCapabilities({
fileServerStatus: { running: true, port: 60005 },
});
expect(result.skills).toEqual(expect.arrayContaining([
expect.objectContaining({
skill_id: "qimingclaw-file-server",
source: "qimingclaw-file-server",
server_id: "qimingclaw-file-server",
transport: "local",
running: true,
description: expect.stringContaining("HTTP 端口：60005"),
tools: expect.arrayContaining(["create_workspace", "upload_file", "download_all_files"]),
tool_count: 18,
recent_calls: [expect.objectContaining({ event_id: "event-5", tool_name: "create_workspace" })],
project_ids: ["project-file"],
}),
expect.objectContaining({
skill_id: "qimingclaw-file-server-tool-create-workspace",
tools: ["create_workspace"],
recent_calls: [expect.objectContaining({ event_id: "event-5" })],
}),
]));
});
});

233
qimingclaw/crates/agent-electron-client/src/main/services/digitalEmployee/skillCatalogService.ts
View File

@@ -1,6 +1,10 @@
import { mcpProxyManager } from "../packages/mcp";
import type { McpServerEntry } from "../packages/mcp";
import { agentService } from "../engines/unifiedAgent";
import { getGuiAgentServerStatus } from "../packages/guiAgentServer";
import { getWindowsMcpStatus } from "../packages/windowsMcp";
import { isWindows } from "../system/shellEnv";
import { getConfiguredPorts } from "../startupPorts";
import { readDigitalEmployeeRuntimeRecords } from "./stateService";
export interface DigitalEmployeeSkillRecentCall {
@@ -31,7 +35,7 @@ export interface DigitalEmployeeSkillCapability {
description: string;
category: string;
capability_level: "core" | "standard" | "specialized" | "experimental";
source: "qimingclaw-mcp" | "qimingclaw-acp";
source: "qimingclaw-mcp" | "qimingclaw-acp" | "qimingclaw-gui" | "qimingclaw-file-server";
server_id: string;
transport: "stdio" | "remote" | "local";
running: boolean;
@@ -69,6 +73,17 @@ interface DigitalEmployeeRuntimeEventLike {
occurredAt: string;
}
interface DigitalEmployeeLocalServiceStatus {
running?: boolean;
port?: number;
error?: string | null;
}
export interface DigitalEmployeeSkillCatalogOptions {
fileServerStatus?: DigitalEmployeeLocalServiceStatus | null;
guiServerStatus?: DigitalEmployeeLocalServiceStatus | null;
}
const ACP_SKILLS: AcpSkillDefinition[] = [
{
skill_id: "qimingclaw-acp-session",
@@ -108,7 +123,48 @@ const ACP_SKILLS: AcpSkillDefinition[] = [
},
];
export function listDigitalEmployeeSkillCapabilities(): {
const GUI_AGENT_TOOLS = [
"gui_execute_task",
"gui_analyze_screen",
"gui_screenshot",
"gui_click",
"gui_double_click",
"gui_move_mouse",
"gui_drag",
"gui_scroll",
"gui_type",
"gui_press_key",
"gui_hotkey",
"gui_cursor_position",
"gui_list_displays",
"gui_find_image",
"gui_wait_for_image",
];
const FILE_SERVER_TOOLS = [
"create_workspace",
"create_workspace_v2",
"push_skills_to_workspace",
"push_skills_to_workspace_v2",
"get_file_list",
"files_update",
"upload_file",
"upload_files",
"download_all_files",
"create_project",
"upload_project",
"get_project_content",
"get_project_content_by_version",
"backup_current_version",
"export_project",
"delete_project",
"upload_attachment_file",
"copy_project",
];
export function listDigitalEmployeeSkillCapabilities(
options: DigitalEmployeeSkillCatalogOptions = {},
): {
generatedAt: string;
skills: DigitalEmployeeSkillCapability[];
} {
@@ -198,6 +254,8 @@ export function listDigitalEmployeeSkillCapabilities(): {
}
skills.push(...buildAcpSkills(runtime.events));
skills.push(...buildGuiAgentSkills(runtime.events, options.guiServerStatus));
skills.push(...buildFileServerSkills(runtime.events, options.fileServerStatus));
return {
generatedAt: new Date().toISOString(),
@@ -205,6 +263,120 @@ export function listDigitalEmployeeSkillCapabilities(): {
};
}
function buildGuiAgentSkills(
events: DigitalEmployeeRuntimeEventLike[],
inputStatus?: DigitalEmployeeLocalServiceStatus | null,
): DigitalEmployeeSkillCapability[] {
const status = normalizeLocalStatus(inputStatus ?? safeGuiServerStatus());
return buildLocalServiceSkills({
source: "qimingclaw-gui",
serverId: "qimingclaw-gui-agent",
aggregateSkillId: "qimingclaw-gui-agent",
aggregateName: "GUI 桌面自动化",
aggregateDescription: [
"通过 qimingclaw GUI Agent 执行截图分析、鼠标键盘动作和自然语言桌面任务。",
status.port ? `MCP 端口：${status.port}。` : null,
status.error ? `当前状态：${status.error}。` : null,
].filter(Boolean).join(" "),
category: "GUI",
capabilityLevel: "specialized",
tools: GUI_AGENT_TOOLS,
status,
matchNeedles: ["gui_agent", "gui-agent", "gui_", "desktop", "screenshot", "mouse", "keyboard", "display"],
toolNamePrefix: "GUI 工具",
}, events);
}
function buildFileServerSkills(
events: DigitalEmployeeRuntimeEventLike[],
inputStatus?: DigitalEmployeeLocalServiceStatus | null,
): DigitalEmployeeSkillCapability[] {
const status = normalizeLocalStatus(inputStatus ?? safeFileServerStatus());
return buildLocalServiceSkills({
source: "qimingclaw-file-server",
serverId: "qimingclaw-file-server",
aggregateSkillId: "qimingclaw-file-server",
aggregateName: "文件服务与工作区交付",
aggregateDescription: [
"通过 qiming-file-server 创建工作区、同步技能包、上传下载文件和导出项目产物。",
status.port ? `HTTP 端口：${status.port}。` : null,
status.error ? `当前状态：${status.error}。` : null,
].filter(Boolean).join(" "),
category: "文件服务",
capabilityLevel: "standard",
tools: FILE_SERVER_TOOLS,
status,
matchNeedles: ["file_server", "file-server", "qiming-file-server", "create-workspace", "workspace", "upload", "download", "artifact"],
toolNamePrefix: "文件工具",
}, events);
}
function buildLocalServiceSkills(
definition: {
source: Extract<DigitalEmployeeSkillCapability["source"], "qimingclaw-gui" | "qimingclaw-file-server">;
serverId: string;
aggregateSkillId: string;
aggregateName: string;
aggregateDescription: string;
category: string;
capabilityLevel: DigitalEmployeeSkillCapability["capability_level"];
tools: string[];
status: Required<Pick<DigitalEmployeeLocalServiceStatus, "running">> & DigitalEmployeeLocalServiceStatus;
matchNeedles: string[];
toolNamePrefix: string;
},
events: DigitalEmployeeRuntimeEventLike[],
): DigitalEmployeeSkillCapability[] {
const governance = buildGovernance(true, [], [], [], []);
const serviceCalls = localRecentCalls(events, definition.tools, definition.matchNeedles, definition.tools[0]);
const skills: DigitalEmployeeSkillCapability[] = [{
skill_id: definition.aggregateSkillId,
name: definition.aggregateName,
version: "1.0.0",
enabled: true,
description: definition.aggregateDescription,
category: definition.category,
capability_level: definition.capabilityLevel,
source: definition.source,
server_id: definition.serverId,
transport: "local",
running: definition.status.running,
governance,
recent_calls: serviceCalls.slice(0, 5),
call_count: serviceCalls.length,
last_called_at: serviceCalls[0]?.occurred_at ?? null,
tools: definition.tools,
tool_count: definition.tools.length,
project_ids: projectIds(serviceCalls),
}];
for (const tool of definition.tools) {
const toolCalls = localRecentCalls(events, [tool], toolNeedles(tool), tool);
skills.push({
skill_id: `${definition.aggregateSkillId}-tool-${slugifySkillPart(tool)}`,
name: `${definition.toolNamePrefix}：${tool}`,
version: "1.0.0",
enabled: true,
description: `由 ${definition.aggregateName} 提供，可被 qimingclaw Agent 编排为本地数字员工动作。`,
category: `${definition.category}工具`,
capability_level: "standard",
source: definition.source,
server_id: definition.serverId,
transport: "local",
running: definition.status.running,
governance,
recent_calls: toolCalls.slice(0, 5),
call_count: toolCalls.length,
last_called_at: toolCalls[0]?.occurred_at ?? null,
tools: [tool],
tool_count: 1,
project_ids: projectIds(toolCalls),
});
}
return skills;
}
function buildAcpSkills(events: DigitalEmployeeRuntimeEventLike[]): DigitalEmployeeSkillCapability[] {
const sessions = safeAgentSessions();
const running = safeAgentReady();
@@ -278,6 +450,63 @@ function includesAny(event: DigitalEmployeeRuntimeEventLike, needles: string[]):
return needles.some((needle) => text.includes(needle.toLowerCase()));
}
function localRecentCalls(
events: DigitalEmployeeRuntimeEventLike[],
tools: string[],
needles: string[],
fallbackToolName: string,
): DigitalEmployeeSkillRecentCall[] {
const toolSet = new Set(tools.map((tool) => tool.toLowerCase()));
return events
.filter((event) => {
const payload = objectRecord(event.payload);
const toolName = stringValue(payload?.toolName ?? payload?.tool_name) || toolNameFromMessage(event.message);
if (toolName && toolSet.has(normalizeToolName(toolName))) return true;
return includesAny(event, needles);
})
.map((event) => toAcpRecentCall(event, fallbackToolName))
.sort((left, right) => right.occurred_at.localeCompare(left.occurred_at));
}
function toolNeedles(tool: string): string[] {
return [
tool,
tool.replace(/_/g, "-"),
tool.replace(/_/g, " "),
];
}
function normalizeToolName(value: string): string {
return value.trim().toLowerCase().replace(/-/g, "_");
}
function normalizeLocalStatus(
value: DigitalEmployeeLocalServiceStatus | null | undefined,
): Required<Pick<DigitalEmployeeLocalServiceStatus, "running">> & DigitalEmployeeLocalServiceStatus {
return {
running: value?.running === true,
port: typeof value?.port === "number" ? value.port : undefined,
error: typeof value?.error === "string" ? value.error : null,
};
}
function safeGuiServerStatus(): DigitalEmployeeLocalServiceStatus {
try {
return isWindows() ? getWindowsMcpStatus() : getGuiAgentServerStatus();
} catch (error) {
return { running: false, error: error instanceof Error ? error.message : String(error) };
}
}
function safeFileServerStatus(): DigitalEmployeeLocalServiceStatus {
try {
const { fileServer } = getConfiguredPorts();
return { running: false, port: fileServer };
} catch (error) {
return { running: false, error: error instanceof Error ? error.message : String(error) };
}
}
function safeAgentReady(): boolean {
try {
return agentService.isReady;

16
qimingclaw/docs/digital-employee-frontend-integration-plan.md
View File

@@ -872,7 +872,8 @@ file server tools
当前客户端实现由 `digitalEmployee:getSkillCapabilities` IPC 输出本地 Skill Catalog，
embedded digital 页面通过 `window.QimingClawBridge.digital.getSkillCapabilities()`
合并到 `/api/skill`。目录已覆盖 MCP server / tool，并把 qimingclaw ACP 内建能力
投射为 Skill；后续再继续扩展 GUI agent tool 和 file server tool。
投射为 Skill；GUI agent tool 和 file server tool 也已作为 qimingclaw 本地技能源
进入目录。
MCP 技能字段已包含：
@@ -899,6 +900,18 @@ qimingclaw-artifact-reporting
这些技能会展示 Agent 引擎、运行状态、活跃会话数、内建 command/tool 列表，
并从本地 digital runtime events 归集最近调用、项目 ID 和会话 ID。
GUI / File Server 技能目录已包含：
```text
qimingclaw-gui-agent
qimingclaw-file-server
gui_execute_task / gui_analyze_screen / gui_click / gui_type / ...
create_workspace / upload_file / download_all_files / export_project / ...
```
这些本地技能会展示对应服务运行状态、端口、工具列表，并从 runtime event 的
`toolName`、`source`、workspace/upload/download/artifact 等线索归集最近调用。
用户可以：
- 查看技能
@@ -907,6 +920,7 @@ qimingclaw-artifact-reporting
- 查看 MCP allow / deny 策略
- 查看 MCP proxy 运行状态
- 查看 ACP 内建能力与最近调用
- 查看 GUI agent / file server 本地工具和最近调用
- 查看输入输出说明
- 查看最近调用记录