chore: initialize qiming workspace repository

qimingclaw/crates/agent-gui-server/src/mcp/atomicTools.ts

@@ -0,0 +1,654 @@
+/**
+ * 14 atomic MCP tool handlers.
+ *
+ * Each handler: validate input → safety check → coordinate resolve → desktop operation → audit → response
+ */
+
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { validateHotkey } from '../safety/hotkeys.js';
+import { resolveCoordinate, type ScreenshotMeta, type DisplayInfo } from '../coordinates/resolver.js';
+import { getModelProfile } from '../coordinates/modelProfiles.js';
+import { captureScreenshot } from '../desktop/screenshot.js';
+import * as mouse from '../desktop/mouse.js';
+import * as keyboard from '../desktop/keyboard.js';
+import * as display from '../desktop/display.js';
+import { findImage, waitForImage } from '../desktop/imageSearch.js';
+import { analyzeScreen } from '../desktop/screenAnalyzer.js';
+import { createModel } from '../agent/taskRunner.js';
+import { logError } from '../utils/logger.js';
+import { SafetyError } from '../utils/errors.js';
+
+const ATOMIC_TOOLS = [
+  {
+    name: 'gui_screenshot',
+    description: 'Capture a screenshot and return as base64 image data. WARNING: Returns raw image data that text-only models cannot interpret. For understanding screen content, use gui_analyze_screen instead. Use this tool only when you need to save or transfer the screenshot file itself.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        displayIndex: { type: 'number', description: 'Display index (default: configured display)' },
+      },
+    },
+  },
+  {
+    name: 'gui_click',
+    description: 'Click at the specified coordinates. Use gui_analyze_screen first to identify element locations, or gui_cursor_position to get current mouse position. Coordinates are in logical pixels (scaled automatically for Retina displays).',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels (0 = left edge of display)' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels (0 = top edge of display)' },
+        button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button (default: left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode: "logical" (default, use logical pixel values), "image-absolute" (divide x/y by image width/height to get 0-1, then multiply by display size), "normalized-1000" (x/y in 0-1000 range, scaled to display size)' },
+      },
+      required: ['x', 'y'],
+    },
+  },
+  {
+    name: 'gui_double_click',
+    description: 'Double-click at the specified coordinates. Typically used to open files or select words. Use gui_analyze_screen first to identify element locations.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels' },
+        button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button (default: left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['x', 'y'],
+    },
+  },
+  {
+    name: 'gui_move_mouse',
+    description: 'Move mouse cursor to the specified coordinates without clicking. Useful for hovering over elements to reveal tooltips or menus.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['x', 'y'],
+    },
+  },
+  {
+    name: 'gui_drag',
+    description: 'Drag from start to end coordinates. Used for dragging files, selecting text, or moving windows. Hold mouse button from start to end position.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        startX: { type: 'number', description: 'Start X coordinate in logical pixels' },
+        startY: { type: 'number', description: 'Start Y coordinate in logical pixels' },
+        endX: { type: 'number', description: 'End X coordinate in logical pixels' },
+        endY: { type: 'number', description: 'End Y coordinate in logical pixels' },
+        button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button (default: left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['startX', 'startY', 'endX', 'endY'],
+    },
+  },
+  {
+    name: 'gui_scroll',
+    description: 'Scroll at the specified position. Positive deltaY scrolls down, negative scrolls up. Used for navigating long pages or lists.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels' },
+        deltaY: { type: 'number', description: 'Vertical scroll amount in "steps" (scroll wheel clicks). Positive = scroll down, negative = scroll up. Typical values: 1-3 for small scroll, 5-10 for larger scroll' },
+        deltaX: { type: 'number', description: 'Horizontal scroll amount in steps (positive = right, negative = left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['x', 'y', 'deltaY'],
+    },
+  },
+  {
+    name: 'gui_type',
+    description: 'Type text at the current cursor position. Automatically handles CJK/non-ASCII text via clipboard paste. Use after clicking on an input field.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        text: { type: 'string', description: 'Text to type (supports all languages including Chinese, Japanese, Korean)' },
+      },
+      required: ['text'],
+    },
+  },
+  {
+    name: 'gui_press_key',
+    description: 'Press a single key. Supported keys: Enter, Tab, Escape, Backspace, Delete, Space, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, Home, End, PageUp, PageDown, F1-F24. Modifiers: Shift, Control/Ctrl, Alt/Option, Meta/Command/Cmd.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        key: { type: 'string', description: 'Key name (e.g. Enter, Tab, Escape, ArrowUp, F1). Supports aliases like Ctrl, Cmd, Opt.' },
+      },
+      required: ['key'],
+    },
+  },
+  {
+    name: 'gui_hotkey',
+    description: 'Press a key combination (shortcut). Key names support aliases: Meta/Command/Cmd/⌘, Control/Ctrl/⌃, Alt/Option/Opt/⌥, Shift/⇧. Example: ["Meta", "C"] for copy, ["Meta", "V"] for paste.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        keys: { type: 'array', items: { type: 'string' }, description: 'Keys to press together in order (e.g. ["Meta", "C"] for copy, ["Alt", "Tab"] for window switcher)' },
+      },
+      required: ['keys'],
+    },
+  },
+  {
+    name: 'gui_cursor_position',
+    description: 'Get current mouse cursor position. Returns both logical coordinates (for use with gui_click) and physical coordinates (raw nut.js values). On Retina displays, logical coords are half of physical.',
+    inputSchema: { type: 'object' as const, properties: {} },
+  },
+  {
+    name: 'gui_list_displays',
+    description: 'List all connected displays with their properties (resolution, scale factor, position). Use to identify which display to target.',
+    inputSchema: { type: 'object' as const, properties: {} },
+  },
+  {
+    name: 'gui_find_image',
+    description: 'Find a template image on screen using image matching. DEPRECATED: For most use cases, prefer gui_analyze_screen with prompt "Find [element]" instead. Returns match location and confidence. Requires base64-encoded template image.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        template: { type: 'string', description: 'Base64-encoded template image to find (PNG/JPEG)' },
+        confidence: { type: 'number', description: 'Match confidence threshold 0-1 (default 0.9, lower for fuzzy matching)' },
+      },
+      required: ['template'],
+    },
+  },
+  {
+    name: 'gui_wait_for_image',
+    description: 'Wait for a template image to appear on screen. DEPRECATED: For most use cases, prefer gui_analyze_screen in a loop instead. Polls until found or timeout.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        template: { type: 'string', description: 'Base64-encoded template image to wait for' },
+        timeout: { type: 'number', description: 'Timeout in milliseconds (default 10000)' },
+        confidence: { type: 'number', description: 'Match confidence threshold 0-1 (default 0.9)' },
+      },
+      required: ['template'],
+    },
+  },
+  {
+    name: 'gui_analyze_screen',
+    description: 'Capture screenshot and analyze with vision model. Returns text description of screen content, UI elements, text, buttons, and their locations. Use this to understand what is on screen before taking action. Preferred over gui_screenshot for text-based models.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        prompt: { type: 'string', description: 'Analysis instruction (e.g., "What buttons are visible?", "Find the search box location and provide coordinates", "Is there an error dialog?", "What is the current active window?")' },
+        displayIndex: { type: 'number', description: 'Display index (default: configured display)' },
+      },
+      required: ['prompt'],
+    },
+  },
+];
+
+/** Cached screenshot dimensions per display (with TTL) */
+interface CachedDimensions {
+  imageWidth: number;
+  imageHeight: number;
+  logicalWidth: number;
+  logicalHeight: number;
+  timestamp: number;
+}
+const screenshotDimensionCache = new Map<number, CachedDimensions>();
+const CACHE_TTL_MS = 5000; // 5 seconds
+
+/** Clear the screenshot dimension cache (for testing) */
+export function clearScreenshotDimensionCache(): void {
+  screenshotDimensionCache.clear();
+}
+
+/** Helper: resolve coordinates for tools that accept x/y + optional coordinateMode */
+async function resolveXY(
+  x: number,
+  y: number,
+  coordinateMode: string | undefined,
+  config: GuiAgentConfig,
+): Promise<{ globalX: number; globalY: number; meta: ScreenshotMeta }> {
+  const profile = getModelProfile(config.model, coordinateMode as any);
+  const disp = await display.getDisplay(config.displayIndex);
+  const displayInfo: DisplayInfo = {
+    origin: disp.origin,
+    bounds: { width: disp.width, height: disp.height },
+    scaleFactor: disp.scaleFactor,
+  };
+
+  if (!coordinateMode) {
+    // No coordinate mode specified — treat as logical coordinates directly
+    // Convert to physical coordinates for nut.js (multiply by scaleFactor)
+    const scaleFactor = disp.scaleFactor ?? 1;
+    const meta: ScreenshotMeta = {
+      imageWidth: disp.width,
+      imageHeight: disp.height,
+      logicalWidth: disp.width,
+      logicalHeight: disp.height,
+    };
+    return {
+      globalX: Math.round(x * scaleFactor) + disp.origin.x * scaleFactor,
+      globalY: Math.round(y * scaleFactor) + disp.origin.y * scaleFactor,
+      meta,
+    };
+  }
+
+  // For coordinate modes (image-absolute, normalized-*), we need actual screenshot dimensions
+  // because models return coordinates based on the image they saw (which may be scaled)
+  // Use cached dimensions if available and fresh
+  const cached = screenshotDimensionCache.get(config.displayIndex);
+  const now = Date.now();
+
+  let meta: ScreenshotMeta;
+  if (cached && (now - cached.timestamp) < CACHE_TTL_MS) {
+    meta = {
+      imageWidth: cached.imageWidth,
+      imageHeight: cached.imageHeight,
+      logicalWidth: cached.logicalWidth,
+      logicalHeight: cached.logicalHeight,
+    };
+  } else {
+    // Capture screenshot to get actual dimensions
+    try {
+      const shot = await captureScreenshot(config.displayIndex, config.jpegQuality);
+      meta = {
+        imageWidth: shot.imageWidth,
+        imageHeight: shot.imageHeight,
+        logicalWidth: shot.logicalWidth,
+        logicalHeight: shot.logicalHeight,
+      };
+      // Cache the dimensions
+      screenshotDimensionCache.set(config.displayIndex, {
+        imageWidth: shot.imageWidth,
+        imageHeight: shot.imageHeight,
+        logicalWidth: shot.logicalWidth,
+        logicalHeight: shot.logicalHeight,
+        timestamp: now,
+      });
+    } catch (err) {
+      // Fallback to display dimensions on screenshot failure
+      logError(`Screenshot failed for coordinate resolution: ${err instanceof Error ? err.message : String(err)}`);
+      meta = {
+        imageWidth: disp.width,
+        imageHeight: disp.height,
+        logicalWidth: disp.width,
+        logicalHeight: disp.height,
+      };
+    }
+  }
+
+  const resolved = resolveCoordinate(x, y, profile, meta, displayInfo);
+  return { globalX: resolved.globalX, globalY: resolved.globalY, meta };
+}
+
+export { ATOMIC_TOOLS };
+
+/**
+ * 获取可用的工具列表
+ * 如果未配置视觉模型，则不包含 gui_analyze_screen
+ */
+export function getAvailableTools(): typeof ATOMIC_TOOLS {
+  const visionModel = process.env.GUI_AGENT_VISION_MODEL;
+  if (!visionModel) {
+    return ATOMIC_TOOLS.filter(t => t.name !== 'gui_analyze_screen');
+  }
+  return ATOMIC_TOOLS;
+}
+
+/** Validate that a value is a finite number */
+function requireNumber(name: string, value: unknown): number {
+  if (typeof value !== 'number' || !isFinite(value)) {
+    throw new Error(`Parameter '${name}' must be a finite number, got: ${value}`);
+  }
+  return value;
+}
+
+/** Validate that a value is a non-empty string */
+function requireString(name: string, value: unknown): string {
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new Error(`Parameter '${name}' must be a non-empty string`);
+  }
+  return value;
+}
+
+/** Validate that a value is a non-empty string array */
+function requireStringArray(name: string, value: unknown): string[] {
+  if (!Array.isArray(value) || value.length === 0 || !value.every(v => typeof v === 'string')) {
+    throw new Error(`Parameter '${name}' must be a non-empty array of strings`);
+  }
+  return value;
+}
+
+export async function handleAtomicToolCall(
+  name: string,
+  args: Record<string, unknown>,
+  config: GuiAgentConfig,
+  auditLog: AuditLog,
+): Promise<{ content: Array<{ type: string; text?: string; data?: string; mimeType?: string }>; isError?: boolean } | null> {
+  // Check if this is an atomic tool
+  if (!ATOMIC_TOOLS.some(t => t.name === name)) {
+    return null; // Not handled by this module
+  }
+
+  const start = Date.now();
+  try {
+    const result = await handleAtomicTool(name, args, config);
+    auditLog.record({ tool: name, args, success: true, durationMs: Date.now() - start });
+    return result;
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    auditLog.record({ tool: name, args, success: false, durationMs: Date.now() - start });
+    logError(`Tool ${name} failed: ${errorMsg}`);
+    return { content: [{ type: 'text', text: `Error: ${errorMsg}` }], isError: true };
+  }
+}
+
+async function handleAtomicTool(name: string, args: Record<string, unknown>, config: GuiAgentConfig) {
+  switch (name) {
+    case 'gui_screenshot': {
+      const idx = (args.displayIndex as number) ?? config.displayIndex;
+      const shot = await captureScreenshot(idx, config.jpegQuality);
+      return {
+        content: [
+          { type: 'image', data: shot.image, mimeType: shot.mimeType },
+          {
+            type: 'text',
+            text: JSON.stringify({
+              success: true,
+              imageWidth: shot.imageWidth,
+              imageHeight: shot.imageHeight,
+              logicalWidth: shot.logicalWidth,
+              logicalHeight: shot.logicalHeight,
+              physicalWidth: shot.physicalWidth,
+              physicalHeight: shot.physicalHeight,
+              scaleFactor: shot.scaleFactor,
+              displayIndex: shot.displayIndex,
+              note: 'Image dimensions are in logical pixels. Use logical coordinates for click operations.',
+            }, null, 2),
+          },
+        ],
+      };
+    }
+
+    case 'gui_click': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const button = (args.button as string) || 'left';
+      const { globalX, globalY, meta } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.click(globalX, globalY, button as any);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'click',
+            button,
+            logicalCoords: { x, y },
+            physicalCoords: { x: globalX, y: globalY },
+            displaySize: { width: meta.logicalWidth, height: meta.logicalHeight },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_double_click': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const button = (args.button as string) || 'left';
+      const { globalX, globalY } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.doubleClick(globalX, globalY, button as any);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'double_click',
+            button,
+            logicalCoords: { x, y },
+            physicalCoords: { x: globalX, y: globalY },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_move_mouse': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const { globalX, globalY } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.moveTo(globalX, globalY);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'move',
+            logicalCoords: { x, y },
+            physicalCoords: { x: globalX, y: globalY },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_drag': {
+      const startX = requireNumber('startX', args.startX);
+      const startY = requireNumber('startY', args.startY);
+      const endX = requireNumber('endX', args.endX);
+      const endY = requireNumber('endY', args.endY);
+      const button = (args.button as string) || 'left';
+      const start = await resolveXY(startX, startY, args.coordinateMode as string, config);
+      const end = await resolveXY(endX, endY, args.coordinateMode as string, config);
+      await mouse.drag(start.globalX, start.globalY, end.globalX, end.globalY, button as any);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'drag',
+            button,
+            start: { logical: { x: startX, y: startY }, physical: { x: start.globalX, y: start.globalY } },
+            end: { logical: { x: endX, y: endY }, physical: { x: end.globalX, y: end.globalY } },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_scroll': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const deltaY = requireNumber('deltaY', args.deltaY);
+      const deltaX = args.deltaX !== undefined ? requireNumber('deltaX', args.deltaX) : 0;
+      const { globalX, globalY } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.scroll(globalX, globalY, deltaY, deltaX);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'scroll',
+            position: { logical: { x, y }, physical: { x: globalX, y: globalY } },
+            scroll: { vertical: deltaY, horizontal: deltaX },
+            direction: deltaY > 0 ? 'down' : deltaY < 0 ? 'up' : 'none',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_type': {
+      const text = requireString('text', args.text);
+      await keyboard.typeText(text);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'type',
+            characterCount: text.length,
+            text: text.length <= 50 ? text : text.substring(0, 50) + '...',
+            method: /[^\x00-\x7F]/.test(text) ? 'clipboard_paste' : 'direct_type',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_press_key': {
+      const key = requireString('key', args.key);
+      await keyboard.pressKey(key);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'press_key',
+            key,
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_hotkey': {
+      const keys = requireStringArray('keys', args.keys);
+      const validation = validateHotkey(keys);
+      if (validation.blocked) {
+        throw new SafetyError(keys, validation.reason!);
+      }
+      await keyboard.hotkey(keys);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'hotkey',
+            keys,
+            combination: keys.join('+'),
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_cursor_position': {
+      const pos = await mouse.getPosition();
+      const disp = await display.getDisplay(config.displayIndex);
+      const scaleFactor = disp.scaleFactor ?? 1;
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            logicalCoords: { x: Math.round(pos.x / scaleFactor), y: Math.round(pos.y / scaleFactor) },
+            physicalCoords: { x: pos.x, y: pos.y },
+            scaleFactor,
+            note: 'Logical coordinates should be used for gui_click operations.',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_list_displays': {
+      const displays = await display.listDisplays();
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            count: displays.length,
+            displays: displays.map(d => ({
+              index: d.index,
+              label: d.label,
+              resolution: { width: d.width, height: d.height },
+              scaleFactor: d.scaleFactor,
+              isPrimary: d.isPrimary,
+              origin: d.origin,
+            })),
+            note: 'Resolution is in logical pixels. Use display index for displayIndex parameter in other tools.',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_find_image': {
+      const template = requireString('template', args.template);
+      const result = await findImage(template, args.confidence as number);
+      const disp = await display.getDisplay(config.displayIndex);
+      const scaleFactor = disp.scaleFactor ?? 1;
+      const region = result.region;
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: result.found,
+            found: result.found,
+            logicalCoords: result.found && region ? {
+              x: Math.round(region.x / scaleFactor),
+              y: Math.round(region.y / scaleFactor),
+              width: Math.round(region.width / scaleFactor),
+              height: Math.round(region.height / scaleFactor),
+            } : null,
+            physicalCoords: region,
+            confidence: result.confidence,
+            note: 'Use logicalCoords.center (x + width/2, y + height/2) for click operations.',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_wait_for_image': {
+      const template = requireString('template', args.template);
+      const result = await waitForImage(template, args.timeout as number, args.confidence as number);
+      const disp = await display.getDisplay(config.displayIndex);
+      const scaleFactor = disp.scaleFactor ?? 1;
+      const region = result.region;
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: result.found,
+            found: result.found,
+            logicalCoords: result.found && region ? {
+              x: Math.round(region.x / scaleFactor),
+              y: Math.round(region.y / scaleFactor),
+              width: Math.round(region.width / scaleFactor),
+              height: Math.round(region.height / scaleFactor),
+            } : null,
+            physicalCoords: region,
+            confidence: result.confidence,
+            timedOut: !result.found,
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_analyze_screen': {
+      const prompt = requireString('prompt', args.prompt);
+      const displayIdx = typeof args.displayIndex === 'number' ? args.displayIndex : config.displayIndex;
+
+      // Create vision model
+      const model = createModel(config.provider, config.apiProtocol, config.model, config.baseUrl);
+      const apiKey = config.apiKey;
+
+      if (!apiKey) {
+        return {
+          content: [{ type: 'text', text: JSON.stringify({ success: false, error: 'API key not configured' }) }],
+          isError: true,
+        };
+      }
+
+      const result = await analyzeScreen(model, apiKey, prompt, displayIdx);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            analysis: result.analysis,
+            imageSize: {
+              width: result.imageWidth,
+              height: result.imageHeight,
+              note: 'Coordinates in analysis reference this image size (logical pixels)',
+            },
+          }, null, 2),
+        }],
+      };
+    }
+
+    default:
+      return { content: [{ type: 'text', text: JSON.stringify({ success: false, error: `Unknown tool: ${name}` }) }], isError: true };
+  }
+}

qimingclaw/crates/agent-gui-server/src/mcp/resources.ts

@@ -0,0 +1,77 @@
+/**
+ * MCP Resources: status, permissions, audit log.
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import {
+  ListResourcesRequestSchema,
+  ReadResourceRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { checkScreenRecordingPermission, checkAccessibilityPermission, getPlatform } from '../utils/platform.js';
+
+const PKG_VERSION = process.env.__GUI_AGENT_PKG_VERSION__ ?? '0.0.0';
+
+const RESOURCES = [
+  {
+    uri: 'gui://status',
+    name: 'GUI Agent Status',
+    description: 'Platform, version, running state',
+    mimeType: 'application/json',
+  },
+  {
+    uri: 'gui://permissions',
+    name: 'GUI Agent Permissions',
+    description: 'Screen recording and accessibility permission status',
+    mimeType: 'application/json',
+  },
+  {
+    uri: 'gui://audit-log',
+    name: 'GUI Agent Audit Log',
+    description: 'Recent tool execution audit entries',
+    mimeType: 'application/json',
+  },
+];
+
+export function registerResources(server: Server, config: GuiAgentConfig, auditLog: AuditLog): void {
+  server.setRequestHandler(ListResourcesRequestSchema, async () => ({
+    resources: RESOURCES,
+  }));
+
+  server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+    const { uri } = request.params;
+
+    switch (uri) {
+      case 'gui://status': {
+        const status = {
+          platform: getPlatform(),
+          version: PKG_VERSION,
+          running: true,
+          transport: config.transport,
+          port: config.port,
+          model: config.model,
+          provider: config.provider,
+        };
+        return { contents: [{ uri, mimeType: 'application/json', text: JSON.stringify(status, null, 2) }] };
+      }
+
+      case 'gui://permissions': {
+        const [screenRecording, accessibility] = await Promise.all([
+          checkScreenRecordingPermission(),
+          checkAccessibilityPermission(),
+        ]);
+        const perms = { screenRecording, accessibility, platform: getPlatform() };
+        return { contents: [{ uri, mimeType: 'application/json', text: JSON.stringify(perms, null, 2) }] };
+      }
+
+      case 'gui://audit-log': {
+        const entries = auditLog.getEntries(100);
+        return { contents: [{ uri, mimeType: 'application/json', text: JSON.stringify(entries, null, 2) }] };
+      }
+
+      default:
+        throw new Error(`Unknown resource: ${uri}`);
+    }
+  });
+}

qimingclaw/crates/agent-gui-server/src/mcp/server.ts

@@ -0,0 +1,225 @@
+/**
+ * MCP Server with dual transport:
+ * - HTTP mode (primary): Streamable HTTP on 127.0.0.1:<port>/mcp
+ * - stdio mode (backup): single StdioServerTransport
+ *
+ * HTTP mode manages per-session Server + Transport instances.
+ */
+
+import * as http from 'http';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { getAvailableTools, handleAtomicToolCall } from './atomicTools.js';
+import { TASK_TOOLS, handleTaskTool } from './taskTools.js';
+import { registerResources } from './resources.js';
+import { logInfo, logWarn, logError } from '../utils/logger.js';
+
+const MCP_SESSION_ID_HEADER = 'mcp-session-id';
+const SESSION_CLEANUP_INTERVAL_MS = 60_000;
+
+interface HttpSession {
+  server: Server;
+  transport: StreamableHTTPServerTransport;
+  lastActivity: number;
+}
+
+function createMcpServer(config: GuiAgentConfig, auditLog: AuditLog): Server {
+  const server = new Server(
+    { name: 'gui-agent-server', version: process.env.__GUI_AGENT_PKG_VERSION__ ?? '0.1.0' },
+    { capabilities: { tools: {}, resources: {} } },
+  );
+
+  // Unified tool registration — combines atomic tools + task tools
+  // 使用动态工具列表（根据视觉模型配置过滤 gui_analyze_screen）
+  const allTools = [...getAvailableTools(), ...TASK_TOOLS];
+
+  server.setRequestHandler(ListToolsRequestSchema, async () => ({
+    tools: allTools,
+  }));
+
+  server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
+    const { name, arguments: args = {} } = request.params;
+
+    // Try atomic tools first
+    const atomicResult = await handleAtomicToolCall(name, args, config, auditLog);
+    if (atomicResult !== null) return atomicResult;
+
+    // Try task tools
+    const taskResult = await handleTaskTool(name, args, config, auditLog, {
+      signal: extra.signal,
+      sendNotification: extra.sendNotification?.bind(extra) as any,
+    });
+    if (taskResult !== null) return taskResult;
+
+    return { content: [{ type: 'text', text: `Unknown tool: ${name}` }], isError: true };
+  });
+
+  registerResources(server, config, auditLog);
+  return server;
+}
+
+export interface GuiAgentServer {
+  start(): Promise<void>;
+  stop(): Promise<void>;
+}
+
+export function createGuiAgentServer(config: GuiAgentConfig): GuiAgentServer {
+  const auditLog = new AuditLog();
+
+  if (config.transport === 'stdio') {
+    return createStdioServer(config, auditLog);
+  }
+  return createHttpServer(config, auditLog);
+}
+
+function createStdioServer(config: GuiAgentConfig, auditLog: AuditLog): GuiAgentServer {
+  const server = createMcpServer(config, auditLog);
+  let transport: StdioServerTransport | null = null;
+
+  return {
+    async start() {
+      transport = new StdioServerTransport();
+      await server.connect(transport);
+      logInfo('MCP Server running on stdio');
+    },
+    async stop() {
+      if (transport) {
+        await server.close();
+        transport = null;
+      }
+    },
+  };
+}
+
+function createHttpServer(config: GuiAgentConfig, auditLog: AuditLog): GuiAgentServer {
+  const sessions = new Map<string, HttpSession>();
+  let httpServer: http.Server | null = null;
+  let cleanupTimer: ReturnType<typeof setInterval> | null = null;
+
+  function getOrCreateSession(sessionId: string | undefined, res: http.ServerResponse): HttpSession | null {
+    // Existing session
+    if (sessionId && sessions.has(sessionId)) {
+      const session = sessions.get(sessionId)!;
+      session.lastActivity = Date.now();
+      return session;
+    }
+
+    // New session — only allowed when no sessionId header (initial request)
+    if (sessionId) {
+      res.writeHead(404, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: 'Session not found' }));
+      return null;
+    }
+
+    const server = createMcpServer(config, auditLog);
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: () => crypto.randomUUID(),
+      onsessioninitialized: (id) => {
+        logInfo(`HTTP session created: ${id}`);
+        sessions.set(id, { server, transport, lastActivity: Date.now() });
+      },
+    });
+
+    server.connect(transport);
+    return { server, transport, lastActivity: Date.now() };
+  }
+
+  async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse) {
+    const url = new URL(req.url ?? '/', `http://${req.headers.host}`);
+
+    if (url.pathname !== '/mcp') {
+      res.writeHead(404);
+      res.end('Not found');
+      return;
+    }
+
+    const sessionId = req.headers[MCP_SESSION_ID_HEADER] as string | undefined;
+
+    if (req.method === 'DELETE') {
+      if (sessionId && sessions.has(sessionId)) {
+        const session = sessions.get(sessionId)!;
+        try { await session.transport.close(); } catch { /* ignore */ }
+        sessions.delete(sessionId);
+        logInfo(`HTTP session closed: ${sessionId}`);
+      }
+      res.writeHead(200);
+      res.end();
+      return;
+    }
+
+    if (req.method === 'GET' || req.method === 'POST') {
+      const session = getOrCreateSession(sessionId, res);
+      if (!session) return;
+
+      try {
+        await session.transport.handleRequest(req, res);
+      } catch (err) {
+        logError(`HTTP request error: ${err instanceof Error ? err.message : String(err)}`);
+        if (!res.headersSent) {
+          res.writeHead(500, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ error: 'Internal server error' }));
+        }
+      }
+      return;
+    }
+
+    res.writeHead(405);
+    res.end('Method not allowed');
+  }
+
+  function cleanupStaleSessions() {
+    const now = Date.now();
+    const staleThreshold = 5 * 60 * 1000; // 5 minutes
+    for (const [id, session] of sessions) {
+      if (now - session.lastActivity > staleThreshold) {
+        logInfo(`Cleaning up stale session: ${id}`);
+        session.transport.close().catch(() => {});
+        sessions.delete(id);
+      }
+    }
+  }
+
+  return {
+    async start() {
+      httpServer = http.createServer(handleRequest);
+
+      await new Promise<void>((resolve, reject) => {
+        httpServer!.listen(config.port, '127.0.0.1', () => {
+          logInfo(`MCP Server running on http://127.0.0.1:${config.port}/mcp`);
+          resolve();
+        });
+        httpServer!.on('error', reject);
+      });
+
+      cleanupTimer = setInterval(cleanupStaleSessions, SESSION_CLEANUP_INTERVAL_MS);
+    },
+    async stop() {
+      if (cleanupTimer) {
+        clearInterval(cleanupTimer);
+        cleanupTimer = null;
+      }
+
+      // Close all sessions
+      await Promise.all(
+        Array.from(sessions.entries()).map(async ([id, session]) => {
+          try { await session.transport.close(); } catch { /* ignore */ }
+        }),
+      );
+      sessions.clear();
+
+      // Close HTTP server
+      if (httpServer) {
+        await new Promise<void>((resolve) => {
+          httpServer!.close(() => resolve());
+        });
+        httpServer = null;
+      }
+
+      logInfo('MCP Server stopped');
+    },
+  };
+}

qimingclaw/crates/agent-gui-server/src/mcp/taskTools.ts

@@ -0,0 +1,154 @@
+/**
+ * gui_execute_task MCP tool handler.
+ *
+ * Provides a high-level "execute a GUI task via natural language" tool
+ * backed by the pi-mono Agent loop. Uses a Mutex to ensure only one
+ * GUI task runs at a time.
+ */
+
+import type { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { createTaskRunner, type ProgressInfo } from '../agent/taskRunner.js';
+import { logInfo, logWarn, logError } from '../utils/logger.js';
+
+export const TASK_TOOLS = [
+  {
+    name: 'gui_execute_task',
+    description:
+      'Execute a GUI automation task described in natural language. ' +
+      'The agent will analyze screenshots and perform mouse/keyboard actions to complete the task. ' +
+      'Only one task can run at a time.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        task: {
+          type: 'string',
+          description: 'Natural language description of the GUI task to execute',
+        },
+        maxSteps: {
+          type: 'number',
+          description: 'Override max steps for this task (default: server config)',
+        },
+      },
+      required: ['task'],
+    },
+  },
+];
+
+/** Mutex — only one GUI task at a time, second caller waits in queue */
+let mutexPromise: Promise<void> = Promise.resolve();
+
+export async function handleTaskTool(
+  name: string,
+  args: Record<string, unknown>,
+  config: GuiAgentConfig,
+  auditLog: AuditLog,
+  extra?: { signal?: AbortSignal; sendNotification?: (notification: { method: string; params: unknown }) => Promise<void> },
+): Promise<{ content: Array<{ type: string; text?: string }>; isError?: boolean } | null> {
+  if (name !== 'gui_execute_task') {
+    return null; // Not handled by this module
+  }
+
+  const taskText = args.task as string;
+  if (!taskText || typeof taskText !== 'string') {
+    return { content: [{ type: 'text', text: 'Error: task parameter is required and must be a string' }], isError: true };
+  }
+
+  // Mutex — wait for previous task to complete (queue, not reject)
+  let releaseMutex: () => void;
+  const prevMutex = mutexPromise;
+  mutexPromise = new Promise<void>((resolve) => { releaseMutex = resolve; });
+
+  // Check if cancelled while waiting for mutex
+  if (extra?.signal?.aborted) {
+    releaseMutex!();
+    return { content: [{ type: 'text', text: 'Task cancelled before start' }], isError: true };
+  }
+
+  await prevMutex;
+
+  // Check cancellation again after acquiring mutex
+  if (extra?.signal?.aborted) {
+    releaseMutex!();
+    return { content: [{ type: 'text', text: 'Task cancelled before start' }], isError: true };
+  }
+
+  // Build config override
+  const taskConfig = { ...config };
+  if (args.maxSteps !== undefined) {
+    taskConfig.maxSteps = Math.min(Math.max(Number(args.maxSteps), 1), 200);
+  }
+
+  const runner = createTaskRunner(taskConfig, auditLog);
+
+  // Create AbortController that merges external signal
+  const taskAbortController = new AbortController();
+  if (extra?.signal) {
+    extra.signal.addEventListener('abort', () => taskAbortController.abort(), { once: true });
+  }
+
+  // Stable progress token for this task — MCP protocol requires same token for all updates
+  const progressToken = `gui_task_${Date.now()}`;
+
+  // Progress callback — sends MCP notifications
+  const onProgress = async (info: ProgressInfo) => {
+    if (extra?.sendNotification) {
+      try {
+        await extra.sendNotification({
+          method: 'notifications/progress',
+          params: {
+            progressToken,
+            progress: info.step,
+            total: info.total,
+            message: info.message,
+          },
+        });
+      } catch {
+        // Progress notification failure is non-fatal
+      }
+    }
+  };
+
+  try {
+    logInfo(`gui_execute_task started: "${taskText.substring(0, 100)}"`);
+    const result = await runner.run(taskText, taskAbortController.signal, onProgress);
+
+    // Build response
+    const content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> = [];
+
+    // Add text result
+    content.push({
+      type: 'text',
+      text: JSON.stringify({
+        success: result.success,
+        result: result.result,
+        error: result.error,
+        stepsCompleted: result.steps.length,
+        steps: result.steps.map(s => ({
+          stepId: s.stepId,
+          tool: s.tool,
+          success: s.success,
+        })),
+      }, null, 2),
+    });
+
+    // Add final screenshot if available
+    if (result.finalScreenshot) {
+      content.push({
+        type: 'image',
+        data: result.finalScreenshot,
+        mimeType: 'image/jpeg',
+      });
+    }
+
+    logInfo(`gui_execute_task completed: success=${result.success}, steps=${result.steps.length}`);
+    return { content, isError: !result.success };
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    logError(`gui_execute_task error: ${errorMsg}`);
+    return { content: [{ type: 'text', text: `Error: ${errorMsg}` }], isError: true };
+  } finally {
+    releaseMutex!();
+  }
+}