chore: initialize qiming workspace repository

qimingclaw/crates/agent-gui-server/src/agent/memoryManager.ts

@@ -0,0 +1,283 @@
+/**
+ * Three-layer memory management with LLM summarization.
+ *
+ * Layers:
+ *   summaryMemory (budget: 2000 chars) — compressed history
+ *   recentMemory  (budget: 500 chars)  — recent step records
+ *   pendingMemory (no budget)          — current step in progress
+ *
+ * Memory text is injected into systemPrompt (not via transformContext).
+ * Screenshot pruning (pruneScreenshots) is used as transformContext hook.
+ */
+
+import { complete } from '@mariozechner/pi-ai';
+import type { Model, Api, AssistantMessage } from '@mariozechner/pi-ai';
+import { logInfo, logDebug, logError } from '../utils/logger.js';
+
+// Type alias for AgentMessage — we only need role and content fields
+interface MessageLike {
+  role: string;
+  content: unknown;
+  [key: string]: unknown;
+}
+
+const MEMORY_SUMMARIZATION_PROMPT = `You are a memory summarization assistant for a GUI automation agent.
+Your task is to condense step-by-step action records into concise memory entries.
+
+Output JSON:
+{
+  "summary": "Concise summary of the actions taken and their outcomes..."
+}
+
+Guidelines:
+- Preserve key information: what was done, what succeeded/failed, current state
+- Remove redundant details and repetitive patterns
+- Keep the summary actionable — the agent needs to know what happened to plan next steps`;
+
+export class MemoryManager {
+  private summaryMemory: string = '';
+  private recentMemory: string = '';
+  private pendingMemory: string = '';
+
+  private recentBudget: number = 500;
+  private summaryBudget: number = 2000;
+  private screenshotKeepCount: number = 3;
+
+  private memoryModel: Model<any>;
+  private apiKey: string;
+
+  constructor(memoryModel: Model<any>, apiKey: string) {
+    this.memoryModel = memoryModel;
+    this.apiKey = apiKey;
+  }
+
+  /**
+   * Record a pending step (currently executing).
+   */
+  addPendingStep(stepId: number, goal: string): void {
+    this.pendingMemory = `Step ${stepId} | Goal: ${goal}`;
+  }
+
+  /**
+   * Finalize a step — move from pending to recent, trigger compression if over budget.
+   */
+  async finalizeStep(stepId: number, evaluation: 'success' | 'failed'): Promise<void> {
+    const entry = `Step ${stepId} | Eval: ${evaluation} | ${this.pendingMemory.replace(`Step ${stepId} | `, '')}`;
+    this.pendingMemory = '';
+
+    if (this.recentMemory) {
+      this.recentMemory += '\n' + entry;
+    } else {
+      this.recentMemory = entry;
+    }
+
+    // Check if recent memory exceeds budget
+    if (this.recentMemory.length > this.recentBudget) {
+      await this.compressRecent();
+    }
+  }
+
+  /**
+   * Compose all three layers into a single text block for systemPrompt injection.
+   */
+  compose(): string {
+    const parts: string[] = [];
+    if (this.summaryMemory) {
+      parts.push(`[Summarized history]\n${this.summaryMemory}`);
+    }
+    if (this.recentMemory) {
+      parts.push(`[Recent steps]\n${this.recentMemory}`);
+    }
+    if (this.pendingMemory) {
+      parts.push(`[Current step]\n${this.pendingMemory}`);
+    }
+    return parts.join('\n\n');
+  }
+
+  /**
+   * Prune screenshots from messages — for use as transformContext hook.
+   *
+   * Keeps last N screenshots, replaces older ones with text placeholders.
+   * This only affects the LLM input, not the Agent's internal message history.
+   */
+  pruneScreenshots<T>(messages: T[]): T[] {
+    // Find all messages with image content
+    const imageIndices: number[] = [];
+    for (let i = 0; i < messages.length; i++) {
+      const msg = messages[i] as any;
+      if (hasImageContent(msg.content)) {
+        imageIndices.push(i);
+      }
+    }
+
+    // If we have fewer images than the keep count, no pruning needed
+    if (imageIndices.length <= this.screenshotKeepCount) {
+      return this.applyTokenHardLimit(messages);
+    }
+
+    // Clone messages array and replace old screenshots
+    const pruned = messages.map((msg, idx) => {
+      if (!imageIndices.includes(idx)) return msg;
+
+      // Keep the most recent N screenshots
+      const imageRank = imageIndices.indexOf(idx);
+      const keepFrom = imageIndices.length - this.screenshotKeepCount;
+      if (imageRank >= keepFrom) return msg;
+
+      // Replace image content with text placeholder
+      return {
+        ...msg,
+        content: replaceImageContent((msg as any).content, `[Screenshot removed - Step ${imageRank + 1}]`),
+      };
+    });
+
+    return this.applyTokenHardLimit(pruned);
+  }
+
+  /**
+   * Token hard limit fallback — estimate total tokens and force-remove
+   * oldest images if exceeding contextWindow * 0.9.
+   */
+  private applyTokenHardLimit<T>(messages: T[]): T[] {
+    const MAX_CONTEXT_TOKENS = 128_000; // conservative default
+    const TOKEN_THRESHOLD = MAX_CONTEXT_TOKENS * 0.9;
+    const TOKENS_PER_IMAGE = 800;
+
+    let totalTokens = 0;
+    const imagePositions: number[] = [];
+
+    for (let i = 0; i < messages.length; i++) {
+      const msg = messages[i] as any;
+      if (!msg.content) continue;
+
+      if (Array.isArray(msg.content)) {
+        for (const c of msg.content) {
+          if (c.type === 'image') {
+            totalTokens += TOKENS_PER_IMAGE;
+            imagePositions.push(i);
+          } else if (c.type === 'text' && c.text) {
+            totalTokens += Math.ceil(c.text.length / 3);
+          }
+        }
+      } else if (typeof msg.content === 'string') {
+        totalTokens += Math.ceil(msg.content.length / 3);
+      }
+    }
+
+    if (totalTokens <= TOKEN_THRESHOLD) {
+      return messages;
+    }
+
+    // Force-remove images from oldest messages until under threshold
+    logDebug(`Token hard limit: estimated ${totalTokens} tokens, threshold ${TOKEN_THRESHOLD}, removing oldest images`);
+    const result = [...messages];
+    for (const idx of imagePositions) {
+      if (totalTokens <= TOKEN_THRESHOLD) break;
+      const msg = result[idx] as any;
+      if (hasImageContent(msg.content)) {
+        result[idx] = {
+          ...msg,
+          content: replaceImageContent(msg.content, '[Screenshot removed - token limit]'),
+        } as T;
+        totalTokens -= TOKENS_PER_IMAGE;
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Compress recent memory into summary via LLM call.
+   */
+  private async compressRecent(): Promise<void> {
+    try {
+      logDebug(`Compressing recent memory (${this.recentMemory.length} chars)`);
+
+      const summary = await this.summarize(this.recentMemory);
+
+      if (this.summaryMemory) {
+        this.summaryMemory += '\n' + summary;
+      } else {
+        this.summaryMemory = summary;
+      }
+      this.recentMemory = '';
+
+      // Check if summary also exceeds budget
+      if (this.summaryMemory.length > this.summaryBudget) {
+        await this.compressSummary();
+      }
+    } catch (err) {
+      logError(`Memory compression failed: ${err instanceof Error ? err.message : String(err)}`);
+      // On failure, keep recent memory as-is rather than losing data
+    }
+  }
+
+  /**
+   * Second-level compression: summarize the summary.
+   */
+  private async compressSummary(): Promise<void> {
+    try {
+      logDebug(`Compressing summary memory (${this.summaryMemory.length} chars)`);
+      const summary = await this.summarize(this.summaryMemory);
+      this.summaryMemory = summary;
+    } catch (err) {
+      logError(`Summary compression failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  /**
+   * Call the memory model to generate a summary.
+   * If apiKey is empty, just returns the original text (no-op compression).
+   */
+  private async summarize(text: string): Promise<string> {
+    // If no API key is configured, skip summarization and return original text
+    if (!this.apiKey) {
+      return text;
+    }
+
+    const result: AssistantMessage = await complete(this.memoryModel, {
+      systemPrompt: MEMORY_SUMMARIZATION_PROMPT,
+      messages: [
+        { role: 'user' as const, content: text, timestamp: Date.now() },
+      ],
+    }, {
+      apiKey: this.apiKey,
+    });
+
+    // Extract text from response
+    const textContent = result.content.find(c => c.type === 'text');
+    if (!textContent || textContent.type !== 'text') {
+      throw new Error('Memory model returned no text content');
+    }
+
+    // Try to parse JSON response
+    try {
+      const parsed = JSON.parse(textContent.text);
+      return parsed.summary || textContent.text;
+    } catch {
+      // If not JSON, use the raw text
+      return textContent.text;
+    }
+  }
+}
+
+// --- Helpers ---
+
+function hasImageContent(content: unknown): boolean {
+  if (Array.isArray(content)) {
+    return content.some(c => c && typeof c === 'object' && 'type' in c && c.type === 'image');
+  }
+  return false;
+}
+
+function replaceImageContent(content: unknown, placeholder: string): unknown {
+  if (Array.isArray(content)) {
+    return content.map(c => {
+      if (c && typeof c === 'object' && 'type' in c && c.type === 'image') {
+        return { type: 'text', text: placeholder };
+      }
+      return c;
+    });
+  }
+  return content;
+}

qimingclaw/crates/agent-gui-server/src/agent/stuckDetector.ts

@@ -0,0 +1,80 @@
+/**
+ * Stuck detection: consecutive screenshot similarity comparison.
+ *
+ * Resizes screenshots to 32x32 thumbnails and compares pixel mean difference.
+ * Consecutive N steps with difference < threshold = stuck.
+ */
+
+import sharp from 'sharp';
+
+const THUMB_SIZE = 32;
+
+export class StuckDetector {
+  private threshold: number;
+  private similarityThreshold: number;
+  private previousThumbnails: Buffer[] = [];
+  private consecutiveSimilar: number = 0;
+
+  constructor(threshold: number = 3, similarityThreshold: number = 0.05) {
+    this.threshold = threshold;
+    this.similarityThreshold = similarityThreshold;
+  }
+
+  /**
+   * Check if the agent is stuck by comparing the latest screenshot
+   * with previous screenshots.
+   *
+   * @param screenshotBase64 - Base64-encoded screenshot
+   * @returns { stuck, consecutiveSimilar }
+   */
+  async check(screenshotBase64: string): Promise<{ stuck: boolean; consecutiveSimilar: number }> {
+    const thumbnail = await this.createThumbnail(screenshotBase64);
+
+    if (this.previousThumbnails.length > 0) {
+      const lastThumb = this.previousThumbnails[this.previousThumbnails.length - 1];
+      const diff = this.computeDifference(thumbnail, lastThumb);
+
+      if (diff < this.similarityThreshold) {
+        this.consecutiveSimilar++;
+      } else {
+        this.consecutiveSimilar = 0;
+      }
+    }
+
+    // Keep only threshold count of thumbnails
+    this.previousThumbnails.push(thumbnail);
+    if (this.previousThumbnails.length > this.threshold + 1) {
+      this.previousThumbnails.shift();
+    }
+
+    return {
+      stuck: this.consecutiveSimilar >= this.threshold,
+      consecutiveSimilar: this.consecutiveSimilar,
+    };
+  }
+
+  reset(): void {
+    this.previousThumbnails = [];
+    this.consecutiveSimilar = 0;
+  }
+
+  private async createThumbnail(base64: string): Promise<Buffer> {
+    const buffer = Buffer.from(base64, 'base64');
+    return sharp(buffer)
+      .resize(THUMB_SIZE, THUMB_SIZE, { fit: 'fill' })
+      .raw()
+      .toBuffer();
+  }
+
+  private computeDifference(a: Buffer, b: Buffer): number {
+    if (a.length !== b.length) return 1;
+
+    let totalDiff = 0;
+    for (let i = 0; i < a.length; i++) {
+      totalDiff += Math.abs(a[i] - b[i]);
+    }
+
+    // Normalize: max per pixel channel is 255, total pixels * channels = a.length
+    return totalDiff / (a.length * 255);
+  }
+}

qimingclaw/crates/agent-gui-server/src/agent/systemPrompt.ts

@@ -0,0 +1,49 @@
+/**
+ * GUI Agent system prompt template.
+ *
+ * Builds the system prompt for the pi-mono Agent, including
+ * role description, tool guidance, and memory injection.
+ */
+
+export function buildSystemPrompt(taskText: string, memoryText: string): string {
+  const memorySection = memoryText
+    ? `\n\n## Previous Actions Memory\n${memoryText}`
+    : '';
+
+  return `You are a GUI automation agent that controls a desktop computer to complete tasks.
+You can see the screen through screenshots and interact using mouse and keyboard.
+
+## Your Task
+${taskText}
+
+## Available Tools
+
+- **computer_screenshot**: Capture the current screen. Use this to see what's on screen before acting.
+- **computer_click**: Click at coordinates (x, y). Use for clicking buttons, links, icons.
+- **computer_type**: Type text. Supports CJK characters and long text.
+- **computer_scroll**: Scroll at position (x, y) with deltaY (positive=down, negative=up).
+- **computer_hotkey**: Press key combinations (e.g. ["Meta", "C"] for copy). Some dangerous combinations are blocked for safety.
+- **computer_wait**: Wait for a specified duration (milliseconds). Use after actions that trigger loading.
+- **computer_done**: Call this when the task is complete. Provide a result description.
+
+## Workflow
+
+1. Start by taking a screenshot to see the current state
+2. Analyze the screenshot to determine the next action
+3. Perform one action at a time (click, type, scroll, etc.)
+4. Take another screenshot to verify the result
+5. Repeat until the task is complete
+6. Call computer_done with a summary of what was accomplished
+
+## Important Guidelines
+
+- Always take a screenshot before your first action and after significant actions
+- Output coordinates in the format your model was trained on
+- Click precisely on UI elements — aim for the center of buttons/links
+- After typing, verify the text appeared correctly
+- If a dialog or popup appears unexpectedly, assess whether to dismiss it or interact with it
+- If you encounter an error or unexpected state, take a screenshot and reassess
+- When the task is complete, call computer_done — do not call any other tools after that
+- If you are stuck and cannot make progress, call computer_done with an error description
+${memorySection}`;
+}

qimingclaw/crates/agent-gui-server/src/agent/taskRunner.ts

@@ -0,0 +1,447 @@
+/**
+ * Agent loop engine using pi-mono Agent class.
+ *
+ * Creates a pi-mono Agent instance configured with GUI tools,
+ * hooks for safety/audit/memory, and manages the task lifecycle.
+ * Does NOT hand-write the loop — pi-mono handles that internally.
+ */
+
+import { Agent } from '@mariozechner/pi-agent-core';
+import { getModel } from '@mariozechner/pi-ai';
+import type { AgentTool, AgentToolResult, AgentEvent } from '@mariozechner/pi-agent-core';
+import type { ImageContent, TextContent, Message, Model, Api } from '@mariozechner/pi-ai';
+import { Type } from '@sinclair/typebox';
+
+import type { GuiAgentConfig } from '../config.js';
+import { buildSystemPrompt } from './systemPrompt.js';
+import { MemoryManager } from './memoryManager.js';
+import { StuckDetector } from './stuckDetector.js';
+import { resolveCoordinate, type ScreenshotMeta, type DisplayInfo } from '../coordinates/resolver.js';
+import { getModelProfile } from '../coordinates/modelProfiles.js';
+import { captureScreenshot, type ScreenshotResult } from '../desktop/screenshot.js';
+import * as mouse from '../desktop/mouse.js';
+import * as desktopKeyboard from '../desktop/keyboard.js';
+import { getDisplay } from '../desktop/display.js';
+import { validateHotkey } from '../safety/hotkeys.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { logInfo, logDebug, logError, logWarn } from '../utils/logger.js';
+
+export interface StepRecord {
+  stepId: number;
+  tool: string;
+  args: Record<string, unknown>;
+  success: boolean;
+  durationMs: number;
+}
+
+export interface TaskResult {
+  success: boolean;
+  result?: string;
+  finalScreenshot?: string;
+  steps: StepRecord[];
+  error?: string;
+}
+
+export interface ProgressInfo {
+  step: number;
+  total: number;
+  status: 'running' | 'done' | 'error' | 'aborted';
+  message?: string;
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Create a pi-mono Model instance.
+ * Built-in providers (anthropic/openai/google) without custom baseUrl use getModel().
+ * Custom providers or those with baseUrl get a manually constructed Model object.
+ */
+export function createModel(
+  provider: string,
+  apiProtocol: 'anthropic' | 'openai',
+  modelId: string,
+  baseUrl?: string,
+): Model<Api> {
+  const builtinProviders = ['anthropic', 'openai', 'google'];
+  if (builtinProviders.includes(provider) && !baseUrl) {
+    return getModel(provider as any, modelId as any);
+  }
+
+  const api = apiProtocol === 'anthropic' ? 'anthropic-messages' : 'openai-completions';
+  return {
+    id: modelId,
+    name: modelId,
+    api,
+    provider,
+    baseUrl,
+    reasoning: false,
+    input: ['text', 'image'],
+    cost: { input: 0, output: 0 },
+    contextWindow: 128000,
+    maxTokens: 8192,
+  } as Model<Api>;
+}
+
+export function createTaskRunner(
+  config: GuiAgentConfig,
+  auditLog: AuditLog,
+) {
+  let currentAgent: Agent | null = null;
+
+  async function run(
+    taskText: string,
+    signal: AbortSignal,
+    onProgress: (info: ProgressInfo) => void,
+  ): Promise<TaskResult> {
+    const steps: StepRecord[] = [];
+    let stepCount = 0;
+    let latestScreenshot: ScreenshotResult | null = null;
+    let taskResult: string | undefined;
+    let pendingMemoryWork: Promise<void> = Promise.resolve();
+
+    // Build model instances
+    const model = createModel(config.provider, config.apiProtocol, config.model, config.baseUrl);
+    const memoryModel = createModel(
+      config.memoryProvider ?? config.provider,
+      config.apiProtocol,
+      config.memoryModel ?? config.model,
+      config.baseUrl,
+    );
+    const memoryManager = new MemoryManager(memoryModel, config.apiKey ?? '');
+    const stuckDetector = new StuckDetector(config.stuckThreshold);
+    const profile = getModelProfile(config.model, config.coordinateMode as any);
+    const displayInfo = await getDisplay(config.displayIndex);
+
+    // Build coordinate resolve helper
+    function resolveXY(x: number, y: number, meta: ScreenshotMeta) {
+      const di: DisplayInfo = {
+        origin: displayInfo.origin,
+        bounds: { width: displayInfo.width, height: displayInfo.height },
+        scaleFactor: displayInfo.scaleFactor,
+      };
+      return resolveCoordinate(x, y, profile, meta, di);
+    }
+
+    function getScreenshotMeta(): ScreenshotMeta {
+      if (latestScreenshot) {
+        return {
+          imageWidth: latestScreenshot.imageWidth,
+          imageHeight: latestScreenshot.imageHeight,
+          logicalWidth: latestScreenshot.logicalWidth,
+          logicalHeight: latestScreenshot.logicalHeight,
+        };
+      }
+      return {
+        imageWidth: displayInfo.width,
+        imageHeight: displayInfo.height,
+        logicalWidth: displayInfo.width,
+        logicalHeight: displayInfo.height,
+      };
+    }
+
+    // --- Define AgentTool[] ---
+    const guiTools: AgentTool[] = [
+      {
+        name: 'computer_screenshot',
+        label: 'Screenshot',
+        description: 'Capture the current screen',
+        parameters: Type.Object({}),
+        execute: async (_toolCallId, _params, _signal) => {
+          const shot = await captureScreenshot(config.displayIndex, config.jpegQuality);
+          latestScreenshot = shot;
+          return {
+            content: [{ type: 'image' as const, data: shot.image, mimeType: shot.mimeType }],
+            details: { imageWidth: shot.imageWidth, imageHeight: shot.imageHeight },
+          };
+        },
+      },
+      {
+        name: 'computer_click',
+        label: 'Click',
+        description: 'Click at coordinates (x, y)',
+        parameters: Type.Object({
+          x: Type.Number({ description: 'X coordinate' }),
+          y: Type.Number({ description: 'Y coordinate' }),
+          button: Type.Optional(Type.String({ description: 'Mouse button: left, right, middle' })),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { x: number; y: number; button?: string };
+          const { globalX, globalY } = resolveXY(params.x, params.y, getScreenshotMeta());
+          await mouse.click(globalX, globalY, params.button as any);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Clicked (${globalX}, ${globalY})` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_type',
+        label: 'Type',
+        description: 'Type text at the current cursor position',
+        parameters: Type.Object({
+          text: Type.String({ description: 'Text to type' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { text: string };
+          await desktopKeyboard.typeText(params.text);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Typed ${params.text.length} characters` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_scroll',
+        label: 'Scroll',
+        description: 'Scroll at coordinates (x, y)',
+        parameters: Type.Object({
+          x: Type.Number({ description: 'X coordinate' }),
+          y: Type.Number({ description: 'Y coordinate' }),
+          deltaY: Type.Number({ description: 'Vertical scroll amount (positive=down)' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { x: number; y: number; deltaY: number };
+          const { globalX, globalY } = resolveXY(params.x, params.y, getScreenshotMeta());
+          await mouse.scroll(globalX, globalY, params.deltaY);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Scrolled at (${globalX}, ${globalY}), dy=${params.deltaY}` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_hotkey',
+        label: 'Hotkey',
+        description: 'Press a key combination',
+        parameters: Type.Object({
+          keys: Type.Array(Type.String(), { description: 'Keys to press together' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { keys: string[] };
+          // Safety check is done in beforeToolCall hook
+          await desktopKeyboard.hotkey(params.keys);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Hotkey: ${params.keys.join('+')}` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_wait',
+        label: 'Wait',
+        description: 'Wait for a specified duration',
+        parameters: Type.Object({
+          ms: Type.Number({ description: 'Duration in milliseconds' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { ms: number };
+          const waitMs = Math.min(params.ms, 10000); // Cap at 10s
+          await delay(waitMs);
+          return {
+            content: [{ type: 'text' as const, text: `Waited ${waitMs}ms` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_done',
+        label: 'Done',
+        description: 'Signal that the task is complete. Provide a result description.',
+        parameters: Type.Object({
+          result: Type.String({ description: 'Task completion result description' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { result: string };
+          taskResult = params.result;
+          return {
+            content: [{ type: 'text' as const, text: params.result }],
+            details: { done: true },
+          };
+        },
+      },
+    ];
+
+    // --- Create Agent ---
+    const agent = new Agent({
+      initialState: {
+        systemPrompt: buildSystemPrompt(taskText, memoryManager.compose()),
+        model,
+        thinkingLevel: 'off',
+        tools: guiTools,
+        messages: [],
+      },
+      toolExecution: 'sequential',
+      transformContext: async (messages, _signal) => {
+        return memoryManager.pruneScreenshots(messages) as any;
+      },
+      convertToLlm: (messages) =>
+        (messages as any[]).filter(m => ['user', 'assistant', 'toolResult'].includes(m.role)) as Message[],
+      beforeToolCall: async ({ toolCall, args }) => {
+        if (toolCall.name === 'computer_hotkey') {
+          const typedArgs = args as { keys: string[] };
+          const validation = validateHotkey(typedArgs.keys);
+          if (validation.blocked) {
+            return { block: true, reason: `Blocked dangerous hotkey: ${validation.reason}` };
+          }
+        }
+        return undefined;
+      },
+      afterToolCall: async ({ toolCall, args, result, isError }) => {
+        auditLog.record({ tool: toolCall.name, args: args as Record<string, unknown>, success: !isError });
+        return undefined;
+      },
+      getApiKey: (_provider: string) => config.apiKey,
+    });
+
+    currentAgent = agent;
+
+    // --- Subscribe to events ---
+    const unsubscribe = agent.subscribe((event: AgentEvent) => {
+      // Record pending step when tool execution starts (visible to LLM during execution)
+      if (event.type === 'tool_execution_start') {
+        const toolName = (event as any).toolCall?.name ?? 'unknown';
+        memoryManager.addPendingStep(stepCount + 1, toolName);
+      }
+
+      if (event.type === 'turn_end') {
+        stepCount++;
+
+        // Record step
+        const toolResults = event.toolResults || [];
+        for (const tr of toolResults) {
+          steps.push({
+            stepId: stepCount,
+            tool: tr.toolName,
+            args: (tr as any).args ?? {},
+            success: !tr.isError,
+            durationMs: (tr as any).durationMs ?? 0,
+          });
+        }
+
+        // Memory management (async, queued)
+        const goal = toolResults.map(tr => tr.toolName).join(', ') || 'LLM response';
+        const evaluation = toolResults.some(tr => tr.isError) ? 'failed' as const : 'success' as const;
+        pendingMemoryWork = memoryManager.finalizeStep(stepCount, evaluation)
+          .then(() => {
+            agent.setSystemPrompt(buildSystemPrompt(taskText, memoryManager.compose()));
+          })
+          .catch(err => {
+            logError(`Memory finalize failed: ${err}`);
+          });
+
+        // Stuck detection (async, fire-and-forget)
+        if (latestScreenshot) {
+          stuckDetector.check(latestScreenshot.image).then(({ stuck }) => {
+            if (stuck) {
+              logWarn(`Agent appears stuck after ${stepCount} steps, aborting`);
+              agent.abort();
+            }
+          }).catch(err => {
+            logError(`Stuck detector error: ${err instanceof Error ? err.message : String(err)}`);
+          });
+        }
+
+        // Progress notification
+        onProgress({
+          step: stepCount,
+          total: config.maxSteps,
+          status: 'running',
+          message: `Step ${stepCount}: ${goal}`,
+        });
+
+        // Max steps check
+        if (stepCount >= config.maxSteps) {
+          logWarn(`Max steps (${config.maxSteps}) reached, aborting`);
+          agent.abort();
+        }
+      }
+    });
+
+    // --- Handle external abort ---
+    const abortHandler = () => agent.abort();
+    signal.addEventListener('abort', abortHandler);
+
+    try {
+      // Capture initial screenshot
+      const initialShot = await captureScreenshot(config.displayIndex, config.jpegQuality);
+      latestScreenshot = initialShot;
+
+      // Start the agent loop
+      logInfo(`Starting task: ${taskText.substring(0, 100)}`);
+      await agent.prompt(taskText, [
+        { type: 'image' as const, data: initialShot.image, mimeType: initialShot.mimeType },
+      ]);
+
+      // Wait for pending memory work
+      await pendingMemoryWork;
+
+      // Determine result
+      const lastMessage = agent.state.messages[agent.state.messages.length - 1];
+      const stopReason = lastMessage && 'stopReason' in lastMessage ? (lastMessage as any).stopReason : undefined;
+
+      if (stopReason === 'aborted') {
+        onProgress({ step: stepCount, total: config.maxSteps, status: 'aborted' });
+        return {
+          success: false,
+          error: 'Task was aborted',
+          steps,
+          finalScreenshot: latestScreenshot?.image,
+        };
+      }
+
+      if (stopReason === 'error') {
+        onProgress({ step: stepCount, total: config.maxSteps, status: 'error', message: 'Agent stopped with error (possible context overflow)' });
+        return {
+          success: false,
+          error: 'Agent stopped with error (possible context overflow)',
+          steps,
+          finalScreenshot: latestScreenshot?.image,
+        };
+      }
+
+      onProgress({ step: stepCount, total: config.maxSteps, status: 'done' });
+      return {
+        success: true,
+        result: taskResult ?? extractTextResult(lastMessage),
+        steps,
+        finalScreenshot: latestScreenshot?.image,
+      };
+    } catch (err) {
+      const errorMsg = err instanceof Error ? err.message : String(err);
+      logError(`Task execution error: ${errorMsg}`);
+      onProgress({ step: stepCount, total: config.maxSteps, status: 'error', message: errorMsg });
+      return {
+        success: false,
+        error: errorMsg,
+        steps,
+        finalScreenshot: latestScreenshot?.image,
+      };
+    } finally {
+      signal.removeEventListener('abort', abortHandler);
+      unsubscribe();
+      currentAgent = null;
+    }
+  }
+
+  function abort(): void {
+    currentAgent?.abort();
+  }
+
+  return { run, abort };
+}
+
+function extractTextResult(message: unknown): string | undefined {
+  if (!message || typeof message !== 'object') return undefined;
+  const msg = message as any;
+  if (Array.isArray(msg.content)) {
+    const text = msg.content.find((c: any) => c.type === 'text');
+    return text?.text;
+  }
+  return undefined;
+}

qimingclaw/crates/agent-gui-server/src/config.ts

@@ -0,0 +1,113 @@
+/**
+ * Unified configuration: parse GUI_AGENT_* environment variables,
+ * validate, and return a type-safe GuiAgentConfig object.
+ *
+ * Missing required fields throw ConfigError immediately (Fail Fast).
+ */
+
+import { ConfigError } from './utils/errors.js';
+import { logInfo } from './utils/logger.js';
+import type { CoordinateMode } from './coordinates/modelProfiles.js';
+
+const VALID_COORDINATE_MODES: readonly string[] = ['image-absolute', 'normalized-1000', 'normalized-999', 'normalized-0-1'];
+
+export interface GuiAgentConfig {
+  /** LLM provider (e.g. 'anthropic', 'openai', 'google', 'zhipu', 'qwen', 'deepseek') */
+  provider: string;
+  /** API protocol: 'anthropic' (x-api-key + /v1/messages) or 'openai' (Bearer + /chat/completions) */
+  apiProtocol: 'anthropic' | 'openai';
+  /** LLM model name */
+  model: string;
+  /** API key for the LLM provider (optional - only required for gui_analyze_screen tool) */
+  apiKey?: string;
+  /** Optional base URL for the LLM API */
+  baseUrl?: string;
+
+  /** Memory model provider (defaults to main provider) */
+  memoryProvider?: string;
+  /** Memory model name (defaults to main model) */
+  memoryModel?: string;
+
+  /** Override coordinate mode (auto-detected from model if empty) */
+  coordinateMode?: CoordinateMode;
+  /** Target display index */
+  displayIndex: number;
+
+  /** JPEG quality for screenshots (1-100) */
+  jpegQuality: number;
+  /** Maximum steps per task */
+  maxSteps: number;
+  /** Delay between steps in ms */
+  stepDelayMs: number;
+  /** Consecutive similar screenshots to trigger stuck detection */
+  stuckThreshold: number;
+
+  /** MCP server transport mode */
+  transport: 'http' | 'stdio';
+  /** HTTP server port */
+  port: number;
+
+  /** Optional log file path */
+  logFile?: string;
+}
+
+function parseIntRange(name: string, value: string | undefined, min: number, max: number, defaultVal: number): number {
+  if (!value) return defaultVal;
+  const n = parseInt(value, 10);
+  if (isNaN(n) || n < min || n > max) {
+    throw new ConfigError(`${name} must be between ${min} and ${max}, got: ${value}`);
+  }
+  return n;
+}
+
+export function loadConfig(overrides?: Partial<GuiAgentConfig>): GuiAgentConfig {
+  const env = process.env;
+
+  // API Key is optional at startup - it's only required when gui_analyze_screen tool is used
+  // (which requires GUI_AGENT_VISION_MODEL to be configured)
+  const apiKey = overrides?.apiKey ?? env.GUI_AGENT_API_KEY;
+
+  const transport = (overrides?.transport ?? env.GUI_AGENT_TRANSPORT ?? 'http') as 'http' | 'stdio';
+  if (transport !== 'http' && transport !== 'stdio') {
+    throw new ConfigError(`GUI_AGENT_TRANSPORT must be "http" or "stdio", got: ${transport}`);
+  }
+
+  const apiProtocol = (overrides?.apiProtocol ?? env.GUI_AGENT_API_PROTOCOL ?? 'anthropic') as 'anthropic' | 'openai';
+  if (apiProtocol !== 'anthropic' && apiProtocol !== 'openai') {
+    throw new ConfigError(`GUI_AGENT_API_PROTOCOL must be "anthropic" or "openai", got: ${apiProtocol}`);
+  }
+
+  const rawCoordinateMode = overrides?.coordinateMode ?? env.GUI_AGENT_COORDINATE_MODE;
+  // 'auto' means auto-detect from model name (via modelProfiles), treat as undefined
+  const effectiveCoordinateMode = rawCoordinateMode === 'auto' ? undefined : rawCoordinateMode;
+  if (effectiveCoordinateMode && !VALID_COORDINATE_MODES.includes(effectiveCoordinateMode)) {
+    throw new ConfigError(`GUI_AGENT_COORDINATE_MODE must be one of: auto, ${VALID_COORDINATE_MODES.join(', ')}, got: ${rawCoordinateMode}`);
+  }
+
+  const config: GuiAgentConfig = {
+    provider: overrides?.provider ?? env.GUI_AGENT_PROVIDER ?? 'anthropic',
+    apiProtocol,
+    model: overrides?.model ?? env.GUI_AGENT_MODEL ?? 'claude-sonnet-4-20250514',
+    apiKey,
+    baseUrl: overrides?.baseUrl ?? env.GUI_AGENT_BASE_URL,
+
+    memoryProvider: overrides?.memoryProvider ?? env.GUI_AGENT_MEMORY_PROVIDER,
+    memoryModel: overrides?.memoryModel ?? env.GUI_AGENT_MEMORY_MODEL,
+
+    coordinateMode: effectiveCoordinateMode as CoordinateMode | undefined,
+    displayIndex: parseIntRange('GUI_AGENT_DISPLAY_INDEX', overrides?.displayIndex?.toString() ?? env.GUI_AGENT_DISPLAY_INDEX, 0, 15, 0),
+
+    jpegQuality: parseIntRange('GUI_AGENT_JPEG_QUALITY', overrides?.jpegQuality?.toString() ?? env.GUI_AGENT_JPEG_QUALITY, 1, 100, 75),
+    maxSteps: parseIntRange('GUI_AGENT_MAX_STEPS', overrides?.maxSteps?.toString() ?? env.GUI_AGENT_MAX_STEPS, 1, 200, 50),
+    stepDelayMs: parseIntRange('GUI_AGENT_STEP_DELAY_MS', overrides?.stepDelayMs?.toString() ?? env.GUI_AGENT_STEP_DELAY_MS, 100, 30000, 1500),
+    stuckThreshold: parseIntRange('GUI_AGENT_STUCK_THRESHOLD', overrides?.stuckThreshold?.toString() ?? env.GUI_AGENT_STUCK_THRESHOLD, 1, 20, 3),
+
+    transport,
+    port: parseIntRange('GUI_AGENT_PORT', overrides?.port?.toString() ?? env.GUI_AGENT_PORT, 1, 65535, 60008),
+
+    logFile: overrides?.logFile ?? env.GUI_AGENT_LOG_FILE,
+  };
+
+  logInfo(`Config loaded: provider=${config.provider}, model=${config.model}, transport=${config.transport}, port=${config.port}`);
+  return config;
+}

qimingclaw/crates/agent-gui-server/src/coordinates/modelProfiles.ts

@@ -0,0 +1,53 @@
+/**
+ * Model configuration table: coordinate modes and coordinate orders.
+ *
+ * Maps model names to their coordinate system properties via regex patterns.
+ */
+
+export type CoordinateMode = 'image-absolute' | 'normalized-1000' | 'normalized-999' | 'normalized-0-1';
+export type CoordinateOrder = 'xy' | 'yx';
+
+export interface ModelProfile {
+  coordinateMode: CoordinateMode;
+  coordinateOrder: CoordinateOrder;
+}
+
+interface ModelRule {
+  pattern: RegExp;
+  profile: ModelProfile;
+}
+
+const MODEL_RULES: ModelRule[] = [
+  { pattern: /^claude-/i, profile: { coordinateMode: 'image-absolute', coordinateOrder: 'xy' } },
+  { pattern: /^gpt-(4o|5)/i, profile: { coordinateMode: 'image-absolute', coordinateOrder: 'xy' } },
+  { pattern: /^gemini/i, profile: { coordinateMode: 'normalized-999', coordinateOrder: 'yx' } },
+  { pattern: /^ui-tars/i, profile: { coordinateMode: 'normalized-1000', coordinateOrder: 'xy' } },
+  { pattern: /^qwen(2\.5)?-vl/i, profile: { coordinateMode: 'image-absolute', coordinateOrder: 'xy' } },
+  { pattern: /^cogagent/i, profile: { coordinateMode: 'image-absolute', coordinateOrder: 'xy' } },
+  { pattern: /^(seeclick|showui)/i, profile: { coordinateMode: 'normalized-0-1', coordinateOrder: 'xy' } },
+];
+
+const FALLBACK_PROFILE: ModelProfile = { coordinateMode: 'image-absolute', coordinateOrder: 'xy' };
+
+/**
+ * Get the coordinate profile for a given model name.
+ *
+ * @param modelName - The model name to match (e.g. 'claude-sonnet-4-20250514', 'gemini-2.5-pro')
+ * @param overrideMode - If provided, overrides the coordinate mode from the matched profile
+ */
+export function getModelProfile(modelName: string, overrideMode?: CoordinateMode): ModelProfile {
+  let profile = FALLBACK_PROFILE;
+
+  for (const rule of MODEL_RULES) {
+    if (rule.pattern.test(modelName)) {
+      profile = rule.profile;
+      break;
+    }
+  }
+
+  if (overrideMode) {
+    return { coordinateMode: overrideMode, coordinateOrder: profile.coordinateOrder };
+  }
+
+  return profile;
+}

qimingclaw/crates/agent-gui-server/src/coordinates/resolver.ts

@@ -0,0 +1,118 @@
+/**
+ * CoordinateResolver: model coordinates → logical coordinates → global coordinates.
+ *
+ * Pure function, zero I/O, highly testable.
+ * Implements the 4-step conversion pipeline.
+ */
+
+import { type CoordinateMode, type CoordinateOrder, type ModelProfile } from './modelProfiles.js';
+import { logWarn } from '../utils/logger.js';
+
+export interface ScreenshotMeta {
+  /** Width of the screenshot image sent to the LLM */
+  imageWidth: number;
+  /** Height of the screenshot image sent to the LLM */
+  imageHeight: number;
+  /** Logical width of the display */
+  logicalWidth: number;
+  /** Logical height of the display */
+  logicalHeight: number;
+}
+
+export interface DisplayInfo {
+  /** Global origin of this display (in logical coordinates) */
+  origin: { x: number; y: number };
+  /** Display bounds (in logical coordinates) */
+  bounds: { width: number; height: number };
+  /** Display scale factor (1 for standard, 2 for Retina/HiDPI) */
+  scaleFactor?: number;
+}
+
+export interface ResolvedCoordinate {
+  /** Global X coordinate (for nut.js) */
+  globalX: number;
+  /** Global Y coordinate (for nut.js) */
+  globalY: number;
+  /** Local X within the display */
+  localX: number;
+  /** Local Y within the display */
+  localY: number;
+  /** Normalized X (0-1) */
+  normalizedX: number;
+  /** Normalized Y (0-1) */
+  normalizedY: number;
+}
+
+/**
+ * Resolve model coordinates to global screen coordinates.
+ *
+ * 4-step pipeline:
+ * 1. Coordinate order correction (yx swap for Gemini)
+ * 2. Normalize to 0-1 based on coordinateMode
+ * 3. Logical coordinate = norm × logicalWidth/Height
+ * 4. Physical coordinate = logical × scaleFactor (for Retina/HiDPI)
+ * 5. Global offset = physical + display.origin × scaleFactor
+ */
+export function resolveCoordinate(
+  modelX: number,
+  modelY: number,
+  profile: ModelProfile,
+  meta: ScreenshotMeta,
+  display: DisplayInfo,
+): ResolvedCoordinate {
+  // Step 1: Coordinate order correction
+  let rawX: number;
+  let rawY: number;
+  if (profile.coordinateOrder === 'yx') {
+    rawX = modelY;
+    rawY = modelX;
+  } else {
+    rawX = modelX;
+    rawY = modelY;
+  }
+
+  // Step 2: Normalize to 0-1
+  const normalizedX = normalize(rawX, profile.coordinateMode, meta.imageWidth);
+  const normalizedY = normalize(rawY, profile.coordinateMode, meta.imageHeight);
+
+  // Step 3: Logical coordinates
+  const localX = Math.round(normalizedX * meta.logicalWidth);
+  const localY = Math.round(normalizedY * meta.logicalHeight);
+
+  // Step 4: Convert to physical coordinates (for Retina/HiDPI displays)
+  // nut.js uses physical pixels, not logical points
+  const scaleFactor = display.scaleFactor ?? 1;
+  const physicalX = localX * scaleFactor;
+  const physicalY = localY * scaleFactor;
+
+  // Step 5: Global offset (in physical coordinates)
+  let globalX = physicalX + display.origin.x * scaleFactor;
+  let globalY = physicalY + display.origin.y * scaleFactor;
+
+  // Boundary clamp (in physical coordinates)
+  const minX = display.origin.x * scaleFactor;
+  const minY = display.origin.y * scaleFactor;
+  const maxX = (display.origin.x + display.bounds.width) * scaleFactor - 1;
+  const maxY = (display.origin.y + display.bounds.height) * scaleFactor - 1;
+
+  if (globalX < minX || globalX > maxX || globalY < minY || globalY > maxY) {
+    logWarn(`Coordinate out of bounds: (${globalX}, ${globalY}) clamped to display [${minX}-${maxX}, ${minY}-${maxY}]`);
+    globalX = Math.max(minX, Math.min(maxX, globalX));
+    globalY = Math.max(minY, Math.min(maxY, globalY));
+  }
+
+  return { globalX, globalY, localX, localY, normalizedX, normalizedY };
+}
+
+function normalize(value: number, mode: CoordinateMode, imageDimension: number): number {
+  switch (mode) {
+    case 'image-absolute':
+      return imageDimension > 0 ? value / imageDimension : 0;
+    case 'normalized-1000':
+      return value / 1000;
+    case 'normalized-999':
+      return value / 999;
+    case 'normalized-0-1':
+      return value;
+  }
+}

qimingclaw/crates/agent-gui-server/src/desktop/clipboard.ts

@@ -0,0 +1,78 @@
+/**
+ * Clipboard operations for CJK paste workflow.
+ *
+ * Uses clipboardy for cross-platform clipboard access
+ * and nut.js for key simulation.
+ */
+
+import { DesktopError } from '../utils/errors.js';
+import { getPlatformPasteKeys } from '../utils/platform.js';
+
+/**
+ * Read current clipboard content.
+ */
+export async function readClipboard(): Promise<string> {
+  try {
+    const { default: clipboard } = await import('clipboardy');
+    return await clipboard.read();
+  } catch (err) {
+    throw new DesktopError('clipboard.read', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+/**
+ * Write text to clipboard.
+ */
+export async function writeClipboard(text: string): Promise<void> {
+  try {
+    const { default: clipboard } = await import('clipboardy');
+    await clipboard.write(text);
+  } catch (err) {
+    throw new DesktopError('clipboard.write', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+/**
+ * Paste text via clipboard:
+ * 1. Backup current clipboard
+ * 2. Write target text
+ * 3. Simulate Cmd/Ctrl+V
+ * 4. Wait 100ms
+ * 5. Restore clipboard (best-effort)
+ */
+export async function pasteText(text: string): Promise<void> {
+  let backup: string | undefined;
+  try {
+    backup = await readClipboard();
+  } catch {
+    // Failed to backup, continue anyway
+  }
+
+  try {
+    await writeClipboard(text);
+
+    const { keyboard, Key } = await import('@nut-tree-fork/nut-js');
+    const pasteKeys = getPlatformPasteKeys();
+
+    const keyObjs = pasteKeys.map(k => {
+      const keyName = k as keyof typeof Key;
+      return Key[keyName];
+    });
+    await keyboard.pressKey(...keyObjs);
+    await keyboard.releaseKey(...[...keyObjs].reverse());
+
+    // Wait for paste to complete
+    await new Promise(resolve => setTimeout(resolve, 100));
+  } catch (err) {
+    throw new DesktopError('clipboard.pasteText', err instanceof Error ? err : new Error(String(err)));
+  }
+
+  // Restore clipboard (best-effort, do not throw)
+  if (backup !== undefined) {
+    try {
+      await writeClipboard(backup);
+    } catch {
+      // Ignore restore failure
+    }
+  }
+}

qimingclaw/crates/agent-gui-server/src/desktop/display.ts

@@ -0,0 +1,185 @@
+/**
+ * Display information provider.
+ *
+ * Uses platform-specific APIs for accurate scaleFactor and multi-display support:
+ * - macOS: CoreGraphics via child_process (system_profiler or screen capture metadata)
+ * - Windows/Linux: nut.js fallback with scaleFactor detection
+ *
+ * Falls back to nut.js single-display when platform APIs are unavailable.
+ */
+
+import { DesktopError } from '../utils/errors.js';
+import { getPlatform } from '../utils/platform.js';
+import { logInfo, logDebug, logWarn } from '../utils/logger.js';
+
+export interface DisplayDescriptor {
+  index: number;
+  label: string;
+  width: number;
+  height: number;
+  scaleFactor: number;
+  isPrimary: boolean;
+  origin: { x: number; y: number };
+}
+
+/**
+ * List all connected displays with accurate metadata.
+ */
+export async function listDisplays(): Promise<DisplayDescriptor[]> {
+  try {
+    const platform = getPlatform();
+
+    if (platform === 'macos') {
+      return await listDisplaysMacOS();
+    }
+
+    // Windows/Linux: use nut.js with scaleFactor detection
+    return await listDisplaysFallback();
+  } catch (err) {
+    logWarn(`Platform-specific display detection failed, using fallback: ${err instanceof Error ? err.message : String(err)}`);
+    return await listDisplaysFallback();
+  }
+}
+
+/**
+ * macOS: Use system_profiler to enumerate displays with scaleFactor.
+ */
+async function listDisplaysMacOS(): Promise<DisplayDescriptor[]> {
+  const { execSync } = await import('child_process');
+
+  try {
+    const output = execSync(
+      'system_profiler SPDisplaysDataType -json',
+      { timeout: 5000, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
+    );
+    const data = JSON.parse(output);
+    const displays: DisplayDescriptor[] = [];
+
+    const gpus = data.SPDisplaysDataType || [];
+    for (const gpu of gpus) {
+      const ndrvs = gpu.spdisplays_ndrvs || [];
+      for (const disp of ndrvs) {
+        // Parse resolution string like "1440 x 900 @ 60.00Hz" or "2560 x 1440"
+        const resStr: string = disp._spdisplays_resolution || disp.spdisplays_resolution || '';
+        const retinaStr: string = disp.spdisplays_retina || '';
+        const isRetina = retinaStr.toLowerCase().includes('yes');
+
+        // Parse pixel resolution
+        const pixelRes: string = disp._spdisplays_pixels || disp.spdisplays_pixels || '';
+        let physicalWidth = 0;
+        let physicalHeight = 0;
+        const pixMatch = pixelRes.match(/(\d+)\s*x\s*(\d+)/);
+        if (pixMatch) {
+          physicalWidth = parseInt(pixMatch[1], 10);
+          physicalHeight = parseInt(pixMatch[2], 10);
+        }
+
+        // Parse logical resolution
+        let logicalWidth = 0;
+        let logicalHeight = 0;
+        const resMatch = resStr.match(/(\d+)\s*x\s*(\d+)/);
+        if (resMatch) {
+          logicalWidth = parseInt(resMatch[1], 10);
+          logicalHeight = parseInt(resMatch[2], 10);
+        }
+
+        // Calculate scaleFactor
+        let scaleFactor = 1;
+        if (physicalWidth > 0 && logicalWidth > 0) {
+          scaleFactor = Math.round(physicalWidth / logicalWidth);
+        } else if (isRetina) {
+          scaleFactor = 2;
+        }
+
+        // Use logical resolution (or fall back to physical if no logical)
+        const width = logicalWidth || physicalWidth;
+        const height = logicalHeight || physicalHeight;
+
+        if (width > 0 && height > 0) {
+          const isPrimary = disp.spdisplays_main === 'spdisplays_yes' ||
+            disp._spdisplays_displayID === '1' ||
+            displays.length === 0;
+
+          displays.push({
+            index: displays.length,
+            label: disp._name || `Display ${displays.length}`,
+            width,
+            height,
+            scaleFactor,
+            isPrimary,
+            origin: { x: 0, y: 0 }, // macOS manages global offsets; nut.js uses logical
+          });
+        }
+      }
+    }
+
+    if (displays.length > 0) {
+      logInfo(`Detected ${displays.length} display(s) via system_profiler`);
+      for (const d of displays) {
+        logDebug(`Display ${d.index}: ${d.width}x${d.height} @${d.scaleFactor}x "${d.label}" primary=${d.isPrimary}`);
+      }
+      return displays;
+    }
+  } catch (err) {
+    logDebug(`system_profiler failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+
+  // Fallback
+  return await listDisplaysFallback();
+}
+
+/**
+ * Fallback: use nut.js for basic display info with scaleFactor detection.
+ */
+async function listDisplaysFallback(): Promise<DisplayDescriptor[]> {
+  const { screen } = await import('@nut-tree-fork/nut-js');
+  const width = await screen.width();
+  const height = await screen.height();
+
+  // Detect scaleFactor: capture a small region and compare physical vs logical
+  let scaleFactor = 1;
+  try {
+    const { Region } = await import('@nut-tree-fork/nut-js');
+    const testRegion = new Region(0, 0, 10, 10);
+    const capture = await screen.grabRegion(testRegion);
+    // If physical capture is larger than requested logical region, we have HiDPI
+    if (capture.width > 10) {
+      scaleFactor = Math.round(capture.width / 10);
+    }
+  } catch {
+    logDebug('scaleFactor detection failed, defaulting to 1');
+  }
+
+  const primary: DisplayDescriptor = {
+    index: 0,
+    label: 'Primary',
+    width,
+    height,
+    scaleFactor,
+    isPrimary: true,
+    origin: { x: 0, y: 0 },
+  };
+
+  logInfo(`Detected display (fallback): ${width}x${height} @${scaleFactor}x`);
+  return [primary];
+}
+
+/**
+ * Get a specific display by index.
+ */
+export async function getDisplay(index: number): Promise<DisplayDescriptor> {
+  const displays = await listDisplays();
+  const display = displays.find(d => d.index === index);
+  if (!display) {
+    throw new DesktopError('getDisplay', new Error(`Display index ${index} not found (${displays.length} displays available)`));
+  }
+  return display;
+}
+
+/**
+ * Get the primary display.
+ */
+export async function getPrimaryDisplay(): Promise<DisplayDescriptor> {
+  const displays = await listDisplays();
+  return displays.find(d => d.isPrimary) ?? displays[0];
+}

qimingclaw/crates/agent-gui-server/src/desktop/imageSearch.ts

@@ -0,0 +1,121 @@
+/**
+ * Image search wrapper around nut.js template matcher.
+ *
+ * Uses nut.js screen.find() with configurable confidence threshold.
+ * Temp file is written once and reused across poll iterations.
+ */
+
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { DesktopError } from '../utils/errors.js';
+
+export interface ImageSearchResult {
+  found: boolean;
+  region?: { x: number; y: number; width: number; height: number };
+  confidence?: number;
+}
+
+export interface WaitForImageResult extends ImageSearchResult {
+  elapsed: number;
+}
+
+/**
+ * Find an image on screen by template matching.
+ */
+export async function findImage(templateBase64: string, confidence: number = 0.9): Promise<ImageSearchResult> {
+  try {
+    const { screen, loadImage } = await import('@nut-tree-fork/nut-js');
+
+    // Write template to temp file (async)
+    const tmpFile = path.join(os.tmpdir(), `gui-agent-template-${Date.now()}-${Math.random().toString(36).slice(2)}.png`);
+    await fs.writeFile(tmpFile, Buffer.from(templateBase64, 'base64'));
+
+    try {
+      const templateImage = await loadImage(tmpFile);
+
+      // Set confidence on the screen finder
+      screen.config.confidence = confidence;
+
+      const result = await screen.find(templateImage);
+      return {
+        found: true,
+        region: {
+          x: result.left,
+          y: result.top,
+          width: result.width,
+          height: result.height,
+        },
+        confidence,
+      };
+    } catch {
+      return { found: false };
+    } finally {
+      try { await fs.unlink(tmpFile); } catch { /* ignore */ }
+    }
+  } catch (err) {
+    if (err instanceof DesktopError) throw err;
+    throw new DesktopError('imageSearch.findImage', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+/**
+ * Find an image using an already-written temp file path.
+ * Used internally by waitForImage to avoid re-writing the file on each poll.
+ */
+async function findImageFromFile(tmpFile: string, confidence: number): Promise<ImageSearchResult> {
+  const { screen, loadImage } = await import('@nut-tree-fork/nut-js');
+  const templateImage = await loadImage(tmpFile);
+  screen.config.confidence = confidence;
+
+  try {
+    const result = await screen.find(templateImage);
+    return {
+      found: true,
+      region: {
+        x: result.left,
+        y: result.top,
+        width: result.width,
+        height: result.height,
+      },
+      confidence,
+    };
+  } catch {
+    return { found: false };
+  }
+}
+
+/**
+ * Wait for an image to appear on screen with timeout.
+ * Writes the template to a temp file once and reuses it across polls.
+ */
+export async function waitForImage(
+  templateBase64: string,
+  timeout: number = 10000,
+  confidence: number = 0.9,
+): Promise<WaitForImageResult> {
+  const start = Date.now();
+  const pollInterval = 500;
+
+  // Write temp file once, reuse across polls
+  const tmpFile = path.join(os.tmpdir(), `gui-agent-wait-${Date.now()}-${Math.random().toString(36).slice(2)}.png`);
+  await fs.writeFile(tmpFile, Buffer.from(templateBase64, 'base64'));
+
+  try {
+    while (Date.now() - start < timeout) {
+      try {
+        const result = await findImageFromFile(tmpFile, confidence);
+        if (result.found) {
+          return { ...result, elapsed: Date.now() - start };
+        }
+      } catch {
+        // Individual poll failure is non-fatal
+      }
+      await new Promise(resolve => setTimeout(resolve, pollInterval));
+    }
+
+    return { found: false, elapsed: Date.now() - start };
+  } finally {
+    try { await fs.unlink(tmpFile); } catch { /* ignore */ }
+  }
+}

qimingclaw/crates/agent-gui-server/src/desktop/keyboard.ts

@@ -0,0 +1,125 @@
+/**
+ * Keyboard operations with CJK/long text routing.
+ *
+ * - Non-ASCII or text > 50 chars → clipboard paste
+ * - Otherwise → nut.js keyboard.type()
+ */
+
+import { DesktopError } from '../utils/errors.js';
+import { pasteText } from './clipboard.js';
+
+const NON_ASCII_RE = /[^\x00-\x7F]/;
+const MAX_TYPE_LENGTH = 50;
+
+/**
+ * Key name aliases mapping to nut.js Key enum names.
+ * LLMs may return various names like "Meta", "Command", "Cmd", "Win", "Super".
+ * nut.js uses: LeftSuper, RightSuper, LeftControl, RightControl, etc.
+ * Arrow keys in nut.js are: Up, Down, Left, Right (not ArrowUp, etc.)
+ */
+const KEY_ALIASES: Record<string, string> = {
+  // macOS Command key aliases → Super
+  Meta: 'LeftSuper',
+  Command: 'LeftSuper',
+  Cmd: 'LeftSuper',
+  '⌘': 'LeftSuper',
+  // Windows/Linux Super/Win key aliases
+  Win: 'LeftSuper',
+  Super: 'LeftSuper',
+  // Control key aliases
+  Control: 'LeftControl',
+  Ctrl: 'LeftControl',
+  '⌃': 'LeftControl',
+  // Alt/Option key aliases
+  Alt: 'LeftAlt',
+  Option: 'LeftAlt',
+  Opt: 'LeftAlt',
+  '⌥': 'LeftAlt',
+  // Shift key aliases
+  Shift: 'LeftShift',
+  '⇧': 'LeftShift',
+  // Arrow key aliases (nut.js uses Up/Down/Left/Right, not ArrowUp/ArrowDown)
+  ArrowUp: 'Up',
+  ArrowDown: 'Down',
+  ArrowLeft: 'Left',
+  ArrowRight: 'Right',
+  // Common alternative names
+  Return: 'Enter',
+  Esc: 'Escape',
+  Del: 'Delete',
+  Ins: 'Insert',
+};
+
+/**
+ * Type text with smart routing:
+ * - CJK/non-ASCII or long text → clipboard paste
+ * - Short ASCII → direct nut.js typing
+ */
+export async function typeText(text: string): Promise<void> {
+  if (NON_ASCII_RE.test(text) || text.length > MAX_TYPE_LENGTH) {
+    await pasteText(text);
+    return;
+  }
+
+  try {
+    const { keyboard } = await import('@nut-tree-fork/nut-js');
+    await keyboard.type(text);
+  } catch (err) {
+    throw new DesktopError('keyboard.typeText', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+/**
+ * Normalize key name using aliases mapping.
+ */
+function normalizeKeyName(key: string): string {
+  // First check aliases
+  if (KEY_ALIASES[key]) {
+    return KEY_ALIASES[key];
+  }
+  // Then return as-is (for F1-F24, ArrowUp, Escape, etc.)
+  return key;
+}
+
+/**
+ * Press a single key by name.
+ */
+export async function pressKey(key: string): Promise<void> {
+  try {
+    const { keyboard, Key } = await import('@nut-tree-fork/nut-js');
+    const normalizedKey = normalizeKeyName(key);
+    const keyObj = Key[normalizedKey as keyof typeof Key];
+    if (keyObj === undefined) {
+      throw new Error(`Unknown key: ${key} (normalized: ${normalizedKey})`);
+    }
+    await keyboard.pressKey(keyObj);
+    await keyboard.releaseKey(keyObj);
+  } catch (err) {
+    throw new DesktopError('keyboard.pressKey', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+/**
+ * Press a key combination (hotkey).
+ *
+ * Note: nut.js pressKey doesn't support multiple keys, but type() does.
+ * For hotkeys, we use keyboard.type(Key1, Key2, ...) which simulates
+ * pressing keys together as a combination.
+ */
+export async function hotkey(keys: string[]): Promise<void> {
+  try {
+    const { keyboard, Key } = await import('@nut-tree-fork/nut-js');
+    const keyObjs = keys.map(k => {
+      const normalizedKey = normalizeKeyName(k);
+      const keyObj = Key[normalizedKey as keyof typeof Key];
+      if (keyObj === undefined) {
+        throw new Error(`Unknown key: ${k} (normalized: ${normalizedKey})`);
+      }
+      return keyObj;
+    });
+    // Use keyboard.type() for hotkeys - it handles key combinations correctly
+    await keyboard.type(...keyObjs);
+  } catch (err) {
+    throw new DesktopError('keyboard.hotkey', err instanceof Error ? err : new Error(String(err)));
+  }
+}

qimingclaw/crates/agent-gui-server/src/desktop/mouse.ts

@@ -0,0 +1,90 @@
+/**
+ * Mouse operations wrapper around nut.js.
+ */
+
+import { DesktopError } from '../utils/errors.js';
+
+type ButtonType = 'left' | 'right' | 'middle';
+
+async function getNutMouse() {
+  const { mouse, Button, Point, straightTo } = await import('@nut-tree-fork/nut-js');
+  return { mouse, Button, Point, straightTo };
+}
+
+function mapButton(button: ButtonType | undefined, Button: any): any {
+  switch (button) {
+    case 'right': return Button.RIGHT;
+    case 'middle': return Button.MIDDLE;
+    default: return Button.LEFT;
+  }
+}
+
+export async function click(x: number, y: number, button?: ButtonType): Promise<void> {
+  try {
+    const { mouse, Button, Point, straightTo } = await getNutMouse();
+    await mouse.move(straightTo(new Point(x, y)));
+    await mouse.click(mapButton(button, Button));
+  } catch (err) {
+    throw new DesktopError('mouse.click', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+export async function doubleClick(x: number, y: number, button?: ButtonType): Promise<void> {
+  try {
+    const { mouse, Button, Point, straightTo } = await getNutMouse();
+    await mouse.move(straightTo(new Point(x, y)));
+    await mouse.doubleClick(mapButton(button, Button));
+  } catch (err) {
+    throw new DesktopError('mouse.doubleClick', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+export async function moveTo(x: number, y: number): Promise<void> {
+  try {
+    const { mouse, Point, straightTo } = await getNutMouse();
+    await mouse.move(straightTo(new Point(x, y)));
+  } catch (err) {
+    throw new DesktopError('mouse.moveTo', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+export async function drag(startX: number, startY: number, endX: number, endY: number, button?: ButtonType): Promise<void> {
+  try {
+    const { mouse, Button, Point, straightTo } = await getNutMouse();
+    await mouse.move(straightTo(new Point(startX, startY)));
+    await mouse.pressButton(mapButton(button, Button));
+    await mouse.move(straightTo(new Point(endX, endY)));
+    await mouse.releaseButton(mapButton(button, Button));
+  } catch (err) {
+    throw new DesktopError('mouse.drag', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+export async function scroll(x: number, y: number, deltaY: number, deltaX?: number): Promise<void> {
+  try {
+    const { mouse, Point, straightTo } = await getNutMouse();
+    await mouse.move(straightTo(new Point(x, y)));
+    if (deltaY > 0) {
+      await mouse.scrollDown(Math.abs(deltaY));
+    } else if (deltaY < 0) {
+      await mouse.scrollUp(Math.abs(deltaY));
+    }
+    if (deltaX && deltaX > 0) {
+      await mouse.scrollRight(Math.abs(deltaX));
+    } else if (deltaX && deltaX < 0) {
+      await mouse.scrollLeft(Math.abs(deltaX));
+    }
+  } catch (err) {
+    throw new DesktopError('mouse.scroll', err instanceof Error ? err : new Error(String(err)));
+  }
+}
+
+export async function getPosition(): Promise<{ x: number; y: number }> {
+  try {
+    const { mouse } = await getNutMouse();
+    const pos = await mouse.getPosition();
+    return { x: pos.x, y: pos.y };
+  } catch (err) {
+    throw new DesktopError('mouse.getPosition', err instanceof Error ? err : new Error(String(err)));
+  }
+}

qimingclaw/crates/agent-gui-server/src/desktop/screenAnalyzer.ts

@@ -0,0 +1,99 @@
+/**
+ * Screen analysis using vision model.
+ *
+ * Captures screenshot and sends to vision model for analysis.
+ */
+
+import { complete } from '@mariozechner/pi-ai';
+import type { Model, Api } from '@mariozechner/pi-ai';
+import { captureScreenshot } from './screenshot.js';
+import { DesktopError } from '../utils/errors.js';
+
+export interface AnalyzeResult {
+  analysis: string;
+  imageWidth: number;
+  imageHeight: number;
+}
+
+const SCREEN_ANALYSIS_SYSTEM_PROMPT = `You are a GUI screen analysis assistant specialized in desktop automation.
+
+Your task is to analyze screenshots and provide actionable information for automation tasks.
+
+When analyzing the screen:
+1. Identify UI elements: buttons, menus, input fields, icons, windows, dialogs
+2. Provide approximate coordinates when asked (as percentages like "top-left quadrant" or pixel estimates)
+3. Describe element states: enabled/disabled, focused, selected, minimized, etc.
+4. Read visible text content accurately
+5. Identify the active application and window focus
+
+Be concise but thorough. Format your response clearly with:
+- Element descriptions
+- Locations (when relevant)
+- States and any actionable details
+
+If the user asks about specific elements, locate them precisely and describe their position relative to the screen or window.`;
+
+/**
+ * Capture screenshot and analyze with vision model.
+ *
+ * @param model - The vision model to use
+ * @param apiKey - API key for the model
+ * @param prompt - Analysis instruction (e.g., "What buttons are visible?")
+ * @param displayIndex - Display to capture (default: 0)
+ */
+export async function analyzeScreen(
+  model: Model<Api>,
+  apiKey: string,
+  prompt: string,
+  displayIndex: number = 0,
+): Promise<AnalyzeResult> {
+  try {
+    // Capture screenshot
+    const screenshot = await captureScreenshot(displayIndex);
+
+    // Build message with image
+    const messages = [
+      {
+        role: 'user' as const,
+        content: [
+          {
+            type: 'image' as const,
+            data: screenshot.image,
+            mimeType: screenshot.mimeType,
+          },
+          {
+            type: 'text' as const,
+            text: prompt,
+          },
+        ],
+        timestamp: Date.now(),
+      },
+    ];
+
+    // Call vision model using pi-ai complete function
+    const response = await complete(model, {
+      systemPrompt: SCREEN_ANALYSIS_SYSTEM_PROMPT,
+      messages,
+    }, {
+      apiKey,
+    });
+
+    // Extract text from response
+    let analysis = '';
+    if (response && response.content) {
+      for (const part of response.content) {
+        if (part.type === 'text') {
+          analysis += part.text;
+        }
+      }
+    }
+
+    return {
+      analysis,
+      imageWidth: screenshot.imageWidth,
+      imageHeight: screenshot.imageHeight,
+    };
+  } catch (err) {
+    throw new DesktopError('screenAnalyzer.analyze', err instanceof Error ? err : new Error(String(err)));
+  }
+}

qimingclaw/crates/agent-gui-server/src/desktop/screenshot.ts

@@ -0,0 +1,113 @@
+/**
+ * Screenshot pipeline: capture → scale → JPEG → base64
+ *
+ * 1. nut.js screen.capture() at physical resolution
+ * 2. sharp resize to logical resolution (absorb scaleFactor)
+ * 3. If longest edge > 1920, proportionally scale to 1920
+ * 4. JPEG encode with configurable quality
+ * 5. Return base64 + metadata
+ */
+
+import sharp from 'sharp';
+import { getDisplay } from './display.js';
+import { DesktopError } from '../utils/errors.js';
+import { logInfo, logDebug } from '../utils/logger.js';
+
+const MAX_LONGEST_EDGE = 1920;
+const MAX_JPEG_BYTES = 1_500_000; // 1.5 MB hard limit for LLM input
+const QUALITY_RETRY_STEPS = [60, 45, 30]; // Fallback quality levels
+
+export interface ScreenshotResult {
+  /** Base64-encoded JPEG image */
+  image: string;
+  mimeType: 'image/jpeg';
+  /** Byte size of the JPEG */
+  imageBytes: number;
+  /** Width of the final image sent to LLM */
+  imageWidth: number;
+  /** Height of the final image sent to LLM */
+  imageHeight: number;
+  /** Logical display width */
+  logicalWidth: number;
+  /** Logical display height */
+  logicalHeight: number;
+  /** Physical capture width */
+  physicalWidth: number;
+  /** Physical capture height */
+  physicalHeight: number;
+  /** Display scale factor */
+  scaleFactor: number;
+  /** Which display was captured */
+  displayIndex: number;
+}
+
+export async function captureScreenshot(displayIndex: number, jpegQuality: number = 75): Promise<ScreenshotResult> {
+  try {
+    const display = await getDisplay(displayIndex);
+    const { screen } = await import('@nut-tree-fork/nut-js');
+
+    // Capture raw RGBA buffer at physical resolution
+    const { Region } = await import('@nut-tree-fork/nut-js');
+    const region = new Region(display.origin.x, display.origin.y, display.width, display.height);
+
+    const captured = await screen.grabRegion(region);
+    const physicalWidth = captured.width;
+    const physicalHeight = captured.height;
+    const rawBuffer = captured.data;
+
+    // Logical resolution (absorb scaleFactor)
+    const scaleFactor = display.scaleFactor;
+    let logicalWidth = Math.round(physicalWidth / scaleFactor);
+    let logicalHeight = Math.round(physicalHeight / scaleFactor);
+
+    // Target dimensions: start with logical
+    let targetWidth = logicalWidth;
+    let targetHeight = logicalHeight;
+
+    // If longest edge > MAX, proportionally scale down
+    const longestEdge = Math.max(targetWidth, targetHeight);
+    if (longestEdge > MAX_LONGEST_EDGE) {
+      const scale = MAX_LONGEST_EDGE / longestEdge;
+      targetWidth = Math.round(targetWidth * scale);
+      targetHeight = Math.round(targetHeight * scale);
+    }
+
+    // Resize + JPEG encode
+    const sharpInstance = sharp(rawBuffer, {
+      raw: { width: physicalWidth, height: physicalHeight, channels: 4 },
+    }).resize(targetWidth, targetHeight, { kernel: 'lanczos3' });
+
+    let jpegBuffer = await sharpInstance.clone().jpeg({ quality: jpegQuality }).toBuffer();
+
+    // Quality degradation retry — if bytes exceed limit, retry with lower quality
+    if (jpegBuffer.length > MAX_JPEG_BYTES) {
+      for (const retryQuality of QUALITY_RETRY_STEPS) {
+        if (retryQuality >= jpegQuality) continue; // Skip if not lower than current
+        logDebug(`Screenshot ${jpegBuffer.length} bytes exceeds ${MAX_JPEG_BYTES}, retrying with quality=${retryQuality}`);
+        jpegBuffer = await sharpInstance.clone().jpeg({ quality: retryQuality }).toBuffer();
+        if (jpegBuffer.length <= MAX_JPEG_BYTES) break;
+      }
+    }
+
+    const base64 = jpegBuffer.toString('base64');
+
+    logDebug(`Screenshot: ${physicalWidth}x${physicalHeight} → ${targetWidth}x${targetHeight}, ${jpegBuffer.length} bytes`);
+
+    return {
+      image: base64,
+      mimeType: 'image/jpeg',
+      imageBytes: jpegBuffer.length,
+      imageWidth: targetWidth,
+      imageHeight: targetHeight,
+      logicalWidth,
+      logicalHeight,
+      physicalWidth,
+      physicalHeight,
+      scaleFactor,
+      displayIndex,
+    };
+  } catch (err) {
+    if (err instanceof DesktopError) throw err;
+    throw new DesktopError('captureScreenshot', err instanceof Error ? err : new Error(String(err)));
+  }
+}

qimingclaw/crates/agent-gui-server/src/index.ts

@@ -0,0 +1,127 @@
+/**
+ * CLI entry point for agent-gui-server.
+ *
+ * Usage:
+ *   agent-gui-server [--port <number>] [--transport <http|stdio>]
+ *
+ * Environment variables:
+ *   GUI_AGENT_API_KEY (required) - LLM API key
+ *   GUI_AGENT_* - See config.ts for all options
+ */
+
+// Disable Node.js warnings (e.g., module type) to keep stderr clean for structured parsing
+process.env.NODE_NO_WARNINGS = '1';
+
+import { loadConfig } from './config.js';
+import { createGuiAgentServer } from './mcp/server.js';
+import { logInfo, logError } from './utils/logger.js';
+
+function parseArgs(argv: string[]): { port?: number; transport?: 'http' | 'stdio' } {
+  const result: { port?: number; transport?: 'http' | 'stdio' } = {};
+
+  for (let i = 2; i < argv.length; i++) {
+    switch (argv[i]) {
+      case '--port':
+        result.port = parseInt(argv[++i], 10);
+        break;
+      case '--transport':
+        result.transport = argv[++i] as 'http' | 'stdio';
+        break;
+      case '--help':
+      case '-h':
+        printUsage();
+        process.exit(0);
+        break;
+    }
+  }
+
+  return result;
+}
+
+function printUsage(): void {
+  console.error(`
+agent-gui-server — GUI Agent MCP Server
+
+Usage:
+  agent-gui-server [options]
+
+Options:
+  --port <number>         HTTP server port (default: 60008)
+  --transport <http|stdio> Transport mode (default: http)
+  --help, -h              Show this help
+
+Environment Variables:
+  GUI_AGENT_API_KEY       LLM API key (optional, only needed for gui_analyze_screen tool)
+  GUI_AGENT_PROVIDER      LLM provider (default: anthropic)
+  GUI_AGENT_MODEL         LLM model (default: claude-sonnet-4-20250514)
+  GUI_AGENT_BASE_URL      Custom API base URL
+  GUI_AGENT_PORT          HTTP port (default: 60008)
+  GUI_AGENT_TRANSPORT     Transport mode: http or stdio (default: http)
+  GUI_AGENT_DISPLAY_INDEX Target display index (default: 0)
+  GUI_AGENT_JPEG_QUALITY  Screenshot JPEG quality 1-100 (default: 75)
+  GUI_AGENT_MAX_STEPS     Max steps per task 1-200 (default: 50)
+  GUI_AGENT_STEP_DELAY_MS Delay between steps 100-30000 (default: 1500)
+  GUI_AGENT_LOG_FILE      Optional log file path
+`);
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+
+  let config;
+  try {
+    config = loadConfig({
+      port: args.port,
+      transport: args.transport,
+    });
+  } catch (err) {
+    const errorObj = {
+      code: 'CONFIG_ERROR',
+      message: err instanceof Error ? err.message : String(err),
+    };
+    process.stderr.write(`GUI_AGENT_ERROR:${JSON.stringify(errorObj)}\n`);
+    process.exit(1);
+  }
+
+  const server = createGuiAgentServer(config);
+
+  // Graceful shutdown
+  let stopping = false;
+  const shutdown = async (signal: string) => {
+    if (stopping) return;
+    stopping = true;
+    logInfo(`Received ${signal}, shutting down...`);
+    try {
+      await server.stop();
+    } catch (err) {
+      logError(`Shutdown error: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    process.exit(0);
+  };
+
+  process.on('SIGINT', () => shutdown('SIGINT'));
+  process.on('SIGTERM', () => shutdown('SIGTERM'));
+  process.on('unhandledRejection', (reason) => {
+    const errorObj = {
+      code: 'UNHANDLED_REJECTION',
+      message: reason instanceof Error ? reason.message : String(reason),
+    };
+    process.stderr.write(`GUI_AGENT_ERROR:${JSON.stringify(errorObj)}\n`);
+    process.exit(1);
+  });
+
+  try {
+    await server.start();
+  } catch (err) {
+    // Output structured error to stderr for easy parsing by parent process
+    // Format: GUI_AGENT_ERROR:<json>
+    const errorObj = {
+      code: 'STARTUP_FAILED',
+      message: err instanceof Error ? err.message : String(err),
+    };
+    process.stderr.write(`GUI_AGENT_ERROR:${JSON.stringify(errorObj)}\n`);
+    process.exit(1);
+  }
+}
+
+main();

qimingclaw/crates/agent-gui-server/src/lib.ts

@@ -0,0 +1,22 @@
+/**
+ * SDK entry point — programmatic API for embedding the GUI Agent MCP Server.
+ */
+
+export { createGuiAgentServer, type GuiAgentServer } from './mcp/server.js';
+export { loadConfig, type GuiAgentConfig } from './config.js';
+export type { ScreenshotResult } from './desktop/screenshot.js';
+export type { DisplayDescriptor } from './desktop/display.js';
+export type { TaskResult, ProgressInfo, StepRecord } from './agent/taskRunner.js';
+
+// Windows-MCP 管理
+export { WindowsMcpManager, healthCheck, waitForReady } from './windowsMcp/index.js';
+export type {
+  WindowsMcpStatus,
+  ProcessConfig,
+  ProcessRunner,
+  StartResult,
+  StopResult,
+  WindowsMcpConfig,
+  HealthCheckOptions,
+  HealthCheckResult,
+} from './windowsMcp/index.js';

qimingclaw/crates/agent-gui-server/src/mcp/atomicTools.ts

@@ -0,0 +1,654 @@
+/**
+ * 14 atomic MCP tool handlers.
+ *
+ * Each handler: validate input → safety check → coordinate resolve → desktop operation → audit → response
+ */
+
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { validateHotkey } from '../safety/hotkeys.js';
+import { resolveCoordinate, type ScreenshotMeta, type DisplayInfo } from '../coordinates/resolver.js';
+import { getModelProfile } from '../coordinates/modelProfiles.js';
+import { captureScreenshot } from '../desktop/screenshot.js';
+import * as mouse from '../desktop/mouse.js';
+import * as keyboard from '../desktop/keyboard.js';
+import * as display from '../desktop/display.js';
+import { findImage, waitForImage } from '../desktop/imageSearch.js';
+import { analyzeScreen } from '../desktop/screenAnalyzer.js';
+import { createModel } from '../agent/taskRunner.js';
+import { logError } from '../utils/logger.js';
+import { SafetyError } from '../utils/errors.js';
+
+const ATOMIC_TOOLS = [
+  {
+    name: 'gui_screenshot',
+    description: 'Capture a screenshot and return as base64 image data. WARNING: Returns raw image data that text-only models cannot interpret. For understanding screen content, use gui_analyze_screen instead. Use this tool only when you need to save or transfer the screenshot file itself.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        displayIndex: { type: 'number', description: 'Display index (default: configured display)' },
+      },
+    },
+  },
+  {
+    name: 'gui_click',
+    description: 'Click at the specified coordinates. Use gui_analyze_screen first to identify element locations, or gui_cursor_position to get current mouse position. Coordinates are in logical pixels (scaled automatically for Retina displays).',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels (0 = left edge of display)' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels (0 = top edge of display)' },
+        button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button (default: left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode: "logical" (default, use logical pixel values), "image-absolute" (divide x/y by image width/height to get 0-1, then multiply by display size), "normalized-1000" (x/y in 0-1000 range, scaled to display size)' },
+      },
+      required: ['x', 'y'],
+    },
+  },
+  {
+    name: 'gui_double_click',
+    description: 'Double-click at the specified coordinates. Typically used to open files or select words. Use gui_analyze_screen first to identify element locations.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels' },
+        button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button (default: left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['x', 'y'],
+    },
+  },
+  {
+    name: 'gui_move_mouse',
+    description: 'Move mouse cursor to the specified coordinates without clicking. Useful for hovering over elements to reveal tooltips or menus.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['x', 'y'],
+    },
+  },
+  {
+    name: 'gui_drag',
+    description: 'Drag from start to end coordinates. Used for dragging files, selecting text, or moving windows. Hold mouse button from start to end position.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        startX: { type: 'number', description: 'Start X coordinate in logical pixels' },
+        startY: { type: 'number', description: 'Start Y coordinate in logical pixels' },
+        endX: { type: 'number', description: 'End X coordinate in logical pixels' },
+        endY: { type: 'number', description: 'End Y coordinate in logical pixels' },
+        button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button (default: left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['startX', 'startY', 'endX', 'endY'],
+    },
+  },
+  {
+    name: 'gui_scroll',
+    description: 'Scroll at the specified position. Positive deltaY scrolls down, negative scrolls up. Used for navigating long pages or lists.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        x: { type: 'number', description: 'X coordinate in logical pixels' },
+        y: { type: 'number', description: 'Y coordinate in logical pixels' },
+        deltaY: { type: 'number', description: 'Vertical scroll amount in "steps" (scroll wheel clicks). Positive = scroll down, negative = scroll up. Typical values: 1-3 for small scroll, 5-10 for larger scroll' },
+        deltaX: { type: 'number', description: 'Horizontal scroll amount in steps (positive = right, negative = left)' },
+        coordinateMode: { type: 'string', description: 'Coordinate mode override' },
+      },
+      required: ['x', 'y', 'deltaY'],
+    },
+  },
+  {
+    name: 'gui_type',
+    description: 'Type text at the current cursor position. Automatically handles CJK/non-ASCII text via clipboard paste. Use after clicking on an input field.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        text: { type: 'string', description: 'Text to type (supports all languages including Chinese, Japanese, Korean)' },
+      },
+      required: ['text'],
+    },
+  },
+  {
+    name: 'gui_press_key',
+    description: 'Press a single key. Supported keys: Enter, Tab, Escape, Backspace, Delete, Space, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, Home, End, PageUp, PageDown, F1-F24. Modifiers: Shift, Control/Ctrl, Alt/Option, Meta/Command/Cmd.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        key: { type: 'string', description: 'Key name (e.g. Enter, Tab, Escape, ArrowUp, F1). Supports aliases like Ctrl, Cmd, Opt.' },
+      },
+      required: ['key'],
+    },
+  },
+  {
+    name: 'gui_hotkey',
+    description: 'Press a key combination (shortcut). Key names support aliases: Meta/Command/Cmd/⌘, Control/Ctrl/⌃, Alt/Option/Opt/⌥, Shift/⇧. Example: ["Meta", "C"] for copy, ["Meta", "V"] for paste.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        keys: { type: 'array', items: { type: 'string' }, description: 'Keys to press together in order (e.g. ["Meta", "C"] for copy, ["Alt", "Tab"] for window switcher)' },
+      },
+      required: ['keys'],
+    },
+  },
+  {
+    name: 'gui_cursor_position',
+    description: 'Get current mouse cursor position. Returns both logical coordinates (for use with gui_click) and physical coordinates (raw nut.js values). On Retina displays, logical coords are half of physical.',
+    inputSchema: { type: 'object' as const, properties: {} },
+  },
+  {
+    name: 'gui_list_displays',
+    description: 'List all connected displays with their properties (resolution, scale factor, position). Use to identify which display to target.',
+    inputSchema: { type: 'object' as const, properties: {} },
+  },
+  {
+    name: 'gui_find_image',
+    description: 'Find a template image on screen using image matching. DEPRECATED: For most use cases, prefer gui_analyze_screen with prompt "Find [element]" instead. Returns match location and confidence. Requires base64-encoded template image.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        template: { type: 'string', description: 'Base64-encoded template image to find (PNG/JPEG)' },
+        confidence: { type: 'number', description: 'Match confidence threshold 0-1 (default 0.9, lower for fuzzy matching)' },
+      },
+      required: ['template'],
+    },
+  },
+  {
+    name: 'gui_wait_for_image',
+    description: 'Wait for a template image to appear on screen. DEPRECATED: For most use cases, prefer gui_analyze_screen in a loop instead. Polls until found or timeout.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        template: { type: 'string', description: 'Base64-encoded template image to wait for' },
+        timeout: { type: 'number', description: 'Timeout in milliseconds (default 10000)' },
+        confidence: { type: 'number', description: 'Match confidence threshold 0-1 (default 0.9)' },
+      },
+      required: ['template'],
+    },
+  },
+  {
+    name: 'gui_analyze_screen',
+    description: 'Capture screenshot and analyze with vision model. Returns text description of screen content, UI elements, text, buttons, and their locations. Use this to understand what is on screen before taking action. Preferred over gui_screenshot for text-based models.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        prompt: { type: 'string', description: 'Analysis instruction (e.g., "What buttons are visible?", "Find the search box location and provide coordinates", "Is there an error dialog?", "What is the current active window?")' },
+        displayIndex: { type: 'number', description: 'Display index (default: configured display)' },
+      },
+      required: ['prompt'],
+    },
+  },
+];
+
+/** Cached screenshot dimensions per display (with TTL) */
+interface CachedDimensions {
+  imageWidth: number;
+  imageHeight: number;
+  logicalWidth: number;
+  logicalHeight: number;
+  timestamp: number;
+}
+const screenshotDimensionCache = new Map<number, CachedDimensions>();
+const CACHE_TTL_MS = 5000; // 5 seconds
+
+/** Clear the screenshot dimension cache (for testing) */
+export function clearScreenshotDimensionCache(): void {
+  screenshotDimensionCache.clear();
+}
+
+/** Helper: resolve coordinates for tools that accept x/y + optional coordinateMode */
+async function resolveXY(
+  x: number,
+  y: number,
+  coordinateMode: string | undefined,
+  config: GuiAgentConfig,
+): Promise<{ globalX: number; globalY: number; meta: ScreenshotMeta }> {
+  const profile = getModelProfile(config.model, coordinateMode as any);
+  const disp = await display.getDisplay(config.displayIndex);
+  const displayInfo: DisplayInfo = {
+    origin: disp.origin,
+    bounds: { width: disp.width, height: disp.height },
+    scaleFactor: disp.scaleFactor,
+  };
+
+  if (!coordinateMode) {
+    // No coordinate mode specified — treat as logical coordinates directly
+    // Convert to physical coordinates for nut.js (multiply by scaleFactor)
+    const scaleFactor = disp.scaleFactor ?? 1;
+    const meta: ScreenshotMeta = {
+      imageWidth: disp.width,
+      imageHeight: disp.height,
+      logicalWidth: disp.width,
+      logicalHeight: disp.height,
+    };
+    return {
+      globalX: Math.round(x * scaleFactor) + disp.origin.x * scaleFactor,
+      globalY: Math.round(y * scaleFactor) + disp.origin.y * scaleFactor,
+      meta,
+    };
+  }
+
+  // For coordinate modes (image-absolute, normalized-*), we need actual screenshot dimensions
+  // because models return coordinates based on the image they saw (which may be scaled)
+  // Use cached dimensions if available and fresh
+  const cached = screenshotDimensionCache.get(config.displayIndex);
+  const now = Date.now();
+
+  let meta: ScreenshotMeta;
+  if (cached && (now - cached.timestamp) < CACHE_TTL_MS) {
+    meta = {
+      imageWidth: cached.imageWidth,
+      imageHeight: cached.imageHeight,
+      logicalWidth: cached.logicalWidth,
+      logicalHeight: cached.logicalHeight,
+    };
+  } else {
+    // Capture screenshot to get actual dimensions
+    try {
+      const shot = await captureScreenshot(config.displayIndex, config.jpegQuality);
+      meta = {
+        imageWidth: shot.imageWidth,
+        imageHeight: shot.imageHeight,
+        logicalWidth: shot.logicalWidth,
+        logicalHeight: shot.logicalHeight,
+      };
+      // Cache the dimensions
+      screenshotDimensionCache.set(config.displayIndex, {
+        imageWidth: shot.imageWidth,
+        imageHeight: shot.imageHeight,
+        logicalWidth: shot.logicalWidth,
+        logicalHeight: shot.logicalHeight,
+        timestamp: now,
+      });
+    } catch (err) {
+      // Fallback to display dimensions on screenshot failure
+      logError(`Screenshot failed for coordinate resolution: ${err instanceof Error ? err.message : String(err)}`);
+      meta = {
+        imageWidth: disp.width,
+        imageHeight: disp.height,
+        logicalWidth: disp.width,
+        logicalHeight: disp.height,
+      };
+    }
+  }
+
+  const resolved = resolveCoordinate(x, y, profile, meta, displayInfo);
+  return { globalX: resolved.globalX, globalY: resolved.globalY, meta };
+}
+
+export { ATOMIC_TOOLS };
+
+/**
+ * 获取可用的工具列表
+ * 如果未配置视觉模型，则不包含 gui_analyze_screen
+ */
+export function getAvailableTools(): typeof ATOMIC_TOOLS {
+  const visionModel = process.env.GUI_AGENT_VISION_MODEL;
+  if (!visionModel) {
+    return ATOMIC_TOOLS.filter(t => t.name !== 'gui_analyze_screen');
+  }
+  return ATOMIC_TOOLS;
+}
+
+/** Validate that a value is a finite number */
+function requireNumber(name: string, value: unknown): number {
+  if (typeof value !== 'number' || !isFinite(value)) {
+    throw new Error(`Parameter '${name}' must be a finite number, got: ${value}`);
+  }
+  return value;
+}
+
+/** Validate that a value is a non-empty string */
+function requireString(name: string, value: unknown): string {
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new Error(`Parameter '${name}' must be a non-empty string`);
+  }
+  return value;
+}
+
+/** Validate that a value is a non-empty string array */
+function requireStringArray(name: string, value: unknown): string[] {
+  if (!Array.isArray(value) || value.length === 0 || !value.every(v => typeof v === 'string')) {
+    throw new Error(`Parameter '${name}' must be a non-empty array of strings`);
+  }
+  return value;
+}
+
+export async function handleAtomicToolCall(
+  name: string,
+  args: Record<string, unknown>,
+  config: GuiAgentConfig,
+  auditLog: AuditLog,
+): Promise<{ content: Array<{ type: string; text?: string; data?: string; mimeType?: string }>; isError?: boolean } | null> {
+  // Check if this is an atomic tool
+  if (!ATOMIC_TOOLS.some(t => t.name === name)) {
+    return null; // Not handled by this module
+  }
+
+  const start = Date.now();
+  try {
+    const result = await handleAtomicTool(name, args, config);
+    auditLog.record({ tool: name, args, success: true, durationMs: Date.now() - start });
+    return result;
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    auditLog.record({ tool: name, args, success: false, durationMs: Date.now() - start });
+    logError(`Tool ${name} failed: ${errorMsg}`);
+    return { content: [{ type: 'text', text: `Error: ${errorMsg}` }], isError: true };
+  }
+}
+
+async function handleAtomicTool(name: string, args: Record<string, unknown>, config: GuiAgentConfig) {
+  switch (name) {
+    case 'gui_screenshot': {
+      const idx = (args.displayIndex as number) ?? config.displayIndex;
+      const shot = await captureScreenshot(idx, config.jpegQuality);
+      return {
+        content: [
+          { type: 'image', data: shot.image, mimeType: shot.mimeType },
+          {
+            type: 'text',
+            text: JSON.stringify({
+              success: true,
+              imageWidth: shot.imageWidth,
+              imageHeight: shot.imageHeight,
+              logicalWidth: shot.logicalWidth,
+              logicalHeight: shot.logicalHeight,
+              physicalWidth: shot.physicalWidth,
+              physicalHeight: shot.physicalHeight,
+              scaleFactor: shot.scaleFactor,
+              displayIndex: shot.displayIndex,
+              note: 'Image dimensions are in logical pixels. Use logical coordinates for click operations.',
+            }, null, 2),
+          },
+        ],
+      };
+    }
+
+    case 'gui_click': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const button = (args.button as string) || 'left';
+      const { globalX, globalY, meta } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.click(globalX, globalY, button as any);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'click',
+            button,
+            logicalCoords: { x, y },
+            physicalCoords: { x: globalX, y: globalY },
+            displaySize: { width: meta.logicalWidth, height: meta.logicalHeight },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_double_click': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const button = (args.button as string) || 'left';
+      const { globalX, globalY } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.doubleClick(globalX, globalY, button as any);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'double_click',
+            button,
+            logicalCoords: { x, y },
+            physicalCoords: { x: globalX, y: globalY },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_move_mouse': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const { globalX, globalY } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.moveTo(globalX, globalY);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'move',
+            logicalCoords: { x, y },
+            physicalCoords: { x: globalX, y: globalY },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_drag': {
+      const startX = requireNumber('startX', args.startX);
+      const startY = requireNumber('startY', args.startY);
+      const endX = requireNumber('endX', args.endX);
+      const endY = requireNumber('endY', args.endY);
+      const button = (args.button as string) || 'left';
+      const start = await resolveXY(startX, startY, args.coordinateMode as string, config);
+      const end = await resolveXY(endX, endY, args.coordinateMode as string, config);
+      await mouse.drag(start.globalX, start.globalY, end.globalX, end.globalY, button as any);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'drag',
+            button,
+            start: { logical: { x: startX, y: startY }, physical: { x: start.globalX, y: start.globalY } },
+            end: { logical: { x: endX, y: endY }, physical: { x: end.globalX, y: end.globalY } },
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_scroll': {
+      const x = requireNumber('x', args.x);
+      const y = requireNumber('y', args.y);
+      const deltaY = requireNumber('deltaY', args.deltaY);
+      const deltaX = args.deltaX !== undefined ? requireNumber('deltaX', args.deltaX) : 0;
+      const { globalX, globalY } = await resolveXY(x, y, args.coordinateMode as string, config);
+      await mouse.scroll(globalX, globalY, deltaY, deltaX);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'scroll',
+            position: { logical: { x, y }, physical: { x: globalX, y: globalY } },
+            scroll: { vertical: deltaY, horizontal: deltaX },
+            direction: deltaY > 0 ? 'down' : deltaY < 0 ? 'up' : 'none',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_type': {
+      const text = requireString('text', args.text);
+      await keyboard.typeText(text);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'type',
+            characterCount: text.length,
+            text: text.length <= 50 ? text : text.substring(0, 50) + '...',
+            method: /[^\x00-\x7F]/.test(text) ? 'clipboard_paste' : 'direct_type',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_press_key': {
+      const key = requireString('key', args.key);
+      await keyboard.pressKey(key);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'press_key',
+            key,
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_hotkey': {
+      const keys = requireStringArray('keys', args.keys);
+      const validation = validateHotkey(keys);
+      if (validation.blocked) {
+        throw new SafetyError(keys, validation.reason!);
+      }
+      await keyboard.hotkey(keys);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            action: 'hotkey',
+            keys,
+            combination: keys.join('+'),
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_cursor_position': {
+      const pos = await mouse.getPosition();
+      const disp = await display.getDisplay(config.displayIndex);
+      const scaleFactor = disp.scaleFactor ?? 1;
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            logicalCoords: { x: Math.round(pos.x / scaleFactor), y: Math.round(pos.y / scaleFactor) },
+            physicalCoords: { x: pos.x, y: pos.y },
+            scaleFactor,
+            note: 'Logical coordinates should be used for gui_click operations.',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_list_displays': {
+      const displays = await display.listDisplays();
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            count: displays.length,
+            displays: displays.map(d => ({
+              index: d.index,
+              label: d.label,
+              resolution: { width: d.width, height: d.height },
+              scaleFactor: d.scaleFactor,
+              isPrimary: d.isPrimary,
+              origin: d.origin,
+            })),
+            note: 'Resolution is in logical pixels. Use display index for displayIndex parameter in other tools.',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_find_image': {
+      const template = requireString('template', args.template);
+      const result = await findImage(template, args.confidence as number);
+      const disp = await display.getDisplay(config.displayIndex);
+      const scaleFactor = disp.scaleFactor ?? 1;
+      const region = result.region;
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: result.found,
+            found: result.found,
+            logicalCoords: result.found && region ? {
+              x: Math.round(region.x / scaleFactor),
+              y: Math.round(region.y / scaleFactor),
+              width: Math.round(region.width / scaleFactor),
+              height: Math.round(region.height / scaleFactor),
+            } : null,
+            physicalCoords: region,
+            confidence: result.confidence,
+            note: 'Use logicalCoords.center (x + width/2, y + height/2) for click operations.',
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_wait_for_image': {
+      const template = requireString('template', args.template);
+      const result = await waitForImage(template, args.timeout as number, args.confidence as number);
+      const disp = await display.getDisplay(config.displayIndex);
+      const scaleFactor = disp.scaleFactor ?? 1;
+      const region = result.region;
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: result.found,
+            found: result.found,
+            logicalCoords: result.found && region ? {
+              x: Math.round(region.x / scaleFactor),
+              y: Math.round(region.y / scaleFactor),
+              width: Math.round(region.width / scaleFactor),
+              height: Math.round(region.height / scaleFactor),
+            } : null,
+            physicalCoords: region,
+            confidence: result.confidence,
+            timedOut: !result.found,
+          }, null, 2),
+        }],
+      };
+    }
+
+    case 'gui_analyze_screen': {
+      const prompt = requireString('prompt', args.prompt);
+      const displayIdx = typeof args.displayIndex === 'number' ? args.displayIndex : config.displayIndex;
+
+      // Create vision model
+      const model = createModel(config.provider, config.apiProtocol, config.model, config.baseUrl);
+      const apiKey = config.apiKey;
+
+      if (!apiKey) {
+        return {
+          content: [{ type: 'text', text: JSON.stringify({ success: false, error: 'API key not configured' }) }],
+          isError: true,
+        };
+      }
+
+      const result = await analyzeScreen(model, apiKey, prompt, displayIdx);
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            analysis: result.analysis,
+            imageSize: {
+              width: result.imageWidth,
+              height: result.imageHeight,
+              note: 'Coordinates in analysis reference this image size (logical pixels)',
+            },
+          }, null, 2),
+        }],
+      };
+    }
+
+    default:
+      return { content: [{ type: 'text', text: JSON.stringify({ success: false, error: `Unknown tool: ${name}` }) }], isError: true };
+  }
+}

qimingclaw/crates/agent-gui-server/src/mcp/resources.ts

@@ -0,0 +1,77 @@
+/**
+ * MCP Resources: status, permissions, audit log.
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import {
+  ListResourcesRequestSchema,
+  ReadResourceRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { checkScreenRecordingPermission, checkAccessibilityPermission, getPlatform } from '../utils/platform.js';
+
+const PKG_VERSION = process.env.__GUI_AGENT_PKG_VERSION__ ?? '0.0.0';
+
+const RESOURCES = [
+  {
+    uri: 'gui://status',
+    name: 'GUI Agent Status',
+    description: 'Platform, version, running state',
+    mimeType: 'application/json',
+  },
+  {
+    uri: 'gui://permissions',
+    name: 'GUI Agent Permissions',
+    description: 'Screen recording and accessibility permission status',
+    mimeType: 'application/json',
+  },
+  {
+    uri: 'gui://audit-log',
+    name: 'GUI Agent Audit Log',
+    description: 'Recent tool execution audit entries',
+    mimeType: 'application/json',
+  },
+];
+
+export function registerResources(server: Server, config: GuiAgentConfig, auditLog: AuditLog): void {
+  server.setRequestHandler(ListResourcesRequestSchema, async () => ({
+    resources: RESOURCES,
+  }));
+
+  server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+    const { uri } = request.params;
+
+    switch (uri) {
+      case 'gui://status': {
+        const status = {
+          platform: getPlatform(),
+          version: PKG_VERSION,
+          running: true,
+          transport: config.transport,
+          port: config.port,
+          model: config.model,
+          provider: config.provider,
+        };
+        return { contents: [{ uri, mimeType: 'application/json', text: JSON.stringify(status, null, 2) }] };
+      }
+
+      case 'gui://permissions': {
+        const [screenRecording, accessibility] = await Promise.all([
+          checkScreenRecordingPermission(),
+          checkAccessibilityPermission(),
+        ]);
+        const perms = { screenRecording, accessibility, platform: getPlatform() };
+        return { contents: [{ uri, mimeType: 'application/json', text: JSON.stringify(perms, null, 2) }] };
+      }
+
+      case 'gui://audit-log': {
+        const entries = auditLog.getEntries(100);
+        return { contents: [{ uri, mimeType: 'application/json', text: JSON.stringify(entries, null, 2) }] };
+      }
+
+      default:
+        throw new Error(`Unknown resource: ${uri}`);
+    }
+  });
+}

qimingclaw/crates/agent-gui-server/src/mcp/server.ts

@@ -0,0 +1,225 @@
+/**
+ * MCP Server with dual transport:
+ * - HTTP mode (primary): Streamable HTTP on 127.0.0.1:<port>/mcp
+ * - stdio mode (backup): single StdioServerTransport
+ *
+ * HTTP mode manages per-session Server + Transport instances.
+ */
+
+import * as http from 'http';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { getAvailableTools, handleAtomicToolCall } from './atomicTools.js';
+import { TASK_TOOLS, handleTaskTool } from './taskTools.js';
+import { registerResources } from './resources.js';
+import { logInfo, logWarn, logError } from '../utils/logger.js';
+
+const MCP_SESSION_ID_HEADER = 'mcp-session-id';
+const SESSION_CLEANUP_INTERVAL_MS = 60_000;
+
+interface HttpSession {
+  server: Server;
+  transport: StreamableHTTPServerTransport;
+  lastActivity: number;
+}
+
+function createMcpServer(config: GuiAgentConfig, auditLog: AuditLog): Server {
+  const server = new Server(
+    { name: 'gui-agent-server', version: process.env.__GUI_AGENT_PKG_VERSION__ ?? '0.1.0' },
+    { capabilities: { tools: {}, resources: {} } },
+  );
+
+  // Unified tool registration — combines atomic tools + task tools
+  // 使用动态工具列表（根据视觉模型配置过滤 gui_analyze_screen）
+  const allTools = [...getAvailableTools(), ...TASK_TOOLS];
+
+  server.setRequestHandler(ListToolsRequestSchema, async () => ({
+    tools: allTools,
+  }));
+
+  server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
+    const { name, arguments: args = {} } = request.params;
+
+    // Try atomic tools first
+    const atomicResult = await handleAtomicToolCall(name, args, config, auditLog);
+    if (atomicResult !== null) return atomicResult;
+
+    // Try task tools
+    const taskResult = await handleTaskTool(name, args, config, auditLog, {
+      signal: extra.signal,
+      sendNotification: extra.sendNotification?.bind(extra) as any,
+    });
+    if (taskResult !== null) return taskResult;
+
+    return { content: [{ type: 'text', text: `Unknown tool: ${name}` }], isError: true };
+  });
+
+  registerResources(server, config, auditLog);
+  return server;
+}
+
+export interface GuiAgentServer {
+  start(): Promise<void>;
+  stop(): Promise<void>;
+}
+
+export function createGuiAgentServer(config: GuiAgentConfig): GuiAgentServer {
+  const auditLog = new AuditLog();
+
+  if (config.transport === 'stdio') {
+    return createStdioServer(config, auditLog);
+  }
+  return createHttpServer(config, auditLog);
+}
+
+function createStdioServer(config: GuiAgentConfig, auditLog: AuditLog): GuiAgentServer {
+  const server = createMcpServer(config, auditLog);
+  let transport: StdioServerTransport | null = null;
+
+  return {
+    async start() {
+      transport = new StdioServerTransport();
+      await server.connect(transport);
+      logInfo('MCP Server running on stdio');
+    },
+    async stop() {
+      if (transport) {
+        await server.close();
+        transport = null;
+      }
+    },
+  };
+}
+
+function createHttpServer(config: GuiAgentConfig, auditLog: AuditLog): GuiAgentServer {
+  const sessions = new Map<string, HttpSession>();
+  let httpServer: http.Server | null = null;
+  let cleanupTimer: ReturnType<typeof setInterval> | null = null;
+
+  function getOrCreateSession(sessionId: string | undefined, res: http.ServerResponse): HttpSession | null {
+    // Existing session
+    if (sessionId && sessions.has(sessionId)) {
+      const session = sessions.get(sessionId)!;
+      session.lastActivity = Date.now();
+      return session;
+    }
+
+    // New session — only allowed when no sessionId header (initial request)
+    if (sessionId) {
+      res.writeHead(404, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: 'Session not found' }));
+      return null;
+    }
+
+    const server = createMcpServer(config, auditLog);
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: () => crypto.randomUUID(),
+      onsessioninitialized: (id) => {
+        logInfo(`HTTP session created: ${id}`);
+        sessions.set(id, { server, transport, lastActivity: Date.now() });
+      },
+    });
+
+    server.connect(transport);
+    return { server, transport, lastActivity: Date.now() };
+  }
+
+  async function handleRequest(req: http.IncomingMessage, res: http.ServerResponse) {
+    const url = new URL(req.url ?? '/', `http://${req.headers.host}`);
+
+    if (url.pathname !== '/mcp') {
+      res.writeHead(404);
+      res.end('Not found');
+      return;
+    }
+
+    const sessionId = req.headers[MCP_SESSION_ID_HEADER] as string | undefined;
+
+    if (req.method === 'DELETE') {
+      if (sessionId && sessions.has(sessionId)) {
+        const session = sessions.get(sessionId)!;
+        try { await session.transport.close(); } catch { /* ignore */ }
+        sessions.delete(sessionId);
+        logInfo(`HTTP session closed: ${sessionId}`);
+      }
+      res.writeHead(200);
+      res.end();
+      return;
+    }
+
+    if (req.method === 'GET' || req.method === 'POST') {
+      const session = getOrCreateSession(sessionId, res);
+      if (!session) return;
+
+      try {
+        await session.transport.handleRequest(req, res);
+      } catch (err) {
+        logError(`HTTP request error: ${err instanceof Error ? err.message : String(err)}`);
+        if (!res.headersSent) {
+          res.writeHead(500, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ error: 'Internal server error' }));
+        }
+      }
+      return;
+    }
+
+    res.writeHead(405);
+    res.end('Method not allowed');
+  }
+
+  function cleanupStaleSessions() {
+    const now = Date.now();
+    const staleThreshold = 5 * 60 * 1000; // 5 minutes
+    for (const [id, session] of sessions) {
+      if (now - session.lastActivity > staleThreshold) {
+        logInfo(`Cleaning up stale session: ${id}`);
+        session.transport.close().catch(() => {});
+        sessions.delete(id);
+      }
+    }
+  }
+
+  return {
+    async start() {
+      httpServer = http.createServer(handleRequest);
+
+      await new Promise<void>((resolve, reject) => {
+        httpServer!.listen(config.port, '127.0.0.1', () => {
+          logInfo(`MCP Server running on http://127.0.0.1:${config.port}/mcp`);
+          resolve();
+        });
+        httpServer!.on('error', reject);
+      });
+
+      cleanupTimer = setInterval(cleanupStaleSessions, SESSION_CLEANUP_INTERVAL_MS);
+    },
+    async stop() {
+      if (cleanupTimer) {
+        clearInterval(cleanupTimer);
+        cleanupTimer = null;
+      }
+
+      // Close all sessions
+      await Promise.all(
+        Array.from(sessions.entries()).map(async ([id, session]) => {
+          try { await session.transport.close(); } catch { /* ignore */ }
+        }),
+      );
+      sessions.clear();
+
+      // Close HTTP server
+      if (httpServer) {
+        await new Promise<void>((resolve) => {
+          httpServer!.close(() => resolve());
+        });
+        httpServer = null;
+      }
+
+      logInfo('MCP Server stopped');
+    },
+  };
+}

qimingclaw/crates/agent-gui-server/src/mcp/taskTools.ts

@@ -0,0 +1,154 @@
+/**
+ * gui_execute_task MCP tool handler.
+ *
+ * Provides a high-level "execute a GUI task via natural language" tool
+ * backed by the pi-mono Agent loop. Uses a Mutex to ensure only one
+ * GUI task runs at a time.
+ */
+
+import type { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import type { GuiAgentConfig } from '../config.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { createTaskRunner, type ProgressInfo } from '../agent/taskRunner.js';
+import { logInfo, logWarn, logError } from '../utils/logger.js';
+
+export const TASK_TOOLS = [
+  {
+    name: 'gui_execute_task',
+    description:
+      'Execute a GUI automation task described in natural language. ' +
+      'The agent will analyze screenshots and perform mouse/keyboard actions to complete the task. ' +
+      'Only one task can run at a time.',
+    inputSchema: {
+      type: 'object' as const,
+      properties: {
+        task: {
+          type: 'string',
+          description: 'Natural language description of the GUI task to execute',
+        },
+        maxSteps: {
+          type: 'number',
+          description: 'Override max steps for this task (default: server config)',
+        },
+      },
+      required: ['task'],
+    },
+  },
+];
+
+/** Mutex — only one GUI task at a time, second caller waits in queue */
+let mutexPromise: Promise<void> = Promise.resolve();
+
+export async function handleTaskTool(
+  name: string,
+  args: Record<string, unknown>,
+  config: GuiAgentConfig,
+  auditLog: AuditLog,
+  extra?: { signal?: AbortSignal; sendNotification?: (notification: { method: string; params: unknown }) => Promise<void> },
+): Promise<{ content: Array<{ type: string; text?: string }>; isError?: boolean } | null> {
+  if (name !== 'gui_execute_task') {
+    return null; // Not handled by this module
+  }
+
+  const taskText = args.task as string;
+  if (!taskText || typeof taskText !== 'string') {
+    return { content: [{ type: 'text', text: 'Error: task parameter is required and must be a string' }], isError: true };
+  }
+
+  // Mutex — wait for previous task to complete (queue, not reject)
+  let releaseMutex: () => void;
+  const prevMutex = mutexPromise;
+  mutexPromise = new Promise<void>((resolve) => { releaseMutex = resolve; });
+
+  // Check if cancelled while waiting for mutex
+  if (extra?.signal?.aborted) {
+    releaseMutex!();
+    return { content: [{ type: 'text', text: 'Task cancelled before start' }], isError: true };
+  }
+
+  await prevMutex;
+
+  // Check cancellation again after acquiring mutex
+  if (extra?.signal?.aborted) {
+    releaseMutex!();
+    return { content: [{ type: 'text', text: 'Task cancelled before start' }], isError: true };
+  }
+
+  // Build config override
+  const taskConfig = { ...config };
+  if (args.maxSteps !== undefined) {
+    taskConfig.maxSteps = Math.min(Math.max(Number(args.maxSteps), 1), 200);
+  }
+
+  const runner = createTaskRunner(taskConfig, auditLog);
+
+  // Create AbortController that merges external signal
+  const taskAbortController = new AbortController();
+  if (extra?.signal) {
+    extra.signal.addEventListener('abort', () => taskAbortController.abort(), { once: true });
+  }
+
+  // Stable progress token for this task — MCP protocol requires same token for all updates
+  const progressToken = `gui_task_${Date.now()}`;
+
+  // Progress callback — sends MCP notifications
+  const onProgress = async (info: ProgressInfo) => {
+    if (extra?.sendNotification) {
+      try {
+        await extra.sendNotification({
+          method: 'notifications/progress',
+          params: {
+            progressToken,
+            progress: info.step,
+            total: info.total,
+            message: info.message,
+          },
+        });
+      } catch {
+        // Progress notification failure is non-fatal
+      }
+    }
+  };
+
+  try {
+    logInfo(`gui_execute_task started: "${taskText.substring(0, 100)}"`);
+    const result = await runner.run(taskText, taskAbortController.signal, onProgress);
+
+    // Build response
+    const content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> = [];
+
+    // Add text result
+    content.push({
+      type: 'text',
+      text: JSON.stringify({
+        success: result.success,
+        result: result.result,
+        error: result.error,
+        stepsCompleted: result.steps.length,
+        steps: result.steps.map(s => ({
+          stepId: s.stepId,
+          tool: s.tool,
+          success: s.success,
+        })),
+      }, null, 2),
+    });
+
+    // Add final screenshot if available
+    if (result.finalScreenshot) {
+      content.push({
+        type: 'image',
+        data: result.finalScreenshot,
+        mimeType: 'image/jpeg',
+      });
+    }
+
+    logInfo(`gui_execute_task completed: success=${result.success}, steps=${result.steps.length}`);
+    return { content, isError: !result.success };
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    logError(`gui_execute_task error: ${errorMsg}`);
+    return { content: [{ type: 'text', text: `Error: ${errorMsg}` }], isError: true };
+  } finally {
+    releaseMutex!();
+  }
+}

qimingclaw/crates/agent-gui-server/src/safety/auditLog.ts

@@ -0,0 +1,55 @@
+/**
+ * Ring buffer audit log for tracking tool executions.
+ */
+
+export interface AuditEntry {
+  timestamp: string;
+  tool: string;
+  args: Record<string, unknown>;
+  success: boolean;
+  durationMs?: number;
+}
+
+const MAX_ENTRIES = 1000;
+
+export class AuditLog {
+  private entries: AuditEntry[] = [];
+  private head: number = 0;
+  private size: number = 0;
+
+  constructor() {
+    this.entries = new Array(MAX_ENTRIES);
+  }
+
+  record(entry: Omit<AuditEntry, 'timestamp'>): void {
+    const full: AuditEntry = {
+      ...entry,
+      timestamp: new Date().toISOString(),
+    };
+    this.entries[this.head] = full;
+    this.head = (this.head + 1) % MAX_ENTRIES;
+    if (this.size < MAX_ENTRIES) this.size++;
+  }
+
+  /**
+   * Get recent entries, most recent first.
+   */
+  getEntries(count?: number): AuditEntry[] {
+    const n = Math.min(count ?? this.size, this.size);
+    const result: AuditEntry[] = [];
+    for (let i = 0; i < n; i++) {
+      const idx = (this.head - 1 - i + MAX_ENTRIES) % MAX_ENTRIES;
+      result.push(this.entries[idx]);
+    }
+    return result;
+  }
+
+  clear(): void {
+    this.head = 0;
+    this.size = 0;
+  }
+
+  get length(): number {
+    return this.size;
+  }
+}

qimingclaw/crates/agent-gui-server/src/safety/hotkeys.ts

@@ -0,0 +1,82 @@
+/**
+ * Dangerous hotkey blacklist with platform-specific rules.
+ */
+
+import { getPlatform, type Platform } from '../utils/platform.js';
+
+export interface HotkeyValidationResult {
+  blocked: boolean;
+  reason?: string;
+}
+
+interface BlacklistEntry {
+  keys: string[];
+  description: string;
+}
+
+const BLACKLISTS: Record<Platform, BlacklistEntry[]> = {
+  macos: [
+    { keys: ['meta', 'q'], description: 'Quit application' },
+    { keys: ['meta', 'w'], description: 'Close window' },
+    { keys: ['meta', 'alt', 'escape'], description: 'Force quit' },
+    { keys: ['meta', 'shift', 'q'], description: 'Log out' },
+    { keys: ['meta', 'alt', 'shift', 'q'], description: 'Force log out' },
+  ],
+  windows: [
+    { keys: ['alt', 'f4'], description: 'Close application' },
+    { keys: ['control', 'alt', 'delete'], description: 'System menu' },
+    { keys: ['meta', 'l'], description: 'Lock screen' },
+  ],
+  linux: [
+    { keys: ['alt', 'f4'], description: 'Close window' },
+    { keys: ['control', 'alt', 'delete'], description: 'System interrupt' },
+    { keys: ['control', 'alt', 'backspace'], description: 'Kill X server' },
+  ],
+};
+
+function normalizeKey(key: string): string {
+  const lower = key.toLowerCase().trim();
+  // Normalize common aliases
+  switch (lower) {
+    case 'cmd':
+    case 'command':
+    case 'super':
+    case 'win':
+      return 'meta';
+    case 'opt':
+    case 'option':
+      return 'alt';
+    case 'ctrl':
+      return 'control';
+    case 'esc':
+      return 'escape';
+    case 'del':
+      return 'delete';
+    default:
+      return lower;
+  }
+}
+
+/**
+ * Validate a hotkey combination against the platform blacklist.
+ */
+export function validateHotkey(keys: string[]): HotkeyValidationResult {
+  const platform = getPlatform();
+  const blacklist = BLACKLISTS[platform];
+  const normalized = keys.map(normalizeKey).sort();
+
+  for (const entry of blacklist) {
+    const entryNormalized = entry.keys.map(normalizeKey).sort();
+    if (
+      normalized.length === entryNormalized.length &&
+      normalized.every((k, i) => k === entryNormalized[i])
+    ) {
+      return {
+        blocked: true,
+        reason: `${entry.description} (${keys.join('+')})`,
+      };
+    }
+  }
+
+  return { blocked: false };
+}

qimingclaw/crates/agent-gui-server/src/utils/errors.ts

@@ -0,0 +1,57 @@
+/**
+ * Structured error types for agent-gui-server
+ */
+
+/** Configuration validation error */
+export class ConfigError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ConfigError';
+  }
+}
+
+/** Desktop automation operation error */
+export class DesktopError extends Error {
+  constructor(
+    public readonly operation: string,
+    public readonly cause: Error,
+  ) {
+    super(`Desktop operation "${operation}" failed: ${cause.message}`);
+    this.name = 'DesktopError';
+  }
+}
+
+/** Coordinate resolution error */
+export class CoordinateError extends Error {
+  constructor(
+    public readonly coordinateMode: string,
+    public readonly rawX: number,
+    public readonly rawY: number,
+  ) {
+    super(`Coordinate resolution failed (mode: ${coordinateMode}, x: ${rawX}, y: ${rawY})`);
+    this.name = 'CoordinateError';
+  }
+}
+
+/** Safety layer blocked operation */
+export class SafetyError extends Error {
+  constructor(
+    public readonly keys: string[],
+    public readonly reason: string,
+  ) {
+    super(`Blocked: ${reason} (keys: ${keys.join('+')})`);
+    this.name = 'SafetyError';
+  }
+}
+
+/** Task execution error */
+export class TaskExecutionError extends Error {
+  constructor(
+    public readonly taskText: string,
+    public readonly step: number,
+    public readonly cause: Error,
+  ) {
+    super(`Task execution failed at step ${step}: ${cause.message}`);
+    this.name = 'TaskExecutionError';
+  }
+}

qimingclaw/crates/agent-gui-server/src/utils/logger.ts

@@ -0,0 +1,104 @@
+/**
+ * Logging utilities — stderr only, stdout is the MCP JSON-RPC channel.
+ *
+ * When the environment variable GUI_AGENT_LOG_FILE is set, log lines are
+ * also appended to that file so the Electron host can tail them into main.log.
+ *
+ * Log rotation: file is named by date (e.g. gui-agent-2026-03-19.log).
+ * A new file is created each day. Old files beyond MAX_LOG_FILES are deleted.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+const logFilePath = process.env.GUI_AGENT_LOG_FILE;
+const MAX_LOG_FILES = 7;
+
+let logStream: fs.WriteStream | null = null;
+let logDir = '';
+let logBaseName = '';
+let logExt = '';
+let currentDateStr = '';
+
+function dateStr(): string {
+  const d = new Date();
+  const pad2 = (n: number) => String(n).padStart(2, '0');
+  return `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}`;
+}
+
+function openLogFile(): void {
+  if (!logFilePath) return;
+
+  const today = dateStr();
+  if (today === currentDateStr && logStream) return;
+
+  if (logStream) {
+    try { logStream.end(); } catch { /* ignore */ }
+  }
+
+  currentDateStr = today;
+  const dated = path.join(logDir, `${logBaseName}-${today}${logExt}`);
+  try {
+    logStream = fs.createWriteStream(dated, { flags: 'a' });
+  } catch {
+    logStream = null;
+  }
+
+  cleanupOldLogs();
+}
+
+function cleanupOldLogs(): void {
+  if (!logDir || !logBaseName) return;
+  try {
+    const prefix = `${logBaseName}-`;
+    const files = fs.readdirSync(logDir)
+      .filter(f => f.startsWith(prefix) && f.endsWith(logExt))
+      .sort()
+      .reverse();
+    for (let i = MAX_LOG_FILES; i < files.length; i++) {
+      try { fs.unlinkSync(path.join(logDir, files[i])); } catch { /* ignore */ }
+    }
+  } catch { /* ignore */ }
+}
+
+if (logFilePath) {
+  try {
+    logDir = path.dirname(logFilePath);
+    if (logDir && !fs.existsSync(logDir)) {
+      fs.mkdirSync(logDir, { recursive: true });
+    }
+    const fullName = path.basename(logFilePath);
+    const dotIdx = fullName.lastIndexOf('.');
+    if (dotIdx > 0) {
+      logBaseName = fullName.substring(0, dotIdx);
+      logExt = fullName.substring(dotIdx);
+    } else {
+      logBaseName = fullName;
+      logExt = '.log';
+    }
+    openLogFile();
+  } catch {
+    // If file creation fails, continue without file logging
+  }
+}
+
+function timestamp(): string {
+  const d = new Date();
+  const pad2 = (n: number) => String(n).padStart(2, '0');
+  const pad3 = (n: number) => String(n).padStart(3, '0');
+  return `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())} ${pad2(d.getHours())}:${pad2(d.getMinutes())}:${pad2(d.getSeconds())}.${pad3(d.getMilliseconds())}`;
+}
+
+export function log(level: string, msg: string): void {
+  const line = `[${timestamp()}] [${level.toLowerCase()}]  [gui-agent] ${msg}\n`;
+  process.stderr.write(line);
+  if (logFilePath) {
+    openLogFile();
+    logStream?.write(line);
+  }
+}
+
+export const logInfo = (msg: string) => log('INFO', msg);
+export const logWarn = (msg: string) => log('WARN', msg);
+export const logError = (msg: string) => log('ERROR', msg);
+export const logDebug = (msg: string) => log('DEBUG', msg);

qimingclaw/crates/agent-gui-server/src/utils/platform.ts

@@ -0,0 +1,57 @@
+/**
+ * Platform detection and permission checking utilities.
+ */
+
+import * as os from 'os';
+
+export type Platform = 'macos' | 'windows' | 'linux';
+
+export function getPlatform(): Platform {
+  switch (os.platform()) {
+    case 'darwin': return 'macos';
+    case 'win32': return 'windows';
+    default: return 'linux';
+  }
+}
+
+/**
+ * Check screen recording permission (macOS only).
+ * On other platforms, always returns true.
+ */
+export async function checkScreenRecordingPermission(): Promise<boolean> {
+  if (getPlatform() !== 'macos') return true;
+  try {
+    // Attempt a test capture to verify permission
+    const { screen } = await import('@nut-tree-fork/nut-js');
+    await screen.width();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Check accessibility permission (macOS only).
+ * On other platforms, always returns true.
+ */
+export async function checkAccessibilityPermission(): Promise<boolean> {
+  if (getPlatform() !== 'macos') return true;
+  try {
+    const { mouse } = await import('@nut-tree-fork/nut-js');
+    await mouse.getPosition();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Get platform-specific paste key combination.
+ * macOS: Meta+V, Windows/Linux: Control+V
+ */
+export function getPlatformPasteKeys(): string[] {
+  if (getPlatform() === 'macos') {
+    return ['Meta', 'V'];
+  }
+  return ['Control', 'V'];
+}

qimingclaw/crates/agent-gui-server/src/windowsMcp/healthCheck.ts

@@ -0,0 +1,127 @@
+/**
+ * Windows-MCP 健康检查
+ *
+ * 使用官方 MCP SDK（Streamable HTTP Transport + Client）完成 initialize 握手，
+ * 并调用 listTools() 确认服务真正可用，与 qiming-mcp-stdio-proxy 针对 HTTP MCP 的判定方式一致。
+ *
+ * 注意：StreamableHTTPClientTransport 在 send() 里会用内部 AbortController 的 signal 覆盖
+ * requestInit.signal，因此超时必须外层 Promise.race，并在 finally 里 client.close() 以 abort 传输层。
+ */
+
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+
+/** 健康检查选项 */
+export interface HealthCheckOptions {
+  /** 端口号 */
+  port: number;
+  /** 超时时间（毫秒），默认 5000 */
+  timeout?: number;
+  /** MCP 端点路径，默认 '/mcp' */
+  endpoint?: string;
+}
+
+/** 健康检查结果 */
+export interface HealthCheckResult {
+  /** 是否健康 */
+  healthy: boolean;
+  /** 响应时间（毫秒） */
+  responseTime?: number;
+  /** 错误信息 */
+  error?: string;
+}
+
+async function withTimeout<T>(promise: Promise<T>, ms: number, message: string): Promise<T> {
+  let timeoutId: ReturnType<typeof setTimeout>;
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeoutId = setTimeout(() => reject(new Error(message)), ms);
+  });
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    clearTimeout(timeoutId!);
+  }
+}
+
+/**
+ * 执行健康检查
+ *
+ * 通过 SDK 建立 MCP 会话并 listTools，避免手写 fetch 解析 SSE/NDJSON。
+ */
+export async function healthCheck(options: HealthCheckOptions): Promise<HealthCheckResult> {
+  const { port, timeout = 5000, endpoint = '/mcp' } = options;
+  const url = new URL(`http://127.0.0.1:${port}${endpoint}`);
+  const startTime = Date.now();
+
+  const client = new Client({
+    name: 'windows-mcp-health-check',
+    version: '1.0.0',
+  });
+
+  try {
+    await withTimeout(
+      (async () => {
+        const transport = new StreamableHTTPClientTransport(url, {
+          requestInit: {
+            headers: {
+              Accept: 'application/json, text/event-stream',
+            },
+          },
+        });
+        await client.connect(transport);
+        await client.listTools();
+      })(),
+      timeout,
+      'Timeout'
+    );
+
+    return { healthy: true, responseTime: Date.now() - startTime };
+  } catch (error) {
+    const responseTime = Date.now() - startTime;
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    if (errorMessage === 'Timeout') {
+      return { healthy: false, error: 'Timeout', responseTime };
+    }
+
+    return { healthy: false, error: errorMessage, responseTime };
+  } finally {
+    try {
+      await client.close();
+    } catch {
+      /* ignore */
+    }
+  }
+}
+
+/**
+ * 等待服务就绪
+ *
+ * 多次尝试健康检查，直到服务就绪或超时。
+ */
+export async function waitForReady(
+  port: number,
+  options?: {
+    /** 总超时时间（毫秒），默认 30000 */
+    timeout?: number;
+    /** 每次检查间隔（毫秒），默认 500 */
+    interval?: number;
+    /** 单次请求超时（毫秒），默认 5000 */
+    requestTimeout?: number;
+    /** MCP 端点路径，默认 '/mcp' */
+    endpoint?: string;
+  }
+): Promise<boolean> {
+  const { timeout = 30000, interval = 500, requestTimeout = 5000, endpoint } = options || {};
+  const startTime = Date.now();
+
+  while (Date.now() - startTime < timeout) {
+    const result = await healthCheck({ port, timeout: requestTimeout, endpoint });
+    if (result.healthy) {
+      return true;
+    }
+    await new Promise((resolve) => setTimeout(resolve, interval));
+  }
+
+  return false;
+}

qimingclaw/crates/agent-gui-server/src/windowsMcp/index.ts

@@ -0,0 +1,15 @@
+/**
+ * Windows-MCP 模块导出
+ */
+
+export { WindowsMcpManager } from './manager.js';
+export { healthCheck, waitForReady } from './healthCheck.js';
+export type { HealthCheckOptions, HealthCheckResult } from './healthCheck.js';
+export type {
+  WindowsMcpStatus,
+  ProcessConfig,
+  ProcessRunner,
+  StartResult,
+  StopResult,
+  WindowsMcpConfig,
+} from './types.js';

qimingclaw/crates/agent-gui-server/src/windowsMcp/manager.ts

@@ -0,0 +1,280 @@
+/**
+ * Windows-MCP 管理器
+ *
+ * 管理 windows-mcp 子进程的生命周期，提供启动、停止、健康检查等功能。
+ *
+ * 设计模式：依赖注入
+ * - 核心逻辑在此模块中实现
+ * - 进程管理由调用方通过 ProcessRunner 接口注入
+ */
+
+import type {
+  WindowsMcpStatus,
+  ProcessConfig,
+  ProcessRunner,
+  StartResult,
+  StopResult,
+  WindowsMcpConfig,
+} from './types.js';
+import { healthCheck, waitForReady } from './healthCheck.js';
+
+/** 默认配置 */
+const DEFAULT_CONFIG: Required<WindowsMcpConfig> = {
+  healthCheckInterval: 30000,
+  startupTimeout: 30000,
+  healthCheckTimeout: 5000,
+  maxRestarts: 3,
+};
+
+/**
+ * Windows-MCP 管理器
+ *
+ * 使用示例：
+ * ```typescript
+ * const manager = new WindowsMcpManager();
+ * manager.setProcessRunner(new MyProcessRunner());
+ *
+ * const result = await manager.start(60020, (port) => ({
+ *   command: '/path/to/uv',
+ *   args: ['tool', 'run', 'windows-mcp', '--transport', 'streamable-http', '--port', port.toString()],
+ *   env: process.env as Record<string, string>,
+ * }));
+ *
+ * if (result.success) {
+ *   console.log('Started on port:', result.port);
+ * }
+ * ```
+ */
+export class WindowsMcpManager {
+  private runner: ProcessRunner | null = null;
+  private port: number = 0;
+  private running: boolean = false;
+  /** 合并并发 start()，避免双实例抢同一端口 */
+  private startInFlight: Promise<StartResult> | null = null;
+  private lastError: string | null = null;
+  private healthCheckTimer: ReturnType<typeof setInterval> | null = null;
+  /** 历史累计重启/失败次数（观测用） */
+  private restartCount: number = 0;
+  /** 连续健康检查失败次数，达到 maxRestarts 后结束子进程释放端口 */
+  private healthFailStreak: number = 0;
+  private config: Required<WindowsMcpConfig>;
+
+  constructor(config?: WindowsMcpConfig) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * 设置进程运行器
+   *
+   * 必须在调用 start() 之前设置。
+   *
+   * @param runner 进程运行器实例
+   */
+  setProcessRunner(runner: ProcessRunner): void {
+    this.runner = runner;
+  }
+
+  /**
+   * 启动 windows-mcp 服务
+   *
+   * @param port 端口号
+   * @param buildConfig 构建进程配置的函数
+   * @returns 启动结果
+   */
+  async start(
+    port: number,
+    buildConfig: (port: number) => ProcessConfig
+  ): Promise<StartResult> {
+    if (this.running) {
+      return { success: true, port: this.port };
+    }
+
+    if (this.startInFlight) {
+      return this.startInFlight;
+    }
+
+    if (!this.runner) {
+      const error = 'ProcessRunner not set. Call setProcessRunner() first.';
+      this.lastError = error;
+      return { success: false, error };
+    }
+
+    this.startInFlight = this.startImpl(port, buildConfig).finally(() => {
+      this.startInFlight = null;
+    });
+    return this.startInFlight;
+  }
+
+  private async startImpl(
+    port: number,
+    buildConfig: (port: number) => ProcessConfig
+  ): Promise<StartResult> {
+    const runner = this.runner;
+    if (!runner) {
+      const error = 'ProcessRunner not set. Call setProcessRunner() first.';
+      this.lastError = error;
+      return { success: false, error };
+    }
+
+    this.lastError = null;
+    const config = buildConfig(port);
+
+    try {
+      // 启动进程
+      const result = await runner.start(config);
+      if (!result.success) {
+        this.lastError = result.error || 'Failed to start process';
+        return { success: false, error: this.lastError };
+      }
+
+      // 等待服务就绪
+      const ready = await waitForReady(port, {
+        timeout: this.config.startupTimeout,
+        requestTimeout: this.config.healthCheckTimeout,
+      });
+
+      if (!ready) {
+        await runner.stop();
+        this.lastError = 'Service failed to become ready within timeout';
+        return { success: false, error: this.lastError };
+      }
+
+      // 成功启动
+      this.port = port;
+      this.running = true;
+      this.restartCount = 0;
+      this.healthFailStreak = 0;
+
+      // 启动健康检查
+      this.startHealthCheck();
+
+      return { success: true, port };
+    } catch (error) {
+      // 若 runner.start 之后、置 running 之前抛错，避免遗留子进程
+      try {
+        await runner.stop();
+      } catch {
+        /* ignore */
+      }
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      this.lastError = errorMessage;
+      this.running = false;
+      this.port = 0;
+      return { success: false, error: errorMessage };
+    }
+  }
+
+  /**
+   * 停止 windows-mcp 服务
+   *
+   * 注意：即使 `this.running === false`（例如健康检查失败时已把标记清掉），仍必须调用
+   * runner.stop()，否则子进程（uv/python）可能仍在监听端口，导致退出 Electron 后残留。
+   */
+  async stop(): Promise<StopResult> {
+    this.stopHealthCheck();
+
+    if (!this.runner) {
+      this.running = false;
+      this.port = 0;
+      return { success: true };
+    }
+
+    try {
+      const result = await this.runner.stop();
+      this.running = false;
+      this.port = 0;
+      return result.success ? { success: true } : result;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      this.running = false;
+      this.port = 0;
+      return { success: false, error: errorMessage };
+    }
+  }
+
+  /**
+   * 重启 windows-mcp 服务
+   */
+  async restart(port: number, buildConfig: (port: number) => ProcessConfig): Promise<StartResult> {
+    await this.stop();
+    return this.start(port, buildConfig);
+  }
+
+  /**
+   * 获取当前状态
+   */
+  getStatus(): WindowsMcpStatus {
+    return {
+      running: this.running,
+      port: this.running ? this.port : undefined,
+      pid: this.runner?.getPid() ?? undefined,
+      error: this.lastError ?? undefined,
+    };
+  }
+
+  /**
+   * 获取 MCP 服务 URL
+   *
+   * @returns URL 或 null（如果未运行）
+   */
+  getMcpUrl(): string | null {
+    if (!this.running || !this.port) {
+      return null;
+    }
+    return `http://127.0.0.1:${this.port}/mcp`;
+  }
+
+  /**
+   * 启动健康检查定时器
+   */
+  private startHealthCheck(): void {
+    this.stopHealthCheck();
+
+    this.healthCheckTimer = setInterval(() => {
+      void (async () => {
+        if (!this.running || !this.port || !this.runner) {
+          return;
+        }
+
+        const port = this.port;
+        const result = await healthCheck({
+          port,
+          timeout: this.config.healthCheckTimeout,
+        });
+
+        if (!result.healthy) {
+          this.lastError = `Health check failed: ${result.error}`;
+          this.healthFailStreak++;
+          if (this.healthFailStreak < this.config.maxRestarts) {
+            return;
+          }
+          // 连续失败达到阈值：结束子进程，避免端口占用且与 stop() 状态一致
+          this.stopHealthCheck();
+          this.running = false;
+          this.port = 0;
+          this.healthFailStreak = 0;
+          this.restartCount++;
+          try {
+            await this.runner.stop();
+          } catch {
+            /* ignore */
+          }
+          return;
+        }
+
+        this.healthFailStreak = 0;
+        this.lastError = null;
+      })();
+    }, this.config.healthCheckInterval);
+  }
+
+  /**
+   * 停止健康检查定时器
+   */
+  private stopHealthCheck(): void {
+    if (this.healthCheckTimer) {
+      clearInterval(this.healthCheckTimer);
+      this.healthCheckTimer = null;
+    }
+  }
+}

qimingclaw/crates/agent-gui-server/src/windowsMcp/types.ts

@@ -0,0 +1,82 @@
+/**
+ * Windows-MCP 类型定义
+ *
+ * 定义 WindowsMcpManager 的接口，由调用方（Electron 客户端）实现具体的进程管理逻辑。
+ */
+
+/** Windows-MCP 服务状态 */
+export interface WindowsMcpStatus {
+  /** 是否正在运行 */
+  running: boolean;
+  /** HTTP 端口号 */
+  port?: number;
+  /** 进程 PID */
+  pid?: number;
+  /** 错误信息（如果启动失败） */
+  error?: string;
+}
+
+/** 进程启动配置（由调用方构建） */
+export interface ProcessConfig {
+  /** uv 可执行文件路径 */
+  command: string;
+  /** 启动参数 */
+  args: string[];
+  /** 环境变量 */
+  env: Record<string, string>;
+  /** 工作目录（可选） */
+  cwd?: string;
+}
+
+/** 启动结果 */
+export interface StartResult {
+  /** 是否成功 */
+  success: boolean;
+  /** 端口号（成功时） */
+  port?: number;
+  /** 错误信息（失败时） */
+  error?: string;
+}
+
+/** 停止结果 */
+export interface StopResult {
+  /** 是否成功 */
+  success: boolean;
+  /** 错误信息（失败时） */
+  error?: string;
+}
+
+/** 进程运行器接口（由调用方实现） */
+export interface ProcessRunner {
+  /**
+   * 启动进程
+   * @param config 进程配置
+   * @returns 启动结果
+   */
+  start(config: ProcessConfig): Promise<StartResult>;
+
+  /**
+   * 停止进程
+   * @returns 停止结果
+   */
+  stop(): Promise<StopResult>;
+
+  /**
+   * 获取进程 PID
+   * @returns PID 或 null
+   */
+  getPid(): number | null;
+}
+
+/** WindowsMcpManager 配置 */
+export interface WindowsMcpConfig {
+  /** 健康检查间隔（毫秒），默认 30000 */
+  healthCheckInterval?: number;
+  /** 启动超时（毫秒），默认 30000 */
+  startupTimeout?: number;
+  /** 健康检查超时（毫秒），默认 5000 */
+  healthCheckTimeout?: number;
+  /** 最大重启次数，默认 3 */
+  /** 连续健康检查失败多少次后结束子进程（防抖，避免单次抖动误杀） */
+  maxRestarts?: number;
+}