chore: initialize qiming workspace repository

qimingclaw/crates/agent-gui-server/src/agent/memoryManager.ts

@@ -0,0 +1,283 @@
+/**
+ * Three-layer memory management with LLM summarization.
+ *
+ * Layers:
+ *   summaryMemory (budget: 2000 chars) — compressed history
+ *   recentMemory  (budget: 500 chars)  — recent step records
+ *   pendingMemory (no budget)          — current step in progress
+ *
+ * Memory text is injected into systemPrompt (not via transformContext).
+ * Screenshot pruning (pruneScreenshots) is used as transformContext hook.
+ */
+
+import { complete } from '@mariozechner/pi-ai';
+import type { Model, Api, AssistantMessage } from '@mariozechner/pi-ai';
+import { logInfo, logDebug, logError } from '../utils/logger.js';
+
+// Type alias for AgentMessage — we only need role and content fields
+interface MessageLike {
+  role: string;
+  content: unknown;
+  [key: string]: unknown;
+}
+
+const MEMORY_SUMMARIZATION_PROMPT = `You are a memory summarization assistant for a GUI automation agent.
+Your task is to condense step-by-step action records into concise memory entries.
+
+Output JSON:
+{
+  "summary": "Concise summary of the actions taken and their outcomes..."
+}
+
+Guidelines:
+- Preserve key information: what was done, what succeeded/failed, current state
+- Remove redundant details and repetitive patterns
+- Keep the summary actionable — the agent needs to know what happened to plan next steps`;
+
+export class MemoryManager {
+  private summaryMemory: string = '';
+  private recentMemory: string = '';
+  private pendingMemory: string = '';
+
+  private recentBudget: number = 500;
+  private summaryBudget: number = 2000;
+  private screenshotKeepCount: number = 3;
+
+  private memoryModel: Model<any>;
+  private apiKey: string;
+
+  constructor(memoryModel: Model<any>, apiKey: string) {
+    this.memoryModel = memoryModel;
+    this.apiKey = apiKey;
+  }
+
+  /**
+   * Record a pending step (currently executing).
+   */
+  addPendingStep(stepId: number, goal: string): void {
+    this.pendingMemory = `Step ${stepId} | Goal: ${goal}`;
+  }
+
+  /**
+   * Finalize a step — move from pending to recent, trigger compression if over budget.
+   */
+  async finalizeStep(stepId: number, evaluation: 'success' | 'failed'): Promise<void> {
+    const entry = `Step ${stepId} | Eval: ${evaluation} | ${this.pendingMemory.replace(`Step ${stepId} | `, '')}`;
+    this.pendingMemory = '';
+
+    if (this.recentMemory) {
+      this.recentMemory += '\n' + entry;
+    } else {
+      this.recentMemory = entry;
+    }
+
+    // Check if recent memory exceeds budget
+    if (this.recentMemory.length > this.recentBudget) {
+      await this.compressRecent();
+    }
+  }
+
+  /**
+   * Compose all three layers into a single text block for systemPrompt injection.
+   */
+  compose(): string {
+    const parts: string[] = [];
+    if (this.summaryMemory) {
+      parts.push(`[Summarized history]\n${this.summaryMemory}`);
+    }
+    if (this.recentMemory) {
+      parts.push(`[Recent steps]\n${this.recentMemory}`);
+    }
+    if (this.pendingMemory) {
+      parts.push(`[Current step]\n${this.pendingMemory}`);
+    }
+    return parts.join('\n\n');
+  }
+
+  /**
+   * Prune screenshots from messages — for use as transformContext hook.
+   *
+   * Keeps last N screenshots, replaces older ones with text placeholders.
+   * This only affects the LLM input, not the Agent's internal message history.
+   */
+  pruneScreenshots<T>(messages: T[]): T[] {
+    // Find all messages with image content
+    const imageIndices: number[] = [];
+    for (let i = 0; i < messages.length; i++) {
+      const msg = messages[i] as any;
+      if (hasImageContent(msg.content)) {
+        imageIndices.push(i);
+      }
+    }
+
+    // If we have fewer images than the keep count, no pruning needed
+    if (imageIndices.length <= this.screenshotKeepCount) {
+      return this.applyTokenHardLimit(messages);
+    }
+
+    // Clone messages array and replace old screenshots
+    const pruned = messages.map((msg, idx) => {
+      if (!imageIndices.includes(idx)) return msg;
+
+      // Keep the most recent N screenshots
+      const imageRank = imageIndices.indexOf(idx);
+      const keepFrom = imageIndices.length - this.screenshotKeepCount;
+      if (imageRank >= keepFrom) return msg;
+
+      // Replace image content with text placeholder
+      return {
+        ...msg,
+        content: replaceImageContent((msg as any).content, `[Screenshot removed - Step ${imageRank + 1}]`),
+      };
+    });
+
+    return this.applyTokenHardLimit(pruned);
+  }
+
+  /**
+   * Token hard limit fallback — estimate total tokens and force-remove
+   * oldest images if exceeding contextWindow * 0.9.
+   */
+  private applyTokenHardLimit<T>(messages: T[]): T[] {
+    const MAX_CONTEXT_TOKENS = 128_000; // conservative default
+    const TOKEN_THRESHOLD = MAX_CONTEXT_TOKENS * 0.9;
+    const TOKENS_PER_IMAGE = 800;
+
+    let totalTokens = 0;
+    const imagePositions: number[] = [];
+
+    for (let i = 0; i < messages.length; i++) {
+      const msg = messages[i] as any;
+      if (!msg.content) continue;
+
+      if (Array.isArray(msg.content)) {
+        for (const c of msg.content) {
+          if (c.type === 'image') {
+            totalTokens += TOKENS_PER_IMAGE;
+            imagePositions.push(i);
+          } else if (c.type === 'text' && c.text) {
+            totalTokens += Math.ceil(c.text.length / 3);
+          }
+        }
+      } else if (typeof msg.content === 'string') {
+        totalTokens += Math.ceil(msg.content.length / 3);
+      }
+    }
+
+    if (totalTokens <= TOKEN_THRESHOLD) {
+      return messages;
+    }
+
+    // Force-remove images from oldest messages until under threshold
+    logDebug(`Token hard limit: estimated ${totalTokens} tokens, threshold ${TOKEN_THRESHOLD}, removing oldest images`);
+    const result = [...messages];
+    for (const idx of imagePositions) {
+      if (totalTokens <= TOKEN_THRESHOLD) break;
+      const msg = result[idx] as any;
+      if (hasImageContent(msg.content)) {
+        result[idx] = {
+          ...msg,
+          content: replaceImageContent(msg.content, '[Screenshot removed - token limit]'),
+        } as T;
+        totalTokens -= TOKENS_PER_IMAGE;
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Compress recent memory into summary via LLM call.
+   */
+  private async compressRecent(): Promise<void> {
+    try {
+      logDebug(`Compressing recent memory (${this.recentMemory.length} chars)`);
+
+      const summary = await this.summarize(this.recentMemory);
+
+      if (this.summaryMemory) {
+        this.summaryMemory += '\n' + summary;
+      } else {
+        this.summaryMemory = summary;
+      }
+      this.recentMemory = '';
+
+      // Check if summary also exceeds budget
+      if (this.summaryMemory.length > this.summaryBudget) {
+        await this.compressSummary();
+      }
+    } catch (err) {
+      logError(`Memory compression failed: ${err instanceof Error ? err.message : String(err)}`);
+      // On failure, keep recent memory as-is rather than losing data
+    }
+  }
+
+  /**
+   * Second-level compression: summarize the summary.
+   */
+  private async compressSummary(): Promise<void> {
+    try {
+      logDebug(`Compressing summary memory (${this.summaryMemory.length} chars)`);
+      const summary = await this.summarize(this.summaryMemory);
+      this.summaryMemory = summary;
+    } catch (err) {
+      logError(`Summary compression failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  /**
+   * Call the memory model to generate a summary.
+   * If apiKey is empty, just returns the original text (no-op compression).
+   */
+  private async summarize(text: string): Promise<string> {
+    // If no API key is configured, skip summarization and return original text
+    if (!this.apiKey) {
+      return text;
+    }
+
+    const result: AssistantMessage = await complete(this.memoryModel, {
+      systemPrompt: MEMORY_SUMMARIZATION_PROMPT,
+      messages: [
+        { role: 'user' as const, content: text, timestamp: Date.now() },
+      ],
+    }, {
+      apiKey: this.apiKey,
+    });
+
+    // Extract text from response
+    const textContent = result.content.find(c => c.type === 'text');
+    if (!textContent || textContent.type !== 'text') {
+      throw new Error('Memory model returned no text content');
+    }
+
+    // Try to parse JSON response
+    try {
+      const parsed = JSON.parse(textContent.text);
+      return parsed.summary || textContent.text;
+    } catch {
+      // If not JSON, use the raw text
+      return textContent.text;
+    }
+  }
+}
+
+// --- Helpers ---
+
+function hasImageContent(content: unknown): boolean {
+  if (Array.isArray(content)) {
+    return content.some(c => c && typeof c === 'object' && 'type' in c && c.type === 'image');
+  }
+  return false;
+}
+
+function replaceImageContent(content: unknown, placeholder: string): unknown {
+  if (Array.isArray(content)) {
+    return content.map(c => {
+      if (c && typeof c === 'object' && 'type' in c && c.type === 'image') {
+        return { type: 'text', text: placeholder };
+      }
+      return c;
+    });
+  }
+  return content;
+}

qimingclaw/crates/agent-gui-server/src/agent/stuckDetector.ts

@@ -0,0 +1,80 @@
+/**
+ * Stuck detection: consecutive screenshot similarity comparison.
+ *
+ * Resizes screenshots to 32x32 thumbnails and compares pixel mean difference.
+ * Consecutive N steps with difference < threshold = stuck.
+ */
+
+import sharp from 'sharp';
+
+const THUMB_SIZE = 32;
+
+export class StuckDetector {
+  private threshold: number;
+  private similarityThreshold: number;
+  private previousThumbnails: Buffer[] = [];
+  private consecutiveSimilar: number = 0;
+
+  constructor(threshold: number = 3, similarityThreshold: number = 0.05) {
+    this.threshold = threshold;
+    this.similarityThreshold = similarityThreshold;
+  }
+
+  /**
+   * Check if the agent is stuck by comparing the latest screenshot
+   * with previous screenshots.
+   *
+   * @param screenshotBase64 - Base64-encoded screenshot
+   * @returns { stuck, consecutiveSimilar }
+   */
+  async check(screenshotBase64: string): Promise<{ stuck: boolean; consecutiveSimilar: number }> {
+    const thumbnail = await this.createThumbnail(screenshotBase64);
+
+    if (this.previousThumbnails.length > 0) {
+      const lastThumb = this.previousThumbnails[this.previousThumbnails.length - 1];
+      const diff = this.computeDifference(thumbnail, lastThumb);
+
+      if (diff < this.similarityThreshold) {
+        this.consecutiveSimilar++;
+      } else {
+        this.consecutiveSimilar = 0;
+      }
+    }
+
+    // Keep only threshold count of thumbnails
+    this.previousThumbnails.push(thumbnail);
+    if (this.previousThumbnails.length > this.threshold + 1) {
+      this.previousThumbnails.shift();
+    }
+
+    return {
+      stuck: this.consecutiveSimilar >= this.threshold,
+      consecutiveSimilar: this.consecutiveSimilar,
+    };
+  }
+
+  reset(): void {
+    this.previousThumbnails = [];
+    this.consecutiveSimilar = 0;
+  }
+
+  private async createThumbnail(base64: string): Promise<Buffer> {
+    const buffer = Buffer.from(base64, 'base64');
+    return sharp(buffer)
+      .resize(THUMB_SIZE, THUMB_SIZE, { fit: 'fill' })
+      .raw()
+      .toBuffer();
+  }
+
+  private computeDifference(a: Buffer, b: Buffer): number {
+    if (a.length !== b.length) return 1;
+
+    let totalDiff = 0;
+    for (let i = 0; i < a.length; i++) {
+      totalDiff += Math.abs(a[i] - b[i]);
+    }
+
+    // Normalize: max per pixel channel is 255, total pixels * channels = a.length
+    return totalDiff / (a.length * 255);
+  }
+}

qimingclaw/crates/agent-gui-server/src/agent/systemPrompt.ts

@@ -0,0 +1,49 @@
+/**
+ * GUI Agent system prompt template.
+ *
+ * Builds the system prompt for the pi-mono Agent, including
+ * role description, tool guidance, and memory injection.
+ */
+
+export function buildSystemPrompt(taskText: string, memoryText: string): string {
+  const memorySection = memoryText
+    ? `\n\n## Previous Actions Memory\n${memoryText}`
+    : '';
+
+  return `You are a GUI automation agent that controls a desktop computer to complete tasks.
+You can see the screen through screenshots and interact using mouse and keyboard.
+
+## Your Task
+${taskText}
+
+## Available Tools
+
+- **computer_screenshot**: Capture the current screen. Use this to see what's on screen before acting.
+- **computer_click**: Click at coordinates (x, y). Use for clicking buttons, links, icons.
+- **computer_type**: Type text. Supports CJK characters and long text.
+- **computer_scroll**: Scroll at position (x, y) with deltaY (positive=down, negative=up).
+- **computer_hotkey**: Press key combinations (e.g. ["Meta", "C"] for copy). Some dangerous combinations are blocked for safety.
+- **computer_wait**: Wait for a specified duration (milliseconds). Use after actions that trigger loading.
+- **computer_done**: Call this when the task is complete. Provide a result description.
+
+## Workflow
+
+1. Start by taking a screenshot to see the current state
+2. Analyze the screenshot to determine the next action
+3. Perform one action at a time (click, type, scroll, etc.)
+4. Take another screenshot to verify the result
+5. Repeat until the task is complete
+6. Call computer_done with a summary of what was accomplished
+
+## Important Guidelines
+
+- Always take a screenshot before your first action and after significant actions
+- Output coordinates in the format your model was trained on
+- Click precisely on UI elements — aim for the center of buttons/links
+- After typing, verify the text appeared correctly
+- If a dialog or popup appears unexpectedly, assess whether to dismiss it or interact with it
+- If you encounter an error or unexpected state, take a screenshot and reassess
+- When the task is complete, call computer_done — do not call any other tools after that
+- If you are stuck and cannot make progress, call computer_done with an error description
+${memorySection}`;
+}

qimingclaw/crates/agent-gui-server/src/agent/taskRunner.ts

@@ -0,0 +1,447 @@
+/**
+ * Agent loop engine using pi-mono Agent class.
+ *
+ * Creates a pi-mono Agent instance configured with GUI tools,
+ * hooks for safety/audit/memory, and manages the task lifecycle.
+ * Does NOT hand-write the loop — pi-mono handles that internally.
+ */
+
+import { Agent } from '@mariozechner/pi-agent-core';
+import { getModel } from '@mariozechner/pi-ai';
+import type { AgentTool, AgentToolResult, AgentEvent } from '@mariozechner/pi-agent-core';
+import type { ImageContent, TextContent, Message, Model, Api } from '@mariozechner/pi-ai';
+import { Type } from '@sinclair/typebox';
+
+import type { GuiAgentConfig } from '../config.js';
+import { buildSystemPrompt } from './systemPrompt.js';
+import { MemoryManager } from './memoryManager.js';
+import { StuckDetector } from './stuckDetector.js';
+import { resolveCoordinate, type ScreenshotMeta, type DisplayInfo } from '../coordinates/resolver.js';
+import { getModelProfile } from '../coordinates/modelProfiles.js';
+import { captureScreenshot, type ScreenshotResult } from '../desktop/screenshot.js';
+import * as mouse from '../desktop/mouse.js';
+import * as desktopKeyboard from '../desktop/keyboard.js';
+import { getDisplay } from '../desktop/display.js';
+import { validateHotkey } from '../safety/hotkeys.js';
+import { AuditLog } from '../safety/auditLog.js';
+import { logInfo, logDebug, logError, logWarn } from '../utils/logger.js';
+
+export interface StepRecord {
+  stepId: number;
+  tool: string;
+  args: Record<string, unknown>;
+  success: boolean;
+  durationMs: number;
+}
+
+export interface TaskResult {
+  success: boolean;
+  result?: string;
+  finalScreenshot?: string;
+  steps: StepRecord[];
+  error?: string;
+}
+
+export interface ProgressInfo {
+  step: number;
+  total: number;
+  status: 'running' | 'done' | 'error' | 'aborted';
+  message?: string;
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Create a pi-mono Model instance.
+ * Built-in providers (anthropic/openai/google) without custom baseUrl use getModel().
+ * Custom providers or those with baseUrl get a manually constructed Model object.
+ */
+export function createModel(
+  provider: string,
+  apiProtocol: 'anthropic' | 'openai',
+  modelId: string,
+  baseUrl?: string,
+): Model<Api> {
+  const builtinProviders = ['anthropic', 'openai', 'google'];
+  if (builtinProviders.includes(provider) && !baseUrl) {
+    return getModel(provider as any, modelId as any);
+  }
+
+  const api = apiProtocol === 'anthropic' ? 'anthropic-messages' : 'openai-completions';
+  return {
+    id: modelId,
+    name: modelId,
+    api,
+    provider,
+    baseUrl,
+    reasoning: false,
+    input: ['text', 'image'],
+    cost: { input: 0, output: 0 },
+    contextWindow: 128000,
+    maxTokens: 8192,
+  } as Model<Api>;
+}
+
+export function createTaskRunner(
+  config: GuiAgentConfig,
+  auditLog: AuditLog,
+) {
+  let currentAgent: Agent | null = null;
+
+  async function run(
+    taskText: string,
+    signal: AbortSignal,
+    onProgress: (info: ProgressInfo) => void,
+  ): Promise<TaskResult> {
+    const steps: StepRecord[] = [];
+    let stepCount = 0;
+    let latestScreenshot: ScreenshotResult | null = null;
+    let taskResult: string | undefined;
+    let pendingMemoryWork: Promise<void> = Promise.resolve();
+
+    // Build model instances
+    const model = createModel(config.provider, config.apiProtocol, config.model, config.baseUrl);
+    const memoryModel = createModel(
+      config.memoryProvider ?? config.provider,
+      config.apiProtocol,
+      config.memoryModel ?? config.model,
+      config.baseUrl,
+    );
+    const memoryManager = new MemoryManager(memoryModel, config.apiKey ?? '');
+    const stuckDetector = new StuckDetector(config.stuckThreshold);
+    const profile = getModelProfile(config.model, config.coordinateMode as any);
+    const displayInfo = await getDisplay(config.displayIndex);
+
+    // Build coordinate resolve helper
+    function resolveXY(x: number, y: number, meta: ScreenshotMeta) {
+      const di: DisplayInfo = {
+        origin: displayInfo.origin,
+        bounds: { width: displayInfo.width, height: displayInfo.height },
+        scaleFactor: displayInfo.scaleFactor,
+      };
+      return resolveCoordinate(x, y, profile, meta, di);
+    }
+
+    function getScreenshotMeta(): ScreenshotMeta {
+      if (latestScreenshot) {
+        return {
+          imageWidth: latestScreenshot.imageWidth,
+          imageHeight: latestScreenshot.imageHeight,
+          logicalWidth: latestScreenshot.logicalWidth,
+          logicalHeight: latestScreenshot.logicalHeight,
+        };
+      }
+      return {
+        imageWidth: displayInfo.width,
+        imageHeight: displayInfo.height,
+        logicalWidth: displayInfo.width,
+        logicalHeight: displayInfo.height,
+      };
+    }
+
+    // --- Define AgentTool[] ---
+    const guiTools: AgentTool[] = [
+      {
+        name: 'computer_screenshot',
+        label: 'Screenshot',
+        description: 'Capture the current screen',
+        parameters: Type.Object({}),
+        execute: async (_toolCallId, _params, _signal) => {
+          const shot = await captureScreenshot(config.displayIndex, config.jpegQuality);
+          latestScreenshot = shot;
+          return {
+            content: [{ type: 'image' as const, data: shot.image, mimeType: shot.mimeType }],
+            details: { imageWidth: shot.imageWidth, imageHeight: shot.imageHeight },
+          };
+        },
+      },
+      {
+        name: 'computer_click',
+        label: 'Click',
+        description: 'Click at coordinates (x, y)',
+        parameters: Type.Object({
+          x: Type.Number({ description: 'X coordinate' }),
+          y: Type.Number({ description: 'Y coordinate' }),
+          button: Type.Optional(Type.String({ description: 'Mouse button: left, right, middle' })),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { x: number; y: number; button?: string };
+          const { globalX, globalY } = resolveXY(params.x, params.y, getScreenshotMeta());
+          await mouse.click(globalX, globalY, params.button as any);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Clicked (${globalX}, ${globalY})` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_type',
+        label: 'Type',
+        description: 'Type text at the current cursor position',
+        parameters: Type.Object({
+          text: Type.String({ description: 'Text to type' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { text: string };
+          await desktopKeyboard.typeText(params.text);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Typed ${params.text.length} characters` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_scroll',
+        label: 'Scroll',
+        description: 'Scroll at coordinates (x, y)',
+        parameters: Type.Object({
+          x: Type.Number({ description: 'X coordinate' }),
+          y: Type.Number({ description: 'Y coordinate' }),
+          deltaY: Type.Number({ description: 'Vertical scroll amount (positive=down)' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { x: number; y: number; deltaY: number };
+          const { globalX, globalY } = resolveXY(params.x, params.y, getScreenshotMeta());
+          await mouse.scroll(globalX, globalY, params.deltaY);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Scrolled at (${globalX}, ${globalY}), dy=${params.deltaY}` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_hotkey',
+        label: 'Hotkey',
+        description: 'Press a key combination',
+        parameters: Type.Object({
+          keys: Type.Array(Type.String(), { description: 'Keys to press together' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { keys: string[] };
+          // Safety check is done in beforeToolCall hook
+          await desktopKeyboard.hotkey(params.keys);
+          await delay(config.stepDelayMs);
+          return {
+            content: [{ type: 'text' as const, text: `Hotkey: ${params.keys.join('+')}` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_wait',
+        label: 'Wait',
+        description: 'Wait for a specified duration',
+        parameters: Type.Object({
+          ms: Type.Number({ description: 'Duration in milliseconds' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { ms: number };
+          const waitMs = Math.min(params.ms, 10000); // Cap at 10s
+          await delay(waitMs);
+          return {
+            content: [{ type: 'text' as const, text: `Waited ${waitMs}ms` }],
+            details: {},
+          };
+        },
+      },
+      {
+        name: 'computer_done',
+        label: 'Done',
+        description: 'Signal that the task is complete. Provide a result description.',
+        parameters: Type.Object({
+          result: Type.String({ description: 'Task completion result description' }),
+        }),
+        execute: async (_toolCallId, rawParams, _signal) => {
+          const params = rawParams as { result: string };
+          taskResult = params.result;
+          return {
+            content: [{ type: 'text' as const, text: params.result }],
+            details: { done: true },
+          };
+        },
+      },
+    ];
+
+    // --- Create Agent ---
+    const agent = new Agent({
+      initialState: {
+        systemPrompt: buildSystemPrompt(taskText, memoryManager.compose()),
+        model,
+        thinkingLevel: 'off',
+        tools: guiTools,
+        messages: [],
+      },
+      toolExecution: 'sequential',
+      transformContext: async (messages, _signal) => {
+        return memoryManager.pruneScreenshots(messages) as any;
+      },
+      convertToLlm: (messages) =>
+        (messages as any[]).filter(m => ['user', 'assistant', 'toolResult'].includes(m.role)) as Message[],
+      beforeToolCall: async ({ toolCall, args }) => {
+        if (toolCall.name === 'computer_hotkey') {
+          const typedArgs = args as { keys: string[] };
+          const validation = validateHotkey(typedArgs.keys);
+          if (validation.blocked) {
+            return { block: true, reason: `Blocked dangerous hotkey: ${validation.reason}` };
+          }
+        }
+        return undefined;
+      },
+      afterToolCall: async ({ toolCall, args, result, isError }) => {
+        auditLog.record({ tool: toolCall.name, args: args as Record<string, unknown>, success: !isError });
+        return undefined;
+      },
+      getApiKey: (_provider: string) => config.apiKey,
+    });
+
+    currentAgent = agent;
+
+    // --- Subscribe to events ---
+    const unsubscribe = agent.subscribe((event: AgentEvent) => {
+      // Record pending step when tool execution starts (visible to LLM during execution)
+      if (event.type === 'tool_execution_start') {
+        const toolName = (event as any).toolCall?.name ?? 'unknown';
+        memoryManager.addPendingStep(stepCount + 1, toolName);
+      }
+
+      if (event.type === 'turn_end') {
+        stepCount++;
+
+        // Record step
+        const toolResults = event.toolResults || [];
+        for (const tr of toolResults) {
+          steps.push({
+            stepId: stepCount,
+            tool: tr.toolName,
+            args: (tr as any).args ?? {},
+            success: !tr.isError,
+            durationMs: (tr as any).durationMs ?? 0,
+          });
+        }
+
+        // Memory management (async, queued)
+        const goal = toolResults.map(tr => tr.toolName).join(', ') || 'LLM response';
+        const evaluation = toolResults.some(tr => tr.isError) ? 'failed' as const : 'success' as const;
+        pendingMemoryWork = memoryManager.finalizeStep(stepCount, evaluation)
+          .then(() => {
+            agent.setSystemPrompt(buildSystemPrompt(taskText, memoryManager.compose()));
+          })
+          .catch(err => {
+            logError(`Memory finalize failed: ${err}`);
+          });
+
+        // Stuck detection (async, fire-and-forget)
+        if (latestScreenshot) {
+          stuckDetector.check(latestScreenshot.image).then(({ stuck }) => {
+            if (stuck) {
+              logWarn(`Agent appears stuck after ${stepCount} steps, aborting`);
+              agent.abort();
+            }
+          }).catch(err => {
+            logError(`Stuck detector error: ${err instanceof Error ? err.message : String(err)}`);
+          });
+        }
+
+        // Progress notification
+        onProgress({
+          step: stepCount,
+          total: config.maxSteps,
+          status: 'running',
+          message: `Step ${stepCount}: ${goal}`,
+        });
+
+        // Max steps check
+        if (stepCount >= config.maxSteps) {
+          logWarn(`Max steps (${config.maxSteps}) reached, aborting`);
+          agent.abort();
+        }
+      }
+    });
+
+    // --- Handle external abort ---
+    const abortHandler = () => agent.abort();
+    signal.addEventListener('abort', abortHandler);
+
+    try {
+      // Capture initial screenshot
+      const initialShot = await captureScreenshot(config.displayIndex, config.jpegQuality);
+      latestScreenshot = initialShot;
+
+      // Start the agent loop
+      logInfo(`Starting task: ${taskText.substring(0, 100)}`);
+      await agent.prompt(taskText, [
+        { type: 'image' as const, data: initialShot.image, mimeType: initialShot.mimeType },
+      ]);
+
+      // Wait for pending memory work
+      await pendingMemoryWork;
+
+      // Determine result
+      const lastMessage = agent.state.messages[agent.state.messages.length - 1];
+      const stopReason = lastMessage && 'stopReason' in lastMessage ? (lastMessage as any).stopReason : undefined;
+
+      if (stopReason === 'aborted') {
+        onProgress({ step: stepCount, total: config.maxSteps, status: 'aborted' });
+        return {
+          success: false,
+          error: 'Task was aborted',
+          steps,
+          finalScreenshot: latestScreenshot?.image,
+        };
+      }
+
+      if (stopReason === 'error') {
+        onProgress({ step: stepCount, total: config.maxSteps, status: 'error', message: 'Agent stopped with error (possible context overflow)' });
+        return {
+          success: false,
+          error: 'Agent stopped with error (possible context overflow)',
+          steps,
+          finalScreenshot: latestScreenshot?.image,
+        };
+      }
+
+      onProgress({ step: stepCount, total: config.maxSteps, status: 'done' });
+      return {
+        success: true,
+        result: taskResult ?? extractTextResult(lastMessage),
+        steps,
+        finalScreenshot: latestScreenshot?.image,
+      };
+    } catch (err) {
+      const errorMsg = err instanceof Error ? err.message : String(err);
+      logError(`Task execution error: ${errorMsg}`);
+      onProgress({ step: stepCount, total: config.maxSteps, status: 'error', message: errorMsg });
+      return {
+        success: false,
+        error: errorMsg,
+        steps,
+        finalScreenshot: latestScreenshot?.image,
+      };
+    } finally {
+      signal.removeEventListener('abort', abortHandler);
+      unsubscribe();
+      currentAgent = null;
+    }
+  }
+
+  function abort(): void {
+    currentAgent?.abort();
+  }
+
+  return { run, abort };
+}
+
+function extractTextResult(message: unknown): string | undefined {
+  if (!message || typeof message !== 'object') return undefined;
+  const msg = message as any;
+  if (Array.isArray(msg.content)) {
+    const text = msg.content.find((c: any) => c.type === 'text');
+    return text?.text;
+  }
+  return undefined;
+}