chore: initialize qiming workspace repository

qimingclaw/crates/agent-gui-server/tests/agent/memoryManager.test.ts

@@ -0,0 +1,145 @@
+/**
+ * Tests for MemoryManager.
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+// Mock pi-ai complete function
+vi.mock('@mariozechner/pi-ai', () => ({
+  complete: vi.fn().mockResolvedValue({
+    content: [{ type: 'text', text: '{"summary": "Steps 1-5: navigated and clicked"}' }],
+  }),
+  getModel: vi.fn().mockReturnValue({ provider: 'mock', model: 'mock-model' }),
+}));
+
+import { MemoryManager } from '../../src/agent/memoryManager.js';
+import { getModel } from '@mariozechner/pi-ai';
+
+describe('MemoryManager', () => {
+  let mm: MemoryManager;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    const model = getModel('anthropic' as any, 'claude-sonnet-4-20250514' as any);
+    mm = new MemoryManager(model, 'test-key');
+  });
+
+  describe('addPendingStep + compose', () => {
+    it('should add pending step and include it in compose', () => {
+      mm.addPendingStep(1, 'Click button');
+      const text = mm.compose();
+      expect(text).toContain('[Current step]');
+      expect(text).toContain('Step 1');
+      expect(text).toContain('Click button');
+    });
+  });
+
+  describe('finalizeStep', () => {
+    it('should move pending to recent on finalize', async () => {
+      mm.addPendingStep(1, 'Click button');
+      await mm.finalizeStep(1, 'success');
+
+      const text = mm.compose();
+      expect(text).toContain('[Recent steps]');
+      expect(text).toContain('Eval: success');
+      expect(text).not.toContain('[Current step]');
+    });
+
+    it('should record failed evaluation', async () => {
+      mm.addPendingStep(1, 'Click button');
+      await mm.finalizeStep(1, 'failed');
+
+      const text = mm.compose();
+      expect(text).toContain('Eval: failed');
+    });
+
+    it('should accumulate multiple steps in recent', async () => {
+      mm.addPendingStep(1, 'Step one');
+      await mm.finalizeStep(1, 'success');
+      mm.addPendingStep(2, 'Step two');
+      await mm.finalizeStep(2, 'success');
+
+      const text = mm.compose();
+      expect(text).toContain('Step 1');
+      expect(text).toContain('Step 2');
+    });
+  });
+
+  describe('compression', () => {
+    it('should trigger compression when recent exceeds budget', async () => {
+      const { complete } = await import('@mariozechner/pi-ai');
+
+      // Add many steps to exceed the 500 char budget
+      for (let i = 1; i <= 20; i++) {
+        mm.addPendingStep(i, `Goal for step ${i}: perform a complex multi-word action description`);
+        await mm.finalizeStep(i, 'success');
+      }
+
+      // complete() should have been called for summarization
+      expect(complete).toHaveBeenCalled();
+
+      // After compression, compose should still work
+      const text = mm.compose();
+      expect(text.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('pruneScreenshots', () => {
+    it('should keep all screenshots when under limit', () => {
+      const messages = [
+        { role: 'user', content: [{ type: 'image', data: 'img1', mimeType: 'image/jpeg' }] },
+        { role: 'user', content: [{ type: 'image', data: 'img2', mimeType: 'image/jpeg' }] },
+      ];
+
+      const result = mm.pruneScreenshots(messages);
+      expect(result).toEqual(messages); // No pruning — under screenshotKeepCount (3)
+    });
+
+    it('should replace old screenshots with placeholders', () => {
+      const messages = [
+        { role: 'user', content: [{ type: 'image', data: 'img1', mimeType: 'image/jpeg' }] },
+        { role: 'user', content: [{ type: 'image', data: 'img2', mimeType: 'image/jpeg' }] },
+        { role: 'user', content: [{ type: 'image', data: 'img3', mimeType: 'image/jpeg' }] },
+        { role: 'user', content: [{ type: 'image', data: 'img4', mimeType: 'image/jpeg' }] },
+        { role: 'user', content: [{ type: 'image', data: 'img5', mimeType: 'image/jpeg' }] },
+      ];
+
+      const result = mm.pruneScreenshots(messages);
+
+      // First 2 should be replaced (5 - 3 = keep from index 2)
+      expect((result[0] as any).content[0].type).toBe('text');
+      expect((result[0] as any).content[0].text).toContain('Screenshot removed');
+      expect((result[1] as any).content[0].type).toBe('text');
+
+      // Last 3 should be kept
+      expect((result[2] as any).content[0].type).toBe('image');
+      expect((result[3] as any).content[0].type).toBe('image');
+      expect((result[4] as any).content[0].type).toBe('image');
+    });
+
+    it('should not modify non-image messages', () => {
+      const messages = [
+        { role: 'user', content: [{ type: 'text', text: 'hello' }] },
+        { role: 'user', content: [{ type: 'image', data: 'img1', mimeType: 'image/jpeg' }] },
+        { role: 'assistant', content: [{ type: 'text', text: 'response' }] },
+      ];
+
+      const result = mm.pruneScreenshots(messages);
+      expect(result).toEqual(messages); // Only 1 image, under limit
+    });
+  });
+
+  describe('compose layers', () => {
+    it('should return empty string when no memory', () => {
+      expect(mm.compose()).toBe('');
+    });
+
+    it('should compose all three layers', () => {
+      // We can't easily set summaryMemory directly, but we can verify
+      // pending + recent compose correctly
+      mm.addPendingStep(1, 'Current action');
+      const text = mm.compose();
+      expect(text).toContain('[Current step]');
+    });
+  });
+});

qimingclaw/crates/agent-gui-server/tests/agent/stuckDetector.test.ts

@@ -0,0 +1,108 @@
+/**
+ * Tests for StuckDetector.
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { StuckDetector } from '../../src/agent/stuckDetector.js';
+import sharp from 'sharp';
+
+/** Create a solid-color 64x64 image as base64 */
+async function createSolidImage(r: number, g: number, b: number): Promise<string> {
+  const buf = await sharp({
+    create: { width: 64, height: 64, channels: 3, background: { r, g, b } },
+  })
+    .png()
+    .toBuffer();
+  return buf.toString('base64');
+}
+
+describe('StuckDetector', () => {
+  let detector: StuckDetector;
+
+  beforeEach(() => {
+    detector = new StuckDetector(3, 0.05);
+  });
+
+  it('should not report stuck on first check', async () => {
+    const img = await createSolidImage(128, 128, 128);
+    const result = await detector.check(img);
+    expect(result.stuck).toBe(false);
+    expect(result.consecutiveSimilar).toBe(0);
+  });
+
+  it('should not report stuck with different images', async () => {
+    const colors = [
+      [255, 0, 0],
+      [0, 255, 0],
+      [0, 0, 255],
+      [255, 255, 0],
+    ] as const;
+
+    for (const [r, g, b] of colors) {
+      const img = await createSolidImage(r, g, b);
+      const result = await detector.check(img);
+      expect(result.stuck).toBe(false);
+    }
+  });
+
+  it('should report stuck after threshold consecutive identical images', async () => {
+    const img = await createSolidImage(100, 100, 100);
+
+    // First check — no previous to compare
+    let result = await detector.check(img);
+    expect(result.stuck).toBe(false);
+
+    // 2nd check — consecutiveSimilar = 1
+    result = await detector.check(img);
+    expect(result.consecutiveSimilar).toBe(1);
+    expect(result.stuck).toBe(false);
+
+    // 3rd check — consecutiveSimilar = 2
+    result = await detector.check(img);
+    expect(result.consecutiveSimilar).toBe(2);
+    expect(result.stuck).toBe(false);
+
+    // 4th check — consecutiveSimilar = 3 >= threshold
+    result = await detector.check(img);
+    expect(result.consecutiveSimilar).toBe(3);
+    expect(result.stuck).toBe(true);
+  });
+
+  it('should reset consecutive count when a different image appears', async () => {
+    const imgA = await createSolidImage(100, 100, 100);
+    const imgB = await createSolidImage(200, 50, 50);
+
+    await detector.check(imgA);
+    await detector.check(imgA); // consecutiveSimilar = 1
+    await detector.check(imgA); // consecutiveSimilar = 2
+
+    // Different image breaks the streak
+    const result = await detector.check(imgB);
+    expect(result.consecutiveSimilar).toBe(0);
+    expect(result.stuck).toBe(false);
+  });
+
+  it('should reset state via reset()', async () => {
+    const img = await createSolidImage(100, 100, 100);
+
+    await detector.check(img);
+    await detector.check(img);
+    await detector.check(img);
+
+    detector.reset();
+
+    const result = await detector.check(img);
+    expect(result.consecutiveSimilar).toBe(0);
+    expect(result.stuck).toBe(false);
+  });
+
+  it('should respect custom threshold', async () => {
+    const customDetector = new StuckDetector(2, 0.05);
+    const img = await createSolidImage(50, 50, 50);
+
+    await customDetector.check(img);
+    await customDetector.check(img); // consecutiveSimilar = 1
+    const result = await customDetector.check(img); // consecutiveSimilar = 2 >= threshold=2
+    expect(result.stuck).toBe(true);
+  });
+});

qimingclaw/crates/agent-gui-server/tests/agent/systemPrompt.test.ts

@@ -0,0 +1,53 @@
+/**
+ * Unit tests for agent/systemPrompt.ts — buildSystemPrompt.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { buildSystemPrompt } from '../../src/agent/systemPrompt.js';
+
+describe('buildSystemPrompt', () => {
+  it('includes the task text', () => {
+    const prompt = buildSystemPrompt('Open Finder', '');
+    expect(prompt).toContain('Open Finder');
+  });
+
+  it('includes all 7 tool names', () => {
+    const prompt = buildSystemPrompt('test', '');
+    expect(prompt).toContain('computer_screenshot');
+    expect(prompt).toContain('computer_click');
+    expect(prompt).toContain('computer_type');
+    expect(prompt).toContain('computer_scroll');
+    expect(prompt).toContain('computer_hotkey');
+    expect(prompt).toContain('computer_wait');
+    expect(prompt).toContain('computer_done');
+  });
+
+  it('includes memory section when memory text is provided', () => {
+    const prompt = buildSystemPrompt('task', 'Step 1: clicked button\nStep 2: typed text');
+    expect(prompt).toContain('Previous Actions Memory');
+    expect(prompt).toContain('Step 1: clicked button');
+    expect(prompt).toContain('Step 2: typed text');
+  });
+
+  it('does not include memory section when memory text is empty', () => {
+    const prompt = buildSystemPrompt('task', '');
+    expect(prompt).not.toContain('Previous Actions Memory');
+  });
+
+  it('contains workflow guidance', () => {
+    const prompt = buildSystemPrompt('task', '');
+    expect(prompt).toContain('Workflow');
+    expect(prompt).toContain('screenshot');
+  });
+
+  it('contains guidelines about popups and errors', () => {
+    const prompt = buildSystemPrompt('task', '');
+    expect(prompt).toContain('dialog');
+    expect(prompt).toContain('popup');
+  });
+
+  it('includes CJK task text correctly', () => {
+    const prompt = buildSystemPrompt('打开访达并创建新文件夹', '');
+    expect(prompt).toContain('打开访达并创建新文件夹');
+  });
+});

qimingclaw/crates/agent-gui-server/tests/agent/taskRunner.test.ts

@@ -0,0 +1,296 @@
+/**
+ * Tests for taskRunner — mock-based unit tests.
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+// Mock all desktop and external dependencies
+vi.mock('@mariozechner/pi-ai', () => ({
+  complete: vi.fn().mockResolvedValue({
+    content: [{ type: 'text', text: '{"summary": "compressed memory"}' }],
+  }),
+  getModel: vi.fn().mockReturnValue({ provider: 'mock', model: 'mock' }),
+}));
+
+vi.mock('@mariozechner/pi-agent-core', () => {
+  const subscribers: Array<(event: any) => void> = [];
+  let aborted = false;
+
+  return {
+    Agent: vi.fn().mockImplementation((config: any) => ({
+      state: {
+        messages: [
+          {
+            role: 'assistant',
+            content: [{ type: 'text', text: 'Task completed successfully' }],
+            stopReason: 'end',
+          },
+        ],
+      },
+      subscribe: vi.fn((fn: any) => {
+        subscribers.push(fn);
+        return () => {};
+      }),
+      prompt: vi.fn(async () => {
+        // Simulate one turn
+        for (const sub of subscribers) {
+          sub({
+            type: 'turn_end',
+            message: { role: 'assistant', content: [{ type: 'text', text: 'done' }] },
+            toolResults: [{ toolName: 'computer_done', isError: false }],
+          });
+        }
+      }),
+      abort: vi.fn(() => { aborted = true; }),
+      setSystemPrompt: vi.fn(),
+    })),
+  };
+});
+
+vi.mock('../../src/desktop/screenshot.js', () => ({
+  captureScreenshot: vi.fn().mockResolvedValue({
+    image: 'base64screenshot',
+    mimeType: 'image/jpeg',
+    imageWidth: 1920,
+    imageHeight: 1080,
+    logicalWidth: 1920,
+    logicalHeight: 1080,
+    physicalWidth: 1920,
+    physicalHeight: 1080,
+    scaleFactor: 1,
+    displayIndex: 0,
+  }),
+}));
+
+vi.mock('../../src/desktop/mouse.js', () => ({
+  click: vi.fn(),
+  doubleClick: vi.fn(),
+  moveTo: vi.fn(),
+  drag: vi.fn(),
+  scroll: vi.fn(),
+  getPosition: vi.fn().mockResolvedValue({ x: 0, y: 0 }),
+}));
+
+vi.mock('../../src/desktop/keyboard.js', () => ({
+  typeText: vi.fn(),
+  pressKey: vi.fn(),
+  hotkey: vi.fn(),
+}));
+
+vi.mock('../../src/desktop/display.js', () => ({
+  getDisplay: vi.fn().mockResolvedValue({
+    index: 0,
+    label: 'Main',
+    width: 1920,
+    height: 1080,
+    scaleFactor: 1,
+    isPrimary: true,
+    origin: { x: 0, y: 0 },
+  }),
+}));
+
+vi.mock('../../src/coordinates/modelProfiles.js', () => ({
+  getModelProfile: vi.fn().mockReturnValue({
+    coordinateMode: 'image-absolute' as const,
+    coordinateOrder: 'xy',
+  }),
+}));
+
+vi.mock('sharp', () => ({
+  default: vi.fn().mockReturnValue({
+    resize: vi.fn().mockReturnThis(),
+    raw: vi.fn().mockReturnThis(),
+    toBuffer: vi.fn().mockResolvedValue(Buffer.alloc(32 * 32 * 3)),
+  }),
+}));
+
+import { createTaskRunner, createModel } from '../../src/agent/taskRunner.js';
+import { AuditLog } from '../../src/safety/auditLog.js';
+import type { GuiAgentConfig } from '../../src/config.js';
+
+function createTestConfig(): GuiAgentConfig {
+  return {
+    provider: 'anthropic',
+    apiProtocol: 'anthropic',
+    model: 'claude-sonnet-4-20250514',
+    apiKey: 'test-key',
+    port: 60008,
+    maxSteps: 50,
+    stepDelayMs: 0,
+    stuckThreshold: 3,
+    jpegQuality: 75,
+    displayIndex: 0,
+    transport: 'http' as const,
+    coordinateMode: 'image-absolute' as const,
+  };
+}
+
+describe('createTaskRunner', () => {
+  let auditLog: AuditLog;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    auditLog = new AuditLog();
+  });
+
+  it('should return run and abort functions', () => {
+    const runner = createTaskRunner(createTestConfig(), auditLog);
+    expect(typeof runner.run).toBe('function');
+    expect(typeof runner.abort).toBe('function');
+  });
+
+  it('should complete a task successfully', async () => {
+    const runner = createTaskRunner(createTestConfig(), auditLog);
+    const controller = new AbortController();
+    const progressCalls: any[] = [];
+
+    const result = await runner.run(
+      'Open Finder',
+      controller.signal,
+      (info) => progressCalls.push(info),
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.finalScreenshot).toBe('base64screenshot');
+    expect(result.steps.length).toBeGreaterThanOrEqual(0);
+  });
+
+  it('should handle abort signal', async () => {
+    const { Agent } = await import('@mariozechner/pi-agent-core');
+    const mockAgent = (Agent as any).mockImplementation((config: any) => ({
+      state: {
+        messages: [
+          {
+            role: 'assistant',
+            content: [{ type: 'text', text: 'aborted' }],
+            stopReason: 'aborted',
+          },
+        ],
+      },
+      subscribe: vi.fn(() => () => {}),
+      prompt: vi.fn(async () => {}),
+      abort: vi.fn(),
+      setSystemPrompt: vi.fn(),
+    }));
+
+    const runner = createTaskRunner(createTestConfig(), auditLog);
+    const controller = new AbortController();
+
+    const result = await runner.run(
+      'Open Finder',
+      controller.signal,
+      () => {},
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error).toBe('Task was aborted');
+  });
+
+  it('should handle errors in task execution', async () => {
+    const { Agent } = await import('@mariozechner/pi-agent-core');
+    (Agent as any).mockImplementation(() => ({
+      state: { messages: [] },
+      subscribe: vi.fn(() => () => {}),
+      prompt: vi.fn(async () => { throw new Error('API connection failed'); }),
+      abort: vi.fn(),
+      setSystemPrompt: vi.fn(),
+    }));
+
+    const runner = createTaskRunner(createTestConfig(), auditLog);
+    const controller = new AbortController();
+
+    const result = await runner.run(
+      'Open Finder',
+      controller.signal,
+      () => {},
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error).toBe('API connection failed');
+  });
+
+  it('should enforce maxSteps', async () => {
+    const { Agent } = await import('@mariozechner/pi-agent-core');
+    let agentAborted = false;
+
+    (Agent as any).mockImplementation((agentConfig: any) => {
+      const subs: any[] = [];
+      return {
+        state: {
+          messages: [
+            { role: 'assistant', content: [{ type: 'text', text: 'done' }], stopReason: 'aborted' },
+          ],
+        },
+        subscribe: vi.fn((fn: any) => { subs.push(fn); return () => {}; }),
+        prompt: vi.fn(async () => {
+          // Simulate maxSteps+1 turns to trigger abort
+          for (let i = 0; i < 5; i++) {
+            for (const sub of subs) {
+              sub({
+                type: 'turn_end',
+                message: { role: 'assistant', content: [] },
+                toolResults: [{ toolName: 'computer_screenshot', isError: false }],
+              });
+            }
+          }
+        }),
+        abort: vi.fn(() => { agentAborted = true; }),
+        setSystemPrompt: vi.fn(),
+      };
+    });
+
+    const config = createTestConfig();
+    config.maxSteps = 3;
+
+    const runner = createTaskRunner(config, auditLog);
+    const controller = new AbortController();
+
+    const result = await runner.run('Open Finder', controller.signal, () => {});
+
+    expect(agentAborted).toBe(true);
+  });
+});
+
+describe('createModel', () => {
+  it('uses getModel for built-in anthropic provider without baseUrl', () => {
+    const model = createModel('anthropic', 'anthropic', 'claude-sonnet-4-20250514');
+    // getModel is mocked to return { provider: 'mock', model: 'mock' }
+    expect(model).toEqual({ provider: 'mock', model: 'mock' });
+  });
+
+  it('uses getModel for built-in openai provider without baseUrl', () => {
+    const model = createModel('openai', 'openai', 'gpt-4o');
+    expect(model).toEqual({ provider: 'mock', model: 'mock' });
+  });
+
+  it('uses getModel for built-in google provider without baseUrl', () => {
+    const model = createModel('google', 'openai', 'gemini-2.5-pro');
+    expect(model).toEqual({ provider: 'mock', model: 'mock' });
+  });
+
+  it('constructs manual Model for custom provider', () => {
+    const model = createModel('zhipu', 'openai', 'glm-4v-plus', 'https://open.bigmodel.cn/api/paas/v4');
+    expect(model.id).toBe('glm-4v-plus');
+    expect(model.name).toBe('glm-4v-plus');
+    expect(model.api).toBe('openai-completions');
+    expect(model.provider).toBe('zhipu');
+    expect(model.baseUrl).toBe('https://open.bigmodel.cn/api/paas/v4');
+  });
+
+  it('constructs manual Model for built-in provider with custom baseUrl', () => {
+    const model = createModel('anthropic', 'anthropic', 'claude-sonnet-4-20250514', 'https://proxy.example.com');
+    expect(model.id).toBe('claude-sonnet-4-20250514');
+    expect(model.api).toBe('anthropic-messages');
+    expect(model.baseUrl).toBe('https://proxy.example.com');
+  });
+
+  it('maps anthropic protocol to anthropic-messages api', () => {
+    const model = createModel('custom', 'anthropic', 'my-model', 'https://custom.api.com');
+    expect(model.api).toBe('anthropic-messages');
+  });
+
+  it('maps openai protocol to openai-completions api', () => {
+    const model = createModel('custom', 'openai', 'my-model', 'https://custom.api.com');
+    expect(model.api).toBe('openai-completions');
+  });
+});