chore: initialize qiming workspace repository

This commit is contained in:
Codex
2026-05-29 14:22:48 +08:00
commit bfd67a0f2c
10750 changed files with 1885711 additions and 0 deletions

View File

@@ -0,0 +1,145 @@
/**
* Tests for MemoryManager.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
// Mock pi-ai complete function
vi.mock('@mariozechner/pi-ai', () => ({
complete: vi.fn().mockResolvedValue({
content: [{ type: 'text', text: '{"summary": "Steps 1-5: navigated and clicked"}' }],
}),
getModel: vi.fn().mockReturnValue({ provider: 'mock', model: 'mock-model' }),
}));
import { MemoryManager } from '../../src/agent/memoryManager.js';
import { getModel } from '@mariozechner/pi-ai';
describe('MemoryManager', () => {
let mm: MemoryManager;
beforeEach(() => {
vi.clearAllMocks();
const model = getModel('anthropic' as any, 'claude-sonnet-4-20250514' as any);
mm = new MemoryManager(model, 'test-key');
});
describe('addPendingStep + compose', () => {
it('should add pending step and include it in compose', () => {
mm.addPendingStep(1, 'Click button');
const text = mm.compose();
expect(text).toContain('[Current step]');
expect(text).toContain('Step 1');
expect(text).toContain('Click button');
});
});
describe('finalizeStep', () => {
it('should move pending to recent on finalize', async () => {
mm.addPendingStep(1, 'Click button');
await mm.finalizeStep(1, 'success');
const text = mm.compose();
expect(text).toContain('[Recent steps]');
expect(text).toContain('Eval: success');
expect(text).not.toContain('[Current step]');
});
it('should record failed evaluation', async () => {
mm.addPendingStep(1, 'Click button');
await mm.finalizeStep(1, 'failed');
const text = mm.compose();
expect(text).toContain('Eval: failed');
});
it('should accumulate multiple steps in recent', async () => {
mm.addPendingStep(1, 'Step one');
await mm.finalizeStep(1, 'success');
mm.addPendingStep(2, 'Step two');
await mm.finalizeStep(2, 'success');
const text = mm.compose();
expect(text).toContain('Step 1');
expect(text).toContain('Step 2');
});
});
describe('compression', () => {
it('should trigger compression when recent exceeds budget', async () => {
const { complete } = await import('@mariozechner/pi-ai');
// Add many steps to exceed the 500 char budget
for (let i = 1; i <= 20; i++) {
mm.addPendingStep(i, `Goal for step ${i}: perform a complex multi-word action description`);
await mm.finalizeStep(i, 'success');
}
// complete() should have been called for summarization
expect(complete).toHaveBeenCalled();
// After compression, compose should still work
const text = mm.compose();
expect(text.length).toBeGreaterThan(0);
});
});
describe('pruneScreenshots', () => {
it('should keep all screenshots when under limit', () => {
const messages = [
{ role: 'user', content: [{ type: 'image', data: 'img1', mimeType: 'image/jpeg' }] },
{ role: 'user', content: [{ type: 'image', data: 'img2', mimeType: 'image/jpeg' }] },
];
const result = mm.pruneScreenshots(messages);
expect(result).toEqual(messages); // No pruning — under screenshotKeepCount (3)
});
it('should replace old screenshots with placeholders', () => {
const messages = [
{ role: 'user', content: [{ type: 'image', data: 'img1', mimeType: 'image/jpeg' }] },
{ role: 'user', content: [{ type: 'image', data: 'img2', mimeType: 'image/jpeg' }] },
{ role: 'user', content: [{ type: 'image', data: 'img3', mimeType: 'image/jpeg' }] },
{ role: 'user', content: [{ type: 'image', data: 'img4', mimeType: 'image/jpeg' }] },
{ role: 'user', content: [{ type: 'image', data: 'img5', mimeType: 'image/jpeg' }] },
];
const result = mm.pruneScreenshots(messages);
// First 2 should be replaced (5 - 3 = keep from index 2)
expect((result[0] as any).content[0].type).toBe('text');
expect((result[0] as any).content[0].text).toContain('Screenshot removed');
expect((result[1] as any).content[0].type).toBe('text');
// Last 3 should be kept
expect((result[2] as any).content[0].type).toBe('image');
expect((result[3] as any).content[0].type).toBe('image');
expect((result[4] as any).content[0].type).toBe('image');
});
it('should not modify non-image messages', () => {
const messages = [
{ role: 'user', content: [{ type: 'text', text: 'hello' }] },
{ role: 'user', content: [{ type: 'image', data: 'img1', mimeType: 'image/jpeg' }] },
{ role: 'assistant', content: [{ type: 'text', text: 'response' }] },
];
const result = mm.pruneScreenshots(messages);
expect(result).toEqual(messages); // Only 1 image, under limit
});
});
describe('compose layers', () => {
it('should return empty string when no memory', () => {
expect(mm.compose()).toBe('');
});
it('should compose all three layers', () => {
// We can't easily set summaryMemory directly, but we can verify
// pending + recent compose correctly
mm.addPendingStep(1, 'Current action');
const text = mm.compose();
expect(text).toContain('[Current step]');
});
});
});

View File

@@ -0,0 +1,108 @@
/**
* Tests for StuckDetector.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { StuckDetector } from '../../src/agent/stuckDetector.js';
import sharp from 'sharp';
/** Create a solid-color 64x64 image as base64 */
async function createSolidImage(r: number, g: number, b: number): Promise<string> {
const buf = await sharp({
create: { width: 64, height: 64, channels: 3, background: { r, g, b } },
})
.png()
.toBuffer();
return buf.toString('base64');
}
describe('StuckDetector', () => {
let detector: StuckDetector;
beforeEach(() => {
detector = new StuckDetector(3, 0.05);
});
it('should not report stuck on first check', async () => {
const img = await createSolidImage(128, 128, 128);
const result = await detector.check(img);
expect(result.stuck).toBe(false);
expect(result.consecutiveSimilar).toBe(0);
});
it('should not report stuck with different images', async () => {
const colors = [
[255, 0, 0],
[0, 255, 0],
[0, 0, 255],
[255, 255, 0],
] as const;
for (const [r, g, b] of colors) {
const img = await createSolidImage(r, g, b);
const result = await detector.check(img);
expect(result.stuck).toBe(false);
}
});
it('should report stuck after threshold consecutive identical images', async () => {
const img = await createSolidImage(100, 100, 100);
// First check — no previous to compare
let result = await detector.check(img);
expect(result.stuck).toBe(false);
// 2nd check — consecutiveSimilar = 1
result = await detector.check(img);
expect(result.consecutiveSimilar).toBe(1);
expect(result.stuck).toBe(false);
// 3rd check — consecutiveSimilar = 2
result = await detector.check(img);
expect(result.consecutiveSimilar).toBe(2);
expect(result.stuck).toBe(false);
// 4th check — consecutiveSimilar = 3 >= threshold
result = await detector.check(img);
expect(result.consecutiveSimilar).toBe(3);
expect(result.stuck).toBe(true);
});
it('should reset consecutive count when a different image appears', async () => {
const imgA = await createSolidImage(100, 100, 100);
const imgB = await createSolidImage(200, 50, 50);
await detector.check(imgA);
await detector.check(imgA); // consecutiveSimilar = 1
await detector.check(imgA); // consecutiveSimilar = 2
// Different image breaks the streak
const result = await detector.check(imgB);
expect(result.consecutiveSimilar).toBe(0);
expect(result.stuck).toBe(false);
});
it('should reset state via reset()', async () => {
const img = await createSolidImage(100, 100, 100);
await detector.check(img);
await detector.check(img);
await detector.check(img);
detector.reset();
const result = await detector.check(img);
expect(result.consecutiveSimilar).toBe(0);
expect(result.stuck).toBe(false);
});
it('should respect custom threshold', async () => {
const customDetector = new StuckDetector(2, 0.05);
const img = await createSolidImage(50, 50, 50);
await customDetector.check(img);
await customDetector.check(img); // consecutiveSimilar = 1
const result = await customDetector.check(img); // consecutiveSimilar = 2 >= threshold=2
expect(result.stuck).toBe(true);
});
});

View File

@@ -0,0 +1,53 @@
/**
* Unit tests for agent/systemPrompt.ts — buildSystemPrompt.
*/
import { describe, it, expect } from 'vitest';
import { buildSystemPrompt } from '../../src/agent/systemPrompt.js';
describe('buildSystemPrompt', () => {
it('includes the task text', () => {
const prompt = buildSystemPrompt('Open Finder', '');
expect(prompt).toContain('Open Finder');
});
it('includes all 7 tool names', () => {
const prompt = buildSystemPrompt('test', '');
expect(prompt).toContain('computer_screenshot');
expect(prompt).toContain('computer_click');
expect(prompt).toContain('computer_type');
expect(prompt).toContain('computer_scroll');
expect(prompt).toContain('computer_hotkey');
expect(prompt).toContain('computer_wait');
expect(prompt).toContain('computer_done');
});
it('includes memory section when memory text is provided', () => {
const prompt = buildSystemPrompt('task', 'Step 1: clicked button\nStep 2: typed text');
expect(prompt).toContain('Previous Actions Memory');
expect(prompt).toContain('Step 1: clicked button');
expect(prompt).toContain('Step 2: typed text');
});
it('does not include memory section when memory text is empty', () => {
const prompt = buildSystemPrompt('task', '');
expect(prompt).not.toContain('Previous Actions Memory');
});
it('contains workflow guidance', () => {
const prompt = buildSystemPrompt('task', '');
expect(prompt).toContain('Workflow');
expect(prompt).toContain('screenshot');
});
it('contains guidelines about popups and errors', () => {
const prompt = buildSystemPrompt('task', '');
expect(prompt).toContain('dialog');
expect(prompt).toContain('popup');
});
it('includes CJK task text correctly', () => {
const prompt = buildSystemPrompt('打开访达并创建新文件夹', '');
expect(prompt).toContain('打开访达并创建新文件夹');
});
});

View File

@@ -0,0 +1,296 @@
/**
* Tests for taskRunner — mock-based unit tests.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
// Mock all desktop and external dependencies
vi.mock('@mariozechner/pi-ai', () => ({
complete: vi.fn().mockResolvedValue({
content: [{ type: 'text', text: '{"summary": "compressed memory"}' }],
}),
getModel: vi.fn().mockReturnValue({ provider: 'mock', model: 'mock' }),
}));
vi.mock('@mariozechner/pi-agent-core', () => {
const subscribers: Array<(event: any) => void> = [];
let aborted = false;
return {
Agent: vi.fn().mockImplementation((config: any) => ({
state: {
messages: [
{
role: 'assistant',
content: [{ type: 'text', text: 'Task completed successfully' }],
stopReason: 'end',
},
],
},
subscribe: vi.fn((fn: any) => {
subscribers.push(fn);
return () => {};
}),
prompt: vi.fn(async () => {
// Simulate one turn
for (const sub of subscribers) {
sub({
type: 'turn_end',
message: { role: 'assistant', content: [{ type: 'text', text: 'done' }] },
toolResults: [{ toolName: 'computer_done', isError: false }],
});
}
}),
abort: vi.fn(() => { aborted = true; }),
setSystemPrompt: vi.fn(),
})),
};
});
vi.mock('../../src/desktop/screenshot.js', () => ({
captureScreenshot: vi.fn().mockResolvedValue({
image: 'base64screenshot',
mimeType: 'image/jpeg',
imageWidth: 1920,
imageHeight: 1080,
logicalWidth: 1920,
logicalHeight: 1080,
physicalWidth: 1920,
physicalHeight: 1080,
scaleFactor: 1,
displayIndex: 0,
}),
}));
vi.mock('../../src/desktop/mouse.js', () => ({
click: vi.fn(),
doubleClick: vi.fn(),
moveTo: vi.fn(),
drag: vi.fn(),
scroll: vi.fn(),
getPosition: vi.fn().mockResolvedValue({ x: 0, y: 0 }),
}));
vi.mock('../../src/desktop/keyboard.js', () => ({
typeText: vi.fn(),
pressKey: vi.fn(),
hotkey: vi.fn(),
}));
vi.mock('../../src/desktop/display.js', () => ({
getDisplay: vi.fn().mockResolvedValue({
index: 0,
label: 'Main',
width: 1920,
height: 1080,
scaleFactor: 1,
isPrimary: true,
origin: { x: 0, y: 0 },
}),
}));
vi.mock('../../src/coordinates/modelProfiles.js', () => ({
getModelProfile: vi.fn().mockReturnValue({
coordinateMode: 'image-absolute' as const,
coordinateOrder: 'xy',
}),
}));
vi.mock('sharp', () => ({
default: vi.fn().mockReturnValue({
resize: vi.fn().mockReturnThis(),
raw: vi.fn().mockReturnThis(),
toBuffer: vi.fn().mockResolvedValue(Buffer.alloc(32 * 32 * 3)),
}),
}));
import { createTaskRunner, createModel } from '../../src/agent/taskRunner.js';
import { AuditLog } from '../../src/safety/auditLog.js';
import type { GuiAgentConfig } from '../../src/config.js';
function createTestConfig(): GuiAgentConfig {
return {
provider: 'anthropic',
apiProtocol: 'anthropic',
model: 'claude-sonnet-4-20250514',
apiKey: 'test-key',
port: 60008,
maxSteps: 50,
stepDelayMs: 0,
stuckThreshold: 3,
jpegQuality: 75,
displayIndex: 0,
transport: 'http' as const,
coordinateMode: 'image-absolute' as const,
};
}
describe('createTaskRunner', () => {
let auditLog: AuditLog;
beforeEach(() => {
vi.clearAllMocks();
auditLog = new AuditLog();
});
it('should return run and abort functions', () => {
const runner = createTaskRunner(createTestConfig(), auditLog);
expect(typeof runner.run).toBe('function');
expect(typeof runner.abort).toBe('function');
});
it('should complete a task successfully', async () => {
const runner = createTaskRunner(createTestConfig(), auditLog);
const controller = new AbortController();
const progressCalls: any[] = [];
const result = await runner.run(
'Open Finder',
controller.signal,
(info) => progressCalls.push(info),
);
expect(result.success).toBe(true);
expect(result.finalScreenshot).toBe('base64screenshot');
expect(result.steps.length).toBeGreaterThanOrEqual(0);
});
it('should handle abort signal', async () => {
const { Agent } = await import('@mariozechner/pi-agent-core');
const mockAgent = (Agent as any).mockImplementation((config: any) => ({
state: {
messages: [
{
role: 'assistant',
content: [{ type: 'text', text: 'aborted' }],
stopReason: 'aborted',
},
],
},
subscribe: vi.fn(() => () => {}),
prompt: vi.fn(async () => {}),
abort: vi.fn(),
setSystemPrompt: vi.fn(),
}));
const runner = createTaskRunner(createTestConfig(), auditLog);
const controller = new AbortController();
const result = await runner.run(
'Open Finder',
controller.signal,
() => {},
);
expect(result.success).toBe(false);
expect(result.error).toBe('Task was aborted');
});
it('should handle errors in task execution', async () => {
const { Agent } = await import('@mariozechner/pi-agent-core');
(Agent as any).mockImplementation(() => ({
state: { messages: [] },
subscribe: vi.fn(() => () => {}),
prompt: vi.fn(async () => { throw new Error('API connection failed'); }),
abort: vi.fn(),
setSystemPrompt: vi.fn(),
}));
const runner = createTaskRunner(createTestConfig(), auditLog);
const controller = new AbortController();
const result = await runner.run(
'Open Finder',
controller.signal,
() => {},
);
expect(result.success).toBe(false);
expect(result.error).toBe('API connection failed');
});
it('should enforce maxSteps', async () => {
const { Agent } = await import('@mariozechner/pi-agent-core');
let agentAborted = false;
(Agent as any).mockImplementation((agentConfig: any) => {
const subs: any[] = [];
return {
state: {
messages: [
{ role: 'assistant', content: [{ type: 'text', text: 'done' }], stopReason: 'aborted' },
],
},
subscribe: vi.fn((fn: any) => { subs.push(fn); return () => {}; }),
prompt: vi.fn(async () => {
// Simulate maxSteps+1 turns to trigger abort
for (let i = 0; i < 5; i++) {
for (const sub of subs) {
sub({
type: 'turn_end',
message: { role: 'assistant', content: [] },
toolResults: [{ toolName: 'computer_screenshot', isError: false }],
});
}
}
}),
abort: vi.fn(() => { agentAborted = true; }),
setSystemPrompt: vi.fn(),
};
});
const config = createTestConfig();
config.maxSteps = 3;
const runner = createTaskRunner(config, auditLog);
const controller = new AbortController();
const result = await runner.run('Open Finder', controller.signal, () => {});
expect(agentAborted).toBe(true);
});
});
describe('createModel', () => {
it('uses getModel for built-in anthropic provider without baseUrl', () => {
const model = createModel('anthropic', 'anthropic', 'claude-sonnet-4-20250514');
// getModel is mocked to return { provider: 'mock', model: 'mock' }
expect(model).toEqual({ provider: 'mock', model: 'mock' });
});
it('uses getModel for built-in openai provider without baseUrl', () => {
const model = createModel('openai', 'openai', 'gpt-4o');
expect(model).toEqual({ provider: 'mock', model: 'mock' });
});
it('uses getModel for built-in google provider without baseUrl', () => {
const model = createModel('google', 'openai', 'gemini-2.5-pro');
expect(model).toEqual({ provider: 'mock', model: 'mock' });
});
it('constructs manual Model for custom provider', () => {
const model = createModel('zhipu', 'openai', 'glm-4v-plus', 'https://open.bigmodel.cn/api/paas/v4');
expect(model.id).toBe('glm-4v-plus');
expect(model.name).toBe('glm-4v-plus');
expect(model.api).toBe('openai-completions');
expect(model.provider).toBe('zhipu');
expect(model.baseUrl).toBe('https://open.bigmodel.cn/api/paas/v4');
});
it('constructs manual Model for built-in provider with custom baseUrl', () => {
const model = createModel('anthropic', 'anthropic', 'claude-sonnet-4-20250514', 'https://proxy.example.com');
expect(model.id).toBe('claude-sonnet-4-20250514');
expect(model.api).toBe('anthropic-messages');
expect(model.baseUrl).toBe('https://proxy.example.com');
});
it('maps anthropic protocol to anthropic-messages api', () => {
const model = createModel('custom', 'anthropic', 'my-model', 'https://custom.api.com');
expect(model.api).toBe('anthropic-messages');
});
it('maps openai protocol to openai-completions api', () => {
const model = createModel('custom', 'openai', 'my-model', 'https://custom.api.com');
expect(model.api).toBe('openai-completions');
});
});

View File

@@ -0,0 +1,199 @@
/**
* Unit tests for config.ts — required field validation, defaults, range checks.
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { loadConfig } from '../../src/config.js';
// Suppress logger output during tests
vi.mock('../../src/utils/logger.js', () => ({
logInfo: vi.fn(),
logWarn: vi.fn(),
logError: vi.fn(),
logDebug: vi.fn(),
}));
describe('loadConfig', () => {
const originalEnv = { ...process.env };
beforeEach(() => {
// Clear all GUI_AGENT_* env vars
Object.keys(process.env).forEach(key => {
if (key.startsWith('GUI_AGENT_')) delete process.env[key];
});
});
afterEach(() => {
process.env = { ...originalEnv };
});
// --- Required field validation ---
it('throws ConfigError when API_KEY is missing', () => {
expect(() => loadConfig()).toThrow('GUI_AGENT_API_KEY is required');
});
it('accepts API_KEY from env', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
const config = loadConfig();
expect(config.apiKey).toBe('test-key');
});
it('accepts API_KEY from overrides', () => {
const config = loadConfig({ apiKey: 'override-key' });
expect(config.apiKey).toBe('override-key');
});
// --- Default values ---
it('has correct defaults', () => {
const config = loadConfig({ apiKey: 'test-key' });
expect(config.provider).toBe('anthropic');
expect(config.apiProtocol).toBe('anthropic');
expect(config.model).toBe('claude-sonnet-4-20250514');
expect(config.port).toBe(60008);
expect(config.transport).toBe('http');
expect(config.maxSteps).toBe(50);
expect(config.stepDelayMs).toBe(1500);
expect(config.stuckThreshold).toBe(3);
expect(config.jpegQuality).toBe(75);
expect(config.displayIndex).toBe(0);
expect(config.coordinateMode).toBeUndefined();
expect(config.baseUrl).toBeUndefined();
expect(config.memoryProvider).toBeUndefined();
expect(config.memoryModel).toBeUndefined();
expect(config.logFile).toBeUndefined();
});
// --- Transport validation ---
it('throws on invalid transport', () => {
expect(() => loadConfig({ apiKey: 'test-key', transport: 'websocket' as any }))
.toThrow('GUI_AGENT_TRANSPORT must be "http" or "stdio"');
});
it('accepts stdio transport', () => {
const config = loadConfig({ apiKey: 'test-key', transport: 'stdio' });
expect(config.transport).toBe('stdio');
});
// --- Numeric range validation ---
it('throws when maxSteps exceeds range', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_MAX_STEPS = '999';
expect(() => loadConfig()).toThrow('GUI_AGENT_MAX_STEPS must be between 1 and 200');
});
it('throws when maxSteps is below range', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_MAX_STEPS = '0';
expect(() => loadConfig()).toThrow('GUI_AGENT_MAX_STEPS must be between 1 and 200');
});
it('throws when jpegQuality exceeds range', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_JPEG_QUALITY = '101';
expect(() => loadConfig()).toThrow('GUI_AGENT_JPEG_QUALITY must be between 1 and 100');
});
it('throws when stepDelayMs exceeds range', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_STEP_DELAY_MS = '50000';
expect(() => loadConfig()).toThrow('GUI_AGENT_STEP_DELAY_MS must be between 100 and 30000');
});
it('throws when port is invalid', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_PORT = '99999';
expect(() => loadConfig()).toThrow('GUI_AGENT_PORT must be between 1 and 65535');
});
it('accepts valid numeric values', () => {
const config = loadConfig({
apiKey: 'test-key',
maxSteps: 100,
jpegQuality: 50,
stepDelayMs: 2000,
port: 8080,
});
expect(config.maxSteps).toBe(100);
expect(config.jpegQuality).toBe(50);
expect(config.stepDelayMs).toBe(2000);
expect(config.port).toBe(8080);
});
// --- apiProtocol validation ---
it('throws on invalid apiProtocol', () => {
expect(() => loadConfig({ apiKey: 'test-key', apiProtocol: 'grpc' as any }))
.toThrow('GUI_AGENT_API_PROTOCOL must be "anthropic" or "openai"');
});
it('accepts openai apiProtocol', () => {
const config = loadConfig({ apiKey: 'test-key', apiProtocol: 'openai' });
expect(config.apiProtocol).toBe('openai');
});
it('reads apiProtocol from env', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_API_PROTOCOL = 'openai';
const config = loadConfig();
expect(config.apiProtocol).toBe('openai');
});
it('overrides take precedence for apiProtocol', () => {
process.env.GUI_AGENT_API_KEY = 'test-key';
process.env.GUI_AGENT_API_PROTOCOL = 'openai';
const config = loadConfig({ apiKey: 'test-key', apiProtocol: 'anthropic' });
expect(config.apiProtocol).toBe('anthropic');
});
// --- coordinateMode validation ---
it('throws on invalid coordinateMode', () => {
expect(() => loadConfig({ apiKey: 'test-key', coordinateMode: 'foobar' as any }))
.toThrow('GUI_AGENT_COORDINATE_MODE must be one of');
});
it('accepts valid coordinateMode', () => {
const config = loadConfig({ apiKey: 'test-key', coordinateMode: 'normalized-1000' });
expect(config.coordinateMode).toBe('normalized-1000');
});
it('treats "auto" coordinateMode as undefined (auto-detect)', () => {
const config = loadConfig({ apiKey: 'test-key', coordinateMode: 'auto' as any });
expect(config.coordinateMode).toBeUndefined();
});
// --- Environment variable overrides ---
it('reads all env vars', () => {
process.env.GUI_AGENT_API_KEY = 'env-key';
process.env.GUI_AGENT_PROVIDER = 'openai';
process.env.GUI_AGENT_MODEL = 'gpt-4o';
process.env.GUI_AGENT_BASE_URL = 'https://example.com';
process.env.GUI_AGENT_MEMORY_PROVIDER = 'anthropic';
process.env.GUI_AGENT_MEMORY_MODEL = 'haiku';
process.env.GUI_AGENT_LOG_FILE = '/tmp/gui.log';
const config = loadConfig();
expect(config.provider).toBe('openai');
expect(config.model).toBe('gpt-4o');
expect(config.baseUrl).toBe('https://example.com');
expect(config.memoryProvider).toBe('anthropic');
expect(config.memoryModel).toBe('haiku');
expect(config.logFile).toBe('/tmp/gui.log');
});
// --- Overrides take precedence over env ---
it('overrides take precedence over env vars', () => {
process.env.GUI_AGENT_API_KEY = 'env-key';
process.env.GUI_AGENT_MODEL = 'env-model';
const config = loadConfig({ apiKey: 'override-key', model: 'override-model' });
expect(config.apiKey).toBe('override-key');
expect(config.model).toBe('override-model');
});
});

View File

@@ -0,0 +1,138 @@
/**
* T2.2 — modelProfiles unit tests
*
* Covers: 7 model rules + fallback + coordinateOrder + override
*/
import { describe, it, expect } from 'vitest';
import { getModelProfile } from '../../src/coordinates/modelProfiles.js';
describe('getModelProfile', () => {
describe('model matching', () => {
it('claude models → image-absolute, xy', () => {
const profile = getModelProfile('claude-sonnet-4-20250514');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('claude-opus → image-absolute, xy', () => {
const profile = getModelProfile('claude-opus-4-1');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('gpt-4o → image-absolute, xy', () => {
const profile = getModelProfile('gpt-4o-2024-05-13');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('gpt-5 → image-absolute, xy', () => {
const profile = getModelProfile('gpt-5-turbo');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('gemini → normalized-999, yx', () => {
const profile = getModelProfile('gemini-2.5-pro');
expect(profile.coordinateMode).toBe('normalized-999');
expect(profile.coordinateOrder).toBe('yx');
});
it('gemini-2.5-flash → normalized-999, yx', () => {
const profile = getModelProfile('gemini-2.5-flash');
expect(profile.coordinateMode).toBe('normalized-999');
expect(profile.coordinateOrder).toBe('yx');
});
it('ui-tars → normalized-1000, xy', () => {
const profile = getModelProfile('ui-tars-1.0');
expect(profile.coordinateMode).toBe('normalized-1000');
expect(profile.coordinateOrder).toBe('xy');
});
it('qwen2.5-vl → image-absolute, xy', () => {
const profile = getModelProfile('qwen2.5-vl-7b');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('qwen-vl → image-absolute, xy', () => {
const profile = getModelProfile('qwen-vl-plus');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('cogagent → image-absolute, xy', () => {
const profile = getModelProfile('cogagent-18b');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('seeclick → normalized-0-1, xy', () => {
const profile = getModelProfile('seeclick-v2');
expect(profile.coordinateMode).toBe('normalized-0-1');
expect(profile.coordinateOrder).toBe('xy');
});
it('showui → normalized-0-1, xy', () => {
const profile = getModelProfile('showui-large');
expect(profile.coordinateMode).toBe('normalized-0-1');
expect(profile.coordinateOrder).toBe('xy');
});
});
describe('fallback', () => {
it('unknown model → image-absolute, xy', () => {
const profile = getModelProfile('some-unknown-model');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
it('empty string → image-absolute, xy', () => {
const profile = getModelProfile('');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('xy');
});
});
describe('coordinateOrder', () => {
it('only Gemini has yx order', () => {
const models = [
'claude-sonnet-4-20250514', 'gpt-4o', 'ui-tars-1.0',
'qwen2.5-vl-7b', 'cogagent-18b', 'seeclick-v2', 'showui-large',
];
for (const model of models) {
expect(getModelProfile(model).coordinateOrder).toBe('xy');
}
expect(getModelProfile('gemini-2.5-pro').coordinateOrder).toBe('yx');
});
});
describe('override mode', () => {
it('overrides coordinateMode but keeps coordinateOrder', () => {
const profile = getModelProfile('gemini-2.5-pro', 'image-absolute');
expect(profile.coordinateMode).toBe('image-absolute');
expect(profile.coordinateOrder).toBe('yx'); // Gemini order preserved
});
it('overrides coordinateMode for Claude', () => {
const profile = getModelProfile('claude-sonnet-4-20250514', 'normalized-1000');
expect(profile.coordinateMode).toBe('normalized-1000');
expect(profile.coordinateOrder).toBe('xy');
});
});
describe('case insensitivity', () => {
it('matches case-insensitively', () => {
const profile = getModelProfile('Claude-Sonnet-4-20250514');
expect(profile.coordinateMode).toBe('image-absolute');
});
it('matches GEMINI uppercase', () => {
const profile = getModelProfile('GEMINI-2.5-PRO');
expect(profile.coordinateMode).toBe('normalized-999');
expect(profile.coordinateOrder).toBe('yx');
});
});
});

View File

@@ -0,0 +1,175 @@
/**
* T2.1 — CoordinateResolver unit tests
*
* Covers: 4 coordinate modes × Gemini yx swap × Retina/HiDPI × multi-display offset × boundary clamp
*/
import { describe, it, expect } from 'vitest';
import { resolveCoordinate, type ScreenshotMeta, type DisplayInfo } from '../../src/coordinates/resolver.js';
import type { ModelProfile } from '../../src/coordinates/modelProfiles.js';
// --- Helpers ---
const primaryDisplay: DisplayInfo = { origin: { x: 0, y: 0 }, bounds: { width: 1920, height: 1080 } };
const secondaryDisplay: DisplayInfo = { origin: { x: -2560, y: 0 }, bounds: { width: 2560, height: 1440 } };
const retinaMeta: ScreenshotMeta = { imageWidth: 1440, imageHeight: 900, logicalWidth: 1440, logicalHeight: 900 };
const fullHdMeta: ScreenshotMeta = { imageWidth: 1920, imageHeight: 1080, logicalWidth: 1920, logicalHeight: 1080 };
const xyProfile = (mode: ModelProfile['coordinateMode']): ModelProfile => ({ coordinateMode: mode, coordinateOrder: 'xy' });
const geminiProfile: ModelProfile = { coordinateMode: 'normalized-999', coordinateOrder: 'yx' };
// --- Tests ---
describe('resolveCoordinate', () => {
describe('image-absolute mode', () => {
it('converts absolute pixel coordinates on 1920x1080', () => {
const result = resolveCoordinate(960, 540, xyProfile('image-absolute'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(960);
expect(result.globalY).toBe(540);
expect(result.normalizedX).toBeCloseTo(0.5, 2);
expect(result.normalizedY).toBeCloseTo(0.5, 2);
});
it('handles top-left corner (0, 0)', () => {
const result = resolveCoordinate(0, 0, xyProfile('image-absolute'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(0);
expect(result.globalY).toBe(0);
});
it('handles bottom-right corner', () => {
const result = resolveCoordinate(1920, 1080, xyProfile('image-absolute'), fullHdMeta, primaryDisplay);
// 1920/1920 * 1920 = 1920, clamped to bounds.width-1 = 1919
expect(result.globalX).toBe(1919);
expect(result.globalY).toBe(1079);
});
});
describe('normalized-1000 mode', () => {
it('converts 500, 500 → center of display', () => {
const result = resolveCoordinate(500, 500, xyProfile('normalized-1000'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(960);
expect(result.globalY).toBe(540);
});
it('converts 0, 0 → top-left', () => {
const result = resolveCoordinate(0, 0, xyProfile('normalized-1000'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(0);
expect(result.globalY).toBe(0);
});
it('converts 1000, 1000 → bottom-right (clamped)', () => {
const result = resolveCoordinate(1000, 1000, xyProfile('normalized-1000'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(1919);
expect(result.globalY).toBe(1079);
});
});
describe('normalized-999 mode', () => {
it('converts 499, 499 → approximate center', () => {
const result = resolveCoordinate(499, 499, xyProfile('normalized-999'), fullHdMeta, primaryDisplay);
// 499/999 ≈ 0.4995 → 0.4995 * 1920 ≈ 959
expect(result.globalX).toBeGreaterThanOrEqual(958);
expect(result.globalX).toBeLessThanOrEqual(960);
});
});
describe('normalized-0-1 mode', () => {
it('converts 0.5, 0.5 → center', () => {
const result = resolveCoordinate(0.5, 0.5, xyProfile('normalized-0-1'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(960);
expect(result.globalY).toBe(540);
});
it('converts 0, 0 → top-left', () => {
const result = resolveCoordinate(0, 0, xyProfile('normalized-0-1'), fullHdMeta, primaryDisplay);
expect(result.globalX).toBe(0);
expect(result.globalY).toBe(0);
});
});
describe('Gemini yx coordinate order swap', () => {
it('swaps yx to xy before normalization', () => {
// Gemini outputs [y, x] = [499, 200] → should become rawX=200, rawY=499
const result = resolveCoordinate(499, 200, geminiProfile, fullHdMeta, primaryDisplay);
// rawX = 200 (from modelY), rawY = 499 (from modelX)
// normX = 200/999 ≈ 0.2002, normY = 499/999 ≈ 0.4995
// localX = 0.2002 * 1920 ≈ 384, localY = 0.4995 * 1080 ≈ 539
expect(result.normalizedX).toBeCloseTo(200 / 999, 3);
expect(result.normalizedY).toBeCloseTo(499 / 999, 3);
expect(result.globalX).toBeGreaterThanOrEqual(383);
expect(result.globalX).toBeLessThanOrEqual(385);
});
it('Gemini center point [500, 500] should still map to center', () => {
// [y=500, x=500] → swap → rawX=500, rawY=500
const result = resolveCoordinate(500, 500, geminiProfile, fullHdMeta, primaryDisplay);
expect(result.normalizedX).toBeCloseTo(500 / 999, 3);
expect(result.normalizedY).toBeCloseTo(500 / 999, 3);
});
});
describe('Retina display (scaleFactor absorbed at screenshot level)', () => {
// Retina: physical 2880x1800, scaleFactor=2, logical 1440x900
// After screenshot pipeline: imageWidth=1440, imageHeight=900 (already logical)
// logicalWidth=1440, logicalHeight=900
it('converts center on Retina display (image-absolute)', () => {
const result = resolveCoordinate(720, 450, xyProfile('image-absolute'), retinaMeta, {
origin: { x: 0, y: 0 },
bounds: { width: 1440, height: 900 },
});
expect(result.globalX).toBe(720);
expect(result.globalY).toBe(450);
expect(result.normalizedX).toBeCloseTo(0.5, 2);
expect(result.normalizedY).toBeCloseTo(0.5, 2);
});
});
describe('HiDPI Windows (scaleFactor=1.5)', () => {
// Physical 2880x1620, scaleFactor=1.5, logical 1920x1080
// Screenshot pipeline outputs imageWidth=1920, imageHeight=1080
const hidpiMeta: ScreenshotMeta = { imageWidth: 1920, imageHeight: 1080, logicalWidth: 1920, logicalHeight: 1080 };
it('converts absolute coords on HiDPI display', () => {
const result = resolveCoordinate(960, 540, xyProfile('image-absolute'), hidpiMeta, primaryDisplay);
expect(result.globalX).toBe(960);
expect(result.globalY).toBe(540);
});
});
describe('multi-display offset', () => {
it('applies negative origin offset for secondary display on left', () => {
// Secondary display at origin (-2560, 0), 2560x1440
const meta: ScreenshotMeta = { imageWidth: 1920, imageHeight: 1080, logicalWidth: 2560, logicalHeight: 1440 };
const result = resolveCoordinate(960, 540, xyProfile('image-absolute'), meta, secondaryDisplay);
// normX = 960/1920 = 0.5, localX = 0.5 * 2560 = 1280
// globalX = 1280 + (-2560) = -1280
expect(result.localX).toBe(1280);
expect(result.globalX).toBe(-1280);
expect(result.localY).toBe(720);
expect(result.globalY).toBe(720);
});
it('primary display has zero offset', () => {
const result = resolveCoordinate(100, 100, xyProfile('image-absolute'), fullHdMeta, primaryDisplay);
expect(result.localX).toBe(result.globalX);
expect(result.localY).toBe(result.globalY);
});
});
describe('boundary clamp', () => {
it('clamps coordinates exceeding display bounds', () => {
// Send coordinates beyond the display
const result = resolveCoordinate(2500, 1500, xyProfile('image-absolute'), fullHdMeta, primaryDisplay);
// normX = 2500/1920 ≈ 1.302, localX = 1.302 * 1920 ≈ 2500 → clamped to 1919
expect(result.globalX).toBe(1919);
expect(result.globalY).toBe(1079);
});
it('clamps negative coordinates', () => {
const result = resolveCoordinate(-100, -100, xyProfile('image-absolute'), fullHdMeta, primaryDisplay);
// normX = -100/1920 < 0, localX < 0 → clamped to 0
expect(result.globalX).toBe(0);
expect(result.globalY).toBe(0);
});
});
});

View File

@@ -0,0 +1,125 @@
/**
* Unit tests for desktop/clipboard.ts — pasteText flow, backup/restore, key order.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const mockRead = vi.fn();
const mockWrite = vi.fn();
vi.mock('clipboardy', () => ({
default: {
read: () => mockRead(),
write: (text: string) => mockWrite(text),
},
}));
const mockPressKey = vi.fn();
const mockReleaseKey = vi.fn();
vi.mock('@nut-tree-fork/nut-js', () => ({
keyboard: {
pressKey: mockPressKey,
releaseKey: mockReleaseKey,
},
Key: {
Meta: 100,
Control: 101,
V: 102,
},
}));
const mockGetPlatformPasteKeys = vi.fn();
vi.mock('../../src/utils/platform.js', () => ({
getPlatformPasteKeys: () => mockGetPlatformPasteKeys(),
}));
import { pasteText, readClipboard, writeClipboard } from '../../src/desktop/clipboard.js';
describe('readClipboard', () => {
beforeEach(() => vi.clearAllMocks());
it('reads clipboard content', async () => {
mockRead.mockResolvedValue('existing content');
const result = await readClipboard();
expect(result).toBe('existing content');
});
});
describe('writeClipboard', () => {
beforeEach(() => vi.clearAllMocks());
it('writes text to clipboard', async () => {
mockWrite.mockResolvedValue(undefined);
await writeClipboard('new text');
expect(mockWrite).toHaveBeenCalledWith('new text');
});
});
describe('pasteText', () => {
beforeEach(() => {
vi.clearAllMocks();
mockGetPlatformPasteKeys.mockReturnValue(['Meta', 'V']);
mockRead.mockResolvedValue('backup-content');
mockWrite.mockResolvedValue(undefined);
mockPressKey.mockResolvedValue(undefined);
mockReleaseKey.mockResolvedValue(undefined);
});
it('follows full paste flow: backup → write → press → wait → restore', async () => {
await pasteText('target text');
// 1. Backup clipboard
expect(mockRead).toHaveBeenCalledTimes(1);
// 2. Write target text
expect(mockWrite).toHaveBeenNthCalledWith(1, 'target text');
// 3. Press paste keys
expect(mockPressKey).toHaveBeenCalledWith(100, 102); // Meta, V
// 4. Release in REVERSE order (V first, then Meta)
expect(mockReleaseKey).toHaveBeenCalledWith(102, 100); // V, Meta
// 5. Restore original clipboard
expect(mockWrite).toHaveBeenNthCalledWith(2, 'backup-content');
});
it('continues if backup read fails', async () => {
mockRead.mockRejectedValue(new Error('clipboard not available'));
await pasteText('text');
// Should still write and paste
expect(mockWrite).toHaveBeenCalledWith('text');
expect(mockPressKey).toHaveBeenCalled();
});
it('does not throw if restore fails', async () => {
mockRead.mockResolvedValue('backup');
// First write succeeds, second (restore) fails
mockWrite
.mockResolvedValueOnce(undefined)
.mockRejectedValueOnce(new Error('restore failed'));
// Should not throw
await expect(pasteText('text')).resolves.toBeUndefined();
});
it('does not attempt restore if backup was undefined', async () => {
mockRead.mockRejectedValue(new Error('no clipboard'));
await pasteText('text');
// Only one write call (the target text), no restore
expect(mockWrite).toHaveBeenCalledTimes(1);
expect(mockWrite).toHaveBeenCalledWith('text');
});
it('uses platform-specific paste keys', async () => {
mockGetPlatformPasteKeys.mockReturnValue(['Control', 'V']);
await pasteText('text');
expect(mockPressKey).toHaveBeenCalledWith(101, 102); // Control, V
expect(mockReleaseKey).toHaveBeenCalledWith(102, 101); // V, Control (reversed)
});
});

View File

@@ -0,0 +1,294 @@
/**
* Unit tests for desktop/display.ts — macOS system_profiler parsing,
* nut.js fallback with scaleFactor detection, getDisplay, getPrimaryDisplay.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
// --- Mock platform ---
const mockGetPlatform = vi.fn();
vi.mock('../../src/utils/platform.js', () => ({
getPlatform: () => mockGetPlatform(),
}));
// --- Mock logger ---
vi.mock('../../src/utils/logger.js', () => ({
logInfo: vi.fn(),
logWarn: vi.fn(),
logError: vi.fn(),
logDebug: vi.fn(),
}));
// --- Mock child_process (for macOS system_profiler) ---
const mockExecSync = vi.fn();
vi.mock('child_process', () => ({
execSync: (...args: any[]) => mockExecSync(...args),
}));
// --- Mock nut.js (for fallback) ---
const mockScreenWidth = vi.fn();
const mockScreenHeight = vi.fn();
const mockGrabRegion = vi.fn();
vi.mock('@nut-tree-fork/nut-js', () => {
class MockRegion {
constructor(
public left: number,
public top: number,
public width: number,
public height: number,
) {}
}
return {
screen: {
width: () => mockScreenWidth(),
height: () => mockScreenHeight(),
grabRegion: (r: any) => mockGrabRegion(r),
},
Region: MockRegion,
};
});
import { listDisplays, getDisplay, getPrimaryDisplay } from '../../src/desktop/display.js';
// --- Helpers: system_profiler JSON fixtures ---
function macSingleDisplay(opts: {
resolution?: string;
pixels?: string;
retina?: string;
name?: string;
main?: string;
} = {}) {
return JSON.stringify({
SPDisplaysDataType: [
{
spdisplays_ndrvs: [
{
_spdisplays_resolution: opts.resolution ?? '1440 x 900 @ 60.00Hz',
_spdisplays_pixels: opts.pixels ?? '2880 x 1800',
spdisplays_retina: opts.retina ?? 'spdisplays_yes',
_name: opts.name ?? 'Built-in Retina Display',
spdisplays_main: opts.main ?? 'spdisplays_yes',
},
],
},
],
});
}
function macDualDisplay() {
return JSON.stringify({
SPDisplaysDataType: [
{
spdisplays_ndrvs: [
{
_spdisplays_resolution: '1440 x 900 @ 60.00Hz',
_spdisplays_pixels: '2880 x 1800',
spdisplays_retina: 'spdisplays_yes',
_name: 'Built-in Retina Display',
spdisplays_main: 'spdisplays_yes',
},
{
_spdisplays_resolution: '2560 x 1440',
_spdisplays_pixels: '2560 x 1440',
spdisplays_retina: 'spdisplays_no',
_name: 'External Monitor',
},
],
},
],
});
}
describe('listDisplays', () => {
beforeEach(() => {
vi.clearAllMocks();
});
// --- macOS via system_profiler ---
describe('macOS (system_profiler)', () => {
beforeEach(() => {
mockGetPlatform.mockReturnValue('macos');
});
it('parses single Retina display', async () => {
mockExecSync.mockReturnValue(macSingleDisplay());
const displays = await listDisplays();
expect(displays).toHaveLength(1);
expect(displays[0]).toMatchObject({
index: 0,
label: 'Built-in Retina Display',
width: 1440,
height: 900,
scaleFactor: 2,
isPrimary: true,
origin: { x: 0, y: 0 },
});
});
it('parses dual display setup', async () => {
mockExecSync.mockReturnValue(macDualDisplay());
const displays = await listDisplays();
expect(displays).toHaveLength(2);
expect(displays[0].label).toBe('Built-in Retina Display');
expect(displays[0].isPrimary).toBe(true);
expect(displays[0].scaleFactor).toBe(2);
expect(displays[1].label).toBe('External Monitor');
expect(displays[1].width).toBe(2560);
expect(displays[1].height).toBe(1440);
expect(displays[1].scaleFactor).toBe(1);
});
it('detects Retina via flag when pixel resolution missing', async () => {
mockExecSync.mockReturnValue(macSingleDisplay({
pixels: '', // no pixel resolution
retina: 'spdisplays_yes',
resolution: '1440 x 900',
}));
const displays = await listDisplays();
expect(displays).toHaveLength(1);
expect(displays[0].scaleFactor).toBe(2); // falls back to retina flag
});
it('defaults scaleFactor 1 for non-Retina without pixel info', async () => {
mockExecSync.mockReturnValue(macSingleDisplay({
pixels: '',
retina: 'spdisplays_no',
resolution: '1920 x 1080',
}));
const displays = await listDisplays();
expect(displays[0].scaleFactor).toBe(1);
});
it('falls back to nut.js when system_profiler throws', async () => {
mockExecSync.mockImplementation(() => { throw new Error('command not found'); });
mockScreenWidth.mockResolvedValue(1920);
mockScreenHeight.mockResolvedValue(1080);
mockGrabRegion.mockResolvedValue({ width: 10 });
const displays = await listDisplays();
expect(displays).toHaveLength(1);
expect(displays[0].width).toBe(1920);
expect(displays[0].scaleFactor).toBe(1);
});
it('falls back when system_profiler returns empty displays', async () => {
mockExecSync.mockReturnValue(JSON.stringify({ SPDisplaysDataType: [] }));
mockScreenWidth.mockResolvedValue(1920);
mockScreenHeight.mockResolvedValue(1080);
mockGrabRegion.mockResolvedValue({ width: 20 });
const displays = await listDisplays();
expect(displays).toHaveLength(1);
expect(displays[0].scaleFactor).toBe(2); // 20/10 = 2
});
it('uses display name from _name field', async () => {
mockExecSync.mockReturnValue(macSingleDisplay({ name: 'LG UltraFine' }));
const displays = await listDisplays();
expect(displays[0].label).toBe('LG UltraFine');
});
});
// --- Windows/Linux fallback ---
describe('fallback (nut.js)', () => {
beforeEach(() => {
mockGetPlatform.mockReturnValue('windows');
});
it('returns single display with correct dimensions', async () => {
mockScreenWidth.mockResolvedValue(1920);
mockScreenHeight.mockResolvedValue(1080);
mockGrabRegion.mockResolvedValue({ width: 10 });
const displays = await listDisplays();
expect(displays).toHaveLength(1);
expect(displays[0]).toMatchObject({
index: 0,
label: 'Primary',
width: 1920,
height: 1080,
scaleFactor: 1,
isPrimary: true,
origin: { x: 0, y: 0 },
});
});
it('detects HiDPI scaleFactor from capture', async () => {
mockScreenWidth.mockResolvedValue(1920);
mockScreenHeight.mockResolvedValue(1080);
mockGrabRegion.mockResolvedValue({ width: 20 }); // 20/10 = 2x
const displays = await listDisplays();
expect(displays[0].scaleFactor).toBe(2);
});
it('defaults scaleFactor to 1 when capture fails', async () => {
mockScreenWidth.mockResolvedValue(1920);
mockScreenHeight.mockResolvedValue(1080);
mockGrabRegion.mockRejectedValue(new Error('no display'));
const displays = await listDisplays();
expect(displays[0].scaleFactor).toBe(1);
});
it('works on Linux', async () => {
mockGetPlatform.mockReturnValue('linux');
mockScreenWidth.mockResolvedValue(2560);
mockScreenHeight.mockResolvedValue(1440);
mockGrabRegion.mockResolvedValue({ width: 10 });
const displays = await listDisplays();
expect(displays[0].width).toBe(2560);
expect(displays[0].height).toBe(1440);
});
});
});
describe('getDisplay', () => {
beforeEach(() => {
vi.clearAllMocks();
mockGetPlatform.mockReturnValue('macos');
mockExecSync.mockReturnValue(macDualDisplay());
});
it('returns display by index', async () => {
const d = await getDisplay(1);
expect(d.index).toBe(1);
expect(d.label).toBe('External Monitor');
});
it('throws DesktopError for out-of-range index', async () => {
await expect(getDisplay(5)).rejects.toThrow('Display index 5 not found');
});
});
describe('getPrimaryDisplay', () => {
beforeEach(() => {
vi.clearAllMocks();
mockGetPlatform.mockReturnValue('macos');
});
it('returns the primary display', async () => {
mockExecSync.mockReturnValue(macDualDisplay());
const d = await getPrimaryDisplay();
expect(d.isPrimary).toBe(true);
expect(d.label).toBe('Built-in Retina Display');
});
it('falls back to first display if none marked primary', async () => {
// Single display with no main flag — first display still returned
mockExecSync.mockReturnValue(macSingleDisplay({ main: '' }));
const d = await getPrimaryDisplay();
// First display is always treated as primary when displays.length === 0 before push
expect(d.index).toBe(0);
});
});

View File

@@ -0,0 +1,133 @@
/**
* Unit tests for desktop/imageSearch.ts — findImage, waitForImage.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
// Mock fs
const mockWriteFile = vi.fn();
const mockUnlink = vi.fn();
vi.mock('fs', () => ({
promises: {
writeFile: (...args: any[]) => mockWriteFile(...args),
unlink: (...args: any[]) => mockUnlink(...args),
},
}));
// Mock nut.js
const mockFind = vi.fn();
const mockLoadImage = vi.fn();
const mockScreenConfig = { confidence: 0.9 };
vi.mock('@nut-tree-fork/nut-js', () => ({
screen: {
find: (...args: any[]) => mockFind(...args),
config: mockScreenConfig,
},
loadImage: (...args: any[]) => mockLoadImage(...args),
}));
import { findImage, waitForImage } from '../../src/desktop/imageSearch.js';
describe('findImage', () => {
beforeEach(() => {
vi.clearAllMocks();
mockWriteFile.mockResolvedValue(undefined);
mockUnlink.mockResolvedValue(undefined);
mockLoadImage.mockResolvedValue('template-obj');
});
it('returns found=true with region when match found', async () => {
mockFind.mockResolvedValue({ left: 10, top: 20, width: 50, height: 30 });
const result = await findImage('dGVzdA==', 0.85);
expect(result.found).toBe(true);
expect(result.region).toEqual({ x: 10, y: 20, width: 50, height: 30 });
expect(result.confidence).toBe(0.85);
expect(mockScreenConfig.confidence).toBe(0.85);
});
it('returns found=false when no match', async () => {
mockFind.mockRejectedValue(new Error('no match'));
const result = await findImage('dGVzdA==');
expect(result.found).toBe(false);
expect(result.region).toBeUndefined();
});
it('writes template to temp file and cleans up', async () => {
mockFind.mockResolvedValue({ left: 0, top: 0, width: 10, height: 10 });
await findImage('dGVzdA==');
expect(mockWriteFile).toHaveBeenCalledTimes(1);
expect(mockUnlink).toHaveBeenCalledTimes(1);
});
it('cleans up temp file even when match fails', async () => {
mockFind.mockRejectedValue(new Error('no match'));
await findImage('dGVzdA==');
expect(mockUnlink).toHaveBeenCalledTimes(1);
});
it('uses default confidence of 0.9', async () => {
mockFind.mockResolvedValue({ left: 0, top: 0, width: 10, height: 10 });
const result = await findImage('dGVzdA==');
expect(result.confidence).toBe(0.9);
});
});
describe('waitForImage', () => {
beforeEach(() => {
vi.clearAllMocks();
mockWriteFile.mockResolvedValue(undefined);
mockUnlink.mockResolvedValue(undefined);
mockLoadImage.mockResolvedValue('template-obj');
});
it('returns found=true with elapsed when image found immediately', async () => {
mockFind.mockResolvedValue({ left: 5, top: 10, width: 20, height: 15 });
const result = await waitForImage('dGVzdA==', 5000, 0.8);
expect(result.found).toBe(true);
expect(result.region).toEqual({ x: 5, y: 10, width: 20, height: 15 });
expect(result.elapsed).toBeDefined();
expect(result.elapsed).toBeGreaterThanOrEqual(0);
});
it('returns found=false with elapsed after timeout', async () => {
mockFind.mockRejectedValue(new Error('no match'));
// Use a very short timeout to avoid slow tests
const result = await waitForImage('dGVzdA==', 100, 0.9);
expect(result.found).toBe(false);
expect(result.elapsed).toBeGreaterThanOrEqual(100);
});
it('cleans up temp file after completion', async () => {
mockFind.mockResolvedValue({ left: 0, top: 0, width: 10, height: 10 });
await waitForImage('dGVzdA==', 5000);
expect(mockUnlink).toHaveBeenCalledTimes(1);
});
it('cleans up temp file even after timeout', async () => {
mockFind.mockRejectedValue(new Error('no match'));
await waitForImage('dGVzdA==', 100);
expect(mockUnlink).toHaveBeenCalledTimes(1);
});
it('writes temp file only once for reuse across polls', async () => {
mockFind.mockRejectedValue(new Error('no match'));
await waitForImage('dGVzdA==', 100);
// Only 1 writeFile call despite potentially multiple polls
expect(mockWriteFile).toHaveBeenCalledTimes(1);
});
});

View File

@@ -0,0 +1,111 @@
/**
* Unit tests for desktop/keyboard.ts — typeText CJK routing, pressKey, hotkey.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const mockType = vi.fn();
const mockPressKey = vi.fn();
const mockReleaseKey = vi.fn();
vi.mock('@nut-tree-fork/nut-js', () => ({
keyboard: {
type: mockType,
pressKey: mockPressKey,
releaseKey: mockReleaseKey,
},
Key: {
Enter: 'Enter_Val',
Tab: 'Tab_Val',
Escape: 'Escape_Val',
LeftSuper: 'LeftSuper_Val', // Meta/Command/Cmd alias
LeftControl: 'LeftControl_Val',
LeftAlt: 'LeftAlt_Val',
LeftShift: 'LeftShift_Val',
C: 'C_Val',
V: 'V_Val',
},
}));
const mockPasteText = vi.fn();
vi.mock('../../src/desktop/clipboard.js', () => ({
pasteText: (...args: any[]) => mockPasteText(...args),
}));
import { typeText, pressKey, hotkey } from '../../src/desktop/keyboard.js';
describe('typeText', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('uses nut.js type for short ASCII text', async () => {
await typeText('hello');
expect(mockType).toHaveBeenCalledWith('hello');
expect(mockPasteText).not.toHaveBeenCalled();
});
it('routes CJK text to clipboard paste', async () => {
await typeText('你好世界');
expect(mockPasteText).toHaveBeenCalledWith('你好世界');
expect(mockType).not.toHaveBeenCalled();
});
it('routes non-ASCII text to clipboard paste', async () => {
await typeText('café');
expect(mockPasteText).toHaveBeenCalledWith('café');
expect(mockType).not.toHaveBeenCalled();
});
it('routes long text (>50 chars) to clipboard paste', async () => {
const longText = 'a'.repeat(51);
await typeText(longText);
expect(mockPasteText).toHaveBeenCalledWith(longText);
expect(mockType).not.toHaveBeenCalled();
});
it('uses nut.js type for exactly 50 ASCII chars', async () => {
const text = 'a'.repeat(50);
await typeText(text);
expect(mockType).toHaveBeenCalledWith(text);
expect(mockPasteText).not.toHaveBeenCalled();
});
it('uses nut.js type for empty string', async () => {
await typeText('');
expect(mockType).toHaveBeenCalledWith('');
expect(mockPasteText).not.toHaveBeenCalled();
});
});
describe('pressKey', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('presses and releases a valid key', async () => {
await pressKey('Enter');
expect(mockPressKey).toHaveBeenCalledWith('Enter_Val');
expect(mockReleaseKey).toHaveBeenCalledWith('Enter_Val');
});
it('throws DesktopError for unknown key', async () => {
await expect(pressKey('UnknownKey')).rejects.toThrow('Unknown key: UnknownKey');
});
});
describe('hotkey', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('uses keyboard.type() for hotkey combinations', async () => {
await hotkey(['Meta', 'C']);
// Meta is aliased to LeftSuper
expect(mockType).toHaveBeenCalledWith('LeftSuper_Val', 'C_Val');
});
it('throws DesktopError for unknown key in combination', async () => {
await expect(hotkey(['Meta', 'UnknownKey'])).rejects.toThrow('Unknown key: UnknownKey');
});
});

View File

@@ -0,0 +1,212 @@
/**
* Unit tests for desktop/mouse.ts — click, doubleClick, moveTo, drag, scroll, getPosition.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const mockMove = vi.fn();
const mockClick = vi.fn();
const mockDoubleClick = vi.fn();
const mockPressButton = vi.fn();
const mockReleaseButton = vi.fn();
const mockScrollDown = vi.fn();
const mockScrollUp = vi.fn();
const mockScrollLeft = vi.fn();
const mockScrollRight = vi.fn();
const mockGetPosition = vi.fn();
const mockStraightTo = vi.fn();
vi.mock('@nut-tree-fork/nut-js', () => {
class MockPoint {
constructor(public x: number, public y: number) {}
}
return {
mouse: {
move: (...args: any[]) => mockMove(...args),
click: (...args: any[]) => mockClick(...args),
doubleClick: (...args: any[]) => mockDoubleClick(...args),
pressButton: (...args: any[]) => mockPressButton(...args),
releaseButton: (...args: any[]) => mockReleaseButton(...args),
scrollDown: (...args: any[]) => mockScrollDown(...args),
scrollUp: (...args: any[]) => mockScrollUp(...args),
scrollLeft: (...args: any[]) => mockScrollLeft(...args),
scrollRight: (...args: any[]) => mockScrollRight(...args),
getPosition: () => mockGetPosition(),
},
Button: { LEFT: 'LEFT', RIGHT: 'RIGHT', MIDDLE: 'MIDDLE' },
Point: MockPoint,
straightTo: (p: any) => mockStraightTo(p),
};
});
import { click, doubleClick, moveTo, drag, scroll, getPosition } from '../../src/desktop/mouse.js';
describe('click', () => {
beforeEach(() => {
vi.clearAllMocks();
mockMove.mockResolvedValue(undefined);
mockClick.mockResolvedValue(undefined);
mockStraightTo.mockImplementation((p: any) => p);
});
it('moves to position and clicks left button by default', async () => {
await click(100, 200);
expect(mockStraightTo).toHaveBeenCalledWith(expect.objectContaining({ x: 100, y: 200 }));
expect(mockMove).toHaveBeenCalled();
expect(mockClick).toHaveBeenCalledWith('LEFT');
});
it('supports right button', async () => {
await click(50, 60, 'right');
expect(mockClick).toHaveBeenCalledWith('RIGHT');
});
it('supports middle button', async () => {
await click(50, 60, 'middle');
expect(mockClick).toHaveBeenCalledWith('MIDDLE');
});
it('wraps errors as DesktopError', async () => {
mockMove.mockRejectedValue(new Error('screen locked'));
await expect(click(0, 0)).rejects.toThrow('mouse.click');
});
});
describe('doubleClick', () => {
beforeEach(() => {
vi.clearAllMocks();
mockMove.mockResolvedValue(undefined);
mockDoubleClick.mockResolvedValue(undefined);
mockStraightTo.mockImplementation((p: any) => p);
});
it('moves and double-clicks', async () => {
await doubleClick(300, 400);
expect(mockMove).toHaveBeenCalled();
expect(mockDoubleClick).toHaveBeenCalledWith('LEFT');
});
it('wraps errors as DesktopError', async () => {
mockDoubleClick.mockRejectedValue(new Error('fail'));
await expect(doubleClick(0, 0)).rejects.toThrow('mouse.doubleClick');
});
});
describe('moveTo', () => {
beforeEach(() => {
vi.clearAllMocks();
mockMove.mockResolvedValue(undefined);
mockStraightTo.mockImplementation((p: any) => p);
});
it('moves mouse to coordinates', async () => {
await moveTo(500, 600);
expect(mockStraightTo).toHaveBeenCalledWith(expect.objectContaining({ x: 500, y: 600 }));
expect(mockMove).toHaveBeenCalled();
});
it('wraps errors as DesktopError', async () => {
mockMove.mockRejectedValue(new Error('fail'));
await expect(moveTo(0, 0)).rejects.toThrow('mouse.moveTo');
});
});
describe('drag', () => {
beforeEach(() => {
vi.clearAllMocks();
mockMove.mockResolvedValue(undefined);
mockPressButton.mockResolvedValue(undefined);
mockReleaseButton.mockResolvedValue(undefined);
mockStraightTo.mockImplementation((p: any) => p);
});
it('moves to start, presses, moves to end, releases', async () => {
await drag(10, 20, 100, 200);
expect(mockMove).toHaveBeenCalledTimes(2);
expect(mockPressButton).toHaveBeenCalledWith('LEFT');
expect(mockReleaseButton).toHaveBeenCalledWith('LEFT');
// First move to start
const firstMove = mockStraightTo.mock.calls[0][0];
expect(firstMove.x).toBe(10);
expect(firstMove.y).toBe(20);
// Second move to end
const secondMove = mockStraightTo.mock.calls[1][0];
expect(secondMove.x).toBe(100);
expect(secondMove.y).toBe(200);
});
it('supports right button drag', async () => {
await drag(0, 0, 50, 50, 'right');
expect(mockPressButton).toHaveBeenCalledWith('RIGHT');
expect(mockReleaseButton).toHaveBeenCalledWith('RIGHT');
});
it('wraps errors as DesktopError', async () => {
mockPressButton.mockRejectedValue(new Error('fail'));
await expect(drag(0, 0, 10, 10)).rejects.toThrow('mouse.drag');
});
});
describe('scroll', () => {
beforeEach(() => {
vi.clearAllMocks();
mockMove.mockResolvedValue(undefined);
mockScrollDown.mockResolvedValue(undefined);
mockScrollUp.mockResolvedValue(undefined);
mockScrollLeft.mockResolvedValue(undefined);
mockScrollRight.mockResolvedValue(undefined);
mockStraightTo.mockImplementation((p: any) => p);
});
it('scrolls down with positive deltaY', async () => {
await scroll(100, 200, 5);
expect(mockMove).toHaveBeenCalled();
expect(mockScrollDown).toHaveBeenCalledWith(5);
expect(mockScrollUp).not.toHaveBeenCalled();
});
it('scrolls up with negative deltaY', async () => {
await scroll(100, 200, -3);
expect(mockScrollUp).toHaveBeenCalledWith(3);
expect(mockScrollDown).not.toHaveBeenCalled();
});
it('does not scroll vertically when deltaY is 0', async () => {
await scroll(100, 200, 0);
expect(mockScrollDown).not.toHaveBeenCalled();
expect(mockScrollUp).not.toHaveBeenCalled();
});
it('scrolls right with positive deltaX', async () => {
await scroll(100, 200, 0, 4);
expect(mockScrollRight).toHaveBeenCalledWith(4);
});
it('scrolls left with negative deltaX', async () => {
await scroll(100, 200, 0, -2);
expect(mockScrollLeft).toHaveBeenCalledWith(2);
});
it('wraps errors as DesktopError', async () => {
mockMove.mockRejectedValue(new Error('fail'));
await expect(scroll(0, 0, 1)).rejects.toThrow('mouse.scroll');
});
});
describe('getPosition', () => {
beforeEach(() => vi.clearAllMocks());
it('returns cursor position', async () => {
mockGetPosition.mockResolvedValue({ x: 350, y: 450 });
const pos = await getPosition();
expect(pos).toEqual({ x: 350, y: 450 });
});
it('wraps errors as DesktopError', async () => {
mockGetPosition.mockRejectedValue(new Error('fail'));
await expect(getPosition()).rejects.toThrow('mouse.getPosition');
});
});

View File

@@ -0,0 +1,156 @@
/**
* T2.3 — Screenshot pipeline unit tests (with mocked nut.js and sharp)
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const mockGrabRegion = vi.fn();
const mockRegion = vi.fn().mockImplementation((left: number, top: number, width: number, height: number) => ({
left, top, width, height,
}));
const mockResize = vi.fn();
const mockJpeg = vi.fn();
const mockToBuffer = vi.fn().mockResolvedValue(Buffer.from('fake-jpeg-data'));
// Build a chainable mock that supports .clone()
function makeChain() {
const chain: any = {
resize: mockResize,
jpeg: mockJpeg,
toBuffer: mockToBuffer,
clone: vi.fn(() => makeChain()),
};
mockResize.mockReturnValue(chain);
mockJpeg.mockReturnValue(chain);
return chain;
}
// Mock sharp
vi.mock('sharp', () => {
return {
default: vi.fn(() => makeChain()),
};
});
// Mock display module
vi.mock('../../src/desktop/display.js', () => ({
getDisplay: vi.fn(),
}));
// Mock nut.js — must handle dynamic import()
vi.mock('@nut-tree-fork/nut-js', () => ({
screen: { grabRegion: mockGrabRegion },
Region: mockRegion,
}));
import { captureScreenshot } from '../../src/desktop/screenshot.js';
import { getDisplay } from '../../src/desktop/display.js';
import sharp from 'sharp';
const mockGetDisplay = vi.mocked(getDisplay);
const mockSharp = vi.mocked(sharp);
function setupDisplay(width: number, height: number, scaleFactor: number = 1) {
mockGetDisplay.mockResolvedValue({
index: 0,
label: 'Test',
width,
height,
scaleFactor,
isPrimary: true,
origin: { x: 0, y: 0 },
});
}
function setupCapture(physicalWidth: number, physicalHeight: number) {
mockGrabRegion.mockResolvedValue({
width: physicalWidth,
height: physicalHeight,
data: Buffer.alloc(physicalWidth * physicalHeight * 4),
});
}
describe('captureScreenshot', () => {
beforeEach(() => {
vi.clearAllMocks();
// Re-set defaults after clearAllMocks
mockToBuffer.mockResolvedValue(Buffer.from('fake-jpeg-data'));
});
it('1080p display: no resize needed', async () => {
setupDisplay(1920, 1080, 1);
setupCapture(1920, 1080);
const result = await captureScreenshot(0, 75);
expect(result.physicalWidth).toBe(1920);
expect(result.physicalHeight).toBe(1080);
expect(result.logicalWidth).toBe(1920);
expect(result.logicalHeight).toBe(1080);
expect(result.imageWidth).toBe(1920);
expect(result.imageHeight).toBe(1080);
expect(result.mimeType).toBe('image/jpeg');
expect(result.displayIndex).toBe(0);
expect(mockResize).toHaveBeenCalledWith(1920, 1080, { kernel: 'lanczos3' });
});
it('Retina: 2880x1800 physical, scaleFactor=2 → 1440x900 (no further resize)', async () => {
setupDisplay(1440, 900, 2);
setupCapture(2880, 1800);
const result = await captureScreenshot(0, 75);
expect(result.physicalWidth).toBe(2880);
expect(result.physicalHeight).toBe(1800);
expect(result.logicalWidth).toBe(1440);
expect(result.logicalHeight).toBe(900);
expect(result.imageWidth).toBe(1440);
expect(result.imageHeight).toBe(900);
expect(result.scaleFactor).toBe(2);
});
it('4K: 3840x2160 physical, scaleFactor=1 → resize to 1920x1080', async () => {
setupDisplay(3840, 2160, 1);
setupCapture(3840, 2160);
const result = await captureScreenshot(0, 75);
expect(result.physicalWidth).toBe(3840);
expect(result.logicalWidth).toBe(3840);
expect(result.imageWidth).toBe(1920);
expect(result.imageHeight).toBe(1080);
});
it('2K: 2560x1440 physical, scaleFactor=1 → resize to 1920x1080', async () => {
setupDisplay(2560, 1440, 1);
setupCapture(2560, 1440);
const result = await captureScreenshot(0, 75);
expect(result.logicalWidth).toBe(2560);
expect(result.imageWidth).toBe(1920);
expect(result.imageHeight).toBe(1080);
});
it('returns valid base64 string', async () => {
setupDisplay(1920, 1080, 1);
setupCapture(1920, 1080);
const result = await captureScreenshot(0, 75);
expect(typeof result.image).toBe('string');
expect(result.image.length).toBeGreaterThan(0);
expect(() => Buffer.from(result.image, 'base64')).not.toThrow();
});
it('passes jpeg quality to sharp', async () => {
setupDisplay(1920, 1080, 1);
setupCapture(1920, 1080);
await captureScreenshot(0, 50);
expect(mockJpeg).toHaveBeenCalledWith({ quality: 50 });
});
});

View File

@@ -0,0 +1,478 @@
/**
* Unit tests for mcp/atomicTools.ts — handleAtomicToolCall, input validation, audit logging, safety.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
// Mock logger
vi.mock('../../src/utils/logger.js', () => ({
logInfo: vi.fn(),
logWarn: vi.fn(),
logError: vi.fn(),
logDebug: vi.fn(),
}));
// Mock desktop modules
const mockClick = vi.fn();
const mockDoubleClick = vi.fn();
const mockMoveTo = vi.fn();
const mockDrag = vi.fn();
const mockScroll = vi.fn();
const mockGetMousePosition = vi.fn();
vi.mock('../../src/desktop/mouse.js', () => ({
click: (...args: any[]) => mockClick(...args),
doubleClick: (...args: any[]) => mockDoubleClick(...args),
moveTo: (...args: any[]) => mockMoveTo(...args),
drag: (...args: any[]) => mockDrag(...args),
scroll: (...args: any[]) => mockScroll(...args),
getPosition: () => mockGetMousePosition(),
}));
const mockTypeText = vi.fn();
const mockPressKey = vi.fn();
const mockHotkey = vi.fn();
vi.mock('../../src/desktop/keyboard.js', () => ({
typeText: (...args: any[]) => mockTypeText(...args),
pressKey: (...args: any[]) => mockPressKey(...args),
hotkey: (...args: any[]) => mockHotkey(...args),
}));
const mockCaptureScreenshot = vi.fn();
vi.mock('../../src/desktop/screenshot.js', () => ({
captureScreenshot: (...args: any[]) => mockCaptureScreenshot(...args),
}));
const mockListDisplays = vi.fn();
const mockGetDisplay = vi.fn();
vi.mock('../../src/desktop/display.js', () => ({
listDisplays: () => mockListDisplays(),
getDisplay: (...args: any[]) => mockGetDisplay(...args),
}));
const mockFindImage = vi.fn();
const mockWaitForImage = vi.fn();
vi.mock('../../src/desktop/imageSearch.js', () => ({
findImage: (...args: any[]) => mockFindImage(...args),
waitForImage: (...args: any[]) => mockWaitForImage(...args),
}));
// Mock safety
const mockValidateHotkey = vi.fn();
vi.mock('../../src/safety/hotkeys.js', () => ({
validateHotkey: (...args: any[]) => mockValidateHotkey(...args),
}));
// Mock coordinates
vi.mock('../../src/coordinates/modelProfiles.js', () => ({
getModelProfile: () => ({ coordinateMode: 'image-absolute', coordinateOrder: 'xy' }),
}));
vi.mock('../../src/coordinates/resolver.js', () => ({
resolveCoordinate: (_x: number, _y: number) => ({ globalX: _x, globalY: _y }),
}));
import { handleAtomicToolCall, ATOMIC_TOOLS, clearScreenshotDimensionCache } from '../../src/mcp/atomicTools.js';
import { AuditLog } from '../../src/safety/auditLog.js';
import type { GuiAgentConfig } from '../../src/config.js';
const baseConfig: GuiAgentConfig = {
apiKey: 'test-key',
provider: 'anthropic',
model: 'claude-sonnet-4-20250514',
port: 60008,
transport: 'http' as const,
maxSteps: 50,
stepDelayMs: 1500,
stuckThreshold: 3,
jpegQuality: 75,
displayIndex: 0,
};
describe('ATOMIC_TOOLS', () => {
it('defines exactly 14 tools', () => {
expect(ATOMIC_TOOLS).toHaveLength(14);
});
it('all tools have name, description, and inputSchema', () => {
for (const tool of ATOMIC_TOOLS) {
expect(tool.name).toBeTruthy();
expect(tool.description).toBeTruthy();
expect(tool.inputSchema).toBeDefined();
}
});
it('includes all expected tool names', () => {
const names = ATOMIC_TOOLS.map(t => t.name);
expect(names).toContain('gui_screenshot');
expect(names).toContain('gui_click');
expect(names).toContain('gui_double_click');
expect(names).toContain('gui_move_mouse');
expect(names).toContain('gui_drag');
expect(names).toContain('gui_scroll');
expect(names).toContain('gui_type');
expect(names).toContain('gui_press_key');
expect(names).toContain('gui_hotkey');
expect(names).toContain('gui_cursor_position');
expect(names).toContain('gui_list_displays');
expect(names).toContain('gui_find_image');
expect(names).toContain('gui_wait_for_image');
expect(names).toContain('gui_analyze_screen');
});
});
describe('handleAtomicToolCall', () => {
let auditLog: AuditLog;
beforeEach(() => {
vi.clearAllMocks();
clearScreenshotDimensionCache();
auditLog = new AuditLog();
// Default mock for getDisplay (needed by coordinate resolution)
mockGetDisplay.mockResolvedValue({
index: 0, label: 'Primary', width: 1920, height: 1080,
scaleFactor: 1, isPrimary: true, origin: { x: 0, y: 0 },
});
// Default mock for captureScreenshot (needed by resolveXY for coordinate modes)
mockCaptureScreenshot.mockResolvedValue({
image: 'base64data',
mimeType: 'image/jpeg',
imageWidth: 1920,
imageHeight: 1080,
logicalWidth: 1920,
logicalHeight: 1080,
physicalWidth: 1920,
physicalHeight: 1080,
scaleFactor: 1,
displayIndex: 0,
});
});
it('returns null for unknown tool names', async () => {
const result = await handleAtomicToolCall('unknown_tool', {}, baseConfig, auditLog);
expect(result).toBeNull();
});
// --- Input validation ---
it('returns error for gui_click with non-number x', async () => {
const result = await handleAtomicToolCall('gui_click', { x: 'abc', y: 100 }, baseConfig, auditLog);
expect(result?.isError).toBe(true);
expect(result?.content[0].text).toContain("'x'");
});
it('returns error for gui_click with NaN y', async () => {
const result = await handleAtomicToolCall('gui_click', { x: 100, y: NaN }, baseConfig, auditLog);
expect(result?.isError).toBe(true);
expect(result?.content[0].text).toContain("'y'");
});
it('returns error for gui_type with empty text', async () => {
const result = await handleAtomicToolCall('gui_type', { text: '' }, baseConfig, auditLog);
expect(result?.isError).toBe(true);
expect(result?.content[0].text).toContain("'text'");
});
it('returns error for gui_hotkey with non-array keys', async () => {
const result = await handleAtomicToolCall('gui_hotkey', { keys: 'Meta' }, baseConfig, auditLog);
expect(result?.isError).toBe(true);
expect(result?.content[0].text).toContain("'keys'");
});
// --- Successful operations ---
it('gui_click calls mouse.click with resolved coordinates', async () => {
mockClick.mockResolvedValue(undefined);
const result = await handleAtomicToolCall('gui_click', { x: 100, y: 200 }, baseConfig, auditLog);
expect(result?.isError).toBeUndefined();
// Without coordinateMode, click uses logical coords directly (no screenshot needed)
expect(mockClick).toHaveBeenCalledWith(100, 200, 'left');
expect(result?.content[0].text).toContain('100');
expect(result?.content[0].text).toContain('200');
});
it('gui_type calls keyboard.typeText', async () => {
mockTypeText.mockResolvedValue(undefined);
const result = await handleAtomicToolCall('gui_type', { text: 'hello' }, baseConfig, auditLog);
expect(mockTypeText).toHaveBeenCalledWith('hello');
const parsed = JSON.parse(result!.content[0].text!);
expect(parsed.characterCount).toBe(5);
});
it('gui_press_key calls keyboard.pressKey', async () => {
mockPressKey.mockResolvedValue(undefined);
const result = await handleAtomicToolCall('gui_press_key', { key: 'Enter' }, baseConfig, auditLog);
expect(mockPressKey).toHaveBeenCalledWith('Enter');
expect(result?.content[0].text).toContain('Enter');
});
it('gui_cursor_position returns position', async () => {
mockGetMousePosition.mockResolvedValue({ x: 350, y: 450 });
const result = await handleAtomicToolCall('gui_cursor_position', {}, baseConfig, auditLog);
const parsed = JSON.parse(result!.content[0].text!);
expect(parsed.logicalCoords).toEqual({ x: 350, y: 450 });
expect(parsed.success).toBe(true);
});
it('gui_list_displays returns display list', async () => {
mockListDisplays.mockResolvedValue([
{ index: 0, label: 'Primary', width: 1920, height: 1080, scaleFactor: 1, isPrimary: true, origin: { x: 0, y: 0 } },
]);
const result = await handleAtomicToolCall('gui_list_displays', {}, baseConfig, auditLog);
const parsed = JSON.parse(result!.content[0].text!);
expect(parsed.count).toBe(1);
expect(parsed.displays[0].label).toBe('Primary');
});
it('gui_screenshot returns image and metadata', async () => {
mockCaptureScreenshot.mockResolvedValue({
image: 'base64data',
mimeType: 'image/jpeg',
imageWidth: 1920,
imageHeight: 1080,
logicalWidth: 1920,
logicalHeight: 1080,
physicalWidth: 3840,
physicalHeight: 2160,
scaleFactor: 2,
displayIndex: 0,
});
const result = await handleAtomicToolCall('gui_screenshot', {}, baseConfig, auditLog);
expect(result?.content).toHaveLength(2);
expect(result?.content[0].type).toBe('image');
expect(result?.content[0].data).toBe('base64data');
const meta = JSON.parse(result!.content[1].text!);
expect(meta.scaleFactor).toBe(2);
});
// --- Safety: hotkey blocking ---
it('gui_hotkey blocks dangerous key combination', async () => {
mockValidateHotkey.mockReturnValue({ blocked: true, reason: 'Quit application' });
const result = await handleAtomicToolCall('gui_hotkey', { keys: ['Meta', 'Q'] }, baseConfig, auditLog);
expect(result?.isError).toBe(true);
expect(result?.content[0].text).toContain('Quit application');
expect(mockHotkey).not.toHaveBeenCalled();
});
it('gui_hotkey allows safe key combination', async () => {
mockValidateHotkey.mockReturnValue({ blocked: false });
mockHotkey.mockResolvedValue(undefined);
const result = await handleAtomicToolCall('gui_hotkey', { keys: ['Meta', 'C'] }, baseConfig, auditLog);
expect(result?.isError).toBeUndefined();
expect(mockHotkey).toHaveBeenCalledWith(['Meta', 'C']);
});
// --- Audit logging ---
it('records successful operation in audit log', async () => {
mockClick.mockResolvedValue(undefined);
await handleAtomicToolCall('gui_click', { x: 100, y: 200 }, baseConfig, auditLog);
const entries = auditLog.getEntries(1);
expect(entries).toHaveLength(1);
expect(entries[0].tool).toBe('gui_click');
expect(entries[0].success).toBe(true);
expect(entries[0].durationMs).toBeDefined();
});
it('records failed operation in audit log', async () => {
mockClick.mockRejectedValue(new Error('click failed'));
await handleAtomicToolCall('gui_click', { x: 100, y: 200 }, baseConfig, auditLog);
const entries = auditLog.getEntries(1);
expect(entries).toHaveLength(1);
expect(entries[0].tool).toBe('gui_click');
expect(entries[0].success).toBe(false);
});
// --- gui_scroll ---
it('gui_scroll calls mouse.scroll with deltaY and optional deltaX', async () => {
mockScroll.mockResolvedValue(undefined);
const result = await handleAtomicToolCall('gui_scroll', { x: 100, y: 200, deltaY: 5, deltaX: 3 }, baseConfig, auditLog);
expect(mockScroll).toHaveBeenCalledWith(100, 200, 5, 3);
expect(result?.isError).toBeUndefined();
});
// --- gui_drag ---
it('gui_drag calls mouse.drag with start and end coordinates', async () => {
mockDrag.mockResolvedValue(undefined);
const result = await handleAtomicToolCall('gui_drag', { startX: 10, startY: 20, endX: 100, endY: 200 }, baseConfig, auditLog);
expect(mockDrag).toHaveBeenCalled();
expect(result?.isError).toBeUndefined();
});
// --- gui_find_image ---
it('gui_find_image calls findImage and returns result', async () => {
mockFindImage.mockResolvedValue({ found: true, region: { x: 10, y: 20, width: 50, height: 30 }, confidence: 0.95 });
const result = await handleAtomicToolCall('gui_find_image', { template: 'base64data' }, baseConfig, auditLog);
const parsed = JSON.parse(result!.content[0].text!);
expect(parsed.found).toBe(true);
expect(parsed.logicalCoords.x).toBe(10);
expect(parsed.logicalCoords.y).toBe(20);
});
});
describe('resolveXY coordinate mode and caching', () => {
let auditLog: AuditLog;
beforeEach(() => {
vi.clearAllMocks();
clearScreenshotDimensionCache();
auditLog = new AuditLog();
mockGetDisplay.mockResolvedValue({
index: 0, label: 'Primary', width: 1920, height: 1080,
scaleFactor: 1, isPrimary: true, origin: { x: 0, y: 0 },
});
mockClick.mockResolvedValue(undefined);
});
it('captures screenshot when coordinateMode is provided', async () => {
mockCaptureScreenshot.mockResolvedValue({
image: 'base64data',
mimeType: 'image/jpeg',
imageWidth: 1920,
imageHeight: 1080,
logicalWidth: 1920,
logicalHeight: 1080,
physicalWidth: 1920,
physicalHeight: 1080,
scaleFactor: 1,
displayIndex: 0,
});
await handleAtomicToolCall('gui_click', { x: 100, y: 200, coordinateMode: 'image-absolute' }, baseConfig, auditLog);
expect(mockCaptureScreenshot).toHaveBeenCalledTimes(1);
});
it('uses cached dimensions for subsequent coordinate mode calls', async () => {
mockCaptureScreenshot.mockResolvedValue({
image: 'base64data',
mimeType: 'image/jpeg',
imageWidth: 1920,
imageHeight: 1080,
logicalWidth: 1920,
logicalHeight: 1080,
physicalWidth: 1920,
physicalHeight: 1080,
scaleFactor: 1,
displayIndex: 0,
});
// First call - should capture screenshot
await handleAtomicToolCall('gui_click', { x: 100, y: 200, coordinateMode: 'image-absolute' }, baseConfig, auditLog);
expect(mockCaptureScreenshot).toHaveBeenCalledTimes(1);
// Second call - should use cached dimensions
await handleAtomicToolCall('gui_click', { x: 150, y: 250, coordinateMode: 'image-absolute' }, baseConfig, auditLog);
expect(mockCaptureScreenshot).toHaveBeenCalledTimes(1); // Still 1, not 2
});
it('captures new screenshot after cache TTL expires', async () => {
vi.useFakeTimers();
mockCaptureScreenshot.mockResolvedValue({
image: 'base64data',
mimeType: 'image/jpeg',
imageWidth: 1920,
imageHeight: 1080,
logicalWidth: 1920,
logicalHeight: 1080,
physicalWidth: 1920,
physicalHeight: 1080,
scaleFactor: 1,
displayIndex: 0,
});
// First call
await handleAtomicToolCall('gui_click', { x: 100, y: 200, coordinateMode: 'image-absolute' }, baseConfig, auditLog);
expect(mockCaptureScreenshot).toHaveBeenCalledTimes(1);
// Advance time past TTL (5001ms)
vi.advanceTimersByTime(5001);
// Second call - should capture new screenshot
await handleAtomicToolCall('gui_click', { x: 150, y: 250, coordinateMode: 'image-absolute' }, baseConfig, auditLog);
expect(mockCaptureScreenshot).toHaveBeenCalledTimes(2);
vi.useRealTimers();
});
it('falls back to display dimensions when screenshot fails', async () => {
mockCaptureScreenshot.mockRejectedValue(new Error('Screenshot failed'));
// Should not throw, should use display dimensions as fallback
const result = await handleAtomicToolCall('gui_click', { x: 100, y: 200, coordinateMode: 'image-absolute' }, baseConfig, auditLog);
expect(result?.isError).toBeUndefined();
expect(mockClick).toHaveBeenCalled();
});
});
describe('Retina display coordinate scaling', () => {
let auditLog: AuditLog;
beforeEach(() => {
vi.clearAllMocks();
clearScreenshotDimensionCache();
auditLog = new AuditLog();
mockClick.mockResolvedValue(undefined);
mockGetDisplay.mockResolvedValue({
index: 0, label: 'Primary', width: 1440, height: 900,
scaleFactor: 2, // Retina display
isPrimary: true, origin: { x: 0, y: 0 },
});
});
it('scales logical coordinates by scaleFactor for Retina displays', async () => {
// logical (100, 200) with scaleFactor 2 should become physical (200, 400)
await handleAtomicToolCall('gui_click', { x: 100, y: 200 }, baseConfig, auditLog);
expect(mockClick).toHaveBeenCalledWith(200, 400, 'left');
});
it('handles non-origin display offset with Retina scaling', async () => {
mockGetDisplay.mockResolvedValue({
index: 1, label: 'External', width: 1920, height: 1080,
scaleFactor: 2, // Retina external display
isPrimary: false, origin: { x: 1440, y: 0 }, // Right of primary
});
// logical (100, 200) + origin (1440, 0) * scaleFactor 2 = physical (3080, 400)
await handleAtomicToolCall('gui_click', { x: 100, y: 200 }, baseConfig, auditLog);
expect(mockClick).toHaveBeenCalledWith(3080, 400, 'left');
});
it('gui_cursor_position returns logical coords divided by scaleFactor', async () => {
mockGetMousePosition.mockResolvedValue({ x: 400, y: 600 }); // Physical coords
const result = await handleAtomicToolCall('gui_cursor_position', {}, baseConfig, auditLog);
const parsed = JSON.parse(result!.content[0].text!);
// Physical (400, 600) / scaleFactor 2 = logical (200, 300)
expect(parsed.logicalCoords).toEqual({ x: 200, y: 300 });
expect(parsed.physicalCoords).toEqual({ x: 400, y: 600 });
expect(parsed.scaleFactor).toBe(2);
});
it('gui_find_image returns logical coords divided by scaleFactor', async () => {
mockFindImage.mockResolvedValue({
found: true,
region: { x: 200, y: 400, width: 100, height: 60 }, // Physical coords
confidence: 0.95,
});
const result = await handleAtomicToolCall('gui_find_image', { template: 'base64data' }, baseConfig, auditLog);
const parsed = JSON.parse(result!.content[0].text!);
// Physical / scaleFactor 2 = logical
expect(parsed.logicalCoords.x).toBe(100);
expect(parsed.logicalCoords.y).toBe(200);
expect(parsed.logicalCoords.width).toBe(50);
expect(parsed.logicalCoords.height).toBe(30);
});
});

View File

@@ -0,0 +1,148 @@
/**
* MCP Integration Test
*
* Tests the full MCP server lifecycle:
* 1. Start server on a random port
* 2. Connect MCP client via Streamable HTTP
* 3. List tools (should return 14)
* 4. Verify tool names
* 5. Stop server
*
* Note: gui_screenshot and gui_execute_task require actual desktop access,
* so we only verify tool listing and schema in this test.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import { createGuiAgentServer, type GuiAgentServer } from '../../src/mcp/server.js';
import type { GuiAgentConfig } from '../../src/config.js';
const TEST_PORT = 60099; // Use a high port unlikely to conflict
function createTestConfig(): GuiAgentConfig {
return {
provider: 'anthropic',
model: 'claude-sonnet-4-20250514',
apiKey: 'test-integration-key',
port: TEST_PORT,
maxSteps: 10,
stepDelayMs: 100,
stuckThreshold: 3,
jpegQuality: 75,
displayIndex: 0,
transport: 'http',
};
}
const EXPECTED_ATOMIC_TOOLS = [
'gui_screenshot',
'gui_click',
'gui_double_click',
'gui_move_mouse',
'gui_drag',
'gui_scroll',
'gui_type',
'gui_press_key',
'gui_hotkey',
'gui_cursor_position',
'gui_list_displays',
'gui_find_image',
'gui_wait_for_image',
'gui_analyze_screen',
];
const EXPECTED_TASK_TOOLS = [
'gui_execute_task',
];
describe('MCP Integration', () => {
let server: GuiAgentServer;
let client: Client;
let transport: StreamableHTTPClientTransport;
beforeAll(async () => {
// Start server
server = createGuiAgentServer(createTestConfig());
await server.start();
// Connect client
transport = new StreamableHTTPClientTransport(
new URL(`http://127.0.0.1:${TEST_PORT}/mcp`),
);
client = new Client({ name: 'test-client', version: '1.0.0' });
await client.connect(transport);
}, 10000);
afterAll(async () => {
try { await client?.close(); } catch { /* ignore */ }
try { await server?.stop(); } catch { /* ignore */ }
}, 10000);
it('should list all 15 tools', async () => {
const result = await client.listTools();
expect(result.tools).toHaveLength(15);
});
it('should include all atomic tools', async () => {
const result = await client.listTools();
const toolNames = result.tools.map(t => t.name);
for (const name of EXPECTED_ATOMIC_TOOLS) {
expect(toolNames).toContain(name);
}
});
it('should include gui_execute_task tool', async () => {
const result = await client.listTools();
const toolNames = result.tools.map(t => t.name);
for (const name of EXPECTED_TASK_TOOLS) {
expect(toolNames).toContain(name);
}
});
it('should have correct schema for gui_execute_task', async () => {
const result = await client.listTools();
const taskTool = result.tools.find(t => t.name === 'gui_execute_task');
expect(taskTool).toBeDefined();
expect(taskTool!.inputSchema).toBeDefined();
expect(taskTool!.inputSchema.properties).toHaveProperty('task');
expect(taskTool!.inputSchema.required).toContain('task');
});
it('should have correct schema for gui_click', async () => {
const result = await client.listTools();
const clickTool = result.tools.find(t => t.name === 'gui_click');
expect(clickTool).toBeDefined();
expect(clickTool!.inputSchema.properties).toHaveProperty('x');
expect(clickTool!.inputSchema.properties).toHaveProperty('y');
expect(clickTool!.inputSchema.required).toContain('x');
expect(clickTool!.inputSchema.required).toContain('y');
});
it('should list resources', async () => {
const result = await client.listResources();
expect(result.resources.length).toBeGreaterThanOrEqual(3);
const uris = result.resources.map(r => r.uri);
expect(uris).toContain('gui://status');
expect(uris).toContain('gui://permissions');
expect(uris).toContain('gui://audit-log');
});
it('should read gui://status resource', async () => {
const result = await client.readResource({ uri: 'gui://status' });
expect(result.contents).toHaveLength(1);
expect(result.contents[0].mimeType).toBe('application/json');
const status = JSON.parse(result.contents[0].text as string);
expect(status).toHaveProperty('platform');
expect(status).toHaveProperty('model');
expect(status.model).toBe('claude-sonnet-4-20250514');
});
it('should read gui://audit-log resource', async () => {
const result = await client.readResource({ uri: 'gui://audit-log' });
expect(result.contents).toHaveLength(1);
const log = JSON.parse(result.contents[0].text as string);
expect(Array.isArray(log)).toBe(true);
});
});

View File

@@ -0,0 +1,138 @@
/**
* Unit tests for mcp/resources.ts — registerResources, 3 resource handlers.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
// Mock platform
const mockGetPlatform = vi.fn();
const mockCheckScreenRecording = vi.fn();
const mockCheckAccessibility = vi.fn();
vi.mock('../../src/utils/platform.js', () => ({
getPlatform: () => mockGetPlatform(),
checkScreenRecordingPermission: () => mockCheckScreenRecording(),
checkAccessibilityPermission: () => mockCheckAccessibility(),
}));
import { registerResources } from '../../src/mcp/resources.js';
import { AuditLog } from '../../src/safety/auditLog.js';
import type { GuiAgentConfig } from '../../src/config.js';
// Minimal Server mock that captures request handlers
class MockServer {
handlers = new Map<any, Function>();
setRequestHandler(schema: any, handler: Function) {
// Use the schema method name or the schema itself as key
this.handlers.set(schema, handler);
}
// Get handler by schema
getHandler(schema: any): Function | undefined {
return this.handlers.get(schema);
}
}
// We need the actual schemas to register & retrieve handlers
// Import them for use as keys
import {
ListResourcesRequestSchema,
ReadResourceRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
const baseConfig: GuiAgentConfig = {
apiKey: 'test-key',
provider: 'anthropic',
model: 'claude-sonnet-4-20250514',
port: 60008,
transport: 'http' as const,
maxSteps: 50,
stepDelayMs: 1500,
stuckThreshold: 3,
jpegQuality: 75,
displayIndex: 0,
};
describe('registerResources', () => {
let server: MockServer;
let auditLog: AuditLog;
beforeEach(() => {
vi.clearAllMocks();
server = new MockServer();
auditLog = new AuditLog();
mockGetPlatform.mockReturnValue('macos');
registerResources(server as any, baseConfig, auditLog);
});
describe('ListResources', () => {
it('returns 3 resources', async () => {
const handler = server.getHandler(ListResourcesRequestSchema)!;
const result = await handler({});
expect(result.resources).toHaveLength(3);
});
it('includes gui://status, gui://permissions, gui://audit-log', async () => {
const handler = server.getHandler(ListResourcesRequestSchema)!;
const result = await handler({});
const uris = result.resources.map((r: any) => r.uri);
expect(uris).toContain('gui://status');
expect(uris).toContain('gui://permissions');
expect(uris).toContain('gui://audit-log');
});
});
describe('ReadResource: gui://status', () => {
it('returns platform, transport, port, model', async () => {
const handler = server.getHandler(ReadResourceRequestSchema)!;
const result = await handler({ params: { uri: 'gui://status' } });
const status = JSON.parse(result.contents[0].text);
expect(status.platform).toBe('macos');
expect(status.transport).toBe('http');
expect(status.port).toBe(60008);
expect(status.model).toBe('claude-sonnet-4-20250514');
expect(status.running).toBe(true);
});
});
describe('ReadResource: gui://permissions', () => {
it('returns screen recording and accessibility status', async () => {
mockCheckScreenRecording.mockResolvedValue(true);
mockCheckAccessibility.mockResolvedValue(false);
const handler = server.getHandler(ReadResourceRequestSchema)!;
const result = await handler({ params: { uri: 'gui://permissions' } });
const perms = JSON.parse(result.contents[0].text);
expect(perms.screenRecording).toBe(true);
expect(perms.accessibility).toBe(false);
expect(perms.platform).toBe('macos');
});
});
describe('ReadResource: gui://audit-log', () => {
it('returns empty entries when no audit records', async () => {
const handler = server.getHandler(ReadResourceRequestSchema)!;
const result = await handler({ params: { uri: 'gui://audit-log' } });
const entries = JSON.parse(result.contents[0].text);
expect(entries).toHaveLength(0);
});
it('returns recorded audit entries', async () => {
auditLog.record({ tool: 'gui_click', args: { x: 100 }, success: true });
auditLog.record({ tool: 'gui_type', args: { text: 'hi' }, success: true });
const handler = server.getHandler(ReadResourceRequestSchema)!;
const result = await handler({ params: { uri: 'gui://audit-log' } });
const entries = JSON.parse(result.contents[0].text);
expect(entries).toHaveLength(2);
expect(entries[0].tool).toBe('gui_type'); // most recent first
});
});
describe('ReadResource: unknown URI', () => {
it('throws for unknown resource URI', async () => {
const handler = server.getHandler(ReadResourceRequestSchema)!;
await expect(handler({ params: { uri: 'gui://unknown' } })).rejects.toThrow('Unknown resource');
});
});
});

View File

@@ -0,0 +1,90 @@
/**
* Unit tests for safety/auditLog.ts — ring buffer, record, getEntries, clear.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { AuditLog } from '../../src/safety/auditLog.js';
describe('AuditLog', () => {
let log: AuditLog;
beforeEach(() => {
log = new AuditLog();
});
it('starts empty', () => {
expect(log.length).toBe(0);
expect(log.getEntries()).toHaveLength(0);
});
it('records entries with auto-generated timestamp', () => {
log.record({ tool: 'gui_click', args: { x: 100, y: 200 }, success: true });
expect(log.length).toBe(1);
const entries = log.getEntries();
expect(entries).toHaveLength(1);
expect(entries[0].tool).toBe('gui_click');
expect(entries[0].args).toEqual({ x: 100, y: 200 });
expect(entries[0].success).toBe(true);
expect(entries[0].timestamp).toBeDefined();
});
it('returns entries most recent first', () => {
log.record({ tool: 'tool_1', args: {}, success: true });
log.record({ tool: 'tool_2', args: {}, success: true });
log.record({ tool: 'tool_3', args: {}, success: true });
const entries = log.getEntries();
expect(entries[0].tool).toBe('tool_3');
expect(entries[1].tool).toBe('tool_2');
expect(entries[2].tool).toBe('tool_1');
});
it('limits entries by count parameter', () => {
for (let i = 0; i < 10; i++) {
log.record({ tool: `tool_${i}`, args: {}, success: true });
}
const entries = log.getEntries(3);
expect(entries).toHaveLength(3);
expect(entries[0].tool).toBe('tool_9');
expect(entries[1].tool).toBe('tool_8');
expect(entries[2].tool).toBe('tool_7');
});
it('wraps around at 1000 entries (ring buffer)', () => {
for (let i = 0; i < 1050; i++) {
log.record({ tool: `tool_${i}`, args: {}, success: true });
}
// Size should cap at 1000
expect(log.length).toBe(1000);
const entries = log.getEntries(1);
expect(entries[0].tool).toBe('tool_1049');
// Oldest entry should be tool_50 (0-49 were overwritten)
const allEntries = log.getEntries();
expect(allEntries).toHaveLength(1000);
expect(allEntries[allEntries.length - 1].tool).toBe('tool_50');
});
it('clears all entries', () => {
for (let i = 0; i < 5; i++) {
log.record({ tool: `tool_${i}`, args: {}, success: true });
}
expect(log.length).toBe(5);
log.clear();
expect(log.length).toBe(0);
expect(log.getEntries()).toHaveLength(0);
});
it('records durationMs when provided', () => {
log.record({ tool: 'gui_screenshot', args: {}, success: true, durationMs: 150 });
const entries = log.getEntries();
expect(entries[0].durationMs).toBe(150);
});
it('works correctly after clear and re-record', () => {
log.record({ tool: 'before_clear', args: {}, success: true });
log.clear();
log.record({ tool: 'after_clear', args: {}, success: true });
expect(log.length).toBe(1);
expect(log.getEntries()[0].tool).toBe('after_clear');
});
});

View File

@@ -0,0 +1,133 @@
/**
* Unit tests for safety/hotkeys.ts — platform blacklists and key normalization.
*/
import { describe, it, expect, vi } from 'vitest';
// Mock platform detection
const mockGetPlatform = vi.fn();
vi.mock('../../src/utils/platform.js', () => ({
getPlatform: () => mockGetPlatform(),
}));
import { validateHotkey } from '../../src/safety/hotkeys.js';
describe('validateHotkey', () => {
describe('macOS blacklist', () => {
it('blocks Cmd+Q', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['Meta', 'Q']);
expect(result.blocked).toBe(true);
expect(result.reason).toContain('Quit application');
});
it('blocks Cmd+W', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['Meta', 'W']);
expect(result.blocked).toBe(true);
expect(result.reason).toContain('Close window');
});
it('blocks Cmd+Opt+Escape', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['Meta', 'Alt', 'Escape']);
expect(result.blocked).toBe(true);
expect(result.reason).toContain('Force quit');
});
it('blocks Cmd+Shift+Q (log out)', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['Meta', 'Shift', 'Q']);
expect(result.blocked).toBe(true);
});
});
describe('Windows blacklist', () => {
it('blocks Alt+F4', () => {
mockGetPlatform.mockReturnValue('windows');
const result = validateHotkey(['Alt', 'F4']);
expect(result.blocked).toBe(true);
expect(result.reason).toContain('Close application');
});
it('blocks Ctrl+Alt+Delete', () => {
mockGetPlatform.mockReturnValue('windows');
const result = validateHotkey(['Control', 'Alt', 'Delete']);
expect(result.blocked).toBe(true);
expect(result.reason).toContain('System menu');
});
});
describe('Linux blacklist', () => {
it('blocks Ctrl+Alt+Delete', () => {
mockGetPlatform.mockReturnValue('linux');
const result = validateHotkey(['Control', 'Alt', 'Delete']);
expect(result.blocked).toBe(true);
});
it('blocks Ctrl+Alt+Backspace', () => {
mockGetPlatform.mockReturnValue('linux');
const result = validateHotkey(['Control', 'Alt', 'Backspace']);
expect(result.blocked).toBe(true);
expect(result.reason).toContain('Kill X server');
});
});
describe('key alias normalization', () => {
it('normalizes cmd to meta', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['cmd', 'q']);
expect(result.blocked).toBe(true);
});
it('normalizes command to meta', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['command', 'q']);
expect(result.blocked).toBe(true);
});
it('normalizes opt/option to alt', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['meta', 'opt', 'esc']);
expect(result.blocked).toBe(true);
});
it('normalizes ctrl to control', () => {
mockGetPlatform.mockReturnValue('windows');
const result = validateHotkey(['ctrl', 'alt', 'del']);
expect(result.blocked).toBe(true);
});
it('is case insensitive', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['META', 'Q']);
expect(result.blocked).toBe(true);
});
});
describe('allowed combinations', () => {
it('allows Cmd+C (copy)', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['Meta', 'C']);
expect(result.blocked).toBe(false);
});
it('allows Ctrl+S (save)', () => {
mockGetPlatform.mockReturnValue('windows');
const result = validateHotkey(['Control', 'S']);
expect(result.blocked).toBe(false);
});
it('allows single keys', () => {
mockGetPlatform.mockReturnValue('macos');
const result = validateHotkey(['Enter']);
expect(result.blocked).toBe(false);
});
it('key order does not matter', () => {
mockGetPlatform.mockReturnValue('macos');
const r1 = validateHotkey(['Q', 'Meta']);
expect(r1.blocked).toBe(true);
});
});
});

View File

@@ -0,0 +1,79 @@
/**
* Unit tests for utils/errors.ts — 5 structured error classes.
*/
import { describe, it, expect } from 'vitest';
import {
ConfigError,
DesktopError,
CoordinateError,
SafetyError,
TaskExecutionError,
} from '../../src/utils/errors.js';
describe('ConfigError', () => {
it('sets name and message', () => {
const err = new ConfigError('API_KEY is required');
expect(err).toBeInstanceOf(Error);
expect(err.name).toBe('ConfigError');
expect(err.message).toBe('API_KEY is required');
});
});
describe('DesktopError', () => {
it('sets name, operation, cause, and formatted message', () => {
const cause = new Error('permission denied');
const err = new DesktopError('mouse.click', cause);
expect(err).toBeInstanceOf(Error);
expect(err.name).toBe('DesktopError');
expect(err.operation).toBe('mouse.click');
expect(err.cause).toBe(cause);
expect(err.message).toContain('mouse.click');
expect(err.message).toContain('permission denied');
});
it('operation is readonly', () => {
const err = new DesktopError('screenshot', new Error('fail'));
expect(err.operation).toBe('screenshot');
});
});
describe('CoordinateError', () => {
it('sets name, coordinateMode, rawX, rawY, and formatted message', () => {
const err = new CoordinateError('normalized-1000', 500, 300);
expect(err).toBeInstanceOf(Error);
expect(err.name).toBe('CoordinateError');
expect(err.coordinateMode).toBe('normalized-1000');
expect(err.rawX).toBe(500);
expect(err.rawY).toBe(300);
expect(err.message).toContain('normalized-1000');
expect(err.message).toContain('500');
expect(err.message).toContain('300');
});
});
describe('SafetyError', () => {
it('sets name, keys, reason, and formatted message', () => {
const err = new SafetyError(['Meta', 'Q'], 'Quit application');
expect(err).toBeInstanceOf(Error);
expect(err.name).toBe('SafetyError');
expect(err.keys).toEqual(['Meta', 'Q']);
expect(err.reason).toBe('Quit application');
expect(err.message).toContain('Meta+Q');
expect(err.message).toContain('Quit application');
});
});
describe('TaskExecutionError', () => {
it('sets name, taskText, step, cause, and formatted message', () => {
const cause = new Error('element not found');
const err = new TaskExecutionError('Open Finder', 3, cause);
expect(err).toBeInstanceOf(Error);
expect(err.name).toBe('TaskExecutionError');
expect(err.taskText).toBe('Open Finder');
expect(err.step).toBe(3);
expect(err.cause).toBe(cause);
expect(err.message).toContain('step 3');
expect(err.message).toContain('element not found');
});
});

View File

@@ -0,0 +1,119 @@
/**
* Unit tests for utils/platform.ts — getPlatform, getPlatformPasteKeys, permission checks.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
// Mock os module
const mockPlatform = vi.fn();
vi.mock('os', () => ({
platform: () => mockPlatform(),
}));
// Mock nut.js for permission checks
const mockScreenWidth = vi.fn();
const mockMouseGetPosition = vi.fn();
vi.mock('@nut-tree-fork/nut-js', () => ({
screen: {
width: () => mockScreenWidth(),
},
mouse: {
getPosition: () => mockMouseGetPosition(),
},
}));
import { getPlatform, getPlatformPasteKeys, checkScreenRecordingPermission, checkAccessibilityPermission } from '../../src/utils/platform.js';
describe('getPlatform', () => {
beforeEach(() => vi.clearAllMocks());
it('returns "macos" for darwin', () => {
mockPlatform.mockReturnValue('darwin');
expect(getPlatform()).toBe('macos');
});
it('returns "windows" for win32', () => {
mockPlatform.mockReturnValue('win32');
expect(getPlatform()).toBe('windows');
});
it('returns "linux" for linux', () => {
mockPlatform.mockReturnValue('linux');
expect(getPlatform()).toBe('linux');
});
it('returns "linux" for unknown platforms', () => {
mockPlatform.mockReturnValue('freebsd');
expect(getPlatform()).toBe('linux');
});
});
describe('getPlatformPasteKeys', () => {
beforeEach(() => vi.clearAllMocks());
it('returns Meta+V on macOS', () => {
mockPlatform.mockReturnValue('darwin');
expect(getPlatformPasteKeys()).toEqual(['Meta', 'V']);
});
it('returns Control+V on Windows', () => {
mockPlatform.mockReturnValue('win32');
expect(getPlatformPasteKeys()).toEqual(['Control', 'V']);
});
it('returns Control+V on Linux', () => {
mockPlatform.mockReturnValue('linux');
expect(getPlatformPasteKeys()).toEqual(['Control', 'V']);
});
});
describe('checkScreenRecordingPermission', () => {
beforeEach(() => vi.clearAllMocks());
it('returns true on non-macOS platforms', async () => {
mockPlatform.mockReturnValue('win32');
const result = await checkScreenRecordingPermission();
expect(result).toBe(true);
expect(mockScreenWidth).not.toHaveBeenCalled();
});
it('returns true on macOS when screen.width succeeds', async () => {
mockPlatform.mockReturnValue('darwin');
mockScreenWidth.mockResolvedValue(1920);
const result = await checkScreenRecordingPermission();
expect(result).toBe(true);
});
it('returns false on macOS when screen.width throws', async () => {
mockPlatform.mockReturnValue('darwin');
mockScreenWidth.mockRejectedValue(new Error('no permission'));
const result = await checkScreenRecordingPermission();
expect(result).toBe(false);
});
});
describe('checkAccessibilityPermission', () => {
beforeEach(() => vi.clearAllMocks());
it('returns true on non-macOS platforms', async () => {
mockPlatform.mockReturnValue('linux');
const result = await checkAccessibilityPermission();
expect(result).toBe(true);
expect(mockMouseGetPosition).not.toHaveBeenCalled();
});
it('returns true on macOS when mouse.getPosition succeeds', async () => {
mockPlatform.mockReturnValue('darwin');
mockMouseGetPosition.mockResolvedValue({ x: 0, y: 0 });
const result = await checkAccessibilityPermission();
expect(result).toBe(true);
});
it('returns false on macOS when mouse.getPosition throws', async () => {
mockPlatform.mockReturnValue('darwin');
mockMouseGetPosition.mockRejectedValue(new Error('no permission'));
const result = await checkAccessibilityPermission();
expect(result).toBe(false);
});
});