From 853404c67a0a919a1406b1fcc8e599795ca8795a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E4=B9=89=E4=BB=91?= Date: Tue, 16 Jun 2026 14:26:08 +0800 Subject: [PATCH] Initialize action topology engine --- .gitignore | 6 + AGENTS.md | 32 + README.md | 63 + docs/design/no-llm-topology-engine.md | 39 + docs/source/chatgpt-share-6a30e583.md | 1869 +++++++++++++++++++++++++ examples/simple.html | 50 + package-lock.json | 616 ++++++++ package.json | 26 + schemas/topology-graph.schema.md | 51 + src/cli.ts | 66 + src/engine.ts | 189 +++ src/hash.ts | 27 + src/locator.ts | 170 +++ src/risk.ts | 51 + src/snapshot.ts | 251 ++++ src/types.ts | 139 ++ tsconfig.json | 19 + 17 files changed, 3664 insertions(+) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 README.md create mode 100644 docs/design/no-llm-topology-engine.md create mode 100644 docs/source/chatgpt-share-6a30e583.md create mode 100644 examples/simple.html create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 schemas/topology-graph.schema.md create mode 100644 src/cli.ts create mode 100644 src/engine.ts create mode 100644 src/hash.ts create mode 100644 src/locator.ts create mode 100644 src/risk.ts create mode 100644 src/snapshot.ts create mode 100644 src/types.ts create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d6cea7c --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +dist/ +artifacts/ +.DS_Store +*.log + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d2b05ad --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,32 @@ +# AGENTS.md + +This project is a focused deterministic website action topology engine. + +## Scope + +Build and maintain only the topology engine: + +- Capture page states. +- Extract actionable elements. +- Generate and verify locator candidates. +- Record action edges and observable effects. +- Emit graph artifacts that other projects can consume. + +Do not turn this repository into a general browser agent platform. LLM planning, natural-language task execution, orchestration, and product UI belong in separate projects. + +## Commands + +- `npm install` — install dependencies. +- `npm run typecheck` — TypeScript type check. +- `npm run build` — compile to `dist/`. +- `npm run scan -- --url ./examples/simple.html --out artifacts/simple-scan.json` — static scan. +- `npm run example` — low-risk exploration of the local example page. + +## Implementation Notes + +- Prefer Playwright user-facing locator strategies first: role, label, placeholder, text, test id. +- Keep CSS locators as fallback candidates, not the primary source of truth. +- Bind elements to states. Do not treat a DOM node as globally meaningful outside its page state. +- Preserve risk gating. High-risk terms such as delete, submit, pay, confirm, and authorize must not be clicked by the default explorer. +- Keep source discussions under `docs/source/` when they drive design decisions. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..7a3a6ac --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Action Topology Engine + +Small deterministic tool for building a website action topology graph. + +The project intentionally does not include an LLM, a generic browser agent, or a large workflow platform. Its job is to observe a site, describe states and actionable elements, generate verifiable locator candidates, and record action edges with observable effects. + +## Current Scope + +- Open a URL or local HTML file with Playwright. +- Capture one page state with DOM and visible-text fingerprints. +- Extract actionable elements: buttons, links, inputs, selects, textareas, roles, tab stops, and click handlers. +- Generate locator candidates with scores. +- Optionally explore low-risk click actions from the entry state. +- Record state nodes, elements, locators, action edges, and basic effects in JSON. + +## Non-Goals + +- No LLM decision making. +- No natural-language task planner. +- No high-risk automatic actions. +- No attempt to fully recover frontend business functions in the first version. + +## Install + +```bash +npm install +``` + +## Commands + +Static scan: + +```bash +npm run scan -- --url ./examples/simple.html --out artifacts/simple-scan.json +``` + +Entry-state low-risk exploration: + +```bash +npm run example +``` + +Build: + +```bash +npm run build +``` + +## Output Model + +The output JSON follows the schema described in [schemas/topology-graph.schema.md](schemas/topology-graph.schema.md). + +Key objects: + +- `StateNode`: URL, route, title, state hashes, modal stack, and entry metadata. +- `ActionableElement`: element semantics, visibility, bounding box, and locator candidates. +- `LocatorCandidate`: Playwright-friendly locator descriptors plus score and verification status. +- `ActionEdge`: action from one state to another through an element, with effects and risk. + +## Source Notes + +The originating ChatGPT share conversation is preserved in [docs/source/chatgpt-share-6a30e583.md](docs/source/chatgpt-share-6a30e583.md). + diff --git a/docs/design/no-llm-topology-engine.md b/docs/design/no-llm-topology-engine.md new file mode 100644 index 0000000..035e1e3 --- /dev/null +++ b/docs/design/no-llm-topology-engine.md @@ -0,0 +1,39 @@ +# No-LLM Topology Engine Design + +## Purpose + +Build a small deterministic engine that maps a website into a verifiable action topology. Other projects can consume the graph for planning, orchestration, or LLM-assisted behavior. This project stays narrow: it observes, indexes, and verifies browser-operable structure. + +## Core Unit + +```text +State A -- action(element, params) --> State B / Effect +``` + +The engine does not index DOM nodes in isolation. It indexes actionable elements inside a state, the locators that can find them, the action that can be applied, and the observable effect. + +## First Implementation Slice + +1. Capture a page state from an entry URL. +2. Extract visible actionable elements. +3. Generate locator candidates and verify uniqueness. +4. Explore a bounded set of low-risk click actions from the entry state. +5. Record resulting states and effects. +6. Emit a graph JSON artifact. + +## Explicit Non-Goals + +- No LLM. +- No natural-language task planner. +- No uncontrolled high-risk actions. +- No deep frontend function reconstruction in the first version. +- No complete arbitrary-site crawler. + +## Risk Policy + +The first explorer only clicks low-risk candidates. Terms such as delete, submit, pay, confirm, authorize, and their Chinese equivalents are classified as high risk and skipped. Form fields are treated as medium risk and are captured but not automatically filled. + +## Storage + +The first version emits JSON. SQLite or Postgres can be added later after the graph shape stabilizes. + diff --git a/docs/source/chatgpt-share-6a30e583.md b/docs/source/chatgpt-share-6a30e583.md new file mode 100644 index 0000000..b833c88 --- /dev/null +++ b/docs/source/chatgpt-share-6a30e583.md @@ -0,0 +1,1869 @@ +# ChatGPT Share: 网站操作项拓扑设计 + +- Source URL: https://chatgpt.com/share/6a30e583-a7d0-83ea-88bd-1299eb78d086 +- Extracted at: 2026-06-16T06:17:36.331Z +- Extraction method: parsed `window.__reactRouterContext.streamController.enqueue(...)` from the shared page HTML. +- Project use: source discussion for a focused deterministic action topology engine. + +## Conversation + +### User Message 2044 + +如何建立一个网站中所有可操作项的拓扑，包含可操作项本身（如按钮，输入框），它的可定位信息（不同框架似乎不一样），前置打开项，所调用函数，相关参数等等。目的是能够在浏览器自动化控制中能够快速实现 + + +### Assistant Message 2002 + +可以建立，而且建议把它设计成一个 **Action Topology Graph，可操作项拓扑图**。核心思想是：**不要只索引 DOM 元素，而是索引“在某个页面状态下，对某个可操作元素执行某个动作，会导致什么结果”**。 + +也就是说，真正有用的基本单元不是： + +```text +button#save +``` + +而是： + +```text +State A -- click(button: Save) --> State B +``` + +其中 `State A` 可能是“首页已登录、左侧菜单已展开、弹窗未打开”，`State B` 可能是“保存成功提示出现”或“发出了 POST /api/save 请求”。 + +--- + +## 1. 拓扑图应该分四层建模 + +### 第一层：页面状态 State + +一个网站不是一个静态页面，而是一组状态。状态可以由以下信息共同定义： + +```json +{ + "stateId": "s_home_menu_open", + "url": "https://example.com/dashboard", + "route": "/dashboard", + "title": "Dashboard", + "framePath": [], + "domHash": "sha256:...", + "axHash": "sha256:...", + "visibleTextHash": "sha256:...", + "modalStack": ["settings-dialog"], + "openedBy": "edge_click_settings", + "preconditionPath": [ + "login", + "click:user-menu", + "click:settings" + ] +} +``` + +这里最重要的是 `preconditionPath`。它表示：要操作这个状态下的元素，浏览器自动化必须先执行哪些前置打开动作。 + +例如： + +```text +点击用户头像 +→ 打开下拉菜单 +→ 点击 Settings +→ 打开设置弹窗 +→ 才能看到 Save 按钮 +``` + +这个 `Save` 按钮的前置路径就不是空的，而是： + +```json +[ + "click:user-avatar", + "click:settings-menu-item" +] +``` + +--- + +### 第二层：可操作元素 Actionable Element + +每个可操作项都应该绑定到某个状态，而不是全局唯一。 + +```json +{ + "elementId": "el_save_button_001", + "stateId": "s_settings_dialog_open", + "kind": "button", + "tag": "button", + "role": "button", + "accessibleName": "Save", + "text": "Save", + "label": null, + "placeholder": null, + "attributes": { + "data-testid": "save-settings", + "type": "submit", + "aria-disabled": "false" + }, + "framePath": [], + "shadowPath": [], + "bbox": { + "x": 824, + "y": 612, + "width": 96, + "height": 36 + }, + "visibility": { + "visible": true, + "enabled": true, + "editable": false, + "receivesEvents": true + }, + "componentHint": { + "framework": "react", + "componentName": "SettingsDialog" + } +} +``` + +建议优先用 **Accessibility Tree + DOM Snapshot** 来抽取元素。Playwright 的定位器体系也建议优先使用面向用户的定位方式，例如 role、label、text、placeholder、test id，而不是脆弱的长 CSS/XPath 链；其文档明确指出 locator 是自动等待和重试能力的核心，并推荐 `getByRole()`、`getByLabel()`、`getByText()`、`getByTestId()` 等方式。citeturn773971view0 + +--- + +### 第三层：定位器 Locator Candidates + +不要只存一个 selector。应该为每个元素存一组候选定位器，并打分。 + +```json +{ + "elementId": "el_save_button_001", + "locators": [ + { + "type": "playwright", + "strategy": "role", + "expression": "page.getByRole('button', { name: 'Save' })", + "score": 0.95, + "reason": "role + accessible name stable" + }, + { + "type": "playwright", + "strategy": "testId", + "expression": "page.getByTestId('save-settings')", + "score": 0.92, + "reason": "explicit test id" + }, + { + "type": "css", + "strategy": "data-testid", + "expression": "[data-testid=\"save-settings\"]", + "score": 0.88 + }, + { + "type": "xpath", + "strategy": "text", + "expression": "//button[normalize-space()='Save']", + "score": 0.55 + } + ] +} +``` + +推荐的定位器优先级： + +```text +1. role + accessible name +2. label / placeholder / alt / title +3. data-testid / data-pw / data-qa +4. 稳定业务属性，例如 name、aria-label、href、type +5. 局部 CSS +6. XPath +7. 坐标点击，仅作为兜底 +``` + +Playwright 文档也特别提醒，CSS 和 XPath 容易跟 DOM 结构绑定，长链 selector 在 DOM 变化时很容易失效，因此应该尽量用更接近用户感知的定位方式，例如 role 或 test id。citeturn773971view0 + +--- + +### 第四层：动作边 Action Edge + +动作边是拓扑的核心。 + +```json +{ + "edgeId": "edge_click_save_settings", + "fromState": "s_settings_dialog_open", + "toState": "s_settings_saved_toast", + "elementId": "el_save_button_001", + "action": { + "type": "click", + "params": {} + }, + "preconditions": [ + "state.url matches /dashboard", + "dialog Settings visible", + "button Save enabled" + ], + "effects": { + "urlChanged": false, + "domChanged": true, + "network": [ + { + "method": "POST", + "url": "/api/settings", + "bodySchema": { + "theme": "string", + "notifications": "boolean" + } + } + ], + "storageChanged": [], + "visibleTextAdded": ["Settings saved"] + }, + "handlerTrace": { + "event": "click", + "functionName": "handleSave", + "source": "src/components/SettingsDialog.tsx:88", + "callStack": [ + "handleSave", + "onClick", + "dispatchEvent" + ] + } +} +``` + +自动化控制时，你真正需要查询的是： + +```text +我要执行“保存设置” +→ 找到对应 Action Edge +→ 取它的 preconditionPath +→ 打开所需状态 +→ 用最高分 locator 执行动作 +→ 验证 expected effects +``` + +--- + +## 2. 采集流程：静态快照 + 动态探索 + 运行时追踪 + +推荐管线如下： + +```text +启动浏览器 + ↓ +注入 instrumentation + ↓ +进入起始 URL + ↓ +采集 State Snapshot + ↓ +抽取可操作元素 + ↓ +生成定位器候选 + ↓ +逐个执行动作 + ↓ +记录状态变化、网络请求、函数调用 + ↓ +生成 Action Edge + ↓ +进入新状态继续 BFS/DFS +``` + +--- + +## 3. 元素发现：不要只靠 querySelectorAll + +基础选择器可以这样开始： + +```js +const actionableSelector = [ + 'button', + 'a[href]', + 'input', + 'textarea', + 'select', + '[role="button"]', + '[role="link"]', + '[role="menuitem"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="tab"]', + '[role="option"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])', + '[onclick]', + '[aria-haspopup]', + '[aria-expanded]' +].join(','); +``` + +但这还不够。现代网站里大量元素是： + +```html +
Submit
+``` + +或者： + +```html +Open +``` + +所以建议同时使用三种信号： + +```text +DOM 信号： +tag、attributes、onclick、tabindex、contenteditable、form 结构 + +Accessibility 信号： +role、accessible name、checked、expanded、disabled、selected + +Layout/visual 信号： +是否可见、是否被遮挡、是否有 bounding box、是否能接收事件 +``` + +Chrome DevTools Protocol 的 `Accessibility.getFullAXTree` 可以拿到完整 Accessibility Tree；`DOMSnapshot.captureSnapshot` 可以返回 DOM、layout、style、iframe、template、shadow DOM flatten 后的信息，并且可以包含 DOM rects。citeturn592949view0 citeturn592949view1 + +--- + +## 4. iframe、Shadow DOM、不同前端框架的定位统一 + +不要让定位器直接依赖具体框架。应该定义一个中间层 `LocatorSpec`，再编译成 Playwright、Selenium 或 CDP 指令。 + +例如： + +```json +{ + "locatorSpec": { + "framePath": [ + { + "strategy": "url", + "value": "https://pay.example.com/*" + } + ], + "shadowPath": [ + { + "host": "payment-widget" + } + ], + "target": { + "role": "button", + "name": "Pay now", + "testId": "pay-now" + } + } +} +``` + +编译成 Playwright： + +```ts +page + .frameLocator('iframe[src*="pay.example.com"]') + .getByRole('button', { name: 'Pay now' }); +``` + +对于 iframe，必须把 `framePath` 存下来。对于 Shadow DOM，也要保存 `shadowHost` 路径。Playwright 的 locator 默认可以穿透 open Shadow DOM，但 XPath 不会穿透 Shadow Root，closed Shadow Root 也不支持直接定位；这类限制要在拓扑里标注。citeturn773971view0 + +--- + +## 5. 如何拿到“所调用函数” + +这里要分三种层次。 + +### 层次一：DOM event listener + +可以用 CDP 的 `DOMDebugger.getEventListeners` 查询某个对象上的 event listeners；也可以用 `DOMDebugger.setEventListenerBreakpoint` 在特定事件上暂停。CDP 的 Debugger domain 会在暂停时给出 call stack，也就是 `callFrames`。citeturn592949view3 citeturn592949view4 citeturn633002view0 + +这可以得到类似： + +```json +{ + "event": "click", + "listener": "bound dispatchDiscreteEvent", + "scriptUrl": "static/js/main.abc123.js", + "line": 12345, + "column": 67 +} +``` + +但在 React、Vue、Angular 里，这一层通常只能看到框架的事件分发函数，不一定直接看到业务函数。 + +--- + +### 层次二：运行时 call stack + +执行动作时，打开 event listener breakpoint： + +```text +click Save +→ debugger paused +→ 读取 callFrames +→ 跳过 react-dom / vue / angular / zone.js 等框架层 +→ 找到第一个业务代码 frame +→ 结合 sourcemap 映射到源码 +``` + +最终得到： + +```json +{ + "businessHandler": { + "functionName": "handleSave", + "sourceFile": "src/features/settings/SettingsForm.tsx", + "line": 88, + "confidence": 0.82 + } +} +``` + +如果代码被压缩、没有 sourcemap，函数名可能只能得到： + +```text +n +t +anonymous +``` + +这时要把置信度降下来，并更多依赖网络请求、DOM 变化和业务文本来描述动作语义。 + +--- + +### 层次三：API / 副作用追踪 + +对自动化更有价值的通常不是“函数名”，而是这个动作导致了什么副作用： + +```text +发起了什么请求？ +请求参数是什么？ +路由是否变化？ +localStorage/sessionStorage/cookie 是否变化？ +是否打开弹窗？ +是否新增 toast？ +是否触发表单校验？ +``` + +建议 hook 这些 API： + +```text +fetch +XMLHttpRequest +HTMLFormElement.prototype.submit +history.pushState +history.replaceState +localStorage.setItem +sessionStorage.setItem +window.open +navigator.clipboard.writeText +``` + +Playwright 的 `page.addInitScript()` 可以在文档创建后、页面自身脚本运行前注入脚本，适合提前 patch `addEventListener`、`fetch`、XHR 等运行时行为。citeturn487582search0 + +--- + +## 6. 参数建模：输入框、下拉框、表单、API body + +输入类元素需要单独建 `ParameterSpec`。 + +```json +{ + "elementId": "el_email_input", + "kind": "input", + "parameter": { + "name": "email", + "inputType": "email", + "required": true, + "pattern": null, + "minLength": null, + "maxLength": 255, + "examples": [ + "user@example.com" + ], + "validationMessages": [ + "Please enter a valid email address" + ] + } +} +``` + +对于表单，可以把多个输入参数合并成一个动作参数： + +```json +{ + "edgeId": "edge_submit_login", + "action": { + "type": "submitForm", + "paramsSchema": { + "email": { + "type": "string", + "format": "email", + "required": true + }, + "password": { + "type": "string", + "minLength": 8, + "required": true + } + } + }, + "effects": { + "network": [ + { + "method": "POST", + "url": "/api/login", + "bodyFields": ["email", "password"] + } + ] + } +} +``` + +参数来源可以按优先级推断： + +```text +1. input type、name、placeholder、label、aria-label +2. required、pattern、min、max、maxlength +3. select/options/radio group/checkbox group +4. 前端校验错误文本 +5. 请求 body 中的字段名 +6. 服务端返回的 validation error +7. 历史成功样本 +``` + +--- + +## 7. 状态探索算法 + +不要无限爬。应该做有边界的状态搜索。 + +```text +Queue = [initialState] + +while Queue not empty: + state = Queue.pop() + snapshot = capture(state) + elements = extractActionables(snapshot) + elements = rank(elements) + + for element in elements: + if shouldSkip(element): + continue + + before = captureFingerprint() + trace = startTrace() + + performAction(element) + + after = captureFingerprint() + effect = diff(before, after, trace) + + edge = buildEdge(state, element, effect) + save(edge) + + if effect.newState and not seen(effect.newState): + Queue.push(effect.newState) +``` + +`shouldSkip` 很关键，否则状态爆炸。建议跳过： + +```text +外链 +登出 / 删除 / 支付 / 提交订单 等高风险动作 +重复列表项中的同构按钮 +分页无限翻页 +文件上传 +下载 +破坏性操作 +需要验证码或 MFA 的动作 +``` + +对于重复结构，例如表格中 100 行都有 “Edit” 按钮，不要存 100 个完全独立节点，而是抽象成模板： + +```json +{ + "template": "table-row-action", + "action": "Edit", + "rowSelector": "row by primary key", + "params": { + "rowId": "dynamic" + } +} +``` + +--- + +## 8. 状态指纹：如何判断“新状态” + +状态不能只看 URL，因为 SPA 经常 URL 不变但弹窗、菜单、tab 已经变化。 + +建议状态指纹由这些组成： + +```json +{ + "canonicalUrl": "/dashboard", + "visibleRoleNamePairs": [ + ["button", "Save"], + ["button", "Cancel"], + ["heading", "Settings"] + ], + "modalSignature": "settings-dialog", + "expandedControls": [ + "user-menu" + ], + "formSignature": [ + "email", + "password" + ], + "networkIdle": true +} +``` + +可以生成： + +```text +stateHash = hash( + canonicalUrl + + visible interactive AX tree + + modal/dialog/menu stack + + selected tab + + major form fields +) +``` + +DOM hash 不能太细，否则每次时间戳、随机 id、广告位变化都会制造伪新状态。 + +--- + +## 9. 最小可行实现架构 + +可以分成五个模块： + +```text +Crawler + 负责打开页面、登录、BFS 探索状态 + +Snapshotter + 负责采集 DOM、AX Tree、layout、frame、shadow 信息 + +Locator Generator + 负责给每个元素生成多种 locator，并评分 + +Tracer + 负责追踪 event listener、call stack、network、storage、route change + +Graph Store + 负责保存 State、Element、Locator、Action Edge、Parameter、Effect +``` + +推荐存储： + +```text +开发期：SQLite / Postgres JSONB +复杂查询：Neo4j / ArangoDB +向量检索：把元素语义、文本、动作描述 embedding 后存入 vector DB +``` + +查询例子： + +```sql +-- 找到“保存设置”相关动作 +SELECT * +FROM action_edges +WHERE semantic_label ILIKE '%save%' + AND effects::text ILIKE '%settings%'; +``` + +图查询例子： + +```cypher +MATCH p = shortestPath( + (:State {name: "dashboard"})-[:ACTION*]->(:Element {accessibleName: "Save"}) +) +RETURN p; +``` + +--- + +## 10. Playwright 执行时如何利用拓扑 + +执行一个任务时： + +```text +用户目标：修改通知设置并保存 + +1. 语义检索 action graph +2. 找到目标 edge: click Save Settings +3. 反查 shortest preconditionPath +4. 逐步打开菜单、弹窗、tab +5. 对目标元素使用最高分 locator +6. 如果失败，尝试下一个 locator +7. 执行动作 +8. 用 expected effects 验证结果 +9. 如果状态漂移，局部重新探索 +``` + +伪代码： + +```ts +async function executeEdge(page, graph, edgeId) { + const edge = graph.getEdge(edgeId); + const path = graph.shortestPathTo(edge.fromState); + + for (const step of path) { + await runAction(page, graph.getEdge(step.edgeId)); + await verifyState(page, step.toState); + } + + const element = graph.getElement(edge.elementId); + const locator = chooseBestLocator(page, element.locators); + + await perform(page, locator, edge.action); + await verifyEffects(page, edge.effects); +} +``` + +Playwright 在执行 `click()`、`fill()` 等动作前会做 actionability 检查，例如元素是否唯一、可见、稳定、可接收事件、enabled/editable 等；这类检查应当成为你的拓扑质量验证标准。citeturn773971view1 + +--- + +## 11. 一个简化版采集代码骨架 + +```ts +import { chromium, Page } from 'playwright'; + +type Actionable = { + tag: string; + role: string | null; + text: string; + name: string | null; + id: string | null; + testId: string | null; + placeholder: string | null; + ariaLabel: string | null; + bbox: { x: number; y: number; width: number; height: number }; +}; + +async function installInstrumentation(page: Page) { + await page.addInitScript(() => { + const store = { + listeners: [], + fetches: [], + xhrs: [], + storage: [], + history: [] + }; + + Object.defineProperty(window, '__actionTopoTrace', { + value: store, + configurable: false + }); + + const originalAddEventListener = EventTarget.prototype.addEventListener; + EventTarget.prototype.addEventListener = function (type, listener, options) { + try { + store.listeners.push({ + type, + listenerName: listener && (listener as any).name, + targetTag: (this as Element).tagName || null, + targetId: (this as Element).id || null, + stack: new Error().stack + }); + } catch {} + + return originalAddEventListener.call(this, type, listener, options); + }; + + const originalFetch = window.fetch; + window.fetch = async function (...args) { + try { + store.fetches.push({ + input: String(args[0]), + init: args[1] || null, + stack: new Error().stack + }); + } catch {} + + return originalFetch.apply(this, args as any); + }; + + const originalPushState = history.pushState; + history.pushState = function (...args) { + try { + store.history.push({ + type: 'pushState', + args, + stack: new Error().stack + }); + } catch {} + + return originalPushState.apply(this, args as any); + }; + }); +} + +async function extractActionables(page: Page): Promise { + return await page.evaluate(() => { + const selector = [ + 'button', + 'a[href]', + 'input', + 'textarea', + 'select', + '[role="button"]', + '[role="link"]', + '[role="menuitem"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="tab"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])', + '[onclick]', + '[aria-haspopup]', + '[aria-expanded]' + ].join(','); + + function isVisible(el: Element) { + const rect = el.getBoundingClientRect(); + const style = window.getComputedStyle(el); + return ( + rect.width > 0 && + rect.height > 0 && + style.visibility !== 'hidden' && + style.display !== 'none' && + Number(style.opacity) > 0 + ); + } + + return Array.from(document.querySelectorAll(selector)) + .filter(isVisible) + .map((el) => { + const rect = el.getBoundingClientRect(); + + return { + tag: el.tagName.toLowerCase(), + role: el.getAttribute('role'), + text: (el.textContent || '').trim().slice(0, 200), + name: el.getAttribute('name'), + id: el.id || null, + testId: + el.getAttribute('data-testid') || + el.getAttribute('data-test') || + el.getAttribute('data-qa'), + placeholder: el.getAttribute('placeholder'), + ariaLabel: el.getAttribute('aria-label'), + bbox: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height + } + }; + }); + }); +} + +async function main() { + const browser = await chromium.launch({ headless: false }); + const context = await browser.newContext(); + const page = await context.newPage(); + + await installInstrumentation(page); + + await page.goto('https://example.com'); + + const cdp = await context.newCDPSession(page); + + const axTree = await cdp.send('Accessibility.getFullAXTree'); + const domSnapshot = await cdp.send('DOMSnapshot.captureSnapshot', { + computedStyles: [ + 'display', + 'visibility', + 'opacity', + 'pointer-events' + ], + includeDOMRects: true, + includePaintOrder: true + }); + + const elements = await extractActionables(page); + + console.log({ + elements, + axNodeCount: axTree.nodes.length, + documentCount: domSnapshot.documents.length + }); + + await browser.close(); +} + +main(); +``` + +Playwright 的 `CDPSession` 可以直接调用 Chrome DevTools Protocol 方法，并订阅协议事件；这适合把 Playwright 的高层自动化能力和 CDP 的底层采集能力结合起来。citeturn576235search0 + +--- + +## 12. 拓扑图中的 Locator 评分公式 + +可以给每个 locator 算一个稳定性分数： + +```text +score = + semanticScore ++ uniquenessScore ++ visibilityScore ++ stabilityScore ++ frameSpecificityScore +- brittlenessPenalty +``` + +示例： + +```text +getByRole('button', { name: 'Save' }) + semanticScore: 0.35 + uniquenessScore: 0.25 + stabilityScore: 0.20 + total: 0.80 + +[data-testid="save-settings"] + contractScore: 0.40 + uniquenessScore: 0.30 + total: 0.90 + +#root > div:nth-child(2) > div > button:nth-child(3) + uniquenessScore: 0.25 + brittlenessPenalty: -0.45 + total: 0.10 +``` + +同时要定期做 locator health check： + +```text +locator 是否还能命中？ +是否唯一？ +命中元素的 role/name/text 是否和原元素一致？ +bbox 是否大致相近？ +点击后 effect 是否一致？ +``` + +--- + +## 13. “前置打开项”应该怎么存 + +对于每个可操作项，存两种前置信息： + +### 结构前置 + +表示它依赖哪个 UI 容器： + +```json +{ + "elementId": "el_save_button", + "requiresVisibleAncestors": [ + { + "kind": "dialog", + "name": "Settings" + }, + { + "kind": "tabPanel", + "name": "Notifications" + } + ] +} +``` + +### 路径前置 + +表示从入口状态怎么走到这里： + +```json +{ + "elementId": "el_save_button", + "shortestOpenPath": [ + { + "action": "click", + "target": "User menu" + }, + { + "action": "click", + "target": "Settings" + }, + { + "action": "click", + "target": "Notifications tab" + } + ] +} +``` + +实际自动化执行时优先用 `shortestOpenPath`；如果路径失效，再用结构前置做局部恢复。 + +--- + +## 14. 最容易踩的坑 + +第一，**“全站所有可操作项”在严格意义上不可穷尽**。搜索框、分页、权限、A/B 实验、用户数据、无限滚动都会产生无穷状态。工程上应当做 bounded crawling、状态归并和模板化。 + +第二，**React/Vue/Angular 的真实业务函数不一定挂在 DOM 节点上**。很多事件是框架委托到根节点的，所以 `getEventListeners(button)` 未必能直接得到 `handleClick`。更可靠的方法是：动作执行时抓 call stack、network、DOM diff、storage diff。 + +第三，**不要把坐标当主定位方式**。坐标可以作为兜底，但 viewport、响应式布局、滚动位置、遮罩层都会让坐标失效。 + +第四，**不要把每次 DOM diff 都当新状态**。动态时间戳、随机 id、广告、埋点元素都会制造噪声。状态归并应基于“可见可交互语义树”，而不是完整 DOM。 + +第五，**不要盲点危险动作**。删除、支付、提交订单、发送邮件、改权限这类动作需要沙盒账户、mock 后端或人工白名单。 + +--- + +## 15. 推荐的最终数据结构 + +一个比较完整的 schema 可以是： + +```text +State + stateId + url + route + title + frameContext + stateHash + visibleAxSignature + modalStack + openedByEdgeId + createdAt + +Element + elementId + stateId + tag + role + accessibleName + text + label + placeholder + attributes + bbox + framePath + shadowPath + componentHint + interactability + +Locator + locatorId + elementId + framework + strategy + expression + score + lastVerifiedAt + failureCount + +ActionEdge + edgeId + fromState + toState + elementId + actionType + actionParams + preconditions + effects + riskLevel + confidence + +HandlerTrace + traceId + edgeId + eventType + listeners + callFrames + businessFunction + sourceMappedLocation + confidence + +NetworkEffect + edgeId + method + url + requestHeaders + requestBodySchema + responseStatus + responseBodySchema + +ParameterSpec + elementId or edgeId + name + type + required + enum + pattern + min + max + examples + validationMessages +``` + +--- + +## 16. 实用落地顺序 + +最小可用版本可以这样做： + +```text +第 1 步：只做已登录状态下的页面状态采集 +第 2 步：抽取 button/link/input/select/textarea +第 3 步：生成 Playwright locators +第 4 步：点击低风险元素，记录 URL/DOM/AX/network diff +第 5 步：把“打开菜单/弹窗/tab”的路径存成 preconditionPath +第 6 步：接入 fetch/XHR/history/storage tracing +第 7 步：加入 call stack 和 sourcemap +第 8 步：做 locator 自愈和状态局部重探索 +``` + +最终你会得到一个可以被自动化系统快速查询的图： + +```text +目标：点击“导出报表” +→ 查询 Action Graph +→ 发现按钮在 Reports 页面 More 菜单里 +→ 自动执行： + 1. goto /reports + 2. click More + 3. click Export +→ 验证： + - 出现 download + - 或请求 GET /api/reports/export +``` + +这就是比传统 Page Object Model 更强的一点：它不是手写固定脚本，而是把网站的可操作结构、前置路径、定位器、函数/API 副作用都建成可检索、可验证、可恢复的拓扑。 + + +### User Message 1449 + +在整个架构里面，你是否建议在里面使用大语言模型。如果是在局域网环境下，那么至少需要多少参数的模型才有效果？因为我们必须考虑，在模型性能不足的情况下，有模型效果会低于没有模型的效果 + + +### Assistant Message 1379 + +建议使用大语言模型，但**不要把它放在浏览器自动化的执行核心里**。在这个架构中，LLM 应该是一个“语义增强层”和“候选生成器”，而不是“事实来源”或“最终执行器”。 + +我会给一个明确结论： + +> **局域网环境下，如果只是做可操作项语义标注，7B/8B 可以试用；如果要参与任务到动作的映射，建议至少 14B；如果要参与多步路径规划、失败恢复、参数推断，建议 32B 起步；如果要做跨网站、长链路、低人工干预的通用浏览器代理，70B 级别才比较稳。** + +如果只能选一个生产起步点： + +> **14B 是最低可用线，32B 是更合理的工程线。低于 7B/8B，不建议让模型参与任何会影响点击、填写、提交的决策。** + +--- + +## 1. 我建议使用 LLM，但只放在这些位置 + +在你前面描述的“网站可操作项拓扑”架构中，确定性系统应该负责： + +```text +DOM / AX Tree 采集 +iframe / shadow DOM 路径 +locator 生成 +actionability 检查 +网络请求追踪 +storage / route / DOM diff +状态图构建 +动作执行 +效果验证 +风险控制 +``` + +LLM 适合负责： + +```text +元素语义理解 +业务意图归类 +自然语言任务到 Action Edge 的匹配 +参数含义推断 +表单字段解释 +相似动作合并 +失败后的候选修复建议 +页面状态摘要 +风险等级初判 +``` + +也就是说，LLM 可以回答： + +```text +“这个按钮大概率是导出报表” +“这个输入框应该对应客户编号” +“用户说的‘下载账单’可能对应 edge_export_invoice” +“这个失败可能是因为菜单没有展开” +``` + +但不应该直接拥有最终权限去做： + +```text +click(x, y) +fill(selector, value) +submit form +delete record +confirm payment +change permission +``` + +最终执行必须由确定性执行器完成，并且要通过 locator 唯一性、可见性、enabled、effect verification 等校验。 + +--- + +## 2. 为什么不能完全不用 LLM？ + +因为纯规则系统在这些地方会很吃力： + +```text +“Export” / “Download” / “导出” / “下载报表” 是同一类动作 +“客户编号” 和 “Customer ID” 是同一类字段 +一个菜单项虽然叫 “More”，但里面真正有 “Export” +按钮文案是图标，没有 text，但周围上下文暗示它是“编辑” +请求 POST /api/report/export 和 UI 上的“下载”语义相关 +``` + +这些都是语义问题，不是 DOM 问题。 + +Web agent 研究也说明，仅把完整 HTML 丢给模型并不好；Mind2Web 论文明确指出真实网页 HTML 经常过大，先过滤候选元素能显著改善 LLM 在网页任务上的效果和效率。你的拓扑图方案本质上就是在做这种“先结构化、再让模型理解”的架构。citeturn694831search1 + +--- + +## 3. 为什么也不能过度依赖 LLM？ + +因为浏览器自动化的错误成本很高。LLM 在网页上最危险的错误通常不是“不知道”，而是**看起来很合理地选错**。 + +典型错误包括： + +```text +把“取消”理解成“确认” +把隐藏菜单里的旧按钮当成当前按钮 +把列表第一行的 Edit 当成目标行的 Edit +把 Disabled 按钮当成可点击按钮 +把“导出模板”当成“导出数据” +误以为某个 modal 已经打开 +幻觉出页面上不存在的按钮 +``` + +WebArena 的实验也说明真实网页任务对自动化代理仍然很难；论文中的最佳 GPT-4-based agent 端到端任务成功率只有 14.41%，远低于人类 78.24%。这不是说 LLM 没用，而是说明**不能把 LLM 当成无约束网页执行器**。citeturn694831academia38 + +所以你的担心是对的： + +> **在模型能力不足、验证不足、错误动作成本较高时，有模型确实可能低于没有模型。** + +--- + +## 4. 建议的 LLM 放置方式 + +我建议把 LLM 做成独立服务： + +```text +Topology Engine + ↓ +Candidate Retriever + ↓ +LLM Semantic Layer + ↓ +Validator / Verifier + ↓ +Deterministic Executor +``` + +LLM 只输出候选，不直接执行。 + +例如用户目标是： + +```text +下载上个月销售报表 +``` + +系统应该这样工作： + +```text +1. 图数据库检索候选 Action Edge +2. LLM 只在候选边中排序，不允许自由编造动作 +3. Validator 检查： + - edge 是否存在 + - locator 是否唯一 + - 前置路径是否可达 + - action 是否低风险 + - 参数是否满足 schema +4. 执行器按图执行 +5. effect verifier 验证是否真的触发下载或请求 +``` + +LLM 输出应该类似： + +```json +{ + "selectedEdgeId": "edge_export_sales_report", + "reason": "用户目标中的下载销售报表与该 edge 的 accessibleName、network effect、页面上下文匹配", + "confidence": 0.84, + "requiredPreconditions": [ + "进入 Reports 页面", + "打开 Sales tab", + "设置日期范围为上个月" + ], + "abstain": false +} +``` + +而不是输出： + +```json +{ + "click": "#app > div:nth-child(3) > button" +} +``` + +后者非常危险。 + +--- + +## 5. 参数量建议：按任务分级 + +### 总表 + +| 使用场景 | 是否建议用 LLM | 最低参数量 | 推荐参数量 | 低于这个规模的风险 | +|---|---:|---:|---:|---| +| 元素语义标注 | 建议 | 7B/8B | 14B | 标签粗糙，但可控 | +| 表单字段含义推断 | 建议 | 7B/8B | 14B/32B | 容易误判业务字段 | +| 自然语言目标 → 已有 Action Edge | 建议 | 14B | 32B | 容易选错相似动作 | +| 多步路径规划 | 谨慎使用 | 32B | 70B | 容易漏前置状态 | +| 自动失败恢复 | 谨慎使用 | 32B | 70B | 容易越修越错 | +| 自主探索未知网页 | 只限沙盒 | 32B | 70B+ | 状态爆炸、误点风险高 | +| 基于截图理解 UI | 需要 VLM | 8B VLM | 32B VLM | 视觉 grounding 不稳 | +| 危险动作：删除、付款、授权 | 不建议全自动 | 70B 也需人工/规则门控 | 70B+ | 不接受模型单独决策 | + +--- + +## 6. 一个实用判断：14B 是“最低可用线”，32B 是“工程舒适线” + +对于你的场景，我会这样定： + +### 7B/8B：只能做辅助标注 + +可以用于： + +```text +给按钮生成中文/英文语义标签 +把相似按钮聚类 +生成候选 description +解释 input 字段 +把 URL / API / 文本摘要成业务对象 +``` + +不建议用于： + +```text +选择最终动作 +复杂路径规划 +失败恢复 +危险操作判断 +``` + +原因是 7B/8B 模型在简单语义上通常够用，但在多约束、多步推理、相似项区分时容易出错。对你的系统来说，7B/8B 的正确用法是： + +```text +LLM enriches graph metadata, but does not control browser. +``` + +--- + +### 14B：可以参与“候选排序”和“意图映射” + +14B 是我认为局域网部署的最低可用线。它可以用于： + +```text +用户目标 → 候选 edge 排序 +元素语义归类 +参数 schema 推断 +菜单/弹窗前置路径解释 +把 trace / network effect 翻译成业务动作 +``` + +但仍然要加硬验证： + +```text +候选 edge 必须来自图数据库 +不能生成不存在的 locator +不能跳过 actionability 检查 +不能执行高风险动作 +confidence 低时必须 abstain +``` + +当前开源模型里，Qwen3-14B 这类 14B 级别模型已经具备比较强的中文、多语言、工具调用和推理能力。Qwen3-14B 模型卡给出的参数量是 14.8B，并且原生上下文长度为 32,768 tokens，可通过 YaRN 扩展到 131,072 tokens。citeturn968736view0 + +--- + +### 32B：更适合生产自动化 + +如果你希望模型真正参与： + +```text +复杂页面状态理解 +多候选动作排序 +失败恢复建议 +表单参数推断 +跨页面 workflow 规划 +工具调用 +``` + +我会建议直接上 32B。 + +Qwen3-32B 模型卡给出的参数量是 32.8B，并强调其 reasoning、instruction-following、agent capabilities、多语言能力和工具集成能力；它同样原生支持 32,768 tokens 上下文，并可通过 YaRN 扩展。citeturn255948view0 + +在你的架构里，32B 的价值不是“能不能聊天”，而是它更可能稳定完成这种任务： + +```text +给定： +- 用户目标 +- 当前 State +- 20 个候选 Action Edge +- 每个 edge 的 accessibleName、role、effect、preconditionPath、riskLevel + +要求： +- 选择最可能的 edge +- 解释选择依据 +- 给出缺失参数 +- 在证据不足时 abstain +``` + +这个任务对 7B/8B 来说容易不稳定，对 14B 勉强可用，对 32B 更像工程上可接受。 + +--- + +### 70B：用于跨网站通用代理和复杂恢复 + +如果目标不是单个业务系统，而是很多不同网站、很多未知页面、长链路任务、低人工干预，那么 70B 级别更合适。 + +70B 级别模型的缺点是部署成本、吞吐、显存压力都更高。以 Llama 3.3 70B 为例，其模型卡显示它是 70B text-only multilingual instruction-tuned model，并支持 128k context。citeturn557352search1 + +但我仍然不建议让 70B 模型绕过验证直接控制浏览器。70B 只是降低错误概率，不改变架构原则。 + +--- + +## 7. 如果涉及截图或视觉 UI，单独考虑 VLM + +如果你能拿到 DOM、Accessibility Tree、layout、network trace，那么**优先不要依赖截图模型**。DOM/AX 通常更可验证、更稳定、更便宜。 + +但有些场景需要视觉模型： + +```text +canvas UI +图标按钮没有文本 +复杂图表区域 +截图式后台系统 +远程桌面或 Citrix +closed shadow root +无法读取语义 DOM 的第三方控件 +``` + +这时需要 VLM。Qwen3-VL 技术报告中提到该系列包含 dense 2B/4B/8B/32B 和 MoE 30B-A3B/235B-A22B 等规模，面向图文/视频和长上下文多模态任务。citeturn621338academia42 + +我的建议是： + +```text +简单截图理解：8B VLM 可以试 +生产级 UI grounding：32B VLM 起步 +高风险点击：不要只靠 VLM 坐标，必须结合 DOM/AX/截图标注/验证 +``` + +--- + +## 8. “模型低于没有模型”的具体触发条件 + +你可以用下面这个标准判断是否应该禁用模型。 + +如果模型出现这些情况，就不应该进入执行链路： + +```text +1. 经常选择不存在的元素 +2. 经常把相似按钮混淆 +3. 无法稳定输出 JSON schema +4. 不会 abstain，证据不足也强行回答 +5. 对危险动作风险识别不稳定 +6. 生成的前置路径不可执行 +7. 延迟过高，导致整体自动化吞吐下降 +8. 修复建议导致更多重试和状态污染 +``` + +尤其是第 4 点很关键。 + +一个弱模型最大的问题不是“不知道”，而是： + +```text +不知道，但仍然给出看起来合理的答案。 +``` + +这种模型对自动化系统是负资产。 + +--- + +## 9. 正确的上线策略：先 shadow mode + +我建议按这四个阶段上线。 + +### 阶段 1：完全不用 LLM，建立 baseline + +先做纯确定性版本： + +```text +采集拓扑 +生成 locator +记录前置路径 +记录 network effect +人工给部分 action edge 打标签 +执行固定 workflow +统计成功率 +``` + +这一步很重要，因为你需要知道“没有模型时”到底能做到什么程度。 + +--- + +### 阶段 2：LLM 只做旁路标注，不影响执行 + +LLM 只生成： + +```text +semanticLabel +businessObject +riskLevelCandidate +parameterMeaning +edgeDescription +``` + +但这些字段不能影响执行。 + +例如： + +```json +{ + "edgeId": "edge_btn_8273", + "llmSemanticLabel": "导出销售报表", + "llmRiskCandidate": "read_only", + "llmConfidence": 0.78 +} +``` + +然后人工或规则抽样评估。 + +--- + +### 阶段 3：LLM 参与候选排序，但执行器强验证 + +用户说： + +```text +导出销售报表 +``` + +系统先用向量检索/关键词/图查询找出 20 个候选 edge，再让 LLM 排序。 + +LLM 不允许生成新 edge，只能从候选中选： + +```json +{ + "allowedEdgeIds": [ + "edge_export_sales", + "edge_export_template", + "edge_download_invoice", + "edge_open_report_menu" + ] +} +``` + +如果 LLM 选了不存在的 edge，直接拒绝。 + +--- + +### 阶段 4：LLM 参与失败恢复，但只能提出修复候选 + +例如执行失败： + +```text +locator 命中 0 个元素 +``` + +LLM 可以建议： + +```text +可能需要先打开 More 菜单 +可能当前 tab 不对 +可能页面语言切换为英文 +可能按钮被移动到右上角菜单 +``` + +但系统仍然要验证： + +```text +这个菜单是否存在？ +打开菜单后目标按钮是否出现？ +新 locator 是否唯一？ +点击后 effect 是否符合原 edge？ +``` + +--- + +## 10. 建议的模型门控机制 + +每次 LLM 输出后都做 gate： + +```text +LLM Output + ↓ +Schema Validator + ↓ +Candidate Existence Check + ↓ +Locator Uniqueness Check + ↓ +Actionability Check + ↓ +Risk Policy Check + ↓ +Effect Verifier + ↓ +Execute or Reject +``` + +可以给每个候选动作计算综合分： + +```text +finalScore = + retrievalScore * 0.25 ++ locatorHealthScore * 0.20 ++ effectMatchScore * 0.25 ++ llmReasoningScore * 0.15 ++ historicalSuccessScore * 0.15 +- riskPenalty +``` + +注意：`llmReasoningScore` 不能占太高。我的建议是不超过 15% 到 25%。 + +LLM 应该是加分项，不是决定项。 + +--- + +## 11. 关键原则：LLM 不写入事实，只写入解释和候选 + +图数据库中应该区分： + +```text +verified fields +derived fields +llm fields +``` + +例如： + +```json +{ + "elementId": "el_123", + "verified": { + "role": "button", + "accessibleName": "Export", + "locator": "page.getByRole('button', { name: 'Export' })", + "framePath": [], + "networkEffect": "GET /api/reports/export" + }, + "llm": { + "semanticLabel": "导出报表", + "businessObject": "report", + "riskGuess": "read_only", + "confidence": 0.82 + } +} +``` + +其中 `verified` 来自 DOM、AX、CDP、Playwright、network trace。 + +`llm` 只能用于检索、排序、解释、候选生成，不能直接覆盖 verified facts。 + +--- + +## 12. 局域网部署时的粗略硬件判断 + +只按推理权重估算，4-bit 量化大致可以这么看： + +| 模型规模 | 4-bit 权重粗略体积 | 工程建议 | +|---:|---:|---| +| 7B/8B | 4–6 GB | 单卡轻量辅助 | +| 14B | 8–10 GB | 最低可用生产线 | +| 32B | 18–24 GB | 推荐生产线 | +| 70B | 40–50 GB+ | 高质量但成本高 | + +实际还要加： + +```text +KV cache +batch size +context length +并发数 +推理框架开销 +显存碎片 +量化格式 +``` + +对于局域网服务，我会优先考虑： + +```text +14B：低成本语义服务 +32B：主力语义/规划服务 +70B：复杂任务、人工助手、疑难恢复 +``` + +可以用统一的 OpenAI-compatible API 封装，例如 vLLM、SGLang、Ollama、llama.cpp server 等。Qwen3 模型卡也给出了 vLLM、SGLang、本地应用和 OpenAI-compatible endpoint 的部署方式。citeturn255948view0 + +--- + +## 13. 我建议的实际组合 + +### 方案 A：保守生产方案 + +```text +确定性拓扑引擎：必须有 +LLM：14B +用途：标注、候选排序、参数解释 +执行：完全由规则和图完成 +``` + +适合： + +```text +企业内网系统 +固定业务流程 +风险较低 +预算有限 +``` + +--- + +### 方案 B：推荐生产方案 + +```text +确定性拓扑引擎：必须有 +LLM：32B +Embedding 模型：单独部署 +可选 reranker：小模型 +用途： + - 目标到 action edge 匹配 + - 多步 workflow 排序 + - 失败恢复建议 + - 参数 schema 推断 +执行： + - 仍由 deterministic executor 完成 +``` + +适合： + +```text +多个业务系统 +页面变化频繁 +需要自然语言驱动 +需要较高成功率 +``` + +--- + +### 方案 C：高能力方案 + +```text +32B 作为在线主力 +70B 作为疑难任务 fallback +8B/14B 用于批量离线标注 +VLM 仅用于 DOM 不可读页面 +``` + +适合： + +```text +跨系统自动化平台 +多租户 +复杂后台 +未知页面较多 +``` + +--- + +## 14. 必须做自己的评测集 + +不要只看通用 benchmark。你的任务很特殊，应该建立自己的 browser automation benchmark。 + +至少准备： + +```text +100–300 个元素语义标注样本 +100–300 个自然语言目标到 edge 的匹配样本 +50–100 个多步 workflow 样本 +50 个失败恢复样本 +20–50 个高风险拒绝样本 +``` + +评估指标： + +```text +schema_valid_rate +top1_edge_accuracy +top3_edge_recall +wrong_action_rate +abstain_precision +abstain_recall +locator_repair_success_rate +effect_verification_success_rate +latency_p50 +latency_p95 +tokens_per_second +human_intervention_rate +``` + +我会设这样的上线门槛： + +```text +schema_valid_rate >= 99% +top3_edge_recall >= 95% +top1_edge_accuracy >= 90% 低风险任务 +wrong_action_rate <= 0.5% 低风险任务 +高风险动作 wrong_action_rate 必须接近 0 +abstain 行为必须稳定 +LLM 方案整体成功率至少比 no-LLM baseline 高 5–10 个百分点 +``` + +如果达不到这些指标，就不要让模型进入执行链路。 + +--- + +## 15. 最重要的工程结论 + +你的系统里，LLM 的正确角色是： + +```text +语义层 +检索增强层 +候选排序层 +解释层 +修复建议层 +``` + +不是： + +```text +DOM 事实来源 +locator 事实来源 +浏览器执行器 +风险最终裁决器 +状态图唯一构建者 +``` + +如果局域网部署，我的建议是： + +```text +MVP：不用 LLM 或只用 7B/8B 做旁路标注 +第一版生产：14B 起步 +推荐生产：32B +复杂通用代理：70B 级别，但仍需强验证 +``` + +所以，针对你的问题，最直接的回答是： + +> **要用 LLM，但要把它关在“可验证候选生成器”的笼子里。14B 是最低可用线，32B 是我会推荐的局域网生产线。低于 7B/8B 的模型，不如不用；低于 14B 的模型，不要参与动作选择；低于 32B 的模型，不要承担复杂路径规划和失败恢复。** + diff --git a/examples/simple.html b/examples/simple.html new file mode 100644 index 0000000..749d45d --- /dev/null +++ b/examples/simple.html @@ -0,0 +1,50 @@ + + + + + + Action Topology Example + + + +

Action Topology Example

+ + Reports + + +

Idle

+ + diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..fe0441f --- /dev/null +++ b/package-lock.json @@ -0,0 +1,616 @@ +{ + "name": "action-topology-engine", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "action-topology-engine", + "version": "0.1.0", + "dependencies": { + "playwright": "^1.61.0" + }, + "bin": { + "action-topology": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^22.15.30", + "tsx": "^4.20.3", + "typescript": "^5.8.3" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "22.19.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.21.tgz", + "integrity": "sha512-VMeFBSCKQKmm2swI2kW51SFusDqekC6q9trBCvJ/JliDchFSuoYYKN7yVNjPthP1HKZcx3U1gI/wTcEBjEFKTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", + "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx": { + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", + "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..d4c7d30 --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "action-topology-engine", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Deterministic website action topology engine built on Playwright.", + "bin": { + "action-topology": "./dist/cli.js" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc --noEmit -p tsconfig.json", + "scan": "tsx src/cli.ts scan", + "explore": "tsx src/cli.ts explore", + "example": "tsx src/cli.ts explore --url ./examples/simple.html --out artifacts/simple-graph.json --max-actions 8" + }, + "dependencies": { + "playwright": "^1.61.0" + }, + "devDependencies": { + "@types/node": "^22.15.30", + "tsx": "^4.20.3", + "typescript": "^5.8.3" + } +} + diff --git a/schemas/topology-graph.schema.md b/schemas/topology-graph.schema.md new file mode 100644 index 0000000..d510261 --- /dev/null +++ b/schemas/topology-graph.schema.md @@ -0,0 +1,51 @@ +# Topology Graph Schema + +This document describes the JSON shape emitted by the engine. It is a practical schema note, not yet a formal JSON Schema file. + +## `TopologyGraph` + +- `schemaVersion`: currently `0.1`. +- `metadata.entryUrl`: entry URL after Playwright navigation. +- `metadata.mode`: `scan` or `explore`. +- `states`: page states. +- `elements`: actionable elements bound to a state. +- `edges`: observed actions from one state to another. + +## `StateNode` + +- `stateId`: stable id derived from URL, DOM hash, visible text hash, and actionable signature hash. +- `url`, `route`, `title`: browser-visible state identity. +- `domHash`: hash of full `document.documentElement.outerHTML`. +- `visibleTextHash`: hash of normalized visible body text. +- `actionableSignatureHash`: hash of actionable element signatures. +- `modalStack`: visible dialogs or modal-like surfaces. +- `openedByEdgeId`: edge that opened this state, if known. + +## `ActionableElement` + +- `elementId`: stable id within the captured state. +- `stateId`: owning state. +- `tag`, `role`, `accessibleName`, `text`, `label`, `placeholder`: semantic identity. +- `attributes`: selected stable attributes such as `data-testid`, `id`, `name`, `type`, `href`, and ARIA fields. +- `bbox`: page-space bounding box. +- `interactability`: visibility, enabled/editable status, and whether the element receives events. +- `locators`: scored locator candidates. + +## `LocatorCandidate` + +- `descriptor`: structured Playwright locator descriptor. +- `expression`: human-readable locator expression. +- `score`: ranking score. +- `verified`: whether it resolved to exactly one element during capture. +- `matchCount`: observed locator match count. + +## `ActionEdge` + +- `fromStateId`, `toStateId`: source and resulting states. +- `elementId`: action target. +- `actionType`: currently focused on `click`, with type support for future `fill`, `select`, and navigation actions. +- `preconditions`: reserved for stored open paths. +- `effects`: URL, visible text, network, download, dialog, storage, or DOM effects. +- `riskLevel`: `low`, `medium`, or `high`. +- `confidence`: confidence in observed edge replayability. + diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..f273eb7 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,66 @@ +#!/usr/bin/env node +import { exploreUrl, scanUrl, type EngineOptions } from "./engine.js"; + +interface ParsedArgs extends EngineOptions { + command: "scan" | "explore"; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const graph = args.command === "scan" ? await scanUrl(args) : await exploreUrl(args); + if (args.out) { + console.error(`Wrote ${graph.states.length} state(s), ${graph.elements.length} element(s), ${graph.edges.length} edge(s) to ${args.out}`); + } +} + +function parseArgs(argv: string[]): ParsedArgs { + const command = argv.shift(); + if (command !== "scan" && command !== "explore") usage(); + + const options: Partial = { command }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--url": + options.url = requireValue(argv, ++index, "--url"); + break; + case "--out": + options.out = requireValue(argv, ++index, "--out"); + break; + case "--storage-state": + options.storageState = requireValue(argv, ++index, "--storage-state"); + break; + case "--max-actions": + options.maxActions = Number(requireValue(argv, ++index, "--max-actions")); + break; + case "--headed": + options.headed = true; + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (!options.url) usage(); + return options as ParsedArgs; +} + +function requireValue(argv: string[], index: number, flag: string): string { + const value = argv[index]; + if (!value) throw new Error(`Missing value for ${flag}`); + return value; +} + +function usage(): never { + console.error(`Usage: + action-topology scan --url [--out graph.json] [--headed] [--storage-state state.json] + action-topology explore --url [--out graph.json] [--max-actions 10] [--headed] [--storage-state state.json] +`); + process.exit(1); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exit(1); +}); + diff --git a/src/engine.ts b/src/engine.ts new file mode 100644 index 0000000..584d25e --- /dev/null +++ b/src/engine.ts @@ -0,0 +1,189 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { chromium, type Browser, type BrowserContext, type Page, type Response } from "playwright"; +import { shortHash } from "./hash.js"; +import { resolveLocator } from "./locator.js"; +import { isLowRiskClickable, classifyClickRisk } from "./risk.js"; +import { snapshotPage } from "./snapshot.js"; +import type { ActionEdge, EffectRecord, TopologyGraph } from "./types.js"; + +export interface EngineOptions { + url: string; + out?: string; + headed?: boolean; + storageState?: string; + maxActions?: number; +} + +export async function scanUrl(options: EngineOptions): Promise { + const browser = await chromium.launch({ headless: !options.headed }); + try { + const context = await newContext(browser, options); + const page = await context.newPage(); + await page.goto(toNavigableUrl(options.url), { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle").catch(() => undefined); + const snapshot = await snapshotPage(page); + const graph: TopologyGraph = { + schemaVersion: "0.1", + metadata: { + entryUrl: page.url(), + generatedAt: new Date().toISOString(), + engineVersion: "0.1.0", + mode: "scan", + }, + states: [snapshot.state], + elements: snapshot.elements, + edges: [], + }; + await maybeWriteGraph(graph, options.out); + await context.close(); + return graph; + } finally { + await browser.close(); + } +} + +export async function exploreUrl(options: EngineOptions): Promise { + const browser = await chromium.launch({ headless: !options.headed }); + try { + const context = await newContext(browser, options); + const entryUrl = toNavigableUrl(options.url); + const page = await context.newPage(); + await page.goto(entryUrl, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle").catch(() => undefined); + + const entrySnapshot = await snapshotPage(page); + const graph: TopologyGraph = { + schemaVersion: "0.1", + metadata: { + entryUrl: page.url(), + generatedAt: new Date().toISOString(), + engineVersion: "0.1.0", + mode: "explore", + }, + states: [entrySnapshot.state], + elements: [...entrySnapshot.elements], + edges: [], + }; + + const candidates = entrySnapshot.elements.filter(isLowRiskClickable).slice(0, options.maxActions ?? 10); + await page.close(); + + for (const element of candidates) { + const actionPage = await context.newPage(); + const networkEvents: EffectRecord[] = []; + actionPage.on("response", (response) => { + const record = responseToEffect(response); + if (record) networkEvents.push(record); + }); + + await actionPage.goto(entryUrl, { waitUntil: "domcontentloaded" }); + await actionPage.waitForLoadState("networkidle").catch(() => undefined); + const beforeUrl = actionPage.url(); + const beforeSnapshot = await snapshotPage(actionPage); + const locator = element.locators.find((candidate) => candidate.verified) ?? element.locators[0]; + + let toStateId: string | null = null; + const effects: EffectRecord[] = []; + if (locator) { + try { + await resolveLocator(actionPage, locator.descriptor).first().click({ timeout: 2000 }); + await actionPage.waitForLoadState("networkidle", { timeout: 2000 }).catch(() => undefined); + const afterSnapshot = await snapshotPage(actionPage); + graph.states = upsertById(graph.states, afterSnapshot.state, "stateId"); + graph.elements.push(...afterSnapshot.elements); + toStateId = afterSnapshot.state.stateId; + effects.push(...diffEffects(beforeUrl, beforeSnapshot.visibleText, actionPage.url(), afterSnapshot.visibleText)); + } catch (error) { + effects.push({ + type: "dom", + description: `Action failed: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + + const edge: ActionEdge = { + edgeId: `edge_${shortHash([entrySnapshot.state.stateId, element.elementId, "click"])}`, + fromStateId: entrySnapshot.state.stateId, + toStateId, + elementId: element.elementId, + actionType: "click", + actionParams: {}, + preconditions: [], + effects: [...effects, ...networkEvents], + parameterSpecs: [], + riskLevel: classifyClickRisk(element), + confidence: locator?.verified ? 0.82 : 0.55, + createdAt: new Date().toISOString(), + }; + graph.edges = upsertById(graph.edges, edge, "edgeId"); + await actionPage.close(); + } + + await maybeWriteGraph(graph, options.out); + await context.close(); + return graph; + } finally { + await browser.close(); + } +} + +async function newContext(browser: Browser, options: EngineOptions): Promise { + return browser.newContext({ + storageState: options.storageState, + viewport: { width: 1440, height: 1000 }, + }); +} + +async function maybeWriteGraph(graph: TopologyGraph, out?: string): Promise { + if (!out) { + console.log(JSON.stringify(graph, null, 2)); + return; + } + await mkdir(path.dirname(out), { recursive: true }); + await writeFile(out, `${JSON.stringify(graph, null, 2)}\n`, "utf8"); +} + +function toNavigableUrl(value: string): string { + if (/^https?:\/\//.test(value) || value.startsWith("file://")) return value; + return pathToFileURL(path.resolve(value)).toString(); +} + +function diffEffects(beforeUrl: string, beforeText: string, afterUrl: string, afterText: string): EffectRecord[] { + const effects: EffectRecord[] = []; + if (beforeUrl !== afterUrl) { + effects.push({ type: "url", before: beforeUrl, after: afterUrl, description: "URL changed after action" }); + } + if (shortHash(beforeText) !== shortHash(afterText)) { + effects.push({ + type: "visibleText", + before: shortHash(beforeText), + after: shortHash(afterText), + description: "Visible text changed after action", + }); + } + return effects; +} + +function responseToEffect(response: Response): EffectRecord | null { + const request = response.request(); + const resourceType = request.resourceType(); + if (!["fetch", "xhr", "document"].includes(resourceType)) return null; + return { + type: "network", + method: request.method(), + url: response.url(), + status: response.status(), + description: `${request.method()} ${response.url()} -> ${response.status()}`, + }; +} + +function upsertById, K extends keyof T>(items: T[], item: T, key: K): T[] { + const index = items.findIndex((existing) => existing[key] === item[key]); + if (index === -1) return [...items, item]; + const next = [...items]; + next[index] = item; + return next; +} + diff --git a/src/hash.ts b/src/hash.ts new file mode 100644 index 0000000..6fc9455 --- /dev/null +++ b/src/hash.ts @@ -0,0 +1,27 @@ +import { createHash } from "node:crypto"; + +export function stableHash(value: unknown): string { + const normalized = typeof value === "string" ? value : stableStringify(value); + return createHash("sha256").update(normalized).digest("hex"); +} + +export function shortHash(value: unknown, length = 12): string { + return stableHash(value).slice(0, length); +} + +function stableStringify(value: unknown): string { + return JSON.stringify(sortValue(value)); +} + +function sortValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortValue); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, item]) => [key, sortValue(item)]), + ); + } + return value; +} + diff --git a/src/locator.ts b/src/locator.ts new file mode 100644 index 0000000..4808d0c --- /dev/null +++ b/src/locator.ts @@ -0,0 +1,170 @@ +import type { Locator, Page } from "playwright"; +import { shortHash } from "./hash.js"; +import type { LocatorCandidate, LocatorDescriptor, RawActionableElement } from "./types.js"; + +export function generateLocators(elementId: string, raw: RawActionableElement): LocatorCandidate[] { + const candidates: Array> = []; + const name = clean(raw.accessibleName); + const label = clean(raw.label); + const placeholder = clean(raw.placeholder); + const text = clean(raw.text); + const testId = raw.attributes["data-testid"] || raw.attributes["data-test-id"] || raw.attributes["data-test"]; + const id = raw.attributes.id; + const nameAttr = raw.attributes.name; + + if (raw.role && name) { + candidates.push({ + framework: "playwright", + strategy: "role", + descriptor: { kind: "role", role: raw.role, name }, + expression: `page.getByRole(${JSON.stringify(raw.role)}, { name: ${JSON.stringify(name)} })`, + score: 0.95, + reason: "role plus accessible name", + }); + } + + if (testId) { + candidates.push({ + framework: "playwright", + strategy: "testId", + descriptor: { kind: "testId", testId }, + expression: `page.getByTestId(${JSON.stringify(testId)})`, + score: 0.92, + reason: "explicit test id", + }); + } + + if (label) { + candidates.push({ + framework: "playwright", + strategy: "label", + descriptor: { kind: "label", label }, + expression: `page.getByLabel(${JSON.stringify(label)})`, + score: 0.9, + reason: "associated label", + }); + } + + if (placeholder) { + candidates.push({ + framework: "playwright", + strategy: "placeholder", + descriptor: { kind: "placeholder", placeholder }, + expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`, + score: 0.86, + reason: "placeholder text", + }); + } + + if (text && text.length <= 80 && ["a", "button"].includes(raw.tag)) { + candidates.push({ + framework: "playwright", + strategy: "text", + descriptor: { kind: "text", text, exact: true }, + expression: `page.getByText(${JSON.stringify(text)}, { exact: true })`, + score: 0.75, + reason: "visible text fallback", + }); + } + + if (id && /^[A-Za-z][A-Za-z0-9_-]*$/.test(id)) { + candidates.push({ + framework: "css", + strategy: "css", + descriptor: { kind: "css", selector: `#${cssEscape(id)}` }, + expression: `page.locator(${JSON.stringify(`#${cssEscape(id)}`)})`, + score: 0.72, + reason: "id selector fallback", + }); + } + + if (nameAttr) { + candidates.push({ + framework: "css", + strategy: "css", + descriptor: { kind: "css", selector: `${raw.tag}[name="${cssString(nameAttr)}"]` }, + expression: `page.locator(${JSON.stringify(`${raw.tag}[name="${cssString(nameAttr)}"]`)})`, + score: 0.68, + reason: "name attribute fallback", + }); + } + + return dedupeCandidates(candidates).map((candidate) => ({ + ...candidate, + locatorId: `loc_${shortHash([elementId, candidate.descriptor])}`, + elementId, + verified: false, + failureCount: 0, + })); +} + +export async function verifyLocators(page: Page, locators: LocatorCandidate[]): Promise { + const verifiedAt = new Date().toISOString(); + const verified: LocatorCandidate[] = []; + + for (const locator of locators) { + try { + const matchCount = await resolveLocator(page, locator.descriptor).count(); + verified.push({ + ...locator, + verified: matchCount === 1, + matchCount, + lastVerifiedAt: verifiedAt, + score: matchCount === 1 ? locator.score : Math.max(0.1, locator.score - 0.35), + }); + } catch { + verified.push({ + ...locator, + verified: false, + matchCount: 0, + lastVerifiedAt: verifiedAt, + score: Math.max(0.1, locator.score - 0.45), + failureCount: locator.failureCount + 1, + }); + } + } + + return verified.sort((a, b) => b.score - a.score); +} + +export function resolveLocator(page: Page, descriptor: LocatorDescriptor): Locator { + switch (descriptor.kind) { + case "role": + return page.getByRole(descriptor.role as never, descriptor.name ? { name: descriptor.name } : undefined); + case "label": + return page.getByLabel(descriptor.label); + case "placeholder": + return page.getByPlaceholder(descriptor.placeholder); + case "text": + return page.getByText(descriptor.text, { exact: descriptor.exact }); + case "testId": + return page.getByTestId(descriptor.testId); + case "css": + return page.locator(descriptor.selector); + } +} + +function dedupeCandidates(candidates: T[]): T[] { + const seen = new Set(); + const deduped: T[] = []; + for (const candidate of candidates) { + if (seen.has(candidate.expression)) continue; + seen.add(candidate.expression); + deduped.push(candidate); + } + return deduped; +} + +function clean(value: string | null | undefined): string | undefined { + const cleaned = value?.replace(/\s+/g, " ").trim(); + return cleaned || undefined; +} + +function cssEscape(value: string): string { + return value.replace(/([!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, "\\$1"); +} + +function cssString(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); +} + diff --git a/src/risk.ts b/src/risk.ts new file mode 100644 index 0000000..d411c34 --- /dev/null +++ b/src/risk.ts @@ -0,0 +1,51 @@ +import type { ActionableElement } from "./types.js"; + +const HIGH_RISK_TERMS = [ + "delete", + "remove", + "destroy", + "drop", + "logout", + "sign out", + "pay", + "purchase", + "checkout", + "submit", + "confirm", + "authorize", + "删除", + "移除", + "注销", + "退出", + "支付", + "购买", + "提交", + "确认", + "授权", +]; + +export function classifyClickRisk(element: ActionableElement): "low" | "medium" | "high" { + const joined = [ + element.accessibleName, + element.text, + element.label, + element.placeholder, + element.attributes.href, + element.attributes.type, + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + + if (HIGH_RISK_TERMS.some((term) => joined.includes(term.toLowerCase()))) return "high"; + if (element.tag === "input" || element.tag === "textarea" || element.tag === "select") return "medium"; + return "low"; +} + +export function isLowRiskClickable(element: ActionableElement): boolean { + if (!element.interactability.visible || !element.interactability.enabled || !element.interactability.receivesEvents) return false; + if (classifyClickRisk(element) !== "low") return false; + const role = element.role; + return element.tag === "button" || element.tag === "a" || role === "button" || role === "link" || role === "tab" || role === "menuitem"; +} + diff --git a/src/snapshot.ts b/src/snapshot.ts new file mode 100644 index 0000000..c2d6c30 --- /dev/null +++ b/src/snapshot.ts @@ -0,0 +1,251 @@ +import type { Page } from "playwright"; +import { shortHash, stableHash } from "./hash.js"; +import { generateLocators, verifyLocators } from "./locator.js"; +import type { ActionableElement, RawActionableElement, SnapshotResult, StateNode } from "./types.js"; + +export async function snapshotPage(page: Page, openedByEdgeId: string | null = null): Promise { + const raw = await page.evaluate(collectPageSnapshot); + const createdAt = new Date().toISOString(); + const stateHash = stableHash({ + url: raw.url, + title: raw.title, + domHash: stableHash(raw.dom), + visibleTextHash: stableHash(raw.visibleText), + actionableSignatureHash: stableHash(raw.elements.map(actionableSignature)), + }); + + const state: StateNode = { + stateId: `state_${shortHash(stateHash)}`, + url: raw.url, + route: raw.route, + title: raw.title, + frameContext: [], + stateHash, + domHash: stableHash(raw.dom), + visibleTextHash: stableHash(raw.visibleText), + actionableSignatureHash: stableHash(raw.elements.map(actionableSignature)), + modalStack: raw.modalStack, + openedByEdgeId, + createdAt, + }; + + const elements: ActionableElement[] = []; + for (const rawElement of raw.elements) { + const elementId = `el_${shortHash([state.stateId, rawElement.uid, actionableSignature(rawElement)])}`; + const locators = await verifyLocators(page, generateLocators(elementId, rawElement)); + elements.push({ + elementId, + stateId: state.stateId, + tag: rawElement.tag, + role: rawElement.role, + accessibleName: rawElement.accessibleName, + text: rawElement.text, + label: rawElement.label, + placeholder: rawElement.placeholder, + attributes: rawElement.attributes, + bbox: rawElement.bbox, + framePath: [], + shadowPath: [], + interactability: rawElement.visibility, + locators, + }); + } + + return { state, elements, visibleText: raw.visibleText }; +} + +interface BrowserSnapshot { + url: string; + route: string; + title: string; + dom: string; + visibleText: string; + modalStack: string[]; + elements: RawActionableElement[]; +} + +function actionableSignature(element: RawActionableElement): unknown { + return { + tag: element.tag, + role: element.role, + accessibleName: element.accessibleName, + text: element.text, + label: element.label, + placeholder: element.placeholder, + attributes: element.attributes, + }; +} + +function collectPageSnapshot(): BrowserSnapshot { + const actionableSelector = [ + "a[href]", + "button", + "input", + "select", + "textarea", + "[role]", + "[onclick]", + "[aria-haspopup]", + "[aria-expanded]", + "[contenteditable='true']", + "[tabindex]:not([tabindex='-1'])", + ].join(","); + + const elements = Array.from(document.querySelectorAll(actionableSelector)) + .filter((element) => isCandidate(element)) + .slice(0, 500) + .map((element, index) => toRawElement(element, index)); + + return { + url: location.href, + route: `${location.pathname}${location.search}`, + title: document.title, + dom: document.documentElement.outerHTML, + visibleText: normalizeText(document.body?.innerText || ""), + modalStack: Array.from(document.querySelectorAll("[role='dialog'], dialog[open], [aria-modal='true']")) + .filter((element) => isVisible(element)) + .map((element) => getAccessibleName(element) || element.id || element.tagName.toLowerCase()), + elements, + }; + + function isCandidate(element: HTMLElement): boolean { + const tag = element.tagName.toLowerCase(); + if (tag === "input" && (element as HTMLInputElement).type === "hidden") return false; + if (!isVisible(element)) return false; + const box = element.getBoundingClientRect(); + return box.width > 0 && box.height > 0; + } + + function toRawElement(element: HTMLElement, index: number): RawActionableElement { + const rect = element.getBoundingClientRect(); + const tag = element.tagName.toLowerCase(); + const attributes = collectAttributes(element); + const text = normalizeText(element.innerText || element.textContent || ""); + const role = attributes.role || inferRole(element); + const label = getLabel(element); + const placeholder = "placeholder" in element ? normalizeNullable(String((element as HTMLInputElement).placeholder || "")) : null; + const accessibleName = normalizeNullable(getAccessibleName(element) || label || placeholder || text); + const centerX = rect.left + rect.width / 2; + const centerY = rect.top + rect.height / 2; + const topAtCenter = document.elementFromPoint(centerX, centerY); + + return { + uid: `${tag}_${index}_${attributes.id || attributes.name || accessibleName || text || ""}`, + tag, + role, + accessibleName, + text: normalizeNullable(text), + label, + placeholder, + attributes, + bbox: { + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + }, + visibility: { + visible: isVisible(element), + enabled: !("disabled" in element && Boolean((element as HTMLButtonElement).disabled)), + editable: isEditable(element), + receivesEvents: topAtCenter === element || Boolean(topAtCenter && element.contains(topAtCenter)), + }, + }; + } + + function collectAttributes(element: HTMLElement): Record { + const keep = new Set([ + "id", + "name", + "type", + "href", + "value", + "role", + "aria-label", + "aria-labelledby", + "aria-expanded", + "aria-haspopup", + "aria-controls", + "aria-disabled", + "data-testid", + "data-test-id", + "data-test", + ]); + const attrs: Record = {}; + for (const attr of Array.from(element.attributes)) { + if (keep.has(attr.name)) attrs[attr.name] = attr.value; + } + return attrs; + } + + function inferRole(element: HTMLElement): string | null { + const tag = element.tagName.toLowerCase(); + if (tag === "button") return "button"; + if (tag === "a" && element.hasAttribute("href")) return "link"; + if (tag === "select") return "combobox"; + if (tag === "textarea") return "textbox"; + if (tag === "input") { + const type = ((element as HTMLInputElement).type || "text").toLowerCase(); + if (["button", "submit", "reset"].includes(type)) return "button"; + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + return "textbox"; + } + return null; + } + + function getAccessibleName(element: HTMLElement): string | null { + const ariaLabel = element.getAttribute("aria-label"); + if (ariaLabel) return normalizeNullable(ariaLabel); + const labelledBy = element.getAttribute("aria-labelledby"); + if (labelledBy) { + const text = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.innerText || document.getElementById(id)?.textContent || "") + .join(" "); + if (text.trim()) return normalizeNullable(text); + } + const title = element.getAttribute("title"); + if (title) return normalizeNullable(title); + const alt = element.getAttribute("alt"); + if (alt) return normalizeNullable(alt); + return normalizeNullable(element.innerText || element.textContent || ""); + } + + function getLabel(element: HTMLElement): string | null { + const id = element.getAttribute("id"); + if (id) { + const explicit = document.querySelector(`label[for="${CSS.escape(id)}"]`); + if (explicit) return normalizeNullable(explicit.innerText || explicit.textContent || ""); + } + const implicit = element.closest("label"); + if (implicit) return normalizeNullable(implicit.innerText || implicit.textContent || ""); + return null; + } + + function isVisible(element: HTMLElement): boolean { + const style = getComputedStyle(element); + if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity) === 0) return false; + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + } + + function isEditable(element: HTMLElement): boolean { + if (element.isContentEditable) return true; + const tag = element.tagName.toLowerCase(); + if (tag === "textarea") return true; + if (tag !== "input") return false; + const type = ((element as HTMLInputElement).type || "text").toLowerCase(); + return !["button", "submit", "reset", "checkbox", "radio", "hidden"].includes(type); + } + + function normalizeText(value: string): string { + return value.replace(/\s+/g, " ").trim(); + } + + function normalizeNullable(value: string): string | null { + const normalized = normalizeText(value); + return normalized || null; + } +} + diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..8a35245 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,139 @@ +export type LocatorDescriptor = + | { kind: "role"; role: string; name?: string } + | { kind: "label"; label: string } + | { kind: "placeholder"; placeholder: string } + | { kind: "text"; text: string; exact: boolean } + | { kind: "testId"; testId: string } + | { kind: "css"; selector: string }; + +export type LocatorFramework = "playwright" | "css"; + +export interface LocatorCandidate { + locatorId: string; + elementId: string; + framework: LocatorFramework; + strategy: LocatorDescriptor["kind"]; + descriptor: LocatorDescriptor; + expression: string; + score: number; + reason: string; + verified: boolean; + lastVerifiedAt?: string; + matchCount?: number; + failureCount: number; +} + +export interface VisibilityInfo { + visible: boolean; + enabled: boolean; + editable: boolean; + receivesEvents: boolean; +} + +export interface BoundingBox { + x: number; + y: number; + width: number; + height: number; +} + +export interface ActionableElement { + elementId: string; + stateId: string; + tag: string; + role: string | null; + accessibleName: string | null; + text: string | null; + label: string | null; + placeholder: string | null; + attributes: Record; + bbox: BoundingBox | null; + framePath: string[]; + shadowPath: string[]; + interactability: VisibilityInfo; + locators: LocatorCandidate[]; +} + +export interface StateNode { + stateId: string; + url: string; + route: string; + title: string; + frameContext: string[]; + stateHash: string; + domHash: string; + visibleTextHash: string; + actionableSignatureHash: string; + modalStack: string[]; + openedByEdgeId: string | null; + createdAt: string; +} + +export type ActionType = "click" | "fill" | "select" | "check" | "uncheck" | "navigate"; + +export interface EffectRecord { + type: "url" | "dom" | "visibleText" | "network" | "download" | "dialog" | "storage"; + before?: string; + after?: string; + method?: string; + url?: string; + status?: number; + description: string; +} + +export interface ParameterSpec { + name: string; + type: "string" | "number" | "boolean" | "enum" | "date" | "unknown"; + required: boolean; + enumValues?: string[]; + pattern?: string; + examples?: string[]; +} + +export interface ActionEdge { + edgeId: string; + fromStateId: string; + toStateId: string | null; + elementId: string; + actionType: ActionType; + actionParams: Record; + preconditions: string[]; + effects: EffectRecord[]; + parameterSpecs: ParameterSpec[]; + riskLevel: "low" | "medium" | "high"; + confidence: number; + createdAt: string; +} + +export interface TopologyGraph { + schemaVersion: "0.1"; + metadata: { + entryUrl: string; + generatedAt: string; + engineVersion: string; + mode: "scan" | "explore"; + }; + states: StateNode[]; + elements: ActionableElement[]; + edges: ActionEdge[]; +} + +export interface RawActionableElement { + uid: string; + tag: string; + role: string | null; + accessibleName: string | null; + text: string | null; + label: string | null; + placeholder: string | null; + attributes: Record; + bbox: BoundingBox | null; + visibility: VisibilityInfo; +} + +export interface SnapshotResult { + state: StateNode; + elements: ActionableElement[]; + visibleText: string; +} + diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..7ea6721 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*.ts"] +} +