Initialize action topology engine
This commit is contained in:
赵义仑
66
src/cli.ts
Normal file
66
src/cli.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env node
|
||||
import { exploreUrl, scanUrl, type EngineOptions } from "./engine.js";
|
||||
|
||||
interface ParsedArgs extends EngineOptions {
|
||||
command: "scan" | "explore";
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const graph = args.command === "scan" ? await scanUrl(args) : await exploreUrl(args);
|
||||
if (args.out) {
|
||||
console.error(`Wrote ${graph.states.length} state(s), ${graph.elements.length} element(s), ${graph.edges.length} edge(s) to ${args.out}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ParsedArgs {
|
||||
const command = argv.shift();
|
||||
if (command !== "scan" && command !== "explore") usage();
|
||||
|
||||
const options: Partial<ParsedArgs> = { command };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
switch (arg) {
|
||||
case "--url":
|
||||
options.url = requireValue(argv, ++index, "--url");
|
||||
break;
|
||||
case "--out":
|
||||
options.out = requireValue(argv, ++index, "--out");
|
||||
break;
|
||||
case "--storage-state":
|
||||
options.storageState = requireValue(argv, ++index, "--storage-state");
|
||||
break;
|
||||
case "--max-actions":
|
||||
options.maxActions = Number(requireValue(argv, ++index, "--max-actions"));
|
||||
break;
|
||||
case "--headed":
|
||||
options.headed = true;
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.url) usage();
|
||||
return options as ParsedArgs;
|
||||
}
|
||||
|
||||
function requireValue(argv: string[], index: number, flag: string): string {
|
||||
const value = argv[index];
|
||||
if (!value) throw new Error(`Missing value for ${flag}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function usage(): never {
|
||||
console.error(`Usage:
|
||||
action-topology scan --url <url-or-file> [--out graph.json] [--headed] [--storage-state state.json]
|
||||
action-topology explore --url <url-or-file> [--out graph.json] [--max-actions 10] [--headed] [--storage-state state.json]
|
||||
`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack || error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
189
src/engine.ts
Normal file
189
src/engine.ts
Normal file
@@ -0,0 +1,189 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { chromium, type Browser, type BrowserContext, type Page, type Response } from "playwright";
|
||||
import { shortHash } from "./hash.js";
|
||||
import { resolveLocator } from "./locator.js";
|
||||
import { isLowRiskClickable, classifyClickRisk } from "./risk.js";
|
||||
import { snapshotPage } from "./snapshot.js";
|
||||
import type { ActionEdge, EffectRecord, TopologyGraph } from "./types.js";
|
||||
|
||||
export interface EngineOptions {
|
||||
url: string;
|
||||
out?: string;
|
||||
headed?: boolean;
|
||||
storageState?: string;
|
||||
maxActions?: number;
|
||||
}
|
||||
|
||||
export async function scanUrl(options: EngineOptions): Promise<TopologyGraph> {
|
||||
const browser = await chromium.launch({ headless: !options.headed });
|
||||
try {
|
||||
const context = await newContext(browser, options);
|
||||
const page = await context.newPage();
|
||||
await page.goto(toNavigableUrl(options.url), { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle").catch(() => undefined);
|
||||
const snapshot = await snapshotPage(page);
|
||||
const graph: TopologyGraph = {
|
||||
schemaVersion: "0.1",
|
||||
metadata: {
|
||||
entryUrl: page.url(),
|
||||
generatedAt: new Date().toISOString(),
|
||||
engineVersion: "0.1.0",
|
||||
mode: "scan",
|
||||
},
|
||||
states: [snapshot.state],
|
||||
elements: snapshot.elements,
|
||||
edges: [],
|
||||
};
|
||||
await maybeWriteGraph(graph, options.out);
|
||||
await context.close();
|
||||
return graph;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function exploreUrl(options: EngineOptions): Promise<TopologyGraph> {
|
||||
const browser = await chromium.launch({ headless: !options.headed });
|
||||
try {
|
||||
const context = await newContext(browser, options);
|
||||
const entryUrl = toNavigableUrl(options.url);
|
||||
const page = await context.newPage();
|
||||
await page.goto(entryUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle").catch(() => undefined);
|
||||
|
||||
const entrySnapshot = await snapshotPage(page);
|
||||
const graph: TopologyGraph = {
|
||||
schemaVersion: "0.1",
|
||||
metadata: {
|
||||
entryUrl: page.url(),
|
||||
generatedAt: new Date().toISOString(),
|
||||
engineVersion: "0.1.0",
|
||||
mode: "explore",
|
||||
},
|
||||
states: [entrySnapshot.state],
|
||||
elements: [...entrySnapshot.elements],
|
||||
edges: [],
|
||||
};
|
||||
|
||||
const candidates = entrySnapshot.elements.filter(isLowRiskClickable).slice(0, options.maxActions ?? 10);
|
||||
await page.close();
|
||||
|
||||
for (const element of candidates) {
|
||||
const actionPage = await context.newPage();
|
||||
const networkEvents: EffectRecord[] = [];
|
||||
actionPage.on("response", (response) => {
|
||||
const record = responseToEffect(response);
|
||||
if (record) networkEvents.push(record);
|
||||
});
|
||||
|
||||
await actionPage.goto(entryUrl, { waitUntil: "domcontentloaded" });
|
||||
await actionPage.waitForLoadState("networkidle").catch(() => undefined);
|
||||
const beforeUrl = actionPage.url();
|
||||
const beforeSnapshot = await snapshotPage(actionPage);
|
||||
const locator = element.locators.find((candidate) => candidate.verified) ?? element.locators[0];
|
||||
|
||||
let toStateId: string | null = null;
|
||||
const effects: EffectRecord[] = [];
|
||||
if (locator) {
|
||||
try {
|
||||
await resolveLocator(actionPage, locator.descriptor).first().click({ timeout: 2000 });
|
||||
await actionPage.waitForLoadState("networkidle", { timeout: 2000 }).catch(() => undefined);
|
||||
const afterSnapshot = await snapshotPage(actionPage);
|
||||
graph.states = upsertById(graph.states, afterSnapshot.state, "stateId");
|
||||
graph.elements.push(...afterSnapshot.elements);
|
||||
toStateId = afterSnapshot.state.stateId;
|
||||
effects.push(...diffEffects(beforeUrl, beforeSnapshot.visibleText, actionPage.url(), afterSnapshot.visibleText));
|
||||
} catch (error) {
|
||||
effects.push({
|
||||
type: "dom",
|
||||
description: `Action failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const edge: ActionEdge = {
|
||||
edgeId: `edge_${shortHash([entrySnapshot.state.stateId, element.elementId, "click"])}`,
|
||||
fromStateId: entrySnapshot.state.stateId,
|
||||
toStateId,
|
||||
elementId: element.elementId,
|
||||
actionType: "click",
|
||||
actionParams: {},
|
||||
preconditions: [],
|
||||
effects: [...effects, ...networkEvents],
|
||||
parameterSpecs: [],
|
||||
riskLevel: classifyClickRisk(element),
|
||||
confidence: locator?.verified ? 0.82 : 0.55,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
graph.edges = upsertById(graph.edges, edge, "edgeId");
|
||||
await actionPage.close();
|
||||
}
|
||||
|
||||
await maybeWriteGraph(graph, options.out);
|
||||
await context.close();
|
||||
return graph;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function newContext(browser: Browser, options: EngineOptions): Promise<BrowserContext> {
|
||||
return browser.newContext({
|
||||
storageState: options.storageState,
|
||||
viewport: { width: 1440, height: 1000 },
|
||||
});
|
||||
}
|
||||
|
||||
async function maybeWriteGraph(graph: TopologyGraph, out?: string): Promise<void> {
|
||||
if (!out) {
|
||||
console.log(JSON.stringify(graph, null, 2));
|
||||
return;
|
||||
}
|
||||
await mkdir(path.dirname(out), { recursive: true });
|
||||
await writeFile(out, `${JSON.stringify(graph, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function toNavigableUrl(value: string): string {
|
||||
if (/^https?:\/\//.test(value) || value.startsWith("file://")) return value;
|
||||
return pathToFileURL(path.resolve(value)).toString();
|
||||
}
|
||||
|
||||
function diffEffects(beforeUrl: string, beforeText: string, afterUrl: string, afterText: string): EffectRecord[] {
|
||||
const effects: EffectRecord[] = [];
|
||||
if (beforeUrl !== afterUrl) {
|
||||
effects.push({ type: "url", before: beforeUrl, after: afterUrl, description: "URL changed after action" });
|
||||
}
|
||||
if (shortHash(beforeText) !== shortHash(afterText)) {
|
||||
effects.push({
|
||||
type: "visibleText",
|
||||
before: shortHash(beforeText),
|
||||
after: shortHash(afterText),
|
||||
description: "Visible text changed after action",
|
||||
});
|
||||
}
|
||||
return effects;
|
||||
}
|
||||
|
||||
function responseToEffect(response: Response): EffectRecord | null {
|
||||
const request = response.request();
|
||||
const resourceType = request.resourceType();
|
||||
if (!["fetch", "xhr", "document"].includes(resourceType)) return null;
|
||||
return {
|
||||
type: "network",
|
||||
method: request.method(),
|
||||
url: response.url(),
|
||||
status: response.status(),
|
||||
description: `${request.method()} ${response.url()} -> ${response.status()}`,
|
||||
};
|
||||
}
|
||||
|
||||
function upsertById<T extends Record<K, string>, K extends keyof T>(items: T[], item: T, key: K): T[] {
|
||||
const index = items.findIndex((existing) => existing[key] === item[key]);
|
||||
if (index === -1) return [...items, item];
|
||||
const next = [...items];
|
||||
next[index] = item;
|
||||
return next;
|
||||
}
|
||||
|
||||
27
src/hash.ts
Normal file
27
src/hash.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export function stableHash(value: unknown): string {
|
||||
const normalized = typeof value === "string" ? value : stableStringify(value);
|
||||
return createHash("sha256").update(normalized).digest("hex");
|
||||
}
|
||||
|
||||
export function shortHash(value: unknown, length = 12): string {
|
||||
return stableHash(value).slice(0, length);
|
||||
}
|
||||
|
||||
function stableStringify(value: unknown): string {
|
||||
return JSON.stringify(sortValue(value));
|
||||
}
|
||||
|
||||
function sortValue(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(sortValue);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, item]) => [key, sortValue(item)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
170
src/locator.ts
Normal file
170
src/locator.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
import type { Locator, Page } from "playwright";
|
||||
import { shortHash } from "./hash.js";
|
||||
import type { LocatorCandidate, LocatorDescriptor, RawActionableElement } from "./types.js";
|
||||
|
||||
export function generateLocators(elementId: string, raw: RawActionableElement): LocatorCandidate[] {
|
||||
const candidates: Array<Omit<LocatorCandidate, "locatorId" | "elementId" | "verified" | "failureCount">> = [];
|
||||
const name = clean(raw.accessibleName);
|
||||
const label = clean(raw.label);
|
||||
const placeholder = clean(raw.placeholder);
|
||||
const text = clean(raw.text);
|
||||
const testId = raw.attributes["data-testid"] || raw.attributes["data-test-id"] || raw.attributes["data-test"];
|
||||
const id = raw.attributes.id;
|
||||
const nameAttr = raw.attributes.name;
|
||||
|
||||
if (raw.role && name) {
|
||||
candidates.push({
|
||||
framework: "playwright",
|
||||
strategy: "role",
|
||||
descriptor: { kind: "role", role: raw.role, name },
|
||||
expression: `page.getByRole(${JSON.stringify(raw.role)}, { name: ${JSON.stringify(name)} })`,
|
||||
score: 0.95,
|
||||
reason: "role plus accessible name",
|
||||
});
|
||||
}
|
||||
|
||||
if (testId) {
|
||||
candidates.push({
|
||||
framework: "playwright",
|
||||
strategy: "testId",
|
||||
descriptor: { kind: "testId", testId },
|
||||
expression: `page.getByTestId(${JSON.stringify(testId)})`,
|
||||
score: 0.92,
|
||||
reason: "explicit test id",
|
||||
});
|
||||
}
|
||||
|
||||
if (label) {
|
||||
candidates.push({
|
||||
framework: "playwright",
|
||||
strategy: "label",
|
||||
descriptor: { kind: "label", label },
|
||||
expression: `page.getByLabel(${JSON.stringify(label)})`,
|
||||
score: 0.9,
|
||||
reason: "associated label",
|
||||
});
|
||||
}
|
||||
|
||||
if (placeholder) {
|
||||
candidates.push({
|
||||
framework: "playwright",
|
||||
strategy: "placeholder",
|
||||
descriptor: { kind: "placeholder", placeholder },
|
||||
expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
|
||||
score: 0.86,
|
||||
reason: "placeholder text",
|
||||
});
|
||||
}
|
||||
|
||||
if (text && text.length <= 80 && ["a", "button"].includes(raw.tag)) {
|
||||
candidates.push({
|
||||
framework: "playwright",
|
||||
strategy: "text",
|
||||
descriptor: { kind: "text", text, exact: true },
|
||||
expression: `page.getByText(${JSON.stringify(text)}, { exact: true })`,
|
||||
score: 0.75,
|
||||
reason: "visible text fallback",
|
||||
});
|
||||
}
|
||||
|
||||
if (id && /^[A-Za-z][A-Za-z0-9_-]*$/.test(id)) {
|
||||
candidates.push({
|
||||
framework: "css",
|
||||
strategy: "css",
|
||||
descriptor: { kind: "css", selector: `#${cssEscape(id)}` },
|
||||
expression: `page.locator(${JSON.stringify(`#${cssEscape(id)}`)})`,
|
||||
score: 0.72,
|
||||
reason: "id selector fallback",
|
||||
});
|
||||
}
|
||||
|
||||
if (nameAttr) {
|
||||
candidates.push({
|
||||
framework: "css",
|
||||
strategy: "css",
|
||||
descriptor: { kind: "css", selector: `${raw.tag}[name="${cssString(nameAttr)}"]` },
|
||||
expression: `page.locator(${JSON.stringify(`${raw.tag}[name="${cssString(nameAttr)}"]`)})`,
|
||||
score: 0.68,
|
||||
reason: "name attribute fallback",
|
||||
});
|
||||
}
|
||||
|
||||
return dedupeCandidates(candidates).map((candidate) => ({
|
||||
...candidate,
|
||||
locatorId: `loc_${shortHash([elementId, candidate.descriptor])}`,
|
||||
elementId,
|
||||
verified: false,
|
||||
failureCount: 0,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function verifyLocators(page: Page, locators: LocatorCandidate[]): Promise<LocatorCandidate[]> {
|
||||
const verifiedAt = new Date().toISOString();
|
||||
const verified: LocatorCandidate[] = [];
|
||||
|
||||
for (const locator of locators) {
|
||||
try {
|
||||
const matchCount = await resolveLocator(page, locator.descriptor).count();
|
||||
verified.push({
|
||||
...locator,
|
||||
verified: matchCount === 1,
|
||||
matchCount,
|
||||
lastVerifiedAt: verifiedAt,
|
||||
score: matchCount === 1 ? locator.score : Math.max(0.1, locator.score - 0.35),
|
||||
});
|
||||
} catch {
|
||||
verified.push({
|
||||
...locator,
|
||||
verified: false,
|
||||
matchCount: 0,
|
||||
lastVerifiedAt: verifiedAt,
|
||||
score: Math.max(0.1, locator.score - 0.45),
|
||||
failureCount: locator.failureCount + 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return verified.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
|
||||
export function resolveLocator(page: Page, descriptor: LocatorDescriptor): Locator {
|
||||
switch (descriptor.kind) {
|
||||
case "role":
|
||||
return page.getByRole(descriptor.role as never, descriptor.name ? { name: descriptor.name } : undefined);
|
||||
case "label":
|
||||
return page.getByLabel(descriptor.label);
|
||||
case "placeholder":
|
||||
return page.getByPlaceholder(descriptor.placeholder);
|
||||
case "text":
|
||||
return page.getByText(descriptor.text, { exact: descriptor.exact });
|
||||
case "testId":
|
||||
return page.getByTestId(descriptor.testId);
|
||||
case "css":
|
||||
return page.locator(descriptor.selector);
|
||||
}
|
||||
}
|
||||
|
||||
function dedupeCandidates<T extends { expression: string }>(candidates: T[]): T[] {
|
||||
const seen = new Set<string>();
|
||||
const deduped: T[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (seen.has(candidate.expression)) continue;
|
||||
seen.add(candidate.expression);
|
||||
deduped.push(candidate);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function clean(value: string | null | undefined): string | undefined {
|
||||
const cleaned = value?.replace(/\s+/g, " ").trim();
|
||||
return cleaned || undefined;
|
||||
}
|
||||
|
||||
function cssEscape(value: string): string {
|
||||
return value.replace(/([!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, "\\$1");
|
||||
}
|
||||
|
||||
function cssString(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
||||
}
|
||||
|
||||
51
src/risk.ts
Normal file
51
src/risk.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import type { ActionableElement } from "./types.js";
|
||||
|
||||
const HIGH_RISK_TERMS = [
|
||||
"delete",
|
||||
"remove",
|
||||
"destroy",
|
||||
"drop",
|
||||
"logout",
|
||||
"sign out",
|
||||
"pay",
|
||||
"purchase",
|
||||
"checkout",
|
||||
"submit",
|
||||
"confirm",
|
||||
"authorize",
|
||||
"删除",
|
||||
"移除",
|
||||
"注销",
|
||||
"退出",
|
||||
"支付",
|
||||
"购买",
|
||||
"提交",
|
||||
"确认",
|
||||
"授权",
|
||||
];
|
||||
|
||||
export function classifyClickRisk(element: ActionableElement): "low" | "medium" | "high" {
|
||||
const joined = [
|
||||
element.accessibleName,
|
||||
element.text,
|
||||
element.label,
|
||||
element.placeholder,
|
||||
element.attributes.href,
|
||||
element.attributes.type,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
|
||||
if (HIGH_RISK_TERMS.some((term) => joined.includes(term.toLowerCase()))) return "high";
|
||||
if (element.tag === "input" || element.tag === "textarea" || element.tag === "select") return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
export function isLowRiskClickable(element: ActionableElement): boolean {
|
||||
if (!element.interactability.visible || !element.interactability.enabled || !element.interactability.receivesEvents) return false;
|
||||
if (classifyClickRisk(element) !== "low") return false;
|
||||
const role = element.role;
|
||||
return element.tag === "button" || element.tag === "a" || role === "button" || role === "link" || role === "tab" || role === "menuitem";
|
||||
}
|
||||
|
||||
251
src/snapshot.ts
Normal file
251
src/snapshot.ts
Normal file
@@ -0,0 +1,251 @@
|
||||
import type { Page } from "playwright";
|
||||
import { shortHash, stableHash } from "./hash.js";
|
||||
import { generateLocators, verifyLocators } from "./locator.js";
|
||||
import type { ActionableElement, RawActionableElement, SnapshotResult, StateNode } from "./types.js";
|
||||
|
||||
export async function snapshotPage(page: Page, openedByEdgeId: string | null = null): Promise<SnapshotResult> {
|
||||
const raw = await page.evaluate(collectPageSnapshot);
|
||||
const createdAt = new Date().toISOString();
|
||||
const stateHash = stableHash({
|
||||
url: raw.url,
|
||||
title: raw.title,
|
||||
domHash: stableHash(raw.dom),
|
||||
visibleTextHash: stableHash(raw.visibleText),
|
||||
actionableSignatureHash: stableHash(raw.elements.map(actionableSignature)),
|
||||
});
|
||||
|
||||
const state: StateNode = {
|
||||
stateId: `state_${shortHash(stateHash)}`,
|
||||
url: raw.url,
|
||||
route: raw.route,
|
||||
title: raw.title,
|
||||
frameContext: [],
|
||||
stateHash,
|
||||
domHash: stableHash(raw.dom),
|
||||
visibleTextHash: stableHash(raw.visibleText),
|
||||
actionableSignatureHash: stableHash(raw.elements.map(actionableSignature)),
|
||||
modalStack: raw.modalStack,
|
||||
openedByEdgeId,
|
||||
createdAt,
|
||||
};
|
||||
|
||||
const elements: ActionableElement[] = [];
|
||||
for (const rawElement of raw.elements) {
|
||||
const elementId = `el_${shortHash([state.stateId, rawElement.uid, actionableSignature(rawElement)])}`;
|
||||
const locators = await verifyLocators(page, generateLocators(elementId, rawElement));
|
||||
elements.push({
|
||||
elementId,
|
||||
stateId: state.stateId,
|
||||
tag: rawElement.tag,
|
||||
role: rawElement.role,
|
||||
accessibleName: rawElement.accessibleName,
|
||||
text: rawElement.text,
|
||||
label: rawElement.label,
|
||||
placeholder: rawElement.placeholder,
|
||||
attributes: rawElement.attributes,
|
||||
bbox: rawElement.bbox,
|
||||
framePath: [],
|
||||
shadowPath: [],
|
||||
interactability: rawElement.visibility,
|
||||
locators,
|
||||
});
|
||||
}
|
||||
|
||||
return { state, elements, visibleText: raw.visibleText };
|
||||
}
|
||||
|
||||
interface BrowserSnapshot {
|
||||
url: string;
|
||||
route: string;
|
||||
title: string;
|
||||
dom: string;
|
||||
visibleText: string;
|
||||
modalStack: string[];
|
||||
elements: RawActionableElement[];
|
||||
}
|
||||
|
||||
function actionableSignature(element: RawActionableElement): unknown {
|
||||
return {
|
||||
tag: element.tag,
|
||||
role: element.role,
|
||||
accessibleName: element.accessibleName,
|
||||
text: element.text,
|
||||
label: element.label,
|
||||
placeholder: element.placeholder,
|
||||
attributes: element.attributes,
|
||||
};
|
||||
}
|
||||
|
||||
function collectPageSnapshot(): BrowserSnapshot {
|
||||
const actionableSelector = [
|
||||
"a[href]",
|
||||
"button",
|
||||
"input",
|
||||
"select",
|
||||
"textarea",
|
||||
"[role]",
|
||||
"[onclick]",
|
||||
"[aria-haspopup]",
|
||||
"[aria-expanded]",
|
||||
"[contenteditable='true']",
|
||||
"[tabindex]:not([tabindex='-1'])",
|
||||
].join(",");
|
||||
|
||||
const elements = Array.from(document.querySelectorAll<HTMLElement>(actionableSelector))
|
||||
.filter((element) => isCandidate(element))
|
||||
.slice(0, 500)
|
||||
.map((element, index) => toRawElement(element, index));
|
||||
|
||||
return {
|
||||
url: location.href,
|
||||
route: `${location.pathname}${location.search}`,
|
||||
title: document.title,
|
||||
dom: document.documentElement.outerHTML,
|
||||
visibleText: normalizeText(document.body?.innerText || ""),
|
||||
modalStack: Array.from(document.querySelectorAll<HTMLElement>("[role='dialog'], dialog[open], [aria-modal='true']"))
|
||||
.filter((element) => isVisible(element))
|
||||
.map((element) => getAccessibleName(element) || element.id || element.tagName.toLowerCase()),
|
||||
elements,
|
||||
};
|
||||
|
||||
function isCandidate(element: HTMLElement): boolean {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag === "input" && (element as HTMLInputElement).type === "hidden") return false;
|
||||
if (!isVisible(element)) return false;
|
||||
const box = element.getBoundingClientRect();
|
||||
return box.width > 0 && box.height > 0;
|
||||
}
|
||||
|
||||
function toRawElement(element: HTMLElement, index: number): RawActionableElement {
|
||||
const rect = element.getBoundingClientRect();
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const attributes = collectAttributes(element);
|
||||
const text = normalizeText(element.innerText || element.textContent || "");
|
||||
const role = attributes.role || inferRole(element);
|
||||
const label = getLabel(element);
|
||||
const placeholder = "placeholder" in element ? normalizeNullable(String((element as HTMLInputElement).placeholder || "")) : null;
|
||||
const accessibleName = normalizeNullable(getAccessibleName(element) || label || placeholder || text);
|
||||
const centerX = rect.left + rect.width / 2;
|
||||
const centerY = rect.top + rect.height / 2;
|
||||
const topAtCenter = document.elementFromPoint(centerX, centerY);
|
||||
|
||||
return {
|
||||
uid: `${tag}_${index}_${attributes.id || attributes.name || accessibleName || text || ""}`,
|
||||
tag,
|
||||
role,
|
||||
accessibleName,
|
||||
text: normalizeNullable(text),
|
||||
label,
|
||||
placeholder,
|
||||
attributes,
|
||||
bbox: {
|
||||
x: Math.round(rect.x),
|
||||
y: Math.round(rect.y),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
},
|
||||
visibility: {
|
||||
visible: isVisible(element),
|
||||
enabled: !("disabled" in element && Boolean((element as HTMLButtonElement).disabled)),
|
||||
editable: isEditable(element),
|
||||
receivesEvents: topAtCenter === element || Boolean(topAtCenter && element.contains(topAtCenter)),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function collectAttributes(element: HTMLElement): Record<string, string> {
|
||||
const keep = new Set([
|
||||
"id",
|
||||
"name",
|
||||
"type",
|
||||
"href",
|
||||
"value",
|
||||
"role",
|
||||
"aria-label",
|
||||
"aria-labelledby",
|
||||
"aria-expanded",
|
||||
"aria-haspopup",
|
||||
"aria-controls",
|
||||
"aria-disabled",
|
||||
"data-testid",
|
||||
"data-test-id",
|
||||
"data-test",
|
||||
]);
|
||||
const attrs: Record<string, string> = {};
|
||||
for (const attr of Array.from(element.attributes)) {
|
||||
if (keep.has(attr.name)) attrs[attr.name] = attr.value;
|
||||
}
|
||||
return attrs;
|
||||
}
|
||||
|
||||
function inferRole(element: HTMLElement): string | null {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "a" && element.hasAttribute("href")) return "link";
|
||||
if (tag === "select") return "combobox";
|
||||
if (tag === "textarea") return "textbox";
|
||||
if (tag === "input") {
|
||||
const type = ((element as HTMLInputElement).type || "text").toLowerCase();
|
||||
if (["button", "submit", "reset"].includes(type)) return "button";
|
||||
if (type === "checkbox") return "checkbox";
|
||||
if (type === "radio") return "radio";
|
||||
return "textbox";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getAccessibleName(element: HTMLElement): string | null {
|
||||
const ariaLabel = element.getAttribute("aria-label");
|
||||
if (ariaLabel) return normalizeNullable(ariaLabel);
|
||||
const labelledBy = element.getAttribute("aria-labelledby");
|
||||
if (labelledBy) {
|
||||
const text = labelledBy
|
||||
.split(/\s+/)
|
||||
.map((id) => document.getElementById(id)?.innerText || document.getElementById(id)?.textContent || "")
|
||||
.join(" ");
|
||||
if (text.trim()) return normalizeNullable(text);
|
||||
}
|
||||
const title = element.getAttribute("title");
|
||||
if (title) return normalizeNullable(title);
|
||||
const alt = element.getAttribute("alt");
|
||||
if (alt) return normalizeNullable(alt);
|
||||
return normalizeNullable(element.innerText || element.textContent || "");
|
||||
}
|
||||
|
||||
function getLabel(element: HTMLElement): string | null {
|
||||
const id = element.getAttribute("id");
|
||||
if (id) {
|
||||
const explicit = document.querySelector<HTMLLabelElement>(`label[for="${CSS.escape(id)}"]`);
|
||||
if (explicit) return normalizeNullable(explicit.innerText || explicit.textContent || "");
|
||||
}
|
||||
const implicit = element.closest("label");
|
||||
if (implicit) return normalizeNullable(implicit.innerText || implicit.textContent || "");
|
||||
return null;
|
||||
}
|
||||
|
||||
function isVisible(element: HTMLElement): boolean {
|
||||
const style = getComputedStyle(element);
|
||||
if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity) === 0) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}
|
||||
|
||||
function isEditable(element: HTMLElement): boolean {
|
||||
if (element.isContentEditable) return true;
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag === "textarea") return true;
|
||||
if (tag !== "input") return false;
|
||||
const type = ((element as HTMLInputElement).type || "text").toLowerCase();
|
||||
return !["button", "submit", "reset", "checkbox", "radio", "hidden"].includes(type);
|
||||
}
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function normalizeNullable(value: string): string | null {
|
||||
const normalized = normalizeText(value);
|
||||
return normalized || null;
|
||||
}
|
||||
}
|
||||
|
||||
139
src/types.ts
Normal file
139
src/types.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
export type LocatorDescriptor =
|
||||
| { kind: "role"; role: string; name?: string }
|
||||
| { kind: "label"; label: string }
|
||||
| { kind: "placeholder"; placeholder: string }
|
||||
| { kind: "text"; text: string; exact: boolean }
|
||||
| { kind: "testId"; testId: string }
|
||||
| { kind: "css"; selector: string };
|
||||
|
||||
export type LocatorFramework = "playwright" | "css";
|
||||
|
||||
export interface LocatorCandidate {
|
||||
locatorId: string;
|
||||
elementId: string;
|
||||
framework: LocatorFramework;
|
||||
strategy: LocatorDescriptor["kind"];
|
||||
descriptor: LocatorDescriptor;
|
||||
expression: string;
|
||||
score: number;
|
||||
reason: string;
|
||||
verified: boolean;
|
||||
lastVerifiedAt?: string;
|
||||
matchCount?: number;
|
||||
failureCount: number;
|
||||
}
|
||||
|
||||
export interface VisibilityInfo {
|
||||
visible: boolean;
|
||||
enabled: boolean;
|
||||
editable: boolean;
|
||||
receivesEvents: boolean;
|
||||
}
|
||||
|
||||
export interface BoundingBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ActionableElement {
|
||||
elementId: string;
|
||||
stateId: string;
|
||||
tag: string;
|
||||
role: string | null;
|
||||
accessibleName: string | null;
|
||||
text: string | null;
|
||||
label: string | null;
|
||||
placeholder: string | null;
|
||||
attributes: Record<string, string>;
|
||||
bbox: BoundingBox | null;
|
||||
framePath: string[];
|
||||
shadowPath: string[];
|
||||
interactability: VisibilityInfo;
|
||||
locators: LocatorCandidate[];
|
||||
}
|
||||
|
||||
export interface StateNode {
|
||||
stateId: string;
|
||||
url: string;
|
||||
route: string;
|
||||
title: string;
|
||||
frameContext: string[];
|
||||
stateHash: string;
|
||||
domHash: string;
|
||||
visibleTextHash: string;
|
||||
actionableSignatureHash: string;
|
||||
modalStack: string[];
|
||||
openedByEdgeId: string | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export type ActionType = "click" | "fill" | "select" | "check" | "uncheck" | "navigate";
|
||||
|
||||
export interface EffectRecord {
|
||||
type: "url" | "dom" | "visibleText" | "network" | "download" | "dialog" | "storage";
|
||||
before?: string;
|
||||
after?: string;
|
||||
method?: string;
|
||||
url?: string;
|
||||
status?: number;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface ParameterSpec {
|
||||
name: string;
|
||||
type: "string" | "number" | "boolean" | "enum" | "date" | "unknown";
|
||||
required: boolean;
|
||||
enumValues?: string[];
|
||||
pattern?: string;
|
||||
examples?: string[];
|
||||
}
|
||||
|
||||
export interface ActionEdge {
|
||||
edgeId: string;
|
||||
fromStateId: string;
|
||||
toStateId: string | null;
|
||||
elementId: string;
|
||||
actionType: ActionType;
|
||||
actionParams: Record<string, unknown>;
|
||||
preconditions: string[];
|
||||
effects: EffectRecord[];
|
||||
parameterSpecs: ParameterSpec[];
|
||||
riskLevel: "low" | "medium" | "high";
|
||||
confidence: number;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface TopologyGraph {
|
||||
schemaVersion: "0.1";
|
||||
metadata: {
|
||||
entryUrl: string;
|
||||
generatedAt: string;
|
||||
engineVersion: string;
|
||||
mode: "scan" | "explore";
|
||||
};
|
||||
states: StateNode[];
|
||||
elements: ActionableElement[];
|
||||
edges: ActionEdge[];
|
||||
}
|
||||
|
||||
export interface RawActionableElement {
|
||||
uid: string;
|
||||
tag: string;
|
||||
role: string | null;
|
||||
accessibleName: string | null;
|
||||
text: string | null;
|
||||
label: string | null;
|
||||
placeholder: string | null;
|
||||
attributes: Record<string, string>;
|
||||
bbox: BoundingBox | null;
|
||||
visibility: VisibilityInfo;
|
||||
}
|
||||
|
||||
export interface SnapshotResult {
|
||||
state: StateNode;
|
||||
elements: ActionableElement[];
|
||||
visibleText: string;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user