From 1ea4c52ad1872bd62050fb36bf985e4b4d460ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E4=B9=89=E4=BB=91?= Date: Tue, 16 Jun 2026 20:45:04 +0800 Subject: [PATCH] feat: verify explored edges by replay --- src/effects.ts | 41 +++++ src/engine.ts | 283 ++++++++++++++++++------------- src/types.ts | 5 +- tests/fixtures/context-risk.html | 8 + tests/integration.test.ts | 122 ++++++++++++- 5 files changed, 337 insertions(+), 122 deletions(-) create mode 100644 src/effects.ts diff --git a/src/effects.ts b/src/effects.ts new file mode 100644 index 0000000..afc0ecb --- /dev/null +++ b/src/effects.ts @@ -0,0 +1,41 @@ +import type { EffectRecord, SnapshotResult } from "./types.js"; + +export function diffSnapshotEffects(args: { + beforeUrl: string; + afterUrl: string; + beforeSnapshot: SnapshotResult; + afterSnapshot: SnapshotResult; +}): EffectRecord[] { + const effects: EffectRecord[] = []; + if (args.beforeUrl !== args.afterUrl) { + effects.push({ type: "url", before: args.beforeUrl, after: args.afterUrl, description: "URL changed after action" }); + } + if (args.beforeSnapshot.state.domHash !== args.afterSnapshot.state.domHash) { + effects.push({ + type: "dom", + before: args.beforeSnapshot.state.domHash, + after: args.afterSnapshot.state.domHash, + description: "DOM hash changed after action", + }); + } + + const beforeVisibleTextHash = args.beforeSnapshot.state.visibleTextHash; + const afterVisibleTextHash = args.afterSnapshot.state.visibleTextHash; + if (beforeVisibleTextHash !== afterVisibleTextHash) { + effects.push({ + type: "visibleText", + before: beforeVisibleTextHash, + after: afterVisibleTextHash, + description: "Visible text changed after action", + }); + } + if (args.beforeSnapshot.state.actionableSignatureHash !== args.afterSnapshot.state.actionableSignatureHash) { + effects.push({ + type: "actionableInventory", + before: args.beforeSnapshot.state.actionableSignatureHash, + after: args.afterSnapshot.state.actionableSignatureHash, + description: "Actionable inventory changed after action", + }); + } + return effects; +} diff --git a/src/engine.ts b/src/engine.ts index 25409b5..2ce8453 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1,13 +1,14 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { chromium, type Browser, type BrowserContext, type Page, type Response } from "playwright"; +import { chromium, type Browser, type BrowserContext, type Page } from "playwright"; +import { diffSnapshotEffects } from "./effects.js"; import { shortHash } from "./hash.js"; import { resolveLocator } from "./locator.js"; import { buildEmptyGraph, defaultViewport } from "./metadata.js"; import { defaultExploreDecision } from "./risk.js"; import { snapshotPage } from "./snapshot.js"; -import type { ActionableElement, ActionEdge, EffectRecord, SnapshotResult, TopologyGraph } from "./types.js"; +import type { ActionableElement, ActionEdge, FailedObservation, LocatorCandidate, SnapshotResult, TopologyGraph } from "./types.js"; export interface EngineOptions { url: string; @@ -17,6 +18,11 @@ export interface EngineOptions { maxActions?: number; } +interface ExploreCandidate { + element: ActionableElement; + locator: LocatorCandidate; +} + export async function scanUrl(options: EngineOptions): Promise { const browser = await chromium.launch({ headless: !options.headed }); try { @@ -44,16 +50,21 @@ export async function scanUrl(options: EngineOptions): Promise { export async function exploreUrl(options: EngineOptions): Promise { const browser = await chromium.launch({ headless: !options.headed }); try { - const context = await newContext(browser, options); - const entryUrl = toNavigableUrl(options.url); - const page = await context.newPage(); - await page.goto(entryUrl, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle").catch(() => undefined); + const requestedEntryUrl = toNavigableUrl(options.url); + const entryContext = await newContext(browser, options); + let entrySnapshot: SnapshotResult; + let entryUrl: string; + try { + const page = await createReplayPage(entryContext, requestedEntryUrl); + entrySnapshot = await snapshotPage(page); + entryUrl = page.url(); + } finally { + await entryContext.close(); + } - const entrySnapshot = await snapshotPage(page); const graph = buildEmptyGraph({ inputUrl: options.url, - entryUrl: page.url(), + entryUrl, mode: "explore", storageState: options.storageState, }); @@ -62,7 +73,7 @@ export async function exploreUrl(options: EngineOptions): Promise recordSkippedActionsForState(graph, entrySnapshot); - const lowRiskCandidates: ActionableElement[] = []; + const lowRiskCandidates: ExploreCandidate[] = []; for (const element of entrySnapshot.elements) { if (!isDefaultClickCandidate(element)) continue; if (!isActionableForClick(element)) { @@ -71,89 +82,72 @@ export async function exploreUrl(options: EngineOptions): Promise } const decision = defaultExploreDecision(element, "click"); if (!decision.allowed) continue; - lowRiskCandidates.push(element); - } - lowRiskCandidates.sort(compareExploreCandidatePriority); - const candidates = lowRiskCandidates.slice(0, options.maxActions ?? 10); - await page.close(); - - for (const element of candidates) { - const actionPage = await context.newPage(); - const networkEvents: EffectRecord[] = []; - actionPage.on("response", (response) => { - const record = responseToEffect(response); - if (record) networkEvents.push(record); - }); - - await actionPage.goto(entryUrl, { waitUntil: "domcontentloaded" }); - await actionPage.waitForLoadState("networkidle").catch(() => undefined); - const beforeUrl = actionPage.url(); - const beforeSnapshot = await snapshotPage(actionPage); - const locator = element.locators.find((candidate) => candidate.verified) ?? element.locators[0]; - const decision = defaultExploreDecision(element, "click"); - - let toStateId: string | null = null; - const effects: EffectRecord[] = []; - if (locator) { - try { - await resolveLocator(actionPage, locator.descriptor).first().click({ timeout: 2000 }); - await actionPage.waitForLoadState("networkidle", { timeout: 2000 }).catch(() => undefined); - const afterSnapshot = await snapshotPage(actionPage); - graph.states = upsertById(graph.states, afterSnapshot.state, "stateId"); - graph.elements.push(...afterSnapshot.elements); - recordSkippedActionsForState(graph, afterSnapshot); - toStateId = afterSnapshot.state.stateId; - effects.push(...diffEffects(beforeUrl, beforeSnapshot.visibleText, actionPage.url(), afterSnapshot.visibleText)); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - effects.push({ - type: "dom", - description: `Action failed: ${message}`, - }); - graph.failedObservations = upsertById( - graph.failedObservations, - { - failedObservationId: `fail_${shortHash([beforeSnapshot.state.stateId, element.elementId, "click", "unexpected_state", message])}`, - stateId: beforeSnapshot.state.stateId, - elementId: element.elementId, - actionType: "click" as const, - reason: "unexpected_state", - message: `Action failed: ${message}`, - evidence: [ - { - level: "observed", - source: "engine.action", - description: "Click action raised before effects could be fully observed", - }, - ], - createdAt: new Date().toISOString(), - }, - "failedObservationId", - ); - } + const locator = findVerifiedUniqueLocator(element); + if (!locator) { + recordLocatorFailure(graph, entrySnapshot, element, locatorFailureReason(element)); + continue; } + lowRiskCandidates.push({ element, locator }); + } + lowRiskCandidates.sort((left, right) => compareExploreCandidatePriority(left.element, right.element)); + const candidates = lowRiskCandidates.slice(0, options.maxActions ?? 10); - const edge: ActionEdge = { - edgeId: `edge_${shortHash([entrySnapshot.state.stateId, element.elementId, "click"])}`, - fromStateId: entrySnapshot.state.stateId, - toStateId, - elementId: element.elementId, - actionType: "click", - actionParams: {}, - preconditions: [], - effects: [...effects, ...networkEvents], - parameterSpecs: [], - riskLevel: decision.risk.level, - riskCategory: decision.risk.category, - confidence: locator?.verified ? 0.82 : 0.55, - createdAt: new Date().toISOString(), - }; - graph.edges = upsertById(graph.edges, edge, "edgeId"); - await actionPage.close(); + for (const { element, locator } of candidates) { + const decision = defaultExploreDecision(element, "click"); + const actionContext = await newContext(browser, options); + try { + const actionPage = await createReplayPage(actionContext, entryUrl); + const beforeUrl = actionPage.url(); + const beforeSnapshot = await snapshotPage(actionPage); + const edgeId = `edge_${shortHash([beforeSnapshot.state.stateId, element.elementId, "click"])}`; + const target = resolveLocator(actionPage, locator.descriptor); + const matchCount = await target.count(); + if (matchCount !== 1) { + recordLocatorFailure(graph, beforeSnapshot, element, matchCount === 0 ? "locator_not_found" : "locator_not_unique", matchCount); + continue; + } + + try { + await target.click({ timeout: 2000, trial: true }); + await target.click({ timeout: 2000 }); + } catch (error) { + recordClickFailure(graph, beforeSnapshot, element, error); + continue; + } + + await actionPage.waitForLoadState("networkidle", { timeout: 2000 }).catch(() => undefined); + const afterSnapshot = await snapshotPage(actionPage, edgeId); + const discoveredState = addStateIfNew(graph, afterSnapshot.state); + graph.elements = upsertManyById(graph.elements, afterSnapshot.elements, "elementId"); + if (discoveredState) recordSkippedActionsForState(graph, afterSnapshot); + + const edge: ActionEdge = { + edgeId, + fromStateId: beforeSnapshot.state.stateId, + toStateId: afterSnapshot.state.stateId, + elementId: element.elementId, + actionType: "click", + actionParams: {}, + preconditions: [], + effects: diffSnapshotEffects({ + beforeUrl, + afterUrl: actionPage.url(), + beforeSnapshot, + afterSnapshot, + }), + parameterSpecs: [], + riskLevel: decision.risk.level, + riskCategory: decision.risk.category, + confidence: 0.82, + createdAt: new Date().toISOString(), + }; + graph.edges = upsertById(graph.edges, edge, "edgeId"); + } finally { + await actionContext.close(); + } } await maybeWriteGraph(graph, options.out); - await context.close(); return graph; } finally { await browser.close(); @@ -168,6 +162,86 @@ async function newContext(browser: Browser, options: EngineOptions): Promise { + const page = await context.newPage(); + await page.goto(entryUrl, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle").catch(() => undefined); + return page; +} + +function addStateIfNew(graph: TopologyGraph, state: SnapshotResult["state"]): boolean { + if (graph.states.some((existing) => existing.stateId === state.stateId)) return false; + graph.states.push(state); + return true; +} + +function findVerifiedUniqueLocator(element: ActionableElement): LocatorCandidate | null { + return element.locators.find((locator) => locator.verified && locator.matchCount === 1) ?? null; +} + +function locatorFailureReason(element: ActionableElement): "locator_not_found" | "locator_not_unique" { + if (element.locators.some((locator) => (locator.matchCount ?? 0) > 1)) return "locator_not_unique"; + return "locator_not_found"; +} + +function recordLocatorFailure( + graph: TopologyGraph, + snapshot: SnapshotResult, + element: ActionableElement, + reason: "locator_not_found" | "locator_not_unique", + matchCount?: number, +): void { + const message = + reason === "locator_not_unique" + ? `Locator matched ${matchCount ?? "multiple"} elements and cannot be clicked safely` + : matchCount === 0 + ? "Locator matched no elements during replay" + : "No verified locator candidate was available for default exploration"; + recordFailedObservation(graph, { + failedObservationId: `fail_${shortHash([snapshot.state.stateId, element.elementId, "click", reason, matchCount ?? "candidate"])}`, + stateId: snapshot.state.stateId, + elementId: element.elementId, + actionType: "click", + reason, + message, + evidence: [ + { + level: "observed", + source: "locator.verify", + description: + matchCount === undefined + ? "Entry-state locator candidates did not contain a verified unique match" + : `Replay locator matched ${matchCount} element(s)`, + }, + ], + createdAt: new Date().toISOString(), + }); +} + +function recordClickFailure(graph: TopologyGraph, snapshot: SnapshotResult, element: ActionableElement, error: unknown): void { + const message = error instanceof Error ? error.message : String(error); + recordFailedObservation(graph, { + failedObservationId: `fail_${shortHash([snapshot.state.stateId, element.elementId, "click", "unexpected_state", message])}`, + stateId: snapshot.state.stateId, + elementId: element.elementId, + actionType: "click", + reason: "unexpected_state", + message: `Action failed: ${message}`, + evidence: [ + { + level: "observed", + source: "engine.action", + description: "Click action raised before effects could be fully observed", + }, + ], + createdAt: new Date().toISOString(), + }); +} + +function recordFailedObservation(graph: TopologyGraph, observation: FailedObservation): void { + graph.failedObservations = upsertById(graph.failedObservations, observation, "failedObservationId"); +} + async function maybeWriteGraph(graph: TopologyGraph, out?: string): Promise { if (!out) { console.log(JSON.stringify(graph, null, 2)); @@ -182,35 +256,6 @@ function toNavigableUrl(value: string): string { return pathToFileURL(path.resolve(value)).toString(); } -function diffEffects(beforeUrl: string, beforeText: string, afterUrl: string, afterText: string): EffectRecord[] { - const effects: EffectRecord[] = []; - if (beforeUrl !== afterUrl) { - effects.push({ type: "url", before: beforeUrl, after: afterUrl, description: "URL changed after action" }); - } - if (shortHash(beforeText) !== shortHash(afterText)) { - effects.push({ - type: "visibleText", - before: shortHash(beforeText), - after: shortHash(afterText), - description: "Visible text changed after action", - }); - } - return effects; -} - -function responseToEffect(response: Response): EffectRecord | null { - const request = response.request(); - const resourceType = request.resourceType(); - if (!["fetch", "xhr", "document"].includes(resourceType)) return null; - return { - type: "network", - method: request.method(), - url: response.url(), - status: response.status(), - description: `${request.method()} ${response.url()} -> ${response.status()}`, - }; -} - function recordSkippedActionsForState(graph: TopologyGraph, snapshot: SnapshotResult): void { for (const element of snapshot.elements) { if (!isDefaultClickCandidate(element)) continue; @@ -308,3 +353,7 @@ function upsertById, K extends keyof T>(items: T[], next[index] = item; return next; } + +function upsertManyById, K extends keyof T>(items: T[], incoming: T[], key: K): T[] { + return incoming.reduce((current, item) => upsertById(current, item, key), items); +} diff --git a/src/types.ts b/src/types.ts index 88e025f..69a05a7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -172,12 +172,9 @@ export interface StateNode { export type ActionType = "click" | "fill" | "select" | "check" | "uncheck" | "navigate"; export interface EffectRecord { - type: "url" | "dom" | "visibleText" | "network" | "download" | "dialog" | "storage"; + type: "url" | "dom" | "visibleText" | "actionableInventory"; before?: string; after?: string; - method?: string; - url?: string; - status?: number; description: string; } diff --git a/tests/fixtures/context-risk.html b/tests/fixtures/context-risk.html index ef01f93..83b9d07 100644 --- a/tests/fixtures/context-risk.html +++ b/tests/fixtures/context-risk.html @@ -11,11 +11,14 @@ button, input, select, a { margin: 4px; padding: 6px 10px; } [role="dialog"] { display: none; border: 1px solid #999; padding: 16px; margin-top: 16px; } [role="dialog"].open { display: block; } + #settings-dialog.open { position: fixed; inset: 24px auto auto 24px; background: white; z-index: 10; } #toast { margin-top: 16px; min-height: 24px; }

Accounts

+ +