feat: verify explored edges by replay

2026-06-16 20:45:04 +08:00
parent 347d68626e
commit 1ea4c52ad1
5 changed files with 337 additions and 122 deletions
--- a/src/effects.ts
+++ b/src/effects.ts
@@ -0,0 +1,41 @@
+import type { EffectRecord, SnapshotResult } from "./types.js";
+
+export function diffSnapshotEffects(args: {
+  beforeUrl: string;
+  afterUrl: string;
+  beforeSnapshot: SnapshotResult;
+  afterSnapshot: SnapshotResult;
+}): EffectRecord[] {
+  const effects: EffectRecord[] = [];
+  if (args.beforeUrl !== args.afterUrl) {
+    effects.push({ type: "url", before: args.beforeUrl, after: args.afterUrl, description: "URL changed after action" });
+  }
+  if (args.beforeSnapshot.state.domHash !== args.afterSnapshot.state.domHash) {
+    effects.push({
+      type: "dom",
+      before: args.beforeSnapshot.state.domHash,
+      after: args.afterSnapshot.state.domHash,
+      description: "DOM hash changed after action",
+    });
+  }
+
+  const beforeVisibleTextHash = args.beforeSnapshot.state.visibleTextHash;
+  const afterVisibleTextHash = args.afterSnapshot.state.visibleTextHash;
+  if (beforeVisibleTextHash !== afterVisibleTextHash) {
+    effects.push({
+      type: "visibleText",
+      before: beforeVisibleTextHash,
+      after: afterVisibleTextHash,
+      description: "Visible text changed after action",
+    });
+  }
+  if (args.beforeSnapshot.state.actionableSignatureHash !== args.afterSnapshot.state.actionableSignatureHash) {
+    effects.push({
+      type: "actionableInventory",
+      before: args.beforeSnapshot.state.actionableSignatureHash,
+      after: args.afterSnapshot.state.actionableSignatureHash,
+      description: "Actionable inventory changed after action",
+    });
+  }
+  return effects;
+}
--- a/src/engine.ts
+++ b/src/engine.ts
@@ -1,13 +1,14 @@
 import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
-import { chromium, type Browser, type BrowserContext, type Page, type Response } from "playwright";
+import { chromium, type Browser, type BrowserContext, type Page } from "playwright";
+import { diffSnapshotEffects } from "./effects.js";
 import { shortHash } from "./hash.js";
 import { resolveLocator } from "./locator.js";
 import { buildEmptyGraph, defaultViewport } from "./metadata.js";
 import { defaultExploreDecision } from "./risk.js";
 import { snapshotPage } from "./snapshot.js";
-import type { ActionableElement, ActionEdge, EffectRecord, SnapshotResult, TopologyGraph } from "./types.js";
+import type { ActionableElement, ActionEdge, FailedObservation, LocatorCandidate, SnapshotResult, TopologyGraph } from "./types.js";

 export interface EngineOptions {
  url: string;
@@ -17,6 +18,11 @@ export interface EngineOptions {
  maxActions?: number;
 }

+interface ExploreCandidate {
+  element: ActionableElement;
+  locator: LocatorCandidate;
+}
+
 export async function scanUrl(options: EngineOptions): Promise<TopologyGraph> {
  const browser = await chromium.launch({ headless: !options.headed });
  try {
@@ -44,16 +50,21 @@ export async function scanUrl(options: EngineOptions): Promise<TopologyGraph> {
 export async function exploreUrl(options: EngineOptions): Promise<TopologyGraph> {
  const browser = await chromium.launch({ headless: !options.headed });
  try {
-    const context = await newContext(browser, options);
-    const entryUrl = toNavigableUrl(options.url);
-    const page = await context.newPage();
-    await page.goto(entryUrl, { waitUntil: "domcontentloaded" });
-    await page.waitForLoadState("networkidle").catch(() => undefined);
+    const requestedEntryUrl = toNavigableUrl(options.url);
+    const entryContext = await newContext(browser, options);
+    let entrySnapshot: SnapshotResult;
+    let entryUrl: string;
+    try {
+      const page = await createReplayPage(entryContext, requestedEntryUrl);
+      entrySnapshot = await snapshotPage(page);
+      entryUrl = page.url();
+    } finally {
+      await entryContext.close();
+    }

-    const entrySnapshot = await snapshotPage(page);
    const graph = buildEmptyGraph({
      inputUrl: options.url,
-      entryUrl: page.url(),
+      entryUrl,
      mode: "explore",
      storageState: options.storageState,
    });
@@ -62,7 +73,7 @@ export async function exploreUrl(options: EngineOptions): Promise<TopologyGraph>

    recordSkippedActionsForState(graph, entrySnapshot);

-    const lowRiskCandidates: ActionableElement[] = [];
+    const lowRiskCandidates: ExploreCandidate[] = [];
    for (const element of entrySnapshot.elements) {
      if (!isDefaultClickCandidate(element)) continue;
      if (!isActionableForClick(element)) {
@@ -71,89 +82,72 @@ export async function exploreUrl(options: EngineOptions): Promise<TopologyGraph>
      }
      const decision = defaultExploreDecision(element, "click");
      if (!decision.allowed) continue;
-      lowRiskCandidates.push(element);
-    }
-    lowRiskCandidates.sort(compareExploreCandidatePriority);
-    const candidates = lowRiskCandidates.slice(0, options.maxActions ?? 10);
-    await page.close();
-
-    for (const element of candidates) {
-      const actionPage = await context.newPage();
-      const networkEvents: EffectRecord[] = [];
-      actionPage.on("response", (response) => {
-        const record = responseToEffect(response);
-        if (record) networkEvents.push(record);
-      });
-
-      await actionPage.goto(entryUrl, { waitUntil: "domcontentloaded" });
-      await actionPage.waitForLoadState("networkidle").catch(() => undefined);
-      const beforeUrl = actionPage.url();
-      const beforeSnapshot = await snapshotPage(actionPage);
-      const locator = element.locators.find((candidate) => candidate.verified) ?? element.locators[0];
-      const decision = defaultExploreDecision(element, "click");
-
-      let toStateId: string | null = null;
-      const effects: EffectRecord[] = [];
-      if (locator) {
-        try {
-          await resolveLocator(actionPage, locator.descriptor).first().click({ timeout: 2000 });
-          await actionPage.waitForLoadState("networkidle", { timeout: 2000 }).catch(() => undefined);
-          const afterSnapshot = await snapshotPage(actionPage);
-          graph.states = upsertById(graph.states, afterSnapshot.state, "stateId");
-          graph.elements.push(...afterSnapshot.elements);
-          recordSkippedActionsForState(graph, afterSnapshot);
-          toStateId = afterSnapshot.state.stateId;
-          effects.push(...diffEffects(beforeUrl, beforeSnapshot.visibleText, actionPage.url(), afterSnapshot.visibleText));
-        } catch (error) {
-          const message = error instanceof Error ? error.message : String(error);
-          effects.push({
-            type: "dom",
-            description: `Action failed: ${message}`,
-          });
-          graph.failedObservations = upsertById(
-            graph.failedObservations,
-            {
-              failedObservationId: `fail_${shortHash([beforeSnapshot.state.stateId, element.elementId, "click", "unexpected_state", message])}`,
-              stateId: beforeSnapshot.state.stateId,
-              elementId: element.elementId,
-              actionType: "click" as const,
-              reason: "unexpected_state",
-              message: `Action failed: ${message}`,
-              evidence: [
-                {
-                  level: "observed",
-                  source: "engine.action",
-                  description: "Click action raised before effects could be fully observed",
-                },
-              ],
-              createdAt: new Date().toISOString(),
-            },
-            "failedObservationId",
-          );
-        }
+      const locator = findVerifiedUniqueLocator(element);
+      if (!locator) {
+        recordLocatorFailure(graph, entrySnapshot, element, locatorFailureReason(element));
+        continue;
      }
+      lowRiskCandidates.push({ element, locator });
+    }
+    lowRiskCandidates.sort((left, right) => compareExploreCandidatePriority(left.element, right.element));
+    const candidates = lowRiskCandidates.slice(0, options.maxActions ?? 10);

-      const edge: ActionEdge = {
-        edgeId: `edge_${shortHash([entrySnapshot.state.stateId, element.elementId, "click"])}`,
-        fromStateId: entrySnapshot.state.stateId,
-        toStateId,
-        elementId: element.elementId,
-        actionType: "click",
-        actionParams: {},
-        preconditions: [],
-        effects: [...effects, ...networkEvents],
-        parameterSpecs: [],
-        riskLevel: decision.risk.level,
-        riskCategory: decision.risk.category,
-        confidence: locator?.verified ? 0.82 : 0.55,
-        createdAt: new Date().toISOString(),
-      };
-      graph.edges = upsertById(graph.edges, edge, "edgeId");
-      await actionPage.close();
+    for (const { element, locator } of candidates) {
+      const decision = defaultExploreDecision(element, "click");
+      const actionContext = await newContext(browser, options);
+      try {
+        const actionPage = await createReplayPage(actionContext, entryUrl);
+        const beforeUrl = actionPage.url();
+        const beforeSnapshot = await snapshotPage(actionPage);
+        const edgeId = `edge_${shortHash([beforeSnapshot.state.stateId, element.elementId, "click"])}`;
+        const target = resolveLocator(actionPage, locator.descriptor);
+        const matchCount = await target.count();
+        if (matchCount !== 1) {
+          recordLocatorFailure(graph, beforeSnapshot, element, matchCount === 0 ? "locator_not_found" : "locator_not_unique", matchCount);
+          continue;
+        }
+
+        try {
+          await target.click({ timeout: 2000, trial: true });
+          await target.click({ timeout: 2000 });
+        } catch (error) {
+          recordClickFailure(graph, beforeSnapshot, element, error);
+          continue;
+        }
+
+        await actionPage.waitForLoadState("networkidle", { timeout: 2000 }).catch(() => undefined);
+        const afterSnapshot = await snapshotPage(actionPage, edgeId);
+        const discoveredState = addStateIfNew(graph, afterSnapshot.state);
+        graph.elements = upsertManyById(graph.elements, afterSnapshot.elements, "elementId");
+        if (discoveredState) recordSkippedActionsForState(graph, afterSnapshot);
+
+        const edge: ActionEdge = {
+          edgeId,
+          fromStateId: beforeSnapshot.state.stateId,
+          toStateId: afterSnapshot.state.stateId,
+          elementId: element.elementId,
+          actionType: "click",
+          actionParams: {},
+          preconditions: [],
+          effects: diffSnapshotEffects({
+            beforeUrl,
+            afterUrl: actionPage.url(),
+            beforeSnapshot,
+            afterSnapshot,
+          }),
+          parameterSpecs: [],
+          riskLevel: decision.risk.level,
+          riskCategory: decision.risk.category,
+          confidence: 0.82,
+          createdAt: new Date().toISOString(),
+        };
+        graph.edges = upsertById(graph.edges, edge, "edgeId");
+      } finally {
+        await actionContext.close();
+      }
    }

    await maybeWriteGraph(graph, options.out);
-    await context.close();
    return graph;
  } finally {
    await browser.close();
@@ -168,6 +162,86 @@ async function newContext(browser: Browser, options: EngineOptions): Promise<Bro
  });
 }

+async function createReplayPage(context: BrowserContext, entryUrl: string): Promise<Page> {
+  const page = await context.newPage();
+  await page.goto(entryUrl, { waitUntil: "domcontentloaded" });
+  await page.waitForLoadState("networkidle").catch(() => undefined);
+  return page;
+}
+
+function addStateIfNew(graph: TopologyGraph, state: SnapshotResult["state"]): boolean {
+  if (graph.states.some((existing) => existing.stateId === state.stateId)) return false;
+  graph.states.push(state);
+  return true;
+}
+
+function findVerifiedUniqueLocator(element: ActionableElement): LocatorCandidate | null {
+  return element.locators.find((locator) => locator.verified && locator.matchCount === 1) ?? null;
+}
+
+function locatorFailureReason(element: ActionableElement): "locator_not_found" | "locator_not_unique" {
+  if (element.locators.some((locator) => (locator.matchCount ?? 0) > 1)) return "locator_not_unique";
+  return "locator_not_found";
+}
+
+function recordLocatorFailure(
+  graph: TopologyGraph,
+  snapshot: SnapshotResult,
+  element: ActionableElement,
+  reason: "locator_not_found" | "locator_not_unique",
+  matchCount?: number,
+): void {
+  const message =
+    reason === "locator_not_unique"
+      ? `Locator matched ${matchCount ?? "multiple"} elements and cannot be clicked safely`
+      : matchCount === 0
+        ? "Locator matched no elements during replay"
+        : "No verified locator candidate was available for default exploration";
+  recordFailedObservation(graph, {
+    failedObservationId: `fail_${shortHash([snapshot.state.stateId, element.elementId, "click", reason, matchCount ?? "candidate"])}`,
+    stateId: snapshot.state.stateId,
+    elementId: element.elementId,
+    actionType: "click",
+    reason,
+    message,
+    evidence: [
+      {
+        level: "observed",
+        source: "locator.verify",
+        description:
+          matchCount === undefined
+            ? "Entry-state locator candidates did not contain a verified unique match"
+            : `Replay locator matched ${matchCount} element(s)`,
+      },
+    ],
+    createdAt: new Date().toISOString(),
+  });
+}
+
+function recordClickFailure(graph: TopologyGraph, snapshot: SnapshotResult, element: ActionableElement, error: unknown): void {
+  const message = error instanceof Error ? error.message : String(error);
+  recordFailedObservation(graph, {
+    failedObservationId: `fail_${shortHash([snapshot.state.stateId, element.elementId, "click", "unexpected_state", message])}`,
+    stateId: snapshot.state.stateId,
+    elementId: element.elementId,
+    actionType: "click",
+    reason: "unexpected_state",
+    message: `Action failed: ${message}`,
+    evidence: [
+      {
+        level: "observed",
+        source: "engine.action",
+        description: "Click action raised before effects could be fully observed",
+      },
+    ],
+    createdAt: new Date().toISOString(),
+  });
+}
+
+function recordFailedObservation(graph: TopologyGraph, observation: FailedObservation): void {
+  graph.failedObservations = upsertById(graph.failedObservations, observation, "failedObservationId");
+}
+
 async function maybeWriteGraph(graph: TopologyGraph, out?: string): Promise<void> {
  if (!out) {
    console.log(JSON.stringify(graph, null, 2));
@@ -182,35 +256,6 @@ function toNavigableUrl(value: string): string {
  return pathToFileURL(path.resolve(value)).toString();
 }

-function diffEffects(beforeUrl: string, beforeText: string, afterUrl: string, afterText: string): EffectRecord[] {
-  const effects: EffectRecord[] = [];
-  if (beforeUrl !== afterUrl) {
-    effects.push({ type: "url", before: beforeUrl, after: afterUrl, description: "URL changed after action" });
-  }
-  if (shortHash(beforeText) !== shortHash(afterText)) {
-    effects.push({
-      type: "visibleText",
-      before: shortHash(beforeText),
-      after: shortHash(afterText),
-      description: "Visible text changed after action",
-    });
-  }
-  return effects;
-}
-
-function responseToEffect(response: Response): EffectRecord | null {
-  const request = response.request();
-  const resourceType = request.resourceType();
-  if (!["fetch", "xhr", "document"].includes(resourceType)) return null;
-  return {
-    type: "network",
-    method: request.method(),
-    url: response.url(),
-    status: response.status(),
-    description: `${request.method()} ${response.url()} -> ${response.status()}`,
-  };
-}
-
 function recordSkippedActionsForState(graph: TopologyGraph, snapshot: SnapshotResult): void {
  for (const element of snapshot.elements) {
    if (!isDefaultClickCandidate(element)) continue;
@@ -308,3 +353,7 @@ function upsertById<T extends Record<K, string>, K extends keyof T>(items: T[],
  next[index] = item;
  return next;
 }
+
+function upsertManyById<T extends Record<K, string>, K extends keyof T>(items: T[], incoming: T[], key: K): T[] {
+  return incoming.reduce((current, item) => upsertById(current, item, key), items);
+}
--- a/src/types.ts
+++ b/src/types.ts
@@ -172,12 +172,9 @@ export interface StateNode {
 export type ActionType = "click" | "fill" | "select" | "check" | "uncheck" | "navigate";

 export interface EffectRecord {
-  type: "url" | "dom" | "visibleText" | "network" | "download" | "dialog" | "storage";
+  type: "url" | "dom" | "visibleText" | "actionableInventory";
  before?: string;
  after?: string;
-  method?: string;
-  url?: string;
-  status?: number;
  description: string;
 }

--- a/tests/fixtures/context-risk.html
+++ b/tests/fixtures/context-risk.html
@@ -11,11 +11,14 @@
      button, input, select, a { margin: 4px; padding: 6px 10px; }
      [role="dialog"] { display: none; border: 1px solid #999; padding: 16px; margin-top: 16px; }
      [role="dialog"].open { display: block; }
+      #settings-dialog.open { position: fixed; inset: 24px auto auto 24px; background: white; z-index: 10; }
      #toast { margin-top: 16px; min-height: 24px; }
    </style>
  </head>
  <body>
    <h1>Accounts</h1>
+    <button id="storage-leak-marker" hidden>Replay storage leaked</button>
+    <button data-testid="remember-filter" onclick="localStorage.setItem('ateReplayLeak', '1')">Remember filter</button>
    <nav aria-label="Primary">
      <a href="#overview" data-testid="overview-link">Overview</a>
      <a href="#reports" data-testid="reports-link">Reports</a>
@@ -78,6 +81,7 @@
    <section aria-label="Duplicate controls">
      <button>Duplicate</button>
      <button>Duplicate</button>
+      <button><span aria-hidden="true"></span></button>
    </section>

    <section role="dialog" aria-modal="true" class="open">
@@ -115,6 +119,10 @@

    <p id="toast" role="status"></p>
    <script>
+      if (localStorage.getItem("ateReplayLeak")) {
+        document.getElementById("storage-leak-marker").hidden = false;
+      }
+
      function openEditor(name) {
        document.getElementById("customer-name").value = name;
        document.getElementById("editor-title").textContent = "Editing " + name;
--- a/tests/integration.test.ts
+++ b/tests/integration.test.ts
@@ -33,6 +33,24 @@ function hasEdgeForAccessibleName(
  });
 }

+function edgeForAccessibleName(
+  graph: {
+    edges: Array<{
+      edgeId: string;
+      elementId: string;
+      toStateId: string | null;
+      effects: Array<{ type: string; before?: string; after?: string; description: string }>;
+    }>;
+    elements: Array<{ elementId: string; accessibleName: string | null }>;
+  },
+  accessibleName: string,
+) {
+  return graph.edges.find((edge) => {
+    const element = graph.elements.find((item) => item.elementId === edge.elementId);
+    return element?.accessibleName === accessibleName;
+  });
+}
+
 test("scan captures a state-local inventory without clicking high-risk actions", async () => {
  const graph = await withTempOutput("scan", (out) => scanUrl({ url: fixturePath, out }));
  assert.equal(graph.states.length, 1);
@@ -198,8 +216,110 @@ test("explore records low-risk edges and keeps high-risk submit actions out of d
  assert.equal(graph.states.length >= 2, true);
 });

+test("explore records edge effects and opened-by state relationships", async () => {
+  const graph = await withTempOutput("explore-effects", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
+  const entryState = graph.states[0];
+  const openSettingsEdge = edgeForAccessibleName(graph, "Open settings");
+
+  assert.ok(entryState);
+  assert.ok(openSettingsEdge);
+  assert.ok(openSettingsEdge.toStateId);
+
+  const openedState = graph.states.find((state) => state.stateId === openSettingsEdge.toStateId);
+  assert.ok(openedState);
+  assert.equal(entryState.openedByEdgeId, null);
+  assert.equal(openedState.openedByEdgeId, openSettingsEdge.edgeId);
+
+  const visibleTextEffect = openSettingsEdge.effects.find((effect) => effect.type === "visibleText");
+  assert.ok(visibleTextEffect);
+  assert.equal(visibleTextEffect.before, entryState.visibleTextHash);
+  assert.equal(visibleTextEffect.after, openedState.visibleTextHash);
+  assert.equal(openSettingsEdge.effects.some((effect) => effect.type === "actionableInventory"), true);
+  assert.equal(openSettingsEdge.effects.some((effect) => effect.description === "Actionable inventory changed after action" && effect.type === "dom"), false);
+});
+
+test("explore replays each v0.2.1 edge from the entry state", async () => {
+  const graph = await withTempOutput("explore-entry-replay", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
+  const entryState = graph.states[0];
+
+  assert.ok(entryState);
+  assert.equal(graph.edges.length > 0, true);
+  assert.equal(graph.edges.every((edge) => edge.fromStateId === entryState.stateId), true);
+});
+
+test("explore isolates replay contexts for storage-writing actions", async () => {
+  const graph = await withTempOutput("explore-storage-isolation", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
+  const entryState = graph.states[0];
+  const rememberFilterEdge = edgeForAccessibleName(graph, "Remember filter");
+  const leakText = "Replay storage leaked";
+
+  assert.ok(entryState);
+  assert.ok(rememberFilterEdge);
+  assert.equal(graph.edges.every((edge) => edge.fromStateId === entryState.stateId), true);
+  assert.equal(
+    graph.states.some(
+      (state) =>
+        state.title.includes(leakText) ||
+        state.url.includes(leakText) ||
+        state.route.includes(leakText) ||
+        state.evidence.some((item) => item.description.includes(leakText)),
+    ),
+    false,
+  );
+  assert.equal(
+    graph.elements.some(
+      (element) =>
+        element.accessibleName === leakText ||
+        element.text?.includes(leakText) ||
+        element.evidence.some((item) => item.description.includes(leakText)),
+    ),
+    false,
+  );
+  assert.equal(
+    graph.edges.some((edge) =>
+      edge.effects.some(
+        (effect) =>
+          effect.description.includes(leakText) || effect.before?.includes(leakText) || effect.after?.includes(leakText),
+      ),
+    ),
+    false,
+  );
+});
+
+test("explore records locator failures instead of edges or fake effects for unsafe low-risk candidates", async () => {
+  const graph = await withTempOutput("explore-locator-failures", (out) => exploreUrl({ url: fixturePath, maxActions: 10, out }));
+  const entryState = graph.states[0];
+  assert.ok(entryState);
+
+  const duplicates = graph.elements.filter((element) => element.stateId === entryState.stateId && element.accessibleName === "Duplicate");
+  const namelessButton = graph.elements.find((element) => element.stateId === entryState.stateId && element.tag === "button" && element.accessibleName === null);
+
+  assert.equal(duplicates.length, 2);
+  assert.equal(hasEdgeForAccessibleName(graph, "Duplicate"), false);
+  assert.ok(namelessButton);
+  assert.equal(graph.edges.some((edge) => edge.elementId === namelessButton.elementId), false);
+  for (const duplicate of duplicates) {
+    assert.equal(
+      graph.failedObservations.some(
+        (failure) => failure.elementId === duplicate.elementId && failure.reason === "locator_not_unique",
+      ),
+      true,
+    );
+  }
+  assert.equal(
+    graph.failedObservations.some(
+      (failure) => failure.elementId === namelessButton.elementId && failure.reason === "locator_not_found",
+    ),
+    true,
+  );
+  assert.equal(
+    graph.edges.some((edge) => edge.effects.some((effect) => effect.description.includes("Action failed:"))),
+    false,
+  );
+});
+
 test("explore records state-local skipped high-risk actions with evidence", async () => {
-  const graph = await withTempOutput("explore-skips", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
+  const graph = await withTempOutput("explore-skips", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
  assert.equal(graph.skippedActions.some((skip) => skip.reason.includes("form_submit")), true);

  const deleteElement = graph.elements.find((element) => element.accessibleName === "Delete account");