feat: verify explored edges by replay

2026-06-16 20:45:04 +08:00
parent 347d68626e
commit 1ea4c52ad1
5 changed files with 337 additions and 122 deletions
--- a/tests/fixtures/context-risk.html
+++ b/tests/fixtures/context-risk.html
@@ -11,11 +11,14 @@
      button, input, select, a { margin: 4px; padding: 6px 10px; }
      [role="dialog"] { display: none; border: 1px solid #999; padding: 16px; margin-top: 16px; }
      [role="dialog"].open { display: block; }
+      #settings-dialog.open { position: fixed; inset: 24px auto auto 24px; background: white; z-index: 10; }
      #toast { margin-top: 16px; min-height: 24px; }
    </style>
  </head>
  <body>
    <h1>Accounts</h1>
+    <button id="storage-leak-marker" hidden>Replay storage leaked</button>
+    <button data-testid="remember-filter" onclick="localStorage.setItem('ateReplayLeak', '1')">Remember filter</button>
    <nav aria-label="Primary">
      <a href="#overview" data-testid="overview-link">Overview</a>
      <a href="#reports" data-testid="reports-link">Reports</a>
@@ -78,6 +81,7 @@
    <section aria-label="Duplicate controls">
      <button>Duplicate</button>
      <button>Duplicate</button>
+      <button><span aria-hidden="true"></span></button>
    </section>

    <section role="dialog" aria-modal="true" class="open">
@@ -115,6 +119,10 @@

    <p id="toast" role="status"></p>
    <script>
+      if (localStorage.getItem("ateReplayLeak")) {
+        document.getElementById("storage-leak-marker").hidden = false;
+      }
+
      function openEditor(name) {
        document.getElementById("customer-name").value = name;
        document.getElementById("editor-title").textContent = "Editing " + name;
--- a/tests/integration.test.ts
+++ b/tests/integration.test.ts
@@ -33,6 +33,24 @@ function hasEdgeForAccessibleName(
  });
 }

+function edgeForAccessibleName(
+  graph: {
+    edges: Array<{
+      edgeId: string;
+      elementId: string;
+      toStateId: string | null;
+      effects: Array<{ type: string; before?: string; after?: string; description: string }>;
+    }>;
+    elements: Array<{ elementId: string; accessibleName: string | null }>;
+  },
+  accessibleName: string,
+) {
+  return graph.edges.find((edge) => {
+    const element = graph.elements.find((item) => item.elementId === edge.elementId);
+    return element?.accessibleName === accessibleName;
+  });
+}
+
 test("scan captures a state-local inventory without clicking high-risk actions", async () => {
  const graph = await withTempOutput("scan", (out) => scanUrl({ url: fixturePath, out }));
  assert.equal(graph.states.length, 1);
@@ -198,8 +216,110 @@ test("explore records low-risk edges and keeps high-risk submit actions out of d
  assert.equal(graph.states.length >= 2, true);
 });

+test("explore records edge effects and opened-by state relationships", async () => {
+  const graph = await withTempOutput("explore-effects", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
+  const entryState = graph.states[0];
+  const openSettingsEdge = edgeForAccessibleName(graph, "Open settings");
+
+  assert.ok(entryState);
+  assert.ok(openSettingsEdge);
+  assert.ok(openSettingsEdge.toStateId);
+
+  const openedState = graph.states.find((state) => state.stateId === openSettingsEdge.toStateId);
+  assert.ok(openedState);
+  assert.equal(entryState.openedByEdgeId, null);
+  assert.equal(openedState.openedByEdgeId, openSettingsEdge.edgeId);
+
+  const visibleTextEffect = openSettingsEdge.effects.find((effect) => effect.type === "visibleText");
+  assert.ok(visibleTextEffect);
+  assert.equal(visibleTextEffect.before, entryState.visibleTextHash);
+  assert.equal(visibleTextEffect.after, openedState.visibleTextHash);
+  assert.equal(openSettingsEdge.effects.some((effect) => effect.type === "actionableInventory"), true);
+  assert.equal(openSettingsEdge.effects.some((effect) => effect.description === "Actionable inventory changed after action" && effect.type === "dom"), false);
+});
+
+test("explore replays each v0.2.1 edge from the entry state", async () => {
+  const graph = await withTempOutput("explore-entry-replay", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
+  const entryState = graph.states[0];
+
+  assert.ok(entryState);
+  assert.equal(graph.edges.length > 0, true);
+  assert.equal(graph.edges.every((edge) => edge.fromStateId === entryState.stateId), true);
+});
+
+test("explore isolates replay contexts for storage-writing actions", async () => {
+  const graph = await withTempOutput("explore-storage-isolation", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
+  const entryState = graph.states[0];
+  const rememberFilterEdge = edgeForAccessibleName(graph, "Remember filter");
+  const leakText = "Replay storage leaked";
+
+  assert.ok(entryState);
+  assert.ok(rememberFilterEdge);
+  assert.equal(graph.edges.every((edge) => edge.fromStateId === entryState.stateId), true);
+  assert.equal(
+    graph.states.some(
+      (state) =>
+        state.title.includes(leakText) ||
+        state.url.includes(leakText) ||
+        state.route.includes(leakText) ||
+        state.evidence.some((item) => item.description.includes(leakText)),
+    ),
+    false,
+  );
+  assert.equal(
+    graph.elements.some(
+      (element) =>
+        element.accessibleName === leakText ||
+        element.text?.includes(leakText) ||
+        element.evidence.some((item) => item.description.includes(leakText)),
+    ),
+    false,
+  );
+  assert.equal(
+    graph.edges.some((edge) =>
+      edge.effects.some(
+        (effect) =>
+          effect.description.includes(leakText) || effect.before?.includes(leakText) || effect.after?.includes(leakText),
+      ),
+    ),
+    false,
+  );
+});
+
+test("explore records locator failures instead of edges or fake effects for unsafe low-risk candidates", async () => {
+  const graph = await withTempOutput("explore-locator-failures", (out) => exploreUrl({ url: fixturePath, maxActions: 10, out }));
+  const entryState = graph.states[0];
+  assert.ok(entryState);
+
+  const duplicates = graph.elements.filter((element) => element.stateId === entryState.stateId && element.accessibleName === "Duplicate");
+  const namelessButton = graph.elements.find((element) => element.stateId === entryState.stateId && element.tag === "button" && element.accessibleName === null);
+
+  assert.equal(duplicates.length, 2);
+  assert.equal(hasEdgeForAccessibleName(graph, "Duplicate"), false);
+  assert.ok(namelessButton);
+  assert.equal(graph.edges.some((edge) => edge.elementId === namelessButton.elementId), false);
+  for (const duplicate of duplicates) {
+    assert.equal(
+      graph.failedObservations.some(
+        (failure) => failure.elementId === duplicate.elementId && failure.reason === "locator_not_unique",
+      ),
+      true,
+    );
+  }
+  assert.equal(
+    graph.failedObservations.some(
+      (failure) => failure.elementId === namelessButton.elementId && failure.reason === "locator_not_found",
+    ),
+    true,
+  );
+  assert.equal(
+    graph.edges.some((edge) => edge.effects.some((effect) => effect.description.includes("Action failed:"))),
+    false,
+  );
+});
+
 test("explore records state-local skipped high-risk actions with evidence", async () => {
-  const graph = await withTempOutput("explore-skips", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
+  const graph = await withTempOutput("explore-skips", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
  assert.equal(graph.skippedActions.some((skip) => skip.reason.includes("form_submit")), true);

  const deleteElement = graph.elements.find((element) => element.accessibleName === "Delete account");