action-topology-engine/tests/integration.test.ts

import assert from "node:assert/strict";
import { test } from "node:test";
import { mkdtemp, rm } from "node:fs/promises";
import { createServer, type Server } from "node:http";
import { tmpdir } from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { exploreUrl, scanUrl } from "../src/engine.js";
import { shortHash } from "../src/hash.js";

const fixturePath = path.resolve("tests/fixtures/context-risk.html");
const fixtureFileUrl = pathToFileURL(fixturePath).href;
const examplePath = path.resolve("examples/simple.html");
const snapshotDiffEffectTypes = ["actionableInventory", "dom", "url", "visibleText"];

async function withTempOutput<T>(name: string, callback: (out: string) => Promise<T>): Promise<T> {
  const dir = await mkdtemp(path.join(tmpdir(), "action-topology-engine-test-"));
  try {
    return await callback(path.join(dir, `${name}.json`));
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
}

async function withServer<T>(handler: Parameters<typeof createServer>[0], callback: (url: string) => Promise<T>): Promise<T> {
  const server = createServer(handler);
  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
  const address = server.address();
  assert.ok(address && typeof address === "object");
  try {
    return await callback(`http://127.0.0.1:${address.port}/`);
  } finally {
    await new Promise<void>((resolve, reject) => {
      (server as Server).close((error) => (error ? reject(error) : resolve()));
    });
  }
}

function hasEdgeForAccessibleName(
  graph: {
    edges: Array<{ elementId: string }>;
    elements: Array<{ elementId: string; accessibleName: string | null }>;
  },
  accessibleName: string,
): boolean {
  return graph.edges.some((edge) => {
    const element = graph.elements.find((item) => item.elementId === edge.elementId);
    return element?.accessibleName === accessibleName;
  });
}

function edgeForAccessibleName(
  graph: {
    edges: Array<{
      edgeId: string;
      elementId: string;
      toStateId: string | null;
      effects: Array<{ type: string; before?: string; after?: string; description: string }>;
    }>;
    elements: Array<{ elementId: string; accessibleName: string | null }>;
  },
  accessibleName: string,
) {
  return graph.edges.find((edge) => {
    const element = graph.elements.find((item) => item.elementId === edge.elementId);
    return element?.accessibleName === accessibleName;
  });
}

test("scan captures a state-local inventory without clicking high-risk actions", async () => {
  const graph = await withTempOutput("scan", (out) => scanUrl({ url: fixturePath, out }));
  assert.equal(graph.states.length, 1);
  assert.equal(graph.edges.length, 0);
  assert.equal(graph.elements.some((element) => element.accessibleName === "Open settings"), true);
  assert.equal(graph.elements.some((element) => element.accessibleName === "Delete account"), false);
});

test("duplicated row actions are verified with semantic locator evidence", async () => {
  const graph = await withTempOutput("duplicate-locators", (out) => scanUrl({ url: fixturePath, out }));
  const editButtons = graph.elements.filter((element) => element.accessibleName === "Edit");
  assert.equal(editButtons.length, 2);
  for (const edit of editButtons) {
    assert.equal(edit.locators.some((locator) => locator.verified && locator.identity?.sameAccessibleName), true);
    assert.equal(edit.locators.some((locator) => locator.scopes.some((scope) => scope.kind === "row")), true);
  }
});

test("graph metadata records generation context and policy versions", async () => {
  const graph = await withTempOutput("metadata", (out) => scanUrl({ url: fixturePath, out }));
  assert.equal(graph.schemaVersion, "0.2");
  assert.equal(graph.metadata.engineVersion, "0.2.2");
  assert.equal(graph.metadata.runtimeContext.browserName, "chromium");
  assert.equal(graph.metadata.runtimeContext.viewport.width, 1440);
  assert.equal(graph.metadata.runtimeContext.viewport.height, 1000);
  assert.equal(graph.metadata.riskPolicyVersion, "risk-policy-v0.2");
  assert.equal(graph.metadata.locatorScoringVersion, "locator-scoring-v0.2");
  assert.equal(typeof graph.metadata.crawlSessionId, "string");
  assert.equal(graph.metadata.inputUrl, fixturePath);
  assert.equal(typeof graph.metadata.inputFingerprint, "string");
});

test("scan records state lifecycle, language, context anchors, and parameter surfaces", async () => {
  const graph = await withTempOutput("evidence", (out) =>
    scanUrl({ url: `${fixtureFileUrl}?utm_source=codex&gclid=ad-click&keep=1#customers`, out }),
  );
  const state = graph.states[0];
  assert.equal(state.lifecycle, "fingerprinted");
  assert.equal(state.language, "en");
  assert.equal(state.viewport.width, 1440);
  assert.equal(state.urlCanonicalKey, `${fixtureFileUrl}?keep=1`);
  assert.equal(state.evidence.some((item) => item.level === "observed" && item.source === "browser.location"), true);
  assert.equal(state.evidence.some((item) => item.level === "derived" && item.source === "snapshot.hash"), true);

  const query = graph.elements.find((element) => element.accessibleName === "Query");
  assert.ok(query);
  assert.equal(query.parameterSurface?.name, "q");
  assert.equal(query.parameterSurface?.required, true);
  assert.equal(query.parameterSurface?.maxLength, 40);
  assert.equal(query.context.form?.name, "Search customers");
  assert.equal(query.evidence.some((item) => item.level === "observed"), true);

  const dialogProbe = graph.elements.find((element) => element.accessibleName === "Dialog probe");
  assert.ok(dialogProbe);
  assert.equal(dialogProbe.context.dialog?.name, null);
  assert.equal(dialogProbe.context.dialog?.text?.includes("noisy descendant copy"), true);

  const shortUnnamedDialog = graph.elements.find(
    (element) => element.tag === "section" && element.role === "dialog" && element.text?.includes("Short volatile copy"),
  );
  assert.ok(shortUnnamedDialog);
  assert.equal(shortUnnamedDialog.context.dialog?.name, null);
  assert.equal(shortUnnamedDialog.context.dialog?.text?.includes("Short volatile copy"), true);
  const shortUnnamedDialogIndex = graph.elements.findIndex((element) => element === shortUnnamedDialog);
  assert.notEqual(shortUnnamedDialogIndex, -1);
  assert.equal(
    shortUnnamedDialog.elementId,
    `el_${shortHash([
      state.stateId,
      shortUnnamedDialogIndex,
      shortUnnamedDialog.tag,
      shortUnnamedDialog.role,
      null,
      null,
      null,
      null,
      shortUnnamedDialog.context.dialog?.name,
      shortUnnamedDialog.context.form?.name,
      shortUnnamedDialog.context.section?.name,
      shortUnnamedDialog.context.landmark?.name,
      shortUnnamedDialog.context.listitem?.name,
      null,
    ])}`,
  );

  const unnamedDialog = graph.elements.find(
    (element) => element.tag === "section" && element.role === "dialog" && element.text?.includes("noisy descendant copy"),
  );
  assert.ok(unnamedDialog);
  const unnamedDialogIndex = graph.elements.findIndex((element) => element === unnamedDialog);
  assert.notEqual(unnamedDialogIndex, -1);
  assert.equal(
    unnamedDialog.elementId,
    `el_${shortHash([
      state.stateId,
      unnamedDialogIndex,
      unnamedDialog.tag,
      unnamedDialog.role,
      null,
      null,
      null,
      null,
      unnamedDialog.context.dialog?.name,
      unnamedDialog.context.form?.name,
      unnamedDialog.context.section?.name,
      unnamedDialog.context.landmark?.name,
      unnamedDialog.context.listitem?.name,
      null,
    ])}`,
  );

  const duplicates = graph.elements.filter((element) => element.accessibleName === "Duplicate");
  assert.equal(duplicates.length, 2);
  assert.notEqual(duplicates[0]?.elementId, duplicates[1]?.elementId);

  const unboundedProbe = graph.elements.find((element) => element.accessibleName === "Unbounded Probe");
  assert.ok(unboundedProbe);
  assert.equal(unboundedProbe.context.form?.name, null);
  assert.equal(unboundedProbe.context.form?.text?.includes("long body of descendant text"), true);

  const mainAction = graph.elements.find((element) => element.accessibleName === "Main action");
  assert.ok(mainAction);
  assert.equal(mainAction.context.section, null);
  assert.equal(mainAction.context.landmark?.name, "Operations");

  const edit = graph.elements.find((element) => element.accessibleName === "Edit");
  assert.ok(edit);
  assert.equal(edit.context.row?.text.includes("Acme Co") || edit.context.row?.text.includes("Beta LLC"), true);
  const edits = graph.elements.filter((element) => element.accessibleName === "Edit");
  assert.equal(edits.length, 2);
  assert.notEqual(edits[0]?.elementId, edits[1]?.elementId);
  assert.notEqual(edits[0]?.context.row?.text, edits[1]?.context.row?.text);
  for (const item of edits) {
    const index = graph.elements.findIndex((element) => element === item);
    assert.notEqual(index, -1);
    assert.equal(
      item.elementId,
      `el_${shortHash([
        state.stateId,
        index,
        item.tag,
        item.role,
        item.attributes["data-testid"] || item.attributes["data-test-id"] || item.attributes["data-test"] || null,
        item.attributes.id || null,
        item.attributes.name || null,
        item.accessibleName,
        item.context.dialog?.name,
        item.context.form?.name,
        item.context.section?.name,
        item.context.landmark?.name,
        item.context.listitem?.name,
        null,
      ])}`,
    );
  }
});

test("explore records low-risk edges and keeps high-risk submit actions out of default execution", async () => {
  const graph = await withTempOutput("explore", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
  assert.equal(graph.edges.some((edge) => edge.riskLevel === "low" && edge.toStateId), true);
  assert.equal(hasEdgeForAccessibleName(graph, "Delete account"), false);
  assert.equal(hasEdgeForAccessibleName(graph, "Submit search"), false);
  assert.equal(graph.states.length >= 2, true);
});

test("explore records edge effects and opened-by state relationships", async () => {
  const graph = await withTempOutput("explore-effects", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
  const entryState = graph.states[0];
  const openSettingsEdge = edgeForAccessibleName(graph, "Open settings");

  assert.ok(entryState);
  assert.ok(openSettingsEdge);
  assert.ok(openSettingsEdge.toStateId);

  const openedState = graph.states.find((state) => state.stateId === openSettingsEdge.toStateId);
  assert.ok(openedState);
  assert.equal(entryState.openedByEdgeId, null);
  assert.equal(openedState.openedByEdgeId, openSettingsEdge.edgeId);

  const visibleTextEffect = openSettingsEdge.effects.find((effect) => effect.type === "visibleText");
  assert.ok(visibleTextEffect);
  assert.equal(visibleTextEffect.before, entryState.visibleTextHash);
  assert.equal(visibleTextEffect.after, openedState.visibleTextHash);
  assert.equal(openSettingsEdge.effects.some((effect) => effect.type === "actionableInventory"), true);
  assert.equal(openSettingsEdge.effects.some((effect) => effect.description === "Actionable inventory changed after action" && effect.type === "dom"), false);
});

test("explore artifact preserves referential integrity and snapshot-diff effect records", async () => {
  const graph = await withTempOutput("explore-integrity", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
  const stateIds = new Set(graph.states.map((state) => state.stateId));
  const elementIds = new Set(graph.elements.map((element) => element.elementId));
  const edgeIds = new Set(graph.edges.map((edge) => edge.edgeId));
  const locatorIds = new Set<string>();
  const skippedActionIds = new Set(graph.skippedActions.map((skip) => skip.skippedActionId));
  const failedObservationIds = new Set(graph.failedObservations.map((failure) => failure.failedObservationId));
  const elementById = new Map(graph.elements.map((element) => [element.elementId, element]));
  const observedEffectTypes = new Set<string>();

  assert.equal(stateIds.size, graph.states.length);
  assert.equal(elementIds.size, graph.elements.length);
  assert.equal(edgeIds.size, graph.edges.length);
  assert.equal(skippedActionIds.size, graph.skippedActions.length);
  assert.equal(failedObservationIds.size, graph.failedObservations.length);

  for (const element of graph.elements) {
    assert.equal(stateIds.has(element.stateId), true);
    for (const locator of element.locators) {
      assert.equal(locator.elementId, element.elementId);
      assert.equal(locatorIds.has(locator.locatorId), false);
      locatorIds.add(locator.locatorId);
    }
  }

  for (const edge of graph.edges) {
    const targetElement = elementById.get(edge.elementId);
    assert.equal(stateIds.has(edge.fromStateId), true);
    if (edge.toStateId !== null) assert.equal(stateIds.has(edge.toStateId), true);
    assert.ok(targetElement);
    assert.equal(targetElement.stateId, edge.fromStateId);
    for (const effect of edge.effects) {
      assert.equal(snapshotDiffEffectTypes.includes(effect.type), true);
      assert.equal(typeof effect.before, "string");
      assert.equal(typeof effect.after, "string");
      observedEffectTypes.add(effect.type);
    }
  }

  assert.deepEqual([...observedEffectTypes].sort(), snapshotDiffEffectTypes);

  for (const state of graph.states) {
    if (state.openedByEdgeId === null) continue;
    const openingEdge = graph.edges.find((edge) => edge.edgeId === state.openedByEdgeId);
    assert.ok(openingEdge);
    assert.equal(edgeIds.has(state.openedByEdgeId), true);
    assert.equal(openingEdge.toStateId, state.stateId);
  }

  for (const skip of graph.skippedActions) {
    const element = graph.elements.find((item) => item.elementId === skip.elementId);
    assert.equal(stateIds.has(skip.stateId), true);
    assert.ok(element);
    assert.equal(skip.stateId, element.stateId);
  }

  for (const failure of graph.failedObservations) {
    if (failure.stateId === null) continue;
    assert.equal(stateIds.has(failure.stateId), true);
    if (failure.elementId === null) continue;
    const element = graph.elements.find((item) => item.elementId === failure.elementId);
    assert.ok(element);
    assert.equal(failure.stateId, element.stateId);
  }
});

test("explore replays each v0.2.1 edge from the entry state", async () => {
  const graph = await withTempOutput("explore-entry-replay", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
  const entryState = graph.states[0];

  assert.ok(entryState);
  assert.equal(graph.edges.length > 0, true);
  assert.equal(graph.edges.every((edge) => edge.fromStateId === entryState.stateId), true);
});

test("explore isolates replay contexts for storage-writing actions", async () => {
  const graph = await withTempOutput("explore-storage-isolation", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
  const entryState = graph.states[0];
  const rememberFilterEdge = edgeForAccessibleName(graph, "Remember filter");
  const leakText = "Replay storage leaked";

  assert.ok(entryState);
  assert.ok(rememberFilterEdge);
  assert.equal(graph.edges.every((edge) => edge.fromStateId === entryState.stateId), true);
  assert.equal(
    graph.states.some(
      (state) =>
        state.title.includes(leakText) ||
        state.url.includes(leakText) ||
        state.route.includes(leakText) ||
        state.evidence.some((item) => item.description.includes(leakText)),
    ),
    false,
  );
  assert.equal(
    graph.elements.some(
      (element) =>
        element.accessibleName === leakText ||
        element.text?.includes(leakText) ||
        element.evidence.some((item) => item.description.includes(leakText)),
    ),
    false,
  );
  assert.equal(
    graph.edges.some((edge) =>
      edge.effects.some(
        (effect) =>
          effect.description.includes(leakText) || effect.before?.includes(leakText) || effect.after?.includes(leakText),
      ),
    ),
    false,
  );
});

test("explore records locator failures instead of edges or fake effects for unsafe low-risk candidates", async () => {
  const graph = await withTempOutput("explore-locator-failures", (out) => exploreUrl({ url: fixturePath, maxActions: 10, out }));
  const entryState = graph.states[0];
  assert.ok(entryState);

  const duplicates = graph.elements.filter((element) => element.stateId === entryState.stateId && element.accessibleName === "Duplicate");
  const namelessButton = graph.elements.find((element) => element.stateId === entryState.stateId && element.tag === "button" && element.accessibleName === null);

  assert.equal(duplicates.length, 2);
  assert.equal(hasEdgeForAccessibleName(graph, "Duplicate"), false);
  assert.ok(namelessButton);
  assert.equal(graph.edges.some((edge) => edge.elementId === namelessButton.elementId), false);
  for (const duplicate of duplicates) {
    assert.equal(
      graph.failedObservations.some(
        (failure) => failure.elementId === duplicate.elementId && failure.reason === "locator_not_unique",
      ),
      true,
    );
  }
  assert.equal(
    graph.failedObservations.some(
      (failure) => failure.elementId === namelessButton.elementId && failure.reason === "locator_not_found",
    ),
    true,
  );
  assert.equal(
    graph.edges.some((edge) => edge.effects.some((effect) => effect.description.includes("Action failed:"))),
    false,
  );
});

test("explore records replay navigation timeout as a failed observation", async () => {
  let requestCount = 0;
  await withServer(
    (_request, response) => {
      requestCount += 1;
      if (requestCount === 1) {
        response.writeHead(200, { "content-type": "text/html; charset=utf-8" });
        response.end("<!doctype html><button>Open panel</button>");
        return;
      }
      // Leave replay navigation hanging so the engine must record a timeout.
    },
    async (url) => {
      const graph = await withTempOutput("explore-navigation-timeout", (out) =>
        exploreUrl({ url, maxActions: 1, navigationTimeoutMs: 50, out }),
      );
      const button = graph.elements.find((element) => element.accessibleName === "Open panel");
      assert.ok(button);
      assert.equal(graph.edges.length, 0);
      assert.equal(
        graph.failedObservations.some(
          (failure) => failure.elementId === button.elementId && failure.reason === "navigation_timeout",
        ),
        true,
      );
    },
  );
});

test("explore records state-local skipped high-risk actions with evidence", async () => {
  const graph = await withTempOutput("explore-skips", (out) => exploreUrl({ url: fixturePath, maxActions: 12, out }));
  assert.equal(graph.skippedActions.some((skip) => skip.reason.includes("form_submit")), true);

  const deleteElement = graph.elements.find((element) => element.accessibleName === "Delete account");
  assert.ok(deleteElement);
  const deleteSkip = graph.skippedActions.find((skip) => skip.elementId === deleteElement.elementId);
  assert.ok(deleteSkip);
  assert.equal(deleteSkip.stateId, deleteElement.stateId);
  assert.equal(deleteSkip.riskCategory, "destructive");
  assert.equal(deleteSkip.riskLevel, "high");

  assert.equal(graph.skippedActions.some((skip) => skip.reason === "allowed:ui_reveal"), false);
  assert.equal(graph.failedObservations.every((failure) => failure.message.length > 0), true);
});

test("disabled low-risk controls are failed observations, not risk skips", async () => {
  const graph = await withTempOutput("explore-disabled", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
  const disabled = graph.elements.find((element) => element.accessibleName === "Open disabled panel");
  assert.ok(disabled);
  assert.equal(graph.skippedActions.some((skip) => skip.elementId === disabled.elementId), false);
  assert.equal(
    graph.failedObservations.some(
      (failure) => failure.elementId === disabled.elementId && failure.reason === "element_disabled",
    ),
    true,
  );
});

test("explore does not record non-click elements as skipped click actions", async () => {
  const graph = await withTempOutput("explore-example-skips", (out) => exploreUrl({ url: examplePath, maxActions: 8, out }));

  const searchInput = graph.elements.find((element) => element.accessibleName === "Search");
  assert.ok(searchInput);
  assert.equal(graph.skippedActions.some((skip) => skip.elementId === searchInput.elementId), false);

  const settingsDialog = graph.elements.find((element) => element.accessibleName === "Settings dialog");
  assert.ok(settingsDialog);
  assert.equal(graph.skippedActions.some((skip) => skip.elementId === settingsDialog.elementId), false);

  const deleteElement = graph.elements.find((element) => element.accessibleName === "Delete account");
  assert.ok(deleteElement);
  assert.equal(graph.skippedActions.some((skip) => skip.elementId === deleteElement.elementId && skip.riskCategory === "destructive"), true);
});

test("disabled high-risk click candidates are failed observations, not risk skips", async () => {
  const graph = await withTempOutput("explore-disabled-high-risk", (out) => exploreUrl({ url: fixturePath, maxActions: 8, out }));
  const disabled = graph.elements.find((element) => element.accessibleName === "Delete disabled account");
  assert.ok(disabled);
  assert.equal(graph.skippedActions.some((skip) => skip.elementId === disabled.elementId), false);
  assert.equal(
    graph.failedObservations.some(
      (failure) => failure.elementId === disabled.elementId && failure.reason === "element_disabled",
    ),
    true,
  );
});