admin/action-topology-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: capture state and element evidence

Browse Source
This commit is contained in:
赵义仑
2026-06-16 19:15:52 +08:00
parent 1324f1882c
commit 581724ea42
6 changed files with 431 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
src/snapshot.ts
View File

@@ -1,7 +1,15 @@
import type { Page } from "playwright";
import { shortHash, stableHash } from "./hash.js";
import { generateLocators, verifyLocators } from "./locator.js";
import type { ActionableElement, RawActionableElement, SnapshotResult, StateNode } from "./types.js";
import type {
ActionableElement,
ContextAnchor,
ElementContext,
ParameterSurface,
RawActionableElement,
SnapshotResult,
StateNode,
} from "./types.js";
export async function snapshotPage(page: Page, openedByEdgeId: string | null = null): Promise<SnapshotResult> {
const raw = await page.evaluate(collectPageSnapshot);
@@ -13,6 +21,7 @@ export async function snapshotPage(page: Page, openedByEdgeId: string | null = n
visibleTextHash: stableHash(raw.visibleText),
actionableSignatureHash: stableHash(raw.elements.map(actionableSignature)),
});
const viewport = raw.viewport;
const state: StateNode = {
stateId: `state_${shortHash(stateHash)}`,
@@ -27,11 +36,38 @@ export async function snapshotPage(page: Page, openedByEdgeId: string | null = n
modalStack: raw.modalStack,
openedByEdgeId,
createdAt,
lifecycle: "fingerprinted",
urlCanonicalKey: raw.urlCanonicalKey,
language: raw.language,
viewport,
evidence: [
{ level: "observed", source: "browser.location", description: "URL and route captured from the page" },
{
level: "derived",
source: "snapshot.hash",
description: "State hashes derived from DOM, visible text, and actionable signatures",
},
],
};
const elements: ActionableElement[] = [];
for (const rawElement of raw.elements) {
const elementId = `el_${shortHash([state.stateId, rawElement.uid, actionableSignature(rawElement)])}`;
const elementId = `el_${shortHash([
state.stateId,
rawElement.candidateIndex,
rawElement.tag,
rawElement.role,
rawElement.attributes["data-testid"] || rawElement.attributes["data-test-id"] || rawElement.attributes["data-test"] || null,
rawElement.attributes.id || null,
rawElement.attributes.name || null,
rawElement.identityText,
rawElement.context.dialog?.name ?? null,
rawElement.context.form?.name ?? null,
rawElement.context.section?.name ?? null,
rawElement.context.landmark?.name ?? null,
rawElement.context.listitem?.name ?? null,
null,
])}`;
const locators = await verifyLocators(page, generateLocators(elementId, rawElement));
elements.push({
elementId,
@@ -48,18 +84,30 @@ export async function snapshotPage(page: Page, openedByEdgeId: string | null = n
shadowPath: [],
interactability: rawElement.visibility,
locators,
context: rawElement.context,
parameterSurface: rawElement.parameterSurface,
evidence: rawElement.evidence,
});
}
return { state, elements, visibleText: raw.visibleText };
}
function boundedIdentityText(value: string | null | undefined): string | null {
const normalized = value?.replace(/\s+/g, " ").trim() ?? "";
if (!normalized || normalized.length > 120) return null;
return normalized;
}
interface BrowserSnapshot {
url: string;
route: string;
title: string;
dom: string;
visibleText: string;
language: string | null;
viewport: { width: number; height: number };
urlCanonicalKey: string;
modalStack: string[];
elements: RawActionableElement[];
}
@@ -102,6 +150,9 @@ function collectPageSnapshot(): BrowserSnapshot {
title: document.title,
dom: document.documentElement.outerHTML,
visibleText: normalizeText(document.body?.innerText || ""),
language: normalizeNullable(document.documentElement.lang || navigator.language || ""),
viewport: { width: window.innerWidth, height: window.innerHeight },
urlCanonicalKey: canonicalUrl(location.href),
modalStack: Array.from(document.querySelectorAll<HTMLElement>("[role='dialog'], dialog[open], [aria-modal='true']"))
.filter((element) => isVisible(element))
.map((element) => getAccessibleName(element) || element.id || element.tagName.toLowerCase()),
@@ -125,15 +176,28 @@ function collectPageSnapshot(): BrowserSnapshot {
const label = getLabel(element);
const placeholder = "placeholder" in element ? normalizeNullable(String((element as HTMLInputElement).placeholder || "")) : null;
const accessibleName = normalizeNullable(getAccessibleName(element) || label || placeholder || text);
const context = getElementContext(element);
const parameterSurface = getParameterSurface(element);
const identityText = getElementIdentityText(element, label, placeholder, text, accessibleName);
const centerX = rect.left + rect.width / 2;
const centerY = rect.top + rect.height / 2;
const topAtCenter = document.elementFromPoint(centerX, centerY);
return {
uid: `${tag}_${index}_${attributes.id || attributes.name || accessibleName || text || ""}`,
uid: `${tag}_${index}_${
attributes.id ||
attributes.name ||
attributes["data-testid"] ||
attributes["data-test-id"] ||
attributes["data-test"] ||
identityText ||
""
}`,
candidateIndex: index,
tag,
role,
accessibleName,
identityText,
text: normalizeNullable(text),
label,
placeholder,
@@ -150,6 +214,132 @@ function collectPageSnapshot(): BrowserSnapshot {
editable: isEditable(element),
receivesEvents: topAtCenter === element || Boolean(topAtCenter && element.contains(topAtCenter)),
},
context,
parameterSurface,
evidence: [
{ level: "observed", source: "dom", description: "Element was visible and matched actionable selector" },
{
level: "derived",
source: "accessibility",
description: "Role, name, label, and placeholder were normalized from DOM evidence",
},
],
};
}
function getElementContext(element: HTMLElement): ElementContext {
return {
dialog: anchor(element.closest<HTMLElement>("[role='dialog'], dialog[open], [aria-modal='true']"), "dialog"),
form: anchor(element.closest<HTMLElement>("form"), "form"),
section: anchor(element.closest<HTMLElement>("section, article"), "section"),
row: anchor(element.closest<HTMLElement>("tr, [role='row']"), "row"),
landmark: anchor(
element.closest<HTMLElement>("nav, aside, header, footer, main, [role='navigation'], [role='main'], [role='complementary']"),
"landmark",
),
listitem: anchor(element.closest<HTMLElement>("li, [role='listitem']"), "listitem"),
};
}
function anchor(element: HTMLElement | null, kind: ContextAnchor["kind"]): ContextAnchor | null {
if (!element) return null;
return {
kind,
name: getAnchorName(element),
text: normalizeNullable((element.innerText || element.textContent || "").slice(0, 240)),
};
}
function getAnchorName(element: HTMLElement): string | null {
return (
boundedAnchorText(element.getAttribute("aria-label")) ??
getAriaLabelledByAnchorName(element) ??
boundedAnchorText(element.getAttribute("title")) ??
boundedAnchorText(element.id) ??
getHeadingAnchorName(element)
);
}
function getAriaLabelledByAnchorName(element: HTMLElement): string | null {
const labelledBy = element.getAttribute("aria-labelledby");
if (!labelledBy) return null;
const text = labelledBy
.split(/\s+/)
.map((id) => document.getElementById(id)?.innerText || document.getElementById(id)?.textContent || "")
.join(" ");
return boundedAnchorText(text);
}
function getHeadingAnchorName(element: HTMLElement): string | null {
const heading = element.querySelector<HTMLElement>("h1, h2, h3, h4, h5, h6, [role='heading']");
if (!heading) return null;
return boundedAnchorText(heading.innerText || heading.textContent || "");
}
function boundedAnchorText(value: string | null | undefined): string | null {
const normalized = normalizeText(value || "");
if (!normalized || normalized.length > 120) return null;
return normalized;
}
function boundedIdentityText(value: string | null | undefined): string | null {
const normalized = normalizeText(value || "");
if (!normalized || normalized.length > 120) return null;
return normalized;
}
function getElementIdentityText(
element: HTMLElement,
label: string | null,
placeholder: string | null,
text: string,
accessibleName: string | null,
): string | null {
return (
boundedIdentityText(element.getAttribute("aria-label")) ??
getAriaLabelledByAnchorName(element) ??
boundedIdentityText(element.getAttribute("title")) ??
boundedIdentityText(label) ??
boundedIdentityText(placeholder) ??
(isContainerLike(element) ? null : boundedIdentityText(accessibleName) ?? boundedIdentityText(text))
);
}
function isContainerLike(element: HTMLElement): boolean {
const tag = element.tagName.toLowerCase();
const role = element.getAttribute("role")?.toLowerCase() || "";
return (
["article", "aside", "dialog", "div", "footer", "form", "header", "li", "main", "nav", "ol", "section", "table", "tbody", "thead", "tr", "ul"].includes(
tag,
) ||
["complementary", "dialog", "form", "grid", "list", "listitem", "main", "navigation", "region", "row", "table"].includes(role)
);
}
function getParameterSurface(element: HTMLElement): ParameterSurface | null {
const tag = element.tagName.toLowerCase();
if (!["input", "select", "textarea"].includes(tag) && !element.isContentEditable) return null;
const input = element as HTMLInputElement;
const select = element as HTMLSelectElement;
return {
name: input.name || element.getAttribute("name"),
inputType: tag === "input" ? (input.type || "text").toLowerCase() : tag,
required: Boolean((input as HTMLInputElement).required),
readonly: Boolean((input as HTMLInputElement).readOnly),
disabled: Boolean((input as HTMLInputElement).disabled),
pattern: input.getAttribute("pattern"),
min: input.getAttribute("min"),
max: input.getAttribute("max"),
maxLength: Number.isFinite(input.maxLength) && input.maxLength >= 0 ? input.maxLength : null,
options:
tag === "select"
? Array.from(select.options).map((option) => ({
label: normalizeText(option.label || option.textContent || ""),
value: option.value,
disabled: option.disabled,
selected: option.selected,
}))
: [],
};
}
@@ -247,5 +437,15 @@ function collectPageSnapshot(): BrowserSnapshot {
const normalized = normalizeText(value);
return normalized || null;
}
}
function canonicalUrl(value: string): string {
const url = new URL(value);
url.hash = "";
const params = Array.from(url.searchParams.entries())
.filter(([key]) => !/^utm_|^fbclid$|^gclid$/.test(key))
.sort(([left], [right]) => left.localeCompare(right));
url.search = "";
for (const [key, item] of params) url.searchParams.append(key, item);
return url.toString();
}
}

41
src/types.ts
View File

@@ -11,6 +11,34 @@ export interface EvidenceRecord {
description: string;
}
export interface ContextAnchor {
kind: "dialog" | "form" | "section" | "row" | "landmark" | "listitem";
name: string | null;
text: string | null;
}
export interface ElementContext {
dialog: ContextAnchor | null;
form: ContextAnchor | null;
section: ContextAnchor | null;
row: ContextAnchor | null;
landmark: ContextAnchor | null;
listitem: ContextAnchor | null;
}
export interface ParameterSurface {
name: string | null;
inputType: string | null;
required: boolean;
readonly: boolean;
disabled: boolean;
pattern: string | null;
min: string | null;
max: string | null;
maxLength: number | null;
options: Array<{ label: string; value: string; disabled: boolean; selected: boolean }>;
}
export interface RuntimeContext {
browserName: "chromium";
viewport: { width: number; height: number };
@@ -73,6 +101,9 @@ export interface ActionableElement {
shadowPath: string[];
interactability: VisibilityInfo;
locators: LocatorCandidate[];
context: ElementContext;
parameterSurface: ParameterSurface | null;
evidence: EvidenceRecord[];
}
export interface StateNode {
@@ -88,6 +119,11 @@ export interface StateNode {
modalStack: string[];
openedByEdgeId: string | null;
createdAt: string;
lifecycle: StateLifecycle;
urlCanonicalKey: string;
language: string | null;
viewport: { width: number; height: number };
evidence: EvidenceRecord[];
}
export type ActionType = "click" | "fill" | "select" | "check" | "uncheck" | "navigate";
@@ -194,15 +230,20 @@ export interface FailedObservation {
export interface RawActionableElement {
uid: string;
candidateIndex: number;
tag: string;
role: string | null;
accessibleName: string | null;
identityText: string | null;
text: string | null;
label: string | null;
placeholder: string | null;
attributes: Record<string, string>;
bbox: BoundingBox | null;
visibility: VisibilityInfo;
context: ElementContext;
parameterSurface: ParameterSurface | null;
evidence: EvidenceRecord[];
}
export interface SnapshotResult {