diff --git a/src/snapshot.ts b/src/snapshot.ts index c2d6c30..8219f3a 100644 --- a/src/snapshot.ts +++ b/src/snapshot.ts @@ -1,7 +1,15 @@ import type { Page } from "playwright"; import { shortHash, stableHash } from "./hash.js"; import { generateLocators, verifyLocators } from "./locator.js"; -import type { ActionableElement, RawActionableElement, SnapshotResult, StateNode } from "./types.js"; +import type { + ActionableElement, + ContextAnchor, + ElementContext, + ParameterSurface, + RawActionableElement, + SnapshotResult, + StateNode, +} from "./types.js"; export async function snapshotPage(page: Page, openedByEdgeId: string | null = null): Promise { const raw = await page.evaluate(collectPageSnapshot); @@ -13,6 +21,7 @@ export async function snapshotPage(page: Page, openedByEdgeId: string | null = n visibleTextHash: stableHash(raw.visibleText), actionableSignatureHash: stableHash(raw.elements.map(actionableSignature)), }); + const viewport = raw.viewport; const state: StateNode = { stateId: `state_${shortHash(stateHash)}`, @@ -27,11 +36,38 @@ export async function snapshotPage(page: Page, openedByEdgeId: string | null = n modalStack: raw.modalStack, openedByEdgeId, createdAt, + lifecycle: "fingerprinted", + urlCanonicalKey: raw.urlCanonicalKey, + language: raw.language, + viewport, + evidence: [ + { level: "observed", source: "browser.location", description: "URL and route captured from the page" }, + { + level: "derived", + source: "snapshot.hash", + description: "State hashes derived from DOM, visible text, and actionable signatures", + }, + ], }; const elements: ActionableElement[] = []; for (const rawElement of raw.elements) { - const elementId = `el_${shortHash([state.stateId, rawElement.uid, actionableSignature(rawElement)])}`; + const elementId = `el_${shortHash([ + state.stateId, + rawElement.candidateIndex, + rawElement.tag, + rawElement.role, + rawElement.attributes["data-testid"] || rawElement.attributes["data-test-id"] || rawElement.attributes["data-test"] || null, + rawElement.attributes.id || null, + rawElement.attributes.name || null, + rawElement.identityText, + rawElement.context.dialog?.name ?? null, + rawElement.context.form?.name ?? null, + rawElement.context.section?.name ?? null, + rawElement.context.landmark?.name ?? null, + rawElement.context.listitem?.name ?? null, + null, + ])}`; const locators = await verifyLocators(page, generateLocators(elementId, rawElement)); elements.push({ elementId, @@ -48,18 +84,30 @@ export async function snapshotPage(page: Page, openedByEdgeId: string | null = n shadowPath: [], interactability: rawElement.visibility, locators, + context: rawElement.context, + parameterSurface: rawElement.parameterSurface, + evidence: rawElement.evidence, }); } return { state, elements, visibleText: raw.visibleText }; } +function boundedIdentityText(value: string | null | undefined): string | null { + const normalized = value?.replace(/\s+/g, " ").trim() ?? ""; + if (!normalized || normalized.length > 120) return null; + return normalized; +} + interface BrowserSnapshot { url: string; route: string; title: string; dom: string; visibleText: string; + language: string | null; + viewport: { width: number; height: number }; + urlCanonicalKey: string; modalStack: string[]; elements: RawActionableElement[]; } @@ -102,6 +150,9 @@ function collectPageSnapshot(): BrowserSnapshot { title: document.title, dom: document.documentElement.outerHTML, visibleText: normalizeText(document.body?.innerText || ""), + language: normalizeNullable(document.documentElement.lang || navigator.language || ""), + viewport: { width: window.innerWidth, height: window.innerHeight }, + urlCanonicalKey: canonicalUrl(location.href), modalStack: Array.from(document.querySelectorAll("[role='dialog'], dialog[open], [aria-modal='true']")) .filter((element) => isVisible(element)) .map((element) => getAccessibleName(element) || element.id || element.tagName.toLowerCase()), @@ -125,15 +176,28 @@ function collectPageSnapshot(): BrowserSnapshot { const label = getLabel(element); const placeholder = "placeholder" in element ? normalizeNullable(String((element as HTMLInputElement).placeholder || "")) : null; const accessibleName = normalizeNullable(getAccessibleName(element) || label || placeholder || text); + const context = getElementContext(element); + const parameterSurface = getParameterSurface(element); + const identityText = getElementIdentityText(element, label, placeholder, text, accessibleName); const centerX = rect.left + rect.width / 2; const centerY = rect.top + rect.height / 2; const topAtCenter = document.elementFromPoint(centerX, centerY); return { - uid: `${tag}_${index}_${attributes.id || attributes.name || accessibleName || text || ""}`, + uid: `${tag}_${index}_${ + attributes.id || + attributes.name || + attributes["data-testid"] || + attributes["data-test-id"] || + attributes["data-test"] || + identityText || + "" + }`, + candidateIndex: index, tag, role, accessibleName, + identityText, text: normalizeNullable(text), label, placeholder, @@ -150,6 +214,132 @@ function collectPageSnapshot(): BrowserSnapshot { editable: isEditable(element), receivesEvents: topAtCenter === element || Boolean(topAtCenter && element.contains(topAtCenter)), }, + context, + parameterSurface, + evidence: [ + { level: "observed", source: "dom", description: "Element was visible and matched actionable selector" }, + { + level: "derived", + source: "accessibility", + description: "Role, name, label, and placeholder were normalized from DOM evidence", + }, + ], + }; + } + + function getElementContext(element: HTMLElement): ElementContext { + return { + dialog: anchor(element.closest("[role='dialog'], dialog[open], [aria-modal='true']"), "dialog"), + form: anchor(element.closest("form"), "form"), + section: anchor(element.closest("section, article"), "section"), + row: anchor(element.closest("tr, [role='row']"), "row"), + landmark: anchor( + element.closest("nav, aside, header, footer, main, [role='navigation'], [role='main'], [role='complementary']"), + "landmark", + ), + listitem: anchor(element.closest("li, [role='listitem']"), "listitem"), + }; + } + + function anchor(element: HTMLElement | null, kind: ContextAnchor["kind"]): ContextAnchor | null { + if (!element) return null; + return { + kind, + name: getAnchorName(element), + text: normalizeNullable((element.innerText || element.textContent || "").slice(0, 240)), + }; + } + + function getAnchorName(element: HTMLElement): string | null { + return ( + boundedAnchorText(element.getAttribute("aria-label")) ?? + getAriaLabelledByAnchorName(element) ?? + boundedAnchorText(element.getAttribute("title")) ?? + boundedAnchorText(element.id) ?? + getHeadingAnchorName(element) + ); + } + + function getAriaLabelledByAnchorName(element: HTMLElement): string | null { + const labelledBy = element.getAttribute("aria-labelledby"); + if (!labelledBy) return null; + const text = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.innerText || document.getElementById(id)?.textContent || "") + .join(" "); + return boundedAnchorText(text); + } + + function getHeadingAnchorName(element: HTMLElement): string | null { + const heading = element.querySelector("h1, h2, h3, h4, h5, h6, [role='heading']"); + if (!heading) return null; + return boundedAnchorText(heading.innerText || heading.textContent || ""); + } + + function boundedAnchorText(value: string | null | undefined): string | null { + const normalized = normalizeText(value || ""); + if (!normalized || normalized.length > 120) return null; + return normalized; + } + + function boundedIdentityText(value: string | null | undefined): string | null { + const normalized = normalizeText(value || ""); + if (!normalized || normalized.length > 120) return null; + return normalized; + } + + function getElementIdentityText( + element: HTMLElement, + label: string | null, + placeholder: string | null, + text: string, + accessibleName: string | null, + ): string | null { + return ( + boundedIdentityText(element.getAttribute("aria-label")) ?? + getAriaLabelledByAnchorName(element) ?? + boundedIdentityText(element.getAttribute("title")) ?? + boundedIdentityText(label) ?? + boundedIdentityText(placeholder) ?? + (isContainerLike(element) ? null : boundedIdentityText(accessibleName) ?? boundedIdentityText(text)) + ); + } + + function isContainerLike(element: HTMLElement): boolean { + const tag = element.tagName.toLowerCase(); + const role = element.getAttribute("role")?.toLowerCase() || ""; + return ( + ["article", "aside", "dialog", "div", "footer", "form", "header", "li", "main", "nav", "ol", "section", "table", "tbody", "thead", "tr", "ul"].includes( + tag, + ) || + ["complementary", "dialog", "form", "grid", "list", "listitem", "main", "navigation", "region", "row", "table"].includes(role) + ); + } + + function getParameterSurface(element: HTMLElement): ParameterSurface | null { + const tag = element.tagName.toLowerCase(); + if (!["input", "select", "textarea"].includes(tag) && !element.isContentEditable) return null; + const input = element as HTMLInputElement; + const select = element as HTMLSelectElement; + return { + name: input.name || element.getAttribute("name"), + inputType: tag === "input" ? (input.type || "text").toLowerCase() : tag, + required: Boolean((input as HTMLInputElement).required), + readonly: Boolean((input as HTMLInputElement).readOnly), + disabled: Boolean((input as HTMLInputElement).disabled), + pattern: input.getAttribute("pattern"), + min: input.getAttribute("min"), + max: input.getAttribute("max"), + maxLength: Number.isFinite(input.maxLength) && input.maxLength >= 0 ? input.maxLength : null, + options: + tag === "select" + ? Array.from(select.options).map((option) => ({ + label: normalizeText(option.label || option.textContent || ""), + value: option.value, + disabled: option.disabled, + selected: option.selected, + })) + : [], }; } @@ -247,5 +437,15 @@ function collectPageSnapshot(): BrowserSnapshot { const normalized = normalizeText(value); return normalized || null; } -} + function canonicalUrl(value: string): string { + const url = new URL(value); + url.hash = ""; + const params = Array.from(url.searchParams.entries()) + .filter(([key]) => !/^utm_|^fbclid$|^gclid$/.test(key)) + .sort(([left], [right]) => left.localeCompare(right)); + url.search = ""; + for (const [key, item] of params) url.searchParams.append(key, item); + return url.toString(); + } +} diff --git a/src/types.ts b/src/types.ts index 4a77f7e..52196ca 100644 --- a/src/types.ts +++ b/src/types.ts @@ -11,6 +11,34 @@ export interface EvidenceRecord { description: string; } +export interface ContextAnchor { + kind: "dialog" | "form" | "section" | "row" | "landmark" | "listitem"; + name: string | null; + text: string | null; +} + +export interface ElementContext { + dialog: ContextAnchor | null; + form: ContextAnchor | null; + section: ContextAnchor | null; + row: ContextAnchor | null; + landmark: ContextAnchor | null; + listitem: ContextAnchor | null; +} + +export interface ParameterSurface { + name: string | null; + inputType: string | null; + required: boolean; + readonly: boolean; + disabled: boolean; + pattern: string | null; + min: string | null; + max: string | null; + maxLength: number | null; + options: Array<{ label: string; value: string; disabled: boolean; selected: boolean }>; +} + export interface RuntimeContext { browserName: "chromium"; viewport: { width: number; height: number }; @@ -73,6 +101,9 @@ export interface ActionableElement { shadowPath: string[]; interactability: VisibilityInfo; locators: LocatorCandidate[]; + context: ElementContext; + parameterSurface: ParameterSurface | null; + evidence: EvidenceRecord[]; } export interface StateNode { @@ -88,6 +119,11 @@ export interface StateNode { modalStack: string[]; openedByEdgeId: string | null; createdAt: string; + lifecycle: StateLifecycle; + urlCanonicalKey: string; + language: string | null; + viewport: { width: number; height: number }; + evidence: EvidenceRecord[]; } export type ActionType = "click" | "fill" | "select" | "check" | "uncheck" | "navigate"; @@ -194,15 +230,20 @@ export interface FailedObservation { export interface RawActionableElement { uid: string; + candidateIndex: number; tag: string; role: string | null; accessibleName: string | null; + identityText: string | null; text: string | null; label: string | null; placeholder: string | null; attributes: Record; bbox: BoundingBox | null; visibility: VisibilityInfo; + context: ElementContext; + parameterSurface: ParameterSurface | null; + evidence: EvidenceRecord[]; } export interface SnapshotResult { diff --git a/tests/fixtures/context-risk.html b/tests/fixtures/context-risk.html index 258b324..c3b9055 100644 --- a/tests/fixtures/context-risk.html +++ b/tests/fixtures/context-risk.html @@ -58,6 +58,42 @@ +
+

+ This unnamed form deliberately contains a long body of descendant text that should remain available as bounded + context evidence but must not be promoted into the anchor name used for element identity. It describes internal + notes, operator workflow details, historical context, and extra explanatory copy that would make a noisy and + unstable identifier if copied into the context anchor name. +

+ +
+ +
+ +
+ +
+ + +
+ + + + +