From 8aa952a491618d98599cf5674c3fb51ec50551f0 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 11:16:53 +0800 Subject: [PATCH 1/6] feat: add a-route uia send connector --- docs/current-status-card.md | 13 + docs/go-mcp-runbook.md | 33 ++- .../2026-07-11-a-route-uia-send.md | 30 +++ .../plans/2026-07-11-a-route-rpa-send.md | 95 +++++++ internal/mcpserver/server.go | 8 +- internal/mcpserver/server_test.go | 57 +++++ internal/tools/isphere_send_message.go | 14 + internal/tools/isphere_send_message_test.go | 64 ++++- internal/tools/send_message_connector.go | 16 +- internal/tools/send_message_uia_adapter.go | 239 ++++++++++++++++++ .../tools/send_message_uia_adapter_test.go | 117 +++++++++ native/ISphereWinHelper/Program.cs | 9 +- native/ISphereWinHelper/UiaSendAction.cs | 214 ++++++++++++++++ scripts/verify-go-mcp.ps1 | 4 +- scripts/verify-win-helper.ps1 | 47 +++- 15 files changed, 941 insertions(+), 19 deletions(-) create mode 100644 docs/source-discovery/2026-07-11-a-route-uia-send.md create mode 100644 docs/superpowers/plans/2026-07-11-a-route-rpa-send.md create mode 100644 internal/tools/send_message_uia_adapter.go create mode 100644 internal/tools/send_message_uia_adapter_test.go create mode 100644 native/ISphereWinHelper/UiaSendAction.cs diff --git a/docs/current-status-card.md b/docs/current-status-card.md index 892b8d9..fbd1609 100644 --- a/docs/current-status-card.md +++ b/docs/current-status-card.md @@ -383,3 +383,16 @@ R14 release-candidate report is complete: - Updated `docs\go-mcp-runbook.md` with the R14 business status and package paths. - Verification passed: package builders/verifiers, `git diff --check`, `scripts\verify-business-goals-smoke.ps1`, `go test ./...`, `go build ./cmd/isphere-mcp`, and `scripts\verify-go-mcp.ps1`. - Final R14 business conclusion: search contacts, search groups, receive messages, and receive-file list are usable; send-message preview and send-file preview are usable; production send, real file download, and production file upload remain evidence-blocked. + + +## A-route UIA send branch + +A-route UIA send branch is in progress on `codex/a-route-rpa-send`: + +- User direction: do not add a product approval gate; implement function first and let the digital employee layer decide when to call it. +- Added plan `docs\superpowers\plans\2026-07-11-a-route-rpa-send.md`. +- Go connector request now carries raw `ContentText` only to the action connector; response and audit remain redacted. +- Added Go `uia-rpa` send connector adapter and env loader. +- Added C# helper op `uia_send_message` that sets the send editor text and invokes the send button through UI Automation. +- Synthetic local verification in `scripts\verify-win-helper.ps1` proves the helper can write to a WinForms send box and trigger a send button marker. +- Default MCP smoke still clears A-route env and remains preview/blocked unless explicitly configured. diff --git a/docs/go-mcp-runbook.md b/docs/go-mcp-runbook.md index 98c33bb..b29013a 100644 --- a/docs/go-mcp-runbook.md +++ b/docs/go-mcp-runbook.md @@ -574,7 +574,38 @@ runs\send-file-sandbox-gate-package.zip They are generated under ignored `runs\` and are not committed. -## 18. Troubleshooting +## 18. A-route UIA send connector + +A-route is the UI Automation fallback for text sending when the B-route sidecar cannot be tested. It is function-first: there is no human approval ID gate inside the connector. The digital employee layer is responsible for deciding when to call production mode. + +Local verification uses a synthetic WinForms window: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-win-helper.ps1 +``` + +Expected output includes helper version `0.5.0` and: + +```json +{"synthetic_uia_send_action":"uia_send_message"} +``` + +To enable A-route in a logged-in desktop session, configure the MCP server environment before startup: + +```powershell +$env:ISPHERE_SEND_CONNECTOR_MODE = "uia_rpa" +$env:ISPHERE_SEND_UIA_HWND = "0x001A0B2C" +$env:ISPHERE_SEND_UIA_EDITOR_AUTOMATION_ID = "rtbSendMessage" +$env:ISPHERE_SEND_UIA_BUTTON_AUTOMATION_ID = "btnSend" +# optional if not using runs\win-helper\ISphereWinHelper.exe +$env:ISPHERE_SEND_UIA_HELPER_PATH = "E:\coding\codex\isphere-ai-bridge\runs\win-helper\ISphereWinHelper.exe" +``` + +Then call `isphere_send_message` with `execution_mode="production"`. The connector will set the send editor text and invoke the send button through UI Automation. Response metadata uses `connector_mode="uia-rpa"`; the audit still stores hashes and metadata, not the raw message body or raw idempotency key. + +Standard `scripts\verify-go-mcp.ps1` clears all A-route env vars so the deterministic smoke remains preview/blocked by default. + +## 19. Troubleshooting - If C# helper build fails, run `scripts\build-win-helper.ps1` directly and check for missing .NET Framework reference assemblies. - If `win_helper_version` fails, rerun `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-win-helper.ps1` first. diff --git a/docs/source-discovery/2026-07-11-a-route-uia-send.md b/docs/source-discovery/2026-07-11-a-route-uia-send.md new file mode 100644 index 0000000..0f9a05c --- /dev/null +++ b/docs/source-discovery/2026-07-11-a-route-uia-send.md @@ -0,0 +1,30 @@ +# A-route UIA Send Implementation Note + +Date: 2026-07-11 +Branch: `codex/a-route-rpa-send` + +## Decision + +The A-route fallback is now function-first. It does not introduce an approval ID gate inside the MCP tool or connector. Digital employee policy can decide when to call `execution_mode="production"`; the connector focuses on performing the UI action when explicitly configured. + +## Implemented local-safe proof + +- `SendMessageConnectorRequest` now carries `ContentText` to the connector only. +- `internal/tools/send_message_uia_adapter.go` maps production send requests to helper op `uia_send_message`. +- `native/ISphereWinHelper/UiaSendAction.cs` sets the send editor text and invokes the send button by UI Automation / Win32 fallback. +- `scripts/verify-win-helper.ps1` uses a synthetic WinForms window to prove write + button invoke without requiring iSphere login. + +## Runtime configuration + +```powershell +$env:ISPHERE_SEND_CONNECTOR_MODE = "uia_rpa" +$env:ISPHERE_SEND_UIA_HWND = "0x001A0B2C" +$env:ISPHERE_SEND_UIA_EDITOR_AUTOMATION_ID = "rtbSendMessage" +$env:ISPHERE_SEND_UIA_BUTTON_AUTOMATION_ID = "btnSend" +``` + +Then call `isphere_send_message` with `execution_mode="production"`. + +## Remaining real-environment work + +The local environment still cannot log in to iSphere. The code path is implemented and locally proven against synthetic UIA, but real iSphere success still needs a logged-in window handle and one real run in the online environment. diff --git a/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md b/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md new file mode 100644 index 0000000..05d4825 --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md @@ -0,0 +1,95 @@ +# A-Route RPA Send Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 在 `codex/a-route-rpa-send` 分支实现 A 方案：通过受控 UI Automation 路线让 `isphere_send_message` 能配置为真实 UI 写入并点击发送。 + +**Architecture:** Go MCP 继续保留稳定 `isphere_send_message` 接口、hash、idempotency、audit。新增 A-route connector 调用 C# `ISphereWinHelper` 的 UIA action op；C# helper 根据窗口句柄、发送框 AutomationId、发送按钮 AutomationId 写入文本并点击。B-route sidecar 仍是主路线，但当前不可测时先实现 A-route 功能兜底。 + +**Tech Stack:** Go 1.23.4, github.com/modelcontextprotocol/go-sdk, Windows PowerShell, .NET Framework C# helper, Windows UI Automation. + +## Global Constraints + +- Repository root: `E:\coding\codex\isphere-ai-bridge`. +- Active branch: `codex/a-route-rpa-send`. +- Do not use `rg`; use `git ls-files`, `ag`, `grep -R`, or PowerShell commands. +- Local machine cannot log in; use synthetic WinForms/UIA verification for local tests. +- User explicitly removed human approval as a product gate for A-route; do not add approval IDs or approval queues in this plan. +- Keep audit redaction: do not store raw message body or raw idempotency key in committed docs/audit tests. +- Real UI action requires explicit env configuration; default MCP behavior remains preview/blocked. + +--- + +## Task 1: Go connector must carry raw content to action connector + +**Files:** +- Modify: `internal/tools/send_message_connector.go` +- Modify: `internal/tools/isphere_send_message.go` +- Modify: `internal/tools/isphere_send_message_test.go` + +**Interfaces:** +- Produces: `SendMessageConnectorRequest.ContentText string` for connector execution only. +- Audit and response still omit raw content. + +- [x] Write failing test proving injected connector receives raw `ContentText` while response/audit do not leak it. +- [x] Run focused test and confirm failure. +- [x] Add `ContentText` to normalized args and connector request. +- [x] Run focused send-message tests. + +## Task 2: Go A-route UIA connector adapter + +**Files:** +- Create: `internal/tools/send_message_uia_adapter.go` +- Create: `internal/tools/send_message_uia_adapter_test.go` + +**Interfaces:** +- Produces: `type UiaSendMessageAdapterConfig struct { HelperPath string; TimeoutSeconds int; Hwnd string; SendEditorAutomationID string; SendButtonAutomationID string; Mode string }`. +- Produces: `NewUiaSendMessageConnector(config UiaSendMessageAdapterConfig, caller UiaHelperCaller) SendMessageConnector`. +- Helper op: `uia_send_message`. + +- [x] Write failing tests for missing config and successful helper call using a fake helper caller. +- [x] Implement adapter request mapping. +- [x] Run focused adapter tests. + +## Task 3: C# WinHelper UIA send action + +**Files:** +- Create: `native/ISphereWinHelper/UiaSendAction.cs` +- Modify: `native/ISphereWinHelper/Program.cs` +- Modify: `scripts/verify-win-helper.ps1` + +**Interfaces:** +- Helper op `uia_send_message` args: `hwnd`, `send_editor_automation_id`, `send_button_automation_id`, `content_text`, `content_sha256`, `target_ref`. +- Helper data: `action_mode="uia_send_message"`, `typed_text=true`, `clicked_ui=true`, `sent_message=true`, `content_sha256`, `target_ref`, `editor_found`, `button_found`. + +- [x] Add synthetic WinForms verification that starts a form with `rtbSendMessage`, `btnSend`, invokes `uia_send_message`, and verifies button-click side effect in a label. +- [x] Implement `UiaSendAction` with ValuePattern/Win32 fallback and InvokePattern/click fallback. +- [x] Run `scripts\verify-win-helper.ps1`. + +## Task 4: Wire A-route connector into MCP env config + +**Files:** +- Modify: `internal/tools/isphere_send_message.go` +- Modify: `internal/mcpserver/server.go` +- Modify: `scripts/verify-go-mcp.ps1` +- Modify: `docs/go-mcp-runbook.md` +- Modify: `docs/current-status-card.md` + +**Interfaces:** +- Env `ISPHERE_SEND_CONNECTOR_MODE=uia_rpa` enables A-route connector. +- Env `ISPHERE_SEND_UIA_HWND`, `ISPHERE_SEND_UIA_EDITOR_AUTOMATION_ID`, `ISPHERE_SEND_UIA_BUTTON_AUTOMATION_ID`, optional `ISPHERE_SEND_UIA_HELPER_PATH` configure the action. +- Default env remains no real send. + +- [x] Add config loader and tests around production connector availability. +- [x] Keep standard smoke deterministic with env cleared. +- [x] Add server env routing test; helper synthetic UIA smoke covers the local write/click action. +- [x] Update docs with minimal usage. + +## Task 5: Verification, commit, push + +- [x] Run `git diff --check`. +- [x] Run `go test ./...`. +- [x] Run `go build ./cmd/isphere-mcp` then remove root binary. +- [x] Run `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-win-helper.ps1`. +- [x] Run `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-go-mcp.ps1`. +- [x] Commit and push branch `codex/a-route-rpa-send`. diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index d5a0193..6396586 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -61,7 +61,7 @@ func NewServerFromEnv() (*mcp.Server, error) { if err != nil { return nil, err } - return NewServerWithSourcesAndMsgLibReceive(source, displaySource, msglibReceiveSource), nil + return NewServerWithSourcesMsgLibReceiveAndSendConnector(source, displaySource, msglibReceiveSource, tools.NewSendMessageConnectorFromEnv()), nil } func msgLibSourcesFromEnv() (tools.DisplayEntitySource, tools.ReceiveMessagesSource, error) { @@ -101,6 +101,10 @@ func NewServerWithSources(source tools.ReceiveMessagesSource, displaySource tool } func NewServerWithSourcesAndMsgLibReceive(source tools.ReceiveMessagesSource, displaySource tools.DisplayEntitySource, msglibReceiveSource tools.ReceiveMessagesSource) *mcp.Server { + return NewServerWithSourcesMsgLibReceiveAndSendConnector(source, displaySource, msglibReceiveSource, nil) +} + +func NewServerWithSourcesMsgLibReceiveAndSendConnector(source tools.ReceiveMessagesSource, displaySource tools.DisplayEntitySource, msglibReceiveSource tools.ReceiveMessagesSource, sendConnector tools.SendMessageConnector) *mcp.Server { server := mcp.NewServer(&mcp.Implementation{ Name: ServerName, Title: ServerTitle, @@ -111,7 +115,7 @@ func NewServerWithSourcesAndMsgLibReceive(source tools.ReceiveMessagesSource, di tools.RegisterISphereContactToolsWithDisplayEntities(server, source, displaySource) tools.RegisterISphereGroupToolsWithDisplayEntities(server, source, displaySource) tools.RegisterISphereFileTools(server, source) - tools.RegisterISphereSendMessageTool(server, nil) + tools.RegisterISphereSendMessageToolWithStateAndConnector(server, nil, nil, sendConnector) tools.RegisterISphereSendFileTool(server) return server } diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index 88271bc..e987308 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -4,7 +4,9 @@ import ( "context" "crypto/cipher" "crypto/des" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "os" "reflect" @@ -283,6 +285,56 @@ func TestNewServerFromEnvUsesPacketLogDirectory(t *testing.T) { } } +func TestNewServerFromEnvWiresUiaRpaSendConnector(t *testing.T) { + clearMsgLibEnvForServerTest(t) + t.Setenv("ISPHERE_SEND_CONNECTOR_MODE", "uia_rpa") + t.Setenv("ISPHERE_SEND_UIA_HELPER_PATH", t.TempDir()+"\\missing-helper.exe") + t.Setenv("ISPHERE_SEND_UIA_HWND", "0x1234") + t.Setenv("ISPHERE_SEND_UIA_EDITOR_AUTOMATION_ID", "rtbSendMessage") + t.Setenv("ISPHERE_SEND_UIA_BUTTON_AUTOMATION_ID", "btnSend") + t.Setenv("ISPHERE_SEND_AUDIT_PATH", t.TempDir()+"\\send-audit.jsonl") + t.Setenv("ISPHERE_SEND_IDEMPOTENCY_PATH", t.TempDir()+"\\send-idempotency.jsonl") + + server, err := NewServerFromEnv() + if err != nil { + t.Fatalf("NewServerFromEnv returned error: %v", err) + } + session, cleanup := connectServerTestSession(t, server) + defer cleanup() + + content := "uia rpa send through env" + callResult, err := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: "isphere_send_message", + Arguments: map[string]any{ + "target_type": "direct", + "target_id": "alice@imopenfire1-lanzhou", + "content_text": content, + "content_sha256": sha256HexForServerTest(content), + "idempotency_key": "idem-uia-env-1", + "execution_mode": "production", + }, + }) + if err != nil { + t.Fatalf("call isphere_send_message: %v", err) + } + if callResult.IsError { + t.Fatalf("send message should return structured connector failure, got error: %+v", callResult) + } + payload, _ := json.Marshal(callResult.StructuredContent) + var decoded struct { + OK bool `json:"ok"` + SendStatus string `json:"send_status"` + ConnectorMode string `json:"connector_mode"` + ErrorCode string `json:"error_code"` + } + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("decode payload %s: %v", payload, err) + } + if decoded.OK || decoded.SendStatus != "failed" || decoded.ConnectorMode != "uia-rpa" || decoded.ErrorCode != "uia_rpa_helper_error" { + t.Fatalf("production send did not route to UIA connector: %s", payload) + } +} + type fakeMsgLibClientForServerTest struct { displayCalls []msglib.DisplayEntitiesOptions listCalls []msglib.ListMessagesOptions @@ -376,3 +428,8 @@ func padPacketLogLineForServerTest(data []byte, blockSize int) []byte { } return out } + +func sha256HexForServerTest(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/tools/isphere_send_message.go b/internal/tools/isphere_send_message.go index a275400..b76400f 100644 --- a/internal/tools/isphere_send_message.go +++ b/internal/tools/isphere_send_message.go @@ -273,6 +273,7 @@ type normalizedSendMessageArgs struct { TargetType string TargetID string TargetRef string + ContentText string ContentLength int ContentSHA256 string IdempotencyKeySHA256 string @@ -308,6 +309,7 @@ func normalizeSendMessageArgs(input SendMessageArgs) (normalizedSendMessageArgs, TargetType: targetType, TargetID: targetID, TargetRef: targetRefPrefix + ":" + targetID, + ContentText: contentText, ContentLength: len(contentText), ContentSHA256: contentHash, IdempotencyKeySHA256: sha256Hex(idempotencyKey), @@ -448,6 +450,12 @@ func applySendMessageConnectorResult(response map[string]any, event *SendMessage response["send_status"] = normalized.Status response["connector_mode"] = normalized.ConnectorMode response["connector_stage"] = normalized.Status + if normalized.ProductionEnabled { + response["production_send_enabled"] = true + } + if len(normalized.SideEffects) > 0 { + response["side_effects"] = normalized.SideEffects + } if normalized.Accepted { response["blocked_reason"] = nil } else if normalized.ErrorMessage != "" { @@ -471,6 +479,9 @@ func applySendMessageConnectorResult(response map[string]any, event *SendMessage if normalized.AckRef != "" { audit["ack_ref"] = normalized.AckRef } + if normalized.ProductionEnabled { + audit["production_send_enabled"] = true + } if normalized.ErrorCode != "" { audit["error_code"] = normalized.ErrorCode } @@ -487,6 +498,9 @@ func applySendMessageConnectorResult(response map[string]any, event *SendMessage event.AckRef = normalized.AckRef event.ErrorCode = normalized.ErrorCode event.ErrorMessage = normalized.ErrorMessage + if len(normalized.SideEffects) > 0 { + event.SideEffects = normalized.SideEffects + } } } diff --git a/internal/tools/isphere_send_message_test.go b/internal/tools/isphere_send_message_test.go index be3cc71..1e278b1 100644 --- a/internal/tools/isphere_send_message_test.go +++ b/internal/tools/isphere_send_message_test.go @@ -385,6 +385,52 @@ func TestISphereSendMessageFakeConnectorFailureAudit(t *testing.T) { } } +func TestISphereSendMessageConnectorReceivesContentTextWithoutLeakingIt(t *testing.T) { + audit := &fakeSendMessageAuditSink{} + idempotency := newFakeSendMessageIdempotencyStore() + connector := &fakeSendMessageConnector{ + result: SendMessageConnectorResult{ + Accepted: true, + Status: "accepted", + AckRef: "fake-content-ack", + ConnectorMode: "fake", + }, + } + session, cleanup := connectToolsTestSession(t, func(server *mcp.Server) { + RegisterISphereSendMessageToolWithStateAndConnector(server, audit, idempotency, connector) + }) + defer cleanup() + + content := "connector needs the raw text to send" + callResult, err := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: ToolNameSendMessage, + Arguments: map[string]any{ + "target_type": "direct", + "target_id": "alice@imopenfire1-lanzhou", + "content_text": content, + "content_sha256": sha256HexForSendMessageTest(content), + "idempotency_key": "idem-connector-content-1", + "execution_mode": "production", + }, + }) + if err != nil { + t.Fatalf("call %s: %v", ToolNameSendMessage, err) + } + payload, _ := json.Marshal(callResult.StructuredContent) + if connector.lastRequest.ContentText != content { + t.Fatalf("connector ContentText = %q, want raw content", connector.lastRequest.ContentText) + } + if strings.Contains(string(payload), content) { + t.Fatalf("structured response leaked raw content: %s", payload) + } + if len(audit.events) != 1 { + t.Fatalf("audit events = %+v, want one", audit.events) + } + if audit.events[0].ContentText != "" || strings.Contains(mustMarshalStringForSendMessageTest(audit.events[0]), content) { + t.Fatalf("audit leaked raw content: %+v", audit.events[0]) + } +} + func TestISphereSendMessageRejectsContentHashMismatch(t *testing.T) { session, cleanup := connectToolsTestSession(t, func(server *mcp.Server) { RegisterISphereSendMessageToolWithState(server, &fakeSendMessageAuditSink{}, newFakeSendMessageIdempotencyStore()) @@ -650,13 +696,15 @@ func (f *fakeSendMessageIdempotencyStore) ReserveSendMessageIdempotency(_ contex } type fakeSendMessageConnector struct { - result SendMessageConnectorResult - err error - calls int + result SendMessageConnectorResult + err error + calls int + lastRequest SendMessageConnectorRequest } -func (f *fakeSendMessageConnector) ExecuteSendMessage(_ context.Context, _ SendMessageConnectorRequest) (SendMessageConnectorResult, error) { +func (f *fakeSendMessageConnector) ExecuteSendMessage(_ context.Context, request SendMessageConnectorRequest) (SendMessageConnectorResult, error) { f.calls++ + f.lastRequest = request return f.result, f.err } @@ -664,3 +712,11 @@ func sha256HexForSendMessageTest(value string) string { sum := sha256.Sum256([]byte(value)) return hex.EncodeToString(sum[:]) } + +func mustMarshalStringForSendMessageTest(value any) string { + payload, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(payload) +} diff --git a/internal/tools/send_message_connector.go b/internal/tools/send_message_connector.go index 153dc9b..f534001 100644 --- a/internal/tools/send_message_connector.go +++ b/internal/tools/send_message_connector.go @@ -13,6 +13,7 @@ type SendMessageConnectorRequest struct { TargetType string TargetID string TargetRef string + ContentText string ContentSHA256 string ContentLength int IdempotencyKeySHA256 string @@ -20,12 +21,14 @@ type SendMessageConnectorRequest struct { } type SendMessageConnectorResult struct { - Accepted bool - Status string - AckRef string - ErrorCode string - ErrorMessage string - ConnectorMode string + Accepted bool + Status string + AckRef string + ErrorCode string + ErrorMessage string + ConnectorMode string + ProductionEnabled bool + SideEffects map[string]any } func sendMessageConnectorRequestFromNormalized(input normalizedSendMessageArgs) SendMessageConnectorRequest { @@ -33,6 +36,7 @@ func sendMessageConnectorRequestFromNormalized(input normalizedSendMessageArgs) TargetType: input.TargetType, TargetID: input.TargetID, TargetRef: input.TargetRef, + ContentText: input.ContentText, ContentSHA256: input.ContentSHA256, ContentLength: input.ContentLength, IdempotencyKeySHA256: input.IdempotencyKeySHA256, diff --git a/internal/tools/send_message_uia_adapter.go b/internal/tools/send_message_uia_adapter.go new file mode 100644 index 0000000..b98b245 --- /dev/null +++ b/internal/tools/send_message_uia_adapter.go @@ -0,0 +1,239 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "isphere-ai-bridge/internal/helperclient" +) + +const ( + uiaSendMessageOp = "uia_send_message" + EnvSendConnectorMode = "ISPHERE_SEND_CONNECTOR_MODE" + EnvSendUIAHelperPath = "ISPHERE_SEND_UIA_HELPER_PATH" + EnvSendUIAHwnd = "ISPHERE_SEND_UIA_HWND" + EnvSendUIAEditorAutomationID = "ISPHERE_SEND_UIA_EDITOR_AUTOMATION_ID" + EnvSendUIButtonAutomationID = "ISPHERE_SEND_UIA_BUTTON_AUTOMATION_ID" + EnvSendUIATimeoutSeconds = "ISPHERE_SEND_UIA_TIMEOUT_SECONDS" +) + +type UiaSendMessageAdapterConfig struct { + HelperPath string + TimeoutSeconds int + Hwnd string + SendEditorAutomationID string + SendButtonAutomationID string + Mode string +} + +type UiaHelperCaller interface { + CallUiaHelper(ctx context.Context, op string, args map[string]any) (uiaHelperResponse, error) +} + +type uiaHelperResponse struct { + OK bool + Data map[string]any + ErrCode string + ErrText string +} + +type helperClientUiaCaller struct { + client helperclient.Client +} + +type uiaSendMessageConnector struct { + config UiaSendMessageAdapterConfig + caller UiaHelperCaller +} + +func NewSendMessageConnectorFromEnv() SendMessageConnector { + mode := strings.TrimSpace(strings.ToLower(os.Getenv(EnvSendConnectorMode))) + if mode != "uia_rpa" && mode != "uia-rpa" { + return nil + } + timeoutSeconds := 10 + if raw := strings.TrimSpace(os.Getenv(EnvSendUIATimeoutSeconds)); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + timeoutSeconds = parsed + } + } + return NewUiaSendMessageConnector(UiaSendMessageAdapterConfig{ + Mode: "enabled", + HelperPath: strings.TrimSpace(os.Getenv(EnvSendUIAHelperPath)), + TimeoutSeconds: timeoutSeconds, + Hwnd: strings.TrimSpace(os.Getenv(EnvSendUIAHwnd)), + SendEditorAutomationID: strings.TrimSpace(os.Getenv(EnvSendUIAEditorAutomationID)), + SendButtonAutomationID: strings.TrimSpace(os.Getenv(EnvSendUIButtonAutomationID)), + }, nil) +} + +func NewUiaSendMessageConnector(config UiaSendMessageAdapterConfig, caller UiaHelperCaller) SendMessageConnector { + config.Mode = strings.TrimSpace(strings.ToLower(config.Mode)) + if config.Mode == "" { + config.Mode = "disabled" + } + if config.SendEditorAutomationID == "" { + config.SendEditorAutomationID = "rtbSendMessage" + } + if config.SendButtonAutomationID == "" { + config.SendButtonAutomationID = "btnSend" + } + if caller == nil { + timeout := 10 * time.Second + if config.TimeoutSeconds > 0 { + timeout = time.Duration(config.TimeoutSeconds) * time.Second + } + caller = helperClientUiaCaller{client: helperclient.Client{HelperPath: config.HelperPath, Timeout: timeout}} + } + return uiaSendMessageConnector{config: config, caller: caller} +} + +func (c uiaSendMessageConnector) ExecuteSendMessage(ctx context.Context, request SendMessageConnectorRequest) (SendMessageConnectorResult, error) { + if c.config.Mode != "enabled" && c.config.Mode != "uia_rpa" { + result := SendMessageConnectorResult{ + Accepted: false, + Status: "blocked", + ErrorCode: "uia_rpa_mode_blocked", + ErrorMessage: "A-route UIA/RPA send adapter is disabled", + ConnectorMode: "uia-rpa-disabled", + } + return result, fmt.Errorf(result.ErrorMessage) + } + if err := validateUiaSendMessageConfig(c.config); err != nil { + result := SendMessageConnectorResult{ + Accepted: false, + Status: "blocked", + ErrorCode: "uia_rpa_config_missing", + ErrorMessage: err.Error(), + ConnectorMode: "uia-rpa", + } + return result, err + } + if err := validateUiaSendMessageRequest(request); err != nil { + return SendMessageConnectorResult{ + Accepted: false, + Status: "failed", + ErrorCode: "uia_rpa_invalid_request", + ErrorMessage: err.Error(), + ConnectorMode: "uia-rpa", + }, nil + } + + args := map[string]any{ + "hwnd": c.config.Hwnd, + "send_editor_automation_id": c.config.SendEditorAutomationID, + "send_button_automation_id": c.config.SendButtonAutomationID, + "target_ref": request.TargetRef, + "content_text": request.ContentText, + "content_sha256": request.ContentSHA256, + "idempotency_key_sha256": request.IdempotencyKeySHA256, + } + response, err := c.caller.CallUiaHelper(ctx, uiaSendMessageOp, args) + if err != nil { + return SendMessageConnectorResult{ + Accepted: false, + Status: "failed", + ErrorCode: "uia_rpa_helper_error", + ErrorMessage: err.Error(), + ConnectorMode: "uia-rpa", + }, err + } + if !response.OK { + code := strings.TrimSpace(response.ErrCode) + if code == "" { + code = "uia_rpa_helper_rejected" + } + message := strings.TrimSpace(response.ErrText) + if message == "" { + message = "UIA helper rejected send action" + } + return SendMessageConnectorResult{ + Accepted: false, + Status: "failed", + ErrorCode: code, + ErrorMessage: message, + ConnectorMode: "uia-rpa", + }, nil + } + ackRef, _ := response.Data["ack_ref"].(string) + if strings.TrimSpace(ackRef) == "" { + ackRef = "uia:" + c.config.Hwnd + ":" + c.config.SendButtonAutomationID + } + return SendMessageConnectorResult{ + Accepted: true, + Status: "accepted", + AckRef: ackRef, + ConnectorMode: "uia-rpa", + ProductionEnabled: true, + SideEffects: map[string]any{ + "sent_message": true, + "typed_text": true, + "clicked_ui": true, + "uploaded_file": false, + "sent_file": false, + "captured_network": false, + "attached_hook": false, + "modified_client_data": false, + }, + }, nil +} + +func validateUiaSendMessageConfig(config UiaSendMessageAdapterConfig) error { + if strings.TrimSpace(config.Hwnd) == "" { + return fmt.Errorf("ISPHERE_SEND_UIA_HWND is required for A-route UIA send") + } + if strings.TrimSpace(config.SendEditorAutomationID) == "" { + return fmt.Errorf("send editor automation id is required") + } + if strings.TrimSpace(config.SendButtonAutomationID) == "" { + return fmt.Errorf("send button automation id is required") + } + return nil +} + +func validateUiaSendMessageRequest(request SendMessageConnectorRequest) error { + if strings.TrimSpace(request.TargetRef) == "" { + return fmt.Errorf("target_ref is required") + } + if strings.TrimSpace(request.ContentText) == "" { + return fmt.Errorf("content_text is required") + } + if strings.TrimSpace(request.ContentSHA256) == "" { + return fmt.Errorf("content_sha256 is required") + } + if strings.TrimSpace(request.IdempotencyKeySHA256) == "" { + return fmt.Errorf("idempotency_key_sha256 is required") + } + if request.ExecutionMode != sendMessageProductionMode { + return fmt.Errorf("execution_mode must be production") + } + return nil +} + +func (c helperClientUiaCaller) CallUiaHelper(ctx context.Context, op string, args map[string]any) (uiaHelperResponse, error) { + response, err := c.client.Call(ctx, op, args) + if err != nil { + return uiaHelperResponse{}, err + } + out := uiaHelperResponse{OK: response != nil && response.OK} + if response != nil { + out.Data = decodeRawData(response.Data) + if response.Error != nil { + out.ErrCode = response.Error.Code + out.ErrText = response.Error.Message + } + } + if out.Data == nil { + out.Data = map[string]any{} + } + return out, nil +} + +func uiaHelperResponseFromRaw(data map[string]json.RawMessage, ok bool) uiaHelperResponse { + return uiaHelperResponse{OK: ok, Data: decodeRawData(data)} +} diff --git a/internal/tools/send_message_uia_adapter_test.go b/internal/tools/send_message_uia_adapter_test.go new file mode 100644 index 0000000..97c630c --- /dev/null +++ b/internal/tools/send_message_uia_adapter_test.go @@ -0,0 +1,117 @@ +package tools + +import ( + "context" + "testing" +) + +func TestUiaSendMessageConnectorRequiresEnabledModeAndWindowConfig(t *testing.T) { + connector := NewUiaSendMessageConnector(UiaSendMessageAdapterConfig{Mode: "disabled"}, &fakeUiaHelperCaller{}) + result, err := connector.ExecuteSendMessage(context.Background(), SendMessageConnectorRequest{ + TargetRef: "contact:alice@imopenfire1-lanzhou", + ContentText: "hello", + ContentSHA256: sha256HexForSendMessageTest("hello"), + IdempotencyKeySHA256: sha256HexForSendMessageTest("idem"), + ExecutionMode: sendMessageProductionMode, + }) + if err == nil { + t.Fatalf("expected disabled connector error") + } + if result.ErrorCode != "uia_rpa_mode_blocked" || result.ConnectorMode != "uia-rpa-disabled" { + t.Fatalf("unexpected result: %+v", result) + } + + connector = NewUiaSendMessageConnector(UiaSendMessageAdapterConfig{Mode: "enabled"}, &fakeUiaHelperCaller{}) + result, err = connector.ExecuteSendMessage(context.Background(), SendMessageConnectorRequest{ + TargetRef: "contact:alice@imopenfire1-lanzhou", + ContentText: "hello", + ContentSHA256: sha256HexForSendMessageTest("hello"), + IdempotencyKeySHA256: sha256HexForSendMessageTest("idem"), + ExecutionMode: sendMessageProductionMode, + }) + if err == nil { + t.Fatalf("expected missing config error") + } + if result.ErrorCode != "uia_rpa_config_missing" { + t.Fatalf("unexpected missing config result: %+v", result) + } +} + +func TestUiaSendMessageConnectorCallsHelperAndMapsAck(t *testing.T) { + caller := &fakeUiaHelperCaller{response: uiaHelperResponse{ + OK: true, + Data: map[string]any{ + "action_mode": "uia_send_message", + "target_ref": "contact:alice@imopenfire1-lanzhou", + "content_sha256": sha256HexForSendMessageTest("hello uia"), + "sent_message": true, + "typed_text": true, + "clicked_ui": true, + "ack_ref": "uia:0x1234:btnSend", + }, + }} + connector := NewUiaSendMessageConnector(UiaSendMessageAdapterConfig{ + Mode: "enabled", + Hwnd: "0x1234", + SendEditorAutomationID: "rtbSendMessage", + SendButtonAutomationID: "btnSend", + }, caller) + result, err := connector.ExecuteSendMessage(context.Background(), SendMessageConnectorRequest{ + TargetRef: "contact:alice@imopenfire1-lanzhou", + ContentText: "hello uia", + ContentSHA256: sha256HexForSendMessageTest("hello uia"), + IdempotencyKeySHA256: sha256HexForSendMessageTest("idem-uia"), + ExecutionMode: sendMessageProductionMode, + }) + if err != nil { + t.Fatalf("ExecuteSendMessage: %v", err) + } + if !result.Accepted || result.Status != "accepted" || result.AckRef != "uia:0x1234:btnSend" || result.ConnectorMode != "uia-rpa" || !result.ProductionEnabled { + t.Fatalf("unexpected result: %+v", result) + } + if caller.op != "uia_send_message" { + t.Fatalf("helper op = %q", caller.op) + } + if caller.args["hwnd"] != "0x1234" || caller.args["send_editor_automation_id"] != "rtbSendMessage" || caller.args["send_button_automation_id"] != "btnSend" { + t.Fatalf("unexpected helper args: %#v", caller.args) + } + if caller.args["content_text"] != "hello uia" || caller.args["content_sha256"] != sha256HexForSendMessageTest("hello uia") { + t.Fatalf("helper did not receive send content/hash: %#v", caller.args) + } +} + +func TestSendMessageConnectorFromEnvBuildsUiaRPAConnector(t *testing.T) { + t.Setenv(EnvSendConnectorMode, "uia_rpa") + t.Setenv(EnvSendUIAHwnd, "0x1234") + t.Setenv(EnvSendUIAEditorAutomationID, "editorA") + t.Setenv(EnvSendUIButtonAutomationID, "buttonA") + t.Setenv(EnvSendUIAHelperPath, t.TempDir()+"\\missing-helper.exe") + + connector := NewSendMessageConnectorFromEnv() + if connector == nil { + t.Fatalf("NewSendMessageConnectorFromEnv returned nil for uia_rpa config") + } + result, err := connector.ExecuteSendMessage(context.Background(), SendMessageConnectorRequest{ + TargetRef: "contact:alice@imopenfire1-lanzhou", + ContentText: "hello", + ContentSHA256: sha256HexForSendMessageTest("hello"), + IdempotencyKeySHA256: sha256HexForSendMessageTest("idem"), + ExecutionMode: sendMessageProductionMode, + }) + if err == nil || result.ErrorCode != "uia_rpa_helper_error" { + t.Fatalf("expected configured connector to reach helper layer and fail on missing helper, got result=%+v err=%v", result, err) + } +} + +type fakeUiaHelperCaller struct { + response uiaHelperResponse + err error + op string + args map[string]any +} + +func (f *fakeUiaHelperCaller) CallUiaHelper(ctx context.Context, op string, args map[string]any) (uiaHelperResponse, error) { + f.op = op + f.args = args + return f.response, f.err +} diff --git a/native/ISphereWinHelper/Program.cs b/native/ISphereWinHelper/Program.cs index 7d1c7fe..366c643 100644 --- a/native/ISphereWinHelper/Program.cs +++ b/native/ISphereWinHelper/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; @@ -53,6 +53,9 @@ namespace ISphereWinHelper case "probe_send_uia_controls": response = SendConnectorPreflight.ProbeSendUiaControls(requestId, op, opArgs); break; + case "uia_send_message": + response = UiaSendAction.SendMessage(requestId, op, opArgs); + break; default: response = HelperProtocol.Failure(requestId, op, "UNSUPPORTED_OP", "unsupported op: " + op); break; @@ -88,7 +91,7 @@ namespace ISphereWinHelper return new Dictionary { { "helper_name", "ISphereWinHelper" }, - { "helper_version", "0.4.0" }, + { "helper_version", "0.5.0" }, { "protocol", HelperProtocol.Protocol }, { "runtime", ".NET Framework " + Environment.Version } }; @@ -113,4 +116,4 @@ namespace ISphereWinHelper }; } } -} +} diff --git a/native/ISphereWinHelper/UiaSendAction.cs b/native/ISphereWinHelper/UiaSendAction.cs new file mode 100644 index 0000000..707c5f8 --- /dev/null +++ b/native/ISphereWinHelper/UiaSendAction.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Windows.Automation; + +namespace ISphereWinHelper +{ + internal static class UiaSendAction + { + private const int WM_SETTEXT = 0x000C; + private const int BM_CLICK = 0x00F5; + + [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern IntPtr SendMessage(IntPtr hWnd, int msg, IntPtr wParam, string lParam); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SendMessage(IntPtr hWnd, int msg, IntPtr wParam, IntPtr lParam); + + public static Dictionary SendMessage(string requestId, string op, Dictionary args) + { + string hwndText = HelperProtocol.GetString(args, "hwnd", ""); + IntPtr hwnd; + if (!WindowScanner.TryParseHwnd(hwndText, out hwnd)) + { + return HelperProtocol.Failure(requestId, op, "WINDOW_NOT_FOUND", "invalid or empty hwnd"); + } + + string editorAutomationId = HelperProtocol.GetString(args, "send_editor_automation_id", "rtbSendMessage"); + string buttonAutomationId = HelperProtocol.GetString(args, "send_button_automation_id", "btnSend"); + string contentText = HelperProtocol.GetString(args, "content_text", ""); + string contentSha256 = HelperProtocol.GetString(args, "content_sha256", "").Trim().ToLowerInvariant(); + string targetRef = HelperProtocol.GetString(args, "target_ref", ""); + + if (string.IsNullOrWhiteSpace(editorAutomationId)) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_CONFIG_MISSING", "send_editor_automation_id is required"); + } + if (string.IsNullOrWhiteSpace(buttonAutomationId)) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_CONFIG_MISSING", "send_button_automation_id is required"); + } + if (string.IsNullOrWhiteSpace(contentText)) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_CONTENT_MISSING", "content_text is required"); + } + if (string.IsNullOrWhiteSpace(contentSha256) || contentSha256 != Sha256Hex(contentText)) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_HASH_MISMATCH", "content_sha256 does not match content_text"); + } + + try + { + AutomationElement root = AutomationElement.FromHandle(hwnd); + if (root == null) + { + return HelperProtocol.Failure(requestId, op, "WINDOW_NOT_FOUND", "no UI Automation element for hwnd"); + } + + AutomationElement editor = FindByAutomationId(root, editorAutomationId); + if (editor == null) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_EDITOR_NOT_FOUND", "send editor control not found: " + editorAutomationId); + } + AutomationElement button = FindByAutomationId(root, buttonAutomationId); + if (button == null) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_BUTTON_NOT_FOUND", "send button control not found: " + buttonAutomationId); + } + + bool typed = SetElementText(editor, contentText); + if (!typed) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_SET_TEXT_FAILED", "could not set send editor text"); + } + + bool clicked = InvokeElement(button); + if (!clicked) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_CLICK_FAILED", "could not invoke send button"); + } + + string ackRef = "uia:" + WindowScanner.FormatHwnd(hwnd) + ":" + buttonAutomationId + ":" + ShortHash(contentSha256); + var data = new Dictionary + { + { "action_mode", "uia_send_message" }, + { "hwnd", WindowScanner.FormatHwnd(hwnd) }, + { "target_ref", targetRef ?? "" }, + { "content_sha256", contentSha256 }, + { "content_length", contentText.Length }, + { "editor_automation_id", editorAutomationId }, + { "button_automation_id", buttonAutomationId }, + { "editor_found", true }, + { "button_found", true }, + { "typed_text", true }, + { "clicked_ui", true }, + { "sent_message", true }, + { "uploaded_file", false }, + { "sent_file", false }, + { "captured_network", false }, + { "attached_hook", false }, + { "modified_client_data", false }, + { "ack_ref", ackRef } + }; + return HelperProtocol.Success(requestId, op, data); + } + catch (Exception ex) + { + return HelperProtocol.Failure(requestId, op, "UIA_SEND_FAILED", ex.Message); + } + } + + private static AutomationElement FindByAutomationId(AutomationElement root, string automationId) + { + if (root == null || string.IsNullOrWhiteSpace(automationId)) + { + return null; + } + try + { + return root.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, automationId)); + } + catch + { + return null; + } + } + + private static bool SetElementText(AutomationElement element, string text) + { + try + { + object pattern; + if (element.TryGetCurrentPattern(ValuePattern.Pattern, out pattern)) + { + ((ValuePattern)pattern).SetValue(text); + return true; + } + } + catch + { + } + + try + { + int nativeHandle = element.Current.NativeWindowHandle; + if (nativeHandle != 0) + { + SendMessage(new IntPtr(nativeHandle), WM_SETTEXT, IntPtr.Zero, text); + return true; + } + } + catch + { + } + return false; + } + + private static bool InvokeElement(AutomationElement element) + { + try + { + object pattern; + if (element.TryGetCurrentPattern(InvokePattern.Pattern, out pattern)) + { + ((InvokePattern)pattern).Invoke(); + return true; + } + } + catch + { + } + + try + { + int nativeHandle = element.Current.NativeWindowHandle; + if (nativeHandle != 0) + { + SendMessage(new IntPtr(nativeHandle), BM_CLICK, IntPtr.Zero, IntPtr.Zero); + return true; + } + } + catch + { + } + return false; + } + + private static string Sha256Hex(string value) + { + using (SHA256 sha = SHA256.Create()) + { + byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(value ?? "")); + StringBuilder sb = new StringBuilder(hash.Length * 2); + foreach (byte b in hash) + { + sb.Append(b.ToString("x2", CultureInfo.InvariantCulture)); + } + return sb.ToString(); + } + } + + private static string ShortHash(string value) + { + if (string.IsNullOrEmpty(value)) + { + return ""; + } + return value.Length <= 12 ? value : value.Substring(0, 12); + } + } +} diff --git a/scripts/verify-go-mcp.ps1 b/scripts/verify-go-mcp.ps1 index e4e43f7..00c402e 100644 --- a/scripts/verify-go-mcp.ps1 +++ b/scripts/verify-go-mcp.ps1 @@ -1,4 +1,4 @@ -$ErrorActionPreference = "Stop" +$ErrorActionPreference = "Stop" Set-StrictMode -Version Latest $repo = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path @@ -100,7 +100,7 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - for _, key := range []string{"ISPHERE_PACKET_LOG_FILE", "ISPHERE_PACKET_LOG_DIR", "ISPHERE_MSGLIB_SIDECAR_EXE", "ISPHERE_MSGLIB_SQLITE_DLL", "ISPHERE_MSGLIB_DB", "ISPHERE_MSGLIB_PASSWORD", "ISPHERE_SEND_FILE_ALLOWED_DIR", "ISPHERE_SEND_FILE_AUDIT_PATH", "ISPHERE_SEND_FILE_IDEMPOTENCY_PATH"} { + for _, key := range []string{"ISPHERE_PACKET_LOG_FILE", "ISPHERE_PACKET_LOG_DIR", "ISPHERE_MSGLIB_SIDECAR_EXE", "ISPHERE_MSGLIB_SQLITE_DLL", "ISPHERE_MSGLIB_DB", "ISPHERE_MSGLIB_PASSWORD", "ISPHERE_SEND_PRODUCTION_ENABLED", "ISPHERE_SEND_CONNECTOR_MODE", "ISPHERE_SEND_UIA_HELPER_PATH", "ISPHERE_SEND_UIA_HWND", "ISPHERE_SEND_UIA_EDITOR_AUTOMATION_ID", "ISPHERE_SEND_UIA_BUTTON_AUTOMATION_ID", "ISPHERE_SEND_UIA_TIMEOUT_SECONDS", "ISPHERE_SEND_FILE_ALLOWED_DIR", "ISPHERE_SEND_FILE_AUDIT_PATH", "ISPHERE_SEND_FILE_IDEMPOTENCY_PATH"} { if err := os.Unsetenv(key); err != nil { fail("server/env", err) } diff --git a/scripts/verify-win-helper.ps1 b/scripts/verify-win-helper.ps1 index 06cd747..774b78c 100644 --- a/scripts/verify-win-helper.ps1 +++ b/scripts/verify-win-helper.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$HelperExe = "runs/win-helper/ISphereWinHelper.exe", [switch]$SkipBuild ) @@ -107,6 +107,7 @@ if ($dumpMissing.ok -or $dumpMissing.error.code -notin @("WINDOW_NOT_FOUND", "UI $title = "Codex WinHelper Verify " + [guid]::NewGuid().ToString("N").Substring(0, 8) $formScriptPath = Join-Path $env:TEMP ("codex-winhelper-form-" + [guid]::NewGuid().ToString("N") + ".ps1") +$sendActionMarkerPath = Join-Path $env:TEMP ("codex-winhelper-send-marker-" + [guid]::NewGuid().ToString("N") + ".json") $formScript = @" Add-Type -AssemblyName System.Windows.Forms `$form = New-Object System.Windows.Forms.Form @@ -137,6 +138,10 @@ Add-Type -AssemblyName System.Windows.Forms `$button.Name = 'btnSend' `$button.Text = 'Send' `$button.Dock = 'Fill' +`$button.Add_Click({ + `$payload = [ordered]@{ clicked = `$true; text = `$send.Text } | ConvertTo-Json -Compress + Set-Content -LiteralPath '$sendActionMarkerPath' -Value `$payload -Encoding UTF8 +}) `$layout.Controls.Add(`$button) `$file = New-Object System.Windows.Forms.Button `$file.Name = 'btnFile' @@ -196,6 +201,42 @@ try { if ($sendUiaProbe.data.safety.sent_message -or $sendUiaProbe.data.safety.uploaded_file -or $sendUiaProbe.data.safety.clicked_ui -or $sendUiaProbe.data.safety.typed_text) { throw "probe_send_uia_controls safety flags failed: $($sendUiaProbe | ConvertTo-Json -Depth 12 -Compress)" } + $sendContent = "Codex synthetic UIA send " + [guid]::NewGuid().ToString("N").Substring(0, 8) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $bytes = [System.Text.Encoding]::UTF8.GetBytes($sendContent) + $sendContentHash = -join ($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") }) + } + finally { + $sha.Dispose() + } + $sendAction = Invoke-HelperJson @{ + protocol = "isphere.helper.v1" + request_id = "verify-uia-send-message" + op = "uia_send_message" + timeout_ms = 5000 + args = @{ + hwnd = $candidate.hwnd + send_editor_automation_id = "rtbSendMessage" + send_button_automation_id = "btnSend" + target_ref = "synthetic:verify" + content_text = $sendContent + content_sha256 = $sendContentHash + } + } + if (-not $sendAction.ok -or $sendAction.data.action_mode -ne "uia_send_message" -or -not $sendAction.data.sent_message -or -not $sendAction.data.typed_text -or -not $sendAction.data.clicked_ui) { + throw "uia_send_message failed: $($sendAction | ConvertTo-Json -Depth 12 -Compress)" + } + for ($i = 0; $i -lt 20 -and -not (Test-Path -LiteralPath $sendActionMarkerPath); $i++) { + Start-Sleep -Milliseconds 100 + } + if (-not (Test-Path -LiteralPath $sendActionMarkerPath)) { + throw "uia_send_message did not trigger synthetic send button marker" + } + $marker = Get-Content -LiteralPath $sendActionMarkerPath -Raw | ConvertFrom-Json + if (-not $marker.clicked -or $marker.text -ne $sendContent) { + throw "uia_send_message marker mismatch: $($marker | ConvertTo-Json -Compress)" + } } finally { if ($process -and -not $process.HasExited) { @@ -204,6 +245,9 @@ finally { if (Test-Path -LiteralPath $formScriptPath) { Remove-Item -LiteralPath $formScriptPath -Force } + if (Test-Path -LiteralPath $sendActionMarkerPath) { + Remove-Item -LiteralPath $sendActionMarkerPath -Force + } } [ordered]@{ @@ -214,5 +258,6 @@ finally { runtime_probe_target_count = $runtimeProbe.data.target_count send_entrypoint_probe_mode = $entrypointProbe.data.probe_mode synthetic_send_uia_route_hint = $sendUiaProbe.data.flags.route_hint + synthetic_uia_send_action = $sendAction.data.action_mode uia_available = $selfCheck.data.uia_available } | ConvertTo-Json -Depth 4 -Compress From d1a1e499ea2ac0086d98753ab929b97f9e2ab254 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 11:56:55 +0800 Subject: [PATCH 2/6] tools: add offline real client window launcher --- scripts/open-offline-real-client-window.ps1 | 268 ++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 scripts/open-offline-real-client-window.ps1 diff --git a/scripts/open-offline-real-client-window.ps1 b/scripts/open-offline-real-client-window.ps1 new file mode 100644 index 0000000..291480c --- /dev/null +++ b/scripts/open-offline-real-client-window.ps1 @@ -0,0 +1,268 @@ +param( + [string]$ArchivePath = "runs/offline-evidence-intake/zyl-qqfile-20260709/archives/zyl.rar", + [string]$ExtractDir = "runs/offline-real-client-window/full", + [ValidateSet("Impp", "iSphere")] + [string]$ClientRoot = "Impp", + [string]$HelperExe = "runs/win-helper/ISphereWinHelper.exe", + [int]$WaitSeconds = 10, + [int]$X = 120, + [int]$Y = 120, + [int]$Width = 270, + [int]$Height = 570, + [switch]$SkipExtract, + [switch]$NoLaunch, + [switch]$KeepExisting, + [switch]$TopMost, + [switch]$NoScreenshot +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$InfoPromptTitle = -join ([char[]](0x4FE1, 0x606F, 0x63D0, 0x793A)) + +function Resolve-RepoPath([string]$PathValue) { + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return $PathValue + } + return (Join-Path $repo $PathValue) +} + +function Find-7Zip { + $candidates = @( + "C:\Program Files\7-Zip\7z.exe", + "C:\Program Files (x86)\7-Zip\7z.exe" + ) + foreach ($candidate in $candidates) { + if (Test-Path -LiteralPath $candidate) { + return $candidate + } + } + $cmd = Get-Command "7z" -ErrorAction SilentlyContinue + if ($cmd) { + return $cmd.Source + } + $cmd = Get-Command "7zr" -ErrorAction SilentlyContinue + if ($cmd) { + return $cmd.Source + } + throw "7-Zip not found. Install 7-Zip or pass -SkipExtract after extracting the archive." +} + +function Invoke-HelperJson([string]$Op, [hashtable]$OpArgs, [string]$RequestId) { + $request = @{ + protocol = "isphere.helper.v1" + request_id = $RequestId + op = $Op + timeout_ms = 5000 + args = $OpArgs + } + $json = $request | ConvertTo-Json -Depth 16 -Compress + $output = $json | & $script:HelperPath --json + try { + return $output | ConvertFrom-Json + } + catch { + throw "helper output was not JSON: $output" + } +} + +function Stop-ExistingOfflineCopies([string]$RootPath) { + $rootFull = [System.IO.Path]::GetFullPath($RootPath).TrimEnd('\') + "\" + $targets = @("IMPlatformClient.exe", "IMPP.ISphere.exe", "IMPlatformClient.Web.exe") + $processes = Get-CimInstance Win32_Process | + Where-Object { $targets -contains $_.Name -and $_.ExecutablePath -and ([System.IO.Path]::GetFullPath($_.ExecutablePath).StartsWith($rootFull, [System.StringComparison]::OrdinalIgnoreCase)) } + foreach ($proc in $processes) { + Stop-Process -Id ([int]$proc.ProcessId) -Force -ErrorAction SilentlyContinue + } +} + +function Add-NativeWindowApi { + Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class ISphereOfflineWindowApi { + [DllImport("user32.dll")] public static extern bool MoveWindow(IntPtr hWnd, int X, int Y, int nWidth, int nHeight, bool bRepaint); + [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); + [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr hWndInsertAfter, int X, int Y, int cx, int cy, uint uFlags); +} +"@ -ErrorAction SilentlyContinue +} + +function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$MainWidth, [int]$MainHeight, [bool]$MakeTopMost) { + $topMostHandle = [IntPtr]::new(-1) + $mainWindow = $null + $promptWindow = $null + foreach ($window in $Windows) { + if ([string]$window.process_name -ne "IMPlatformClient") { + continue + } + $hwndValue = [Convert]::ToInt64(($window.hwnd -replace "^0x", ""), 16) + $hwnd = [IntPtr]::new($hwndValue) + [ISphereOfflineWindowApi]::ShowWindow($hwnd, 9) | Out-Null + if ([string]$window.title -eq $InfoPromptTitle) { + [ISphereOfflineWindowApi]::MoveWindow($hwnd, ($MainX + $MainWidth + 40), ($MainY + 40), 300, 190, $true) | Out-Null + [ISphereOfflineWindowApi]::SetForegroundWindow($hwnd) | Out-Null + $promptWindow = $window + } + else { + if ($MakeTopMost) { + [ISphereOfflineWindowApi]::SetWindowPos($hwnd, $topMostHandle, $MainX, $MainY, $MainWidth, $MainHeight, 0x0040) | Out-Null + } + else { + [ISphereOfflineWindowApi]::MoveWindow($hwnd, $MainX, $MainY, $MainWidth, $MainHeight, $true) | Out-Null + } + [ISphereOfflineWindowApi]::SetForegroundWindow($hwnd) | Out-Null + $mainWindow = $window + } + } + return @{ + main = $mainWindow + prompt = $promptWindow + } +} + +function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY, [int]$CaptureWidth, [int]$CaptureHeight) { + Add-Type -AssemblyName System.Drawing + $bitmap = New-Object System.Drawing.Bitmap $CaptureWidth, $CaptureHeight + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + try { + $graphics.CopyFromScreen(($ScreenX - 30), ($ScreenY - 30), 0, 0, $bitmap.Size) + $bitmap.Save($OutputPath, [System.Drawing.Imaging.ImageFormat]::Png) + } + finally { + $graphics.Dispose() + $bitmap.Dispose() + } +} + +$archiveFull = Resolve-RepoPath $ArchivePath +$extractFull = Resolve-RepoPath $ExtractDir +$script:HelperPath = Resolve-RepoPath $HelperExe +$runOut = Join-Path $repo "runs\offline-real-client-window" +New-Item -ItemType Directory -Force -Path $runOut | Out-Null + +if (-not (Test-Path -LiteralPath $script:HelperPath)) { + throw "WinHelper not found: $script:HelperPath. Run scripts\build-win-helper.ps1 first." +} + +$clientDir = Join-Path $extractFull ("zyl\" + $ClientRoot) +$clientExe = Join-Path $clientDir "IMPlatformClient.exe" +$clientConfig = Join-Path $clientDir "IMPlatformClient.exe.config" +$requiredDependency = Join-Path $clientDir "Utilities.Lib.Base.dll" + +if (-not $SkipExtract -and (-not (Test-Path -LiteralPath $clientExe) -or -not (Test-Path -LiteralPath $clientConfig) -or -not (Test-Path -LiteralPath $requiredDependency))) { + if (-not (Test-Path -LiteralPath $archiveFull)) { + throw "archive not found: $archiveFull" + } + New-Item -ItemType Directory -Force -Path $extractFull | Out-Null + $sevenZip = Find-7Zip + & $sevenZip x $archiveFull "-o$extractFull" "zyl\Impp\*" "zyl\iSphere\*" -y | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "7-Zip extraction failed with exit code $LASTEXITCODE" + } +} + +if (-not (Test-Path -LiteralPath $clientExe)) { + throw "client exe not found after extraction: $clientExe" +} + +Add-NativeWindowApi + +if (-not $KeepExisting) { + Stop-ExistingOfflineCopies -RootPath $extractFull + Start-Sleep -Milliseconds 500 +} + +$process = $null +if (-not $NoLaunch) { + $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -PassThru +} + +$deadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) +$scan = $null +$windows = @() +do { + Start-Sleep -Milliseconds 500 + $scan = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-scan" + if ($scan.ok -and $scan.data.windows) { + $windows = @($scan.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $process) -or $_.pid -eq $process.Id) + }) + } +} while ($windows.Count -eq 0 -and (Get-Date) -lt $deadline) + +if ($windows.Count -eq 0) { + throw "no visible IMPlatformClient window found. Last scan: $($scan | ConvertTo-Json -Depth 12 -Compress)" +} + +$moveResult = Move-ClientWindows -Windows $windows -MainX $X -MainY $Y -MainWidth $Width -MainHeight $Height -MakeTopMost ([bool]$TopMost) +Start-Sleep -Seconds 1 + +$scanAfterMove = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-scan-after-move" +$windowsAfterMove = @($scanAfterMove.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $process) -or $_.pid -eq $process.Id) +}) + +# The offline client can raise a late "信息提示" dialog after the login window is +# already visible. Move it to the right side so the login window remains usable +# for RPA probing, then refresh the final window list. +$latePromptWindows = @($windowsAfterMove | Where-Object { [string]$_.title -eq $InfoPromptTitle }) +if ($latePromptWindows.Count -gt 0) { + foreach ($latePromptWindow in $latePromptWindows) { + $promptHwndValue = [Convert]::ToInt64(($latePromptWindow.hwnd -replace "^0x", ""), 16) + $promptHwnd = [IntPtr]::new($promptHwndValue) + [ISphereOfflineWindowApi]::ShowWindow($promptHwnd, 9) | Out-Null + [ISphereOfflineWindowApi]::MoveWindow($promptHwnd, ($X + $Width + 40), ($Y + 40), 300, 190, $true) | Out-Null + [ISphereOfflineWindowApi]::SetForegroundWindow($promptHwnd) | Out-Null + } + Start-Sleep -Milliseconds 500 + $scanAfterMove = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-scan-after-prompt-move" + $windowsAfterMove = @($scanAfterMove.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $process) -or $_.pid -eq $process.Id) + }) +} + +$mainWindowMatches = @($windowsAfterMove | Where-Object { [string]$_.title -ne $InfoPromptTitle }) +$promptWindowMatches = @($windowsAfterMove | Where-Object { [string]$_.title -eq $InfoPromptTitle }) +$mainWindow = if ($mainWindowMatches.Count -gt 0) { $mainWindowMatches[0] } else { $null } +$promptWindow = if ($promptWindowMatches.Count -gt 0) { $promptWindowMatches[0] } else { $null } + +$dumpFile = $null +$dump = $null +if ($mainWindow) { + $dump = Invoke-HelperJson -Op "dump_uia" -OpArgs @{ + hwnd = $mainWindow.hwnd + max_depth = 8 + max_children = 200 + include_text = $true + } -RequestId "offline-real-client-dump-uia" + $dumpFile = Join-Path $runOut ("uia-dump-open-offline-real-client-{0}.json" -f ((Get-Date).ToString("yyyyMMdd-HHmmss"))) + $dump | ConvertTo-Json -Depth 40 | Set-Content -LiteralPath $dumpFile -Encoding UTF8 +} + +$screenshotFile = $null +if (-not $NoScreenshot) { + $screenshotFile = Join-Path $runOut ("open-offline-real-client-{0}.png" -f ((Get-Date).ToString("yyyyMMdd-HHmmss"))) + $captureWidth = if ($promptWindow) { $Width + 420 } else { $Width + 80 } + Save-WindowScreenshot -OutputPath $screenshotFile -ScreenX $X -ScreenY $Y -CaptureWidth $captureWidth -CaptureHeight ($Height + 80) +} + +[pscustomobject]@{ + ok = $true + archive = $archiveFull + extracted_dir = $extractFull + client_root = $ClientRoot + client_exe = $clientExe + process_id = if ($process) { $process.Id } else { $null } + alive = if ($process) { $null -ne (Get-Process -Id $process.Id -ErrorAction SilentlyContinue) } else { $null } + main_window = $mainWindow + prompt_window = $promptWindow + uia_root = if ($dump -and $dump.ok) { $dump.data.root } else { $null } + uia_dump_file = $dumpFile + screenshot_file = $screenshotFile + windows = $windowsAfterMove +} | ConvertTo-Json -Depth 24 From e8b4110cd20ff3bf005ba25f28cabe03f27a97e4 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 15:11:03 +0800 Subject: [PATCH 3/6] tools: add user-silent offline rpa window mode --- .../2026-07-11-a-route-user-silent-rpa.md | 129 ++++++++++++++++++ scripts/open-offline-real-client-window.ps1 | 87 ++++++++++-- 2 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md diff --git a/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md b/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md new file mode 100644 index 0000000..04eefa6 --- /dev/null +++ b/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md @@ -0,0 +1,129 @@ +# A-route RPA 用户无感运行说明 + +日期：2026-07-11 + +## 目标 + +A-route 当前看重窗口和 UIA 控件，而不是背后的网络逻辑。这里的“用户无感”指： + +1. 不遮挡当前用户桌面。 +2. 不抢焦点。 +3. 不接管鼠标键盘。 +4. 数字员工仍然能通过 HWND/UIA 找到目标窗口和控件。 +5. 所有动作可记录、可复现、可回收。 + +## 当前已实现 + +脚本： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -UserSilent +``` + +效果： + +- 从完整离线客户端目录启动 `IMPlatformClient.exe`。 +- 使用 `-WindowStyle Minimized` 降低启动阶段闪窗概率。 +- 找到真实 `IMPlatformClient` 窗口后移动到屏幕外： + - 主窗：`x=-32000, y=-32000` + - 提示窗：同样移到屏幕外 +- 设置 `SWP_NOACTIVATE`，不调用前台激活。 +- 不生成屏幕截图，避免因为窗口在屏幕外得到无意义图片。 +- 继续输出 UIA dump，供 RPA 选择器验证。 + +本轮验证结果： + +- 新启动进程：`IMPlatformClient.exe` +- PID：`20060` +- 主窗口：`0x340330` +- UIA 根控件： + - `automation_id=frmLogin` + - `framework_id=WinForm` + - `class_name=WindowsForms10.Window.8.app.0.d3a00f_r7_ad1` +- UIA dump 仍可读，即使窗口处于屏幕外： + - `is_offscreen=true` + - 子控件 `skinState`、`skinLoadPanel` 可见于控件树 + +## 推荐无感等级 + +### L1：当前用户会话内屏幕外运行 + +适合当前开发机验证： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -UserSilent +``` + +优点： + +- 实现最快。 +- 不影响当前屏幕。 +- UIA 读控件可继续工作。 + +限制： + +- 如果某些发送动作必须依赖真实鼠标点击或 OCR 截图，屏幕外模式不适合。 +- 发送路径应优先使用 `ValuePattern`、`InvokePattern`、`WM_SETTEXT`、`BM_CLICK` 等 HWND/UIA/Win32 方式。 + +### L2：当前用户会话内可见但不激活 + +适合排查窗口状态： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -WindowMode Visible -NoActivate +``` + +优点： + +- 人能看到窗口。 +- 尽量不抢焦点。 + +限制： + +- 仍可能遮挡用户桌面。 +- 只适合调试，不适合长期运行。 + +### L3：独立 Windows 用户会话运行 + +这是生产上更稳的“真正无感”方案： + +- 给数字员工单独建一个 Windows 用户。 +- 在该用户会话里登录 iSphere。 +- RPA/WinHelper/MCP 都运行在同一个独立会话。 +- 业务用户使用自己的桌面，看不到数字员工窗口。 + +优点： + +- 不影响业务用户桌面。 +- 不抢业务用户焦点。 +- 可以保留真实可见窗口，兼容 OCR/坐标兜底。 + +限制： + +- 需要部署层面的账户和会话管理。 +- WinHelper 必须和目标窗口在同一交互式桌面会话里运行。 + +### L4：不用 RPA，走 B-route/API/sidecar + +这是最终最无感的方向： + +- 没窗口。 +- 不依赖桌面。 +- 不受焦点、分辨率、UI 改版影响。 + +但当前分支是 A-route RPA，所以本轮先把 L1/L2 做出来。 + +## 结论 + +当前已经具备 L1 无感基础能力： + +- 真实客户端可打开。 +- 可屏幕外运行。 +- 不需要前台焦点。 +- UIA 控件树仍可读取。 + +下一轮应把发送/搜索动作约束在非焦点方式上： + +1. 优先 UIA `ValuePattern`/`InvokePattern`。 +2. 其次 HWND 消息 `WM_SETTEXT`/`BM_CLICK`。 +3. 最后才考虑坐标/OCR；坐标/OCR 只能放到独立 Windows 用户会话里做，不能放当前用户桌面长期运行。 diff --git a/scripts/open-offline-real-client-window.ps1 b/scripts/open-offline-real-client-window.ps1 index 291480c..3677a6e 100644 --- a/scripts/open-offline-real-client-window.ps1 +++ b/scripts/open-offline-real-client-window.ps1 @@ -9,10 +9,14 @@ param( [int]$Y = 120, [int]$Width = 270, [int]$Height = 570, + [ValidateSet("Visible", "Offscreen", "Minimized")] + [string]$WindowMode = "Visible", [switch]$SkipExtract, [switch]$NoLaunch, [switch]$KeepExisting, [switch]$TopMost, + [switch]$NoActivate, + [switch]$UserSilent, [switch]$NoScreenshot ) @@ -20,6 +24,20 @@ $ErrorActionPreference = "Stop" $repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") $InfoPromptTitle = -join ([char[]](0x4FE1, 0x606F, 0x63D0, 0x793A)) +if ($UserSilent) { + $WindowMode = "Offscreen" + $NoActivate = $true + $NoScreenshot = $true + $TopMost = $false +} + +$effectiveX = $X +$effectiveY = $Y +if ($WindowMode -eq "Offscreen") { + $effectiveX = -32000 + $effectiveY = -32000 +} + function Resolve-RepoPath([string]$PathValue) { if ([System.IO.Path]::IsPathRooted($PathValue)) { return $PathValue @@ -89,8 +107,11 @@ public static class ISphereOfflineWindowApi { "@ -ErrorAction SilentlyContinue } -function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$MainWidth, [int]$MainHeight, [bool]$MakeTopMost) { +function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$MainWidth, [int]$MainHeight, [bool]$MakeTopMost, [bool]$Activate, [string]$Mode) { + $hwndTop = [IntPtr]::Zero $topMostHandle = [IntPtr]::new(-1) + $swpShowWindow = 0x0040 + $swpNoActivate = 0x0010 $mainWindow = $null $promptWindow = $null foreach ($window in $Windows) { @@ -101,18 +122,32 @@ function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$ $hwnd = [IntPtr]::new($hwndValue) [ISphereOfflineWindowApi]::ShowWindow($hwnd, 9) | Out-Null if ([string]$window.title -eq $InfoPromptTitle) { - [ISphereOfflineWindowApi]::MoveWindow($hwnd, ($MainX + $MainWidth + 40), ($MainY + 40), 300, 190, $true) | Out-Null - [ISphereOfflineWindowApi]::SetForegroundWindow($hwnd) | Out-Null + $promptX = if ($Mode -eq "Offscreen") { $MainX } else { $MainX + $MainWidth + 40 } + $promptY = if ($Mode -eq "Offscreen") { $MainY + $MainHeight + 40 } else { $MainY + 40 } + $flags = $swpShowWindow + if (-not $Activate) { $flags = $flags -bor $swpNoActivate } + [ISphereOfflineWindowApi]::SetWindowPos($hwnd, $hwndTop, $promptX, $promptY, 300, 190, [uint32]$flags) | Out-Null + if ($Activate) { + [ISphereOfflineWindowApi]::SetForegroundWindow($hwnd) | Out-Null + } + if ($Mode -eq "Minimized") { + [ISphereOfflineWindowApi]::ShowWindow($hwnd, 6) | Out-Null + } $promptWindow = $window } else { - if ($MakeTopMost) { - [ISphereOfflineWindowApi]::SetWindowPos($hwnd, $topMostHandle, $MainX, $MainY, $MainWidth, $MainHeight, 0x0040) | Out-Null + $insertAfter = if ($MakeTopMost) { $topMostHandle } else { $hwndTop } + $flags = $swpShowWindow + if (-not $Activate) { + $flags = $flags -bor $swpNoActivate } - else { - [ISphereOfflineWindowApi]::MoveWindow($hwnd, $MainX, $MainY, $MainWidth, $MainHeight, $true) | Out-Null + [ISphereOfflineWindowApi]::SetWindowPos($hwnd, $insertAfter, $MainX, $MainY, $MainWidth, $MainHeight, [uint32]$flags) | Out-Null + if ($Activate) { + [ISphereOfflineWindowApi]::SetForegroundWindow($hwnd) | Out-Null + } + if ($Mode -eq "Minimized") { + [ISphereOfflineWindowApi]::ShowWindow($hwnd, 6) | Out-Null } - [ISphereOfflineWindowApi]::SetForegroundWindow($hwnd) | Out-Null $mainWindow = $window } } @@ -176,7 +211,12 @@ if (-not $KeepExisting) { $process = $null if (-not $NoLaunch) { - $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -PassThru + if ($UserSilent -or $WindowMode -ne "Visible" -or $NoActivate) { + $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -WindowStyle Minimized -PassThru + } + else { + $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -PassThru + } } $deadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) @@ -197,7 +237,8 @@ if ($windows.Count -eq 0) { throw "no visible IMPlatformClient window found. Last scan: $($scan | ConvertTo-Json -Depth 12 -Compress)" } -$moveResult = Move-ClientWindows -Windows $windows -MainX $X -MainY $Y -MainWidth $Width -MainHeight $Height -MakeTopMost ([bool]$TopMost) +$activateWindows = -not [bool]$NoActivate +$moveResult = Move-ClientWindows -Windows $windows -MainX $effectiveX -MainY $effectiveY -MainWidth $Width -MainHeight $Height -MakeTopMost ([bool]$TopMost) -Activate $activateWindows -Mode $WindowMode Start-Sleep -Seconds 1 $scanAfterMove = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-scan-after-move" @@ -215,8 +256,17 @@ if ($latePromptWindows.Count -gt 0) { $promptHwndValue = [Convert]::ToInt64(($latePromptWindow.hwnd -replace "^0x", ""), 16) $promptHwnd = [IntPtr]::new($promptHwndValue) [ISphereOfflineWindowApi]::ShowWindow($promptHwnd, 9) | Out-Null - [ISphereOfflineWindowApi]::MoveWindow($promptHwnd, ($X + $Width + 40), ($Y + 40), 300, 190, $true) | Out-Null - [ISphereOfflineWindowApi]::SetForegroundWindow($promptHwnd) | Out-Null + $latePromptX = if ($WindowMode -eq "Offscreen") { $effectiveX } else { $effectiveX + $Width + 40 } + $latePromptY = if ($WindowMode -eq "Offscreen") { $effectiveY + $Height + 40 } else { $effectiveY + 40 } + $latePromptFlags = 0x0040 + if (-not $activateWindows) { $latePromptFlags = $latePromptFlags -bor 0x0010 } + [ISphereOfflineWindowApi]::SetWindowPos($promptHwnd, [IntPtr]::Zero, $latePromptX, $latePromptY, 300, 190, [uint32]$latePromptFlags) | Out-Null + if ($activateWindows) { + [ISphereOfflineWindowApi]::SetForegroundWindow($promptHwnd) | Out-Null + } + if ($WindowMode -eq "Minimized") { + [ISphereOfflineWindowApi]::ShowWindow($promptHwnd, 6) | Out-Null + } } Start-Sleep -Milliseconds 500 $scanAfterMove = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-scan-after-prompt-move" @@ -245,14 +295,23 @@ if ($mainWindow) { } $screenshotFile = $null -if (-not $NoScreenshot) { +if (-not $NoScreenshot -and $WindowMode -eq "Visible") { $screenshotFile = Join-Path $runOut ("open-offline-real-client-{0}.png" -f ((Get-Date).ToString("yyyyMMdd-HHmmss"))) $captureWidth = if ($promptWindow) { $Width + 420 } else { $Width + 80 } - Save-WindowScreenshot -OutputPath $screenshotFile -ScreenX $X -ScreenY $Y -CaptureWidth $captureWidth -CaptureHeight ($Height + 80) + Save-WindowScreenshot -OutputPath $screenshotFile -ScreenX $effectiveX -ScreenY $effectiveY -CaptureWidth $captureWidth -CaptureHeight ($Height + 80) } [pscustomobject]@{ ok = $true + user_silent = [bool]$UserSilent + window_mode = $WindowMode + no_activate = [bool]$NoActivate + effective_bounds = @{ + x = $effectiveX + y = $effectiveY + width = $Width + height = $Height + } archive = $archiveFull extracted_dir = $extractFull client_root = $ClientRoot From c2a1ca560f6dc0faab91bd9d27f2302a32e2e517 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 20:04:34 +0800 Subject: [PATCH 4/6] tools: add offline rpa window watchdog --- .../2026-07-11-a-route-user-silent-rpa.md | 46 +++++++ scripts/open-offline-real-client-window.ps1 | 124 +++++++++++++++++- ...en-offline-real-client-window-watchdog.ps1 | 45 +++++++ 3 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 scripts/test-open-offline-real-client-window-watchdog.ps1 diff --git a/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md b/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md index 04eefa6..14f0fa6 100644 --- a/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md +++ b/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md @@ -31,6 +31,52 @@ powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-b - 不生成屏幕截图，避免因为窗口在屏幕外得到无意义图片。 - 继续输出 UIA dump，供 RPA 选择器验证。 +## 抗干扰能力 + +新增守护模式： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -WindowMode Visible -WatchSeconds 30 -PollIntervalMs 500 +``` + +也可和无感模式一起使用： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -UserSilent -WatchSeconds 30 -PollIntervalMs 500 +``` + +当前守护逻辑： + +- 用户手动移动主窗口：检测到位置/大小偏离后自动移回目标位置。 +- 用户手动移动提示窗：检测到位置/大小偏离后自动移回目标位置。 +- 用户关闭主窗口/杀掉客户端进程：检测到主窗口消失后自动重新启动离线客户端。 +- 每次守护输出： + - `watch_iteration_count` + - `repair_count` + - `relaunch_count` + - `recovery_actions` + +本轮实际验证： + +1. 模拟移动窗口到 `760,260,310x610`： + - 守护输出 `repair_count=1` + - `recovery_actions=["repair_window_placement"]` + - 最终窗口恢复到 `120,120,270x570` + +2. 模拟关闭客户端进程： + - 被关闭 PID：`11592` + - 守护输出 `relaunch_count=1` + - 新启动 PID：`10504` + - `recovery_actions=["relaunch_missing_main_window","repair_after_relaunch"]` + - 最终 UIA 根控件仍为 `frmLogin` + +抗干扰强度结论： + +- 对“用户移动窗口”：强。 +- 对“用户关闭窗口/进程”：中强，可自动重启。 +- 对“用户在关键发送瞬间抢焦点/操作同一窗口”：当前还不是强，需要动作级锁定和发送前后校验。 +- 对“应用崩溃/服务端不可用”：只能自动拉起窗口；业务登录/聊天能力仍受真实服务端影响。 + 本轮验证结果： - 新启动进程：`IMPlatformClient.exe` diff --git a/scripts/open-offline-real-client-window.ps1 b/scripts/open-offline-real-client-window.ps1 index 3677a6e..8fb5c67 100644 --- a/scripts/open-offline-real-client-window.ps1 +++ b/scripts/open-offline-real-client-window.ps1 @@ -9,6 +9,8 @@ param( [int]$Y = 120, [int]$Width = 270, [int]$Height = 570, + [int]$WatchSeconds = 0, + [int]$PollIntervalMs = 500, [ValidateSet("Visible", "Offscreen", "Minimized")] [string]$WindowMode = "Visible", [switch]$SkipExtract, @@ -107,6 +109,19 @@ public static class ISphereOfflineWindowApi { "@ -ErrorAction SilentlyContinue } +function Start-OfflineClient { + if ($script:UserSilentMode -or $script:WindowModeValue -ne "Visible" -or $script:NoActivateMode) { + return Start-Process -FilePath $script:ClientExePath -WorkingDirectory $script:ClientDirPath -WindowStyle Minimized -PassThru + } + return Start-Process -FilePath $script:ClientExePath -WorkingDirectory $script:ClientDirPath -PassThru +} + +function Relaunch-OfflineClient { + Stop-ExistingOfflineCopies -RootPath $script:ExtractRootPath + Start-Sleep -Milliseconds 500 + return Start-OfflineClient +} + function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$MainWidth, [int]$MainHeight, [bool]$MakeTopMost, [bool]$Activate, [string]$Mode) { $hwndTop = [IntPtr]::Zero $topMostHandle = [IntPtr]::new(-1) @@ -157,6 +172,32 @@ function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$ } } +function Get-ClientWindowsForProcess([object]$Process) { + $scanResult = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-watch-scan" + if (-not $scanResult.ok -or -not $scanResult.data.windows) { + return @() + } + return @($scanResult.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $Process) -or $_.pid -eq $Process.Id) + }) +} + +function Test-WindowNear([object]$Window, [int]$ExpectedX, [int]$ExpectedY, [int]$ExpectedWidth, [int]$ExpectedHeight) { + if (-not $Window -or -not $Window.bounds) { + return $false + } + $tolerance = 3 + return ([Math]::Abs([int]$Window.bounds.x - $ExpectedX) -le $tolerance) -and + ([Math]::Abs([int]$Window.bounds.y - $ExpectedY) -le $tolerance) -and + ([Math]::Abs([int]$Window.bounds.width - $ExpectedWidth) -le $tolerance) -and + ([Math]::Abs([int]$Window.bounds.height - $ExpectedHeight) -le $tolerance) +} + +function Repair-ClientWindowPlacement([object[]]$Windows) { + Move-ClientWindows -Windows $Windows -MainX $script:EffectiveX -MainY $script:EffectiveY -MainWidth $script:DesiredWidth -MainHeight $script:DesiredHeight -MakeTopMost ([bool]$script:TopMostMode) -Activate (-not [bool]$script:NoActivateMode) -Mode $script:WindowModeValue | Out-Null +} + function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY, [int]$CaptureWidth, [int]$CaptureHeight) { Add-Type -AssemblyName System.Drawing $bitmap = New-Object System.Drawing.Bitmap $CaptureWidth, $CaptureHeight @@ -174,6 +215,7 @@ function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY $archiveFull = Resolve-RepoPath $ArchivePath $extractFull = Resolve-RepoPath $ExtractDir $script:HelperPath = Resolve-RepoPath $HelperExe +$script:ExtractRootPath = $extractFull $runOut = Join-Path $repo "runs\offline-real-client-window" New-Item -ItemType Directory -Force -Path $runOut | Out-Null @@ -185,6 +227,16 @@ $clientDir = Join-Path $extractFull ("zyl\" + $ClientRoot) $clientExe = Join-Path $clientDir "IMPlatformClient.exe" $clientConfig = Join-Path $clientDir "IMPlatformClient.exe.config" $requiredDependency = Join-Path $clientDir "Utilities.Lib.Base.dll" +$script:ClientDirPath = $clientDir +$script:ClientExePath = $clientExe +$script:UserSilentMode = [bool]$UserSilent +$script:NoActivateMode = [bool]$NoActivate +$script:WindowModeValue = $WindowMode +$script:EffectiveX = $effectiveX +$script:EffectiveY = $effectiveY +$script:DesiredWidth = $Width +$script:DesiredHeight = $Height +$script:TopMostMode = [bool]$TopMost if (-not $SkipExtract -and (-not (Test-Path -LiteralPath $clientExe) -or -not (Test-Path -LiteralPath $clientConfig) -or -not (Test-Path -LiteralPath $requiredDependency))) { if (-not (Test-Path -LiteralPath $archiveFull)) { @@ -211,12 +263,7 @@ if (-not $KeepExisting) { $process = $null if (-not $NoLaunch) { - if ($UserSilent -or $WindowMode -ne "Visible" -or $NoActivate) { - $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -WindowStyle Minimized -PassThru - } - else { - $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -PassThru - } + $process = Start-OfflineClient } $deadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) @@ -276,6 +323,65 @@ if ($latePromptWindows.Count -gt 0) { }) } +$watchIterationCount = 0 +$repairCount = 0 +$relaunchCount = 0 +$recoveryActions = @() + +if ($WatchSeconds -gt 0) { + $safePollIntervalMs = [Math]::Max(100, $PollIntervalMs) + $watchDeadline = (Get-Date).AddSeconds($WatchSeconds) + while ((Get-Date) -lt $watchDeadline) { + Start-Sleep -Milliseconds $safePollIntervalMs + $watchIterationCount++ + $watchedWindows = Get-ClientWindowsForProcess -Process $process + $watchedMain = @($watchedWindows | Where-Object { [string]$_.title -ne $InfoPromptTitle }) | Select-Object -First 1 + $watchedPrompt = @($watchedWindows | Where-Object { [string]$_.title -eq $InfoPromptTitle }) | Select-Object -First 1 + + if (-not $watchedMain) { + if (-not $NoLaunch) { + $process = Relaunch-OfflineClient + $relaunchCount++ + $recoveryActions += "relaunch_missing_main_window" + + $relaunchDeadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) + do { + Start-Sleep -Milliseconds $safePollIntervalMs + $watchedWindows = Get-ClientWindowsForProcess -Process $process + $watchedMain = @($watchedWindows | Where-Object { [string]$_.title -ne $InfoPromptTitle }) | Select-Object -First 1 + } while (-not $watchedMain -and (Get-Date) -lt $relaunchDeadline) + + if ($watchedWindows.Count -gt 0) { + Repair-ClientWindowPlacement -Windows $watchedWindows + $repairCount++ + $recoveryActions += "repair_after_relaunch" + } + } + continue + } + + $expectedPromptX = if ($WindowMode -eq "Offscreen") { $effectiveX } else { $effectiveX + $Width + 40 } + $expectedPromptY = if ($WindowMode -eq "Offscreen") { $effectiveY + $Height + 40 } else { $effectiveY + 40 } + + $needsRepair = -not (Test-WindowNear -Window $watchedMain -ExpectedX $effectiveX -ExpectedY $effectiveY -ExpectedWidth $Width -ExpectedHeight $Height) + if ($watchedPrompt) { + $needsRepair = $needsRepair -or -not (Test-WindowNear -Window $watchedPrompt -ExpectedX $expectedPromptX -ExpectedY $expectedPromptY -ExpectedWidth 300 -ExpectedHeight 190) + } + + if ($needsRepair) { + Repair-ClientWindowPlacement -Windows $watchedWindows + $repairCount++ + $recoveryActions += "repair_window_placement" + } + } + + $scanAfterMove = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-final-scan-after-watch" + $windowsAfterMove = @($scanAfterMove.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $process) -or $_.pid -eq $process.Id) + }) +} + $mainWindowMatches = @($windowsAfterMove | Where-Object { [string]$_.title -ne $InfoPromptTitle }) $promptWindowMatches = @($windowsAfterMove | Where-Object { [string]$_.title -eq $InfoPromptTitle }) $mainWindow = if ($mainWindowMatches.Count -gt 0) { $mainWindowMatches[0] } else { $null } @@ -306,6 +412,12 @@ if (-not $NoScreenshot -and $WindowMode -eq "Visible") { user_silent = [bool]$UserSilent window_mode = $WindowMode no_activate = [bool]$NoActivate + watch_seconds = $WatchSeconds + poll_interval_ms = $PollIntervalMs + watch_iteration_count = $watchIterationCount + relaunch_count = $relaunchCount + repair_count = $repairCount + recovery_actions = $recoveryActions effective_bounds = @{ x = $effectiveX y = $effectiveY diff --git a/scripts/test-open-offline-real-client-window-watchdog.ps1 b/scripts/test-open-offline-real-client-window-watchdog.ps1 new file mode 100644 index 0000000..7ccb481 --- /dev/null +++ b/scripts/test-open-offline-real-client-window-watchdog.ps1 @@ -0,0 +1,45 @@ +param( + [string]$ScriptPath = "scripts/open-offline-real-client-window.ps1" +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$target = if ([System.IO.Path]::IsPathRooted($ScriptPath)) { + $ScriptPath +} +else { + Join-Path $repo $ScriptPath +} + +if (-not (Test-Path -LiteralPath $target)) { + throw "script not found: $target" +} + +$text = Get-Content -LiteralPath $target -Raw + +$requiredPatterns = @( + @{ Name = "WatchSeconds parameter"; Pattern = '\[int\]\$WatchSeconds' }, + @{ Name = "PollIntervalMs parameter"; Pattern = '\[int\]\$PollIntervalMs' }, + @{ Name = "watch loop output"; Pattern = 'watch_iteration_count' }, + @{ Name = "relaunch counter output"; Pattern = 'relaunch_count' }, + @{ Name = "move repair counter output"; Pattern = 'repair_count' }, + @{ Name = "closed window recovery"; Pattern = 'Relaunch-OfflineClient' }, + @{ Name = "moved window recovery"; Pattern = 'Repair-ClientWindowPlacement' } +) + +$missing = @() +foreach ($item in $requiredPatterns) { + if ($text -notmatch $item.Pattern) { + $missing += $item.Name + } +} + +if ($missing.Count -gt 0) { + throw "watchdog contract missing: $($missing -join ', ')" +} + +[pscustomobject]@{ + ok = $true + script = $target + checked = $requiredPatterns.Count +} | ConvertTo-Json -Compress From a06fe60eb2217ef18ec903dbaea56481d06becdb Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 20:21:41 +0800 Subject: [PATCH 5/6] tools: add offline chat window probe --- ...-07-11-a-route-open-conversation-window.md | 120 ++++++ .../plans/2026-07-11-a-route-rpa-send.md | 20 + scripts/open-offline-chat-window-probe.ps1 | 371 ++++++++++++++++++ .../test-open-offline-chat-window-probe.ps1 | 62 +++ 4 files changed, 573 insertions(+) create mode 100644 docs/source-discovery/2026-07-11-a-route-open-conversation-window.md create mode 100644 scripts/open-offline-chat-window-probe.ps1 create mode 100644 scripts/test-open-offline-chat-window-probe.ps1 diff --git a/docs/source-discovery/2026-07-11-a-route-open-conversation-window.md b/docs/source-discovery/2026-07-11-a-route-open-conversation-window.md new file mode 100644 index 0000000..02f8ccd --- /dev/null +++ b/docs/source-discovery/2026-07-11-a-route-open-conversation-window.md @@ -0,0 +1,120 @@ +# A-route open_conversation 窗口验证说明 + +日期：2026-07-11 +分支：`codex/a-route-rpa-send` + +## 本轮目标 + +用户明确当前先不要求真实发送消息，只要求“打开那个会话窗口”，用来验证数字员工是否能： + +1. 识别会话窗口。 +2. 定位发送编辑框。 +3. 定位发送按钮/文件按钮。 +4. 执行一次可审计的点击。 + +本地环境没有 iSphere 服务端，无法登录，所以真实会话窗口不能通过正常业务路径打开。 + +## 已实现：离线会话窗口探针 + +新增脚本： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-chat-window-probe.ps1 -ProbeClick +``` + +它会打开一个本地 WinForms 会话窗口探针，不连接服务端，不发送真实消息。关键 UIA 标识对齐真实会话窗口/既有 RPA 选择器： + +| 目标 | AutomationId / Name | +| --- | --- | +| 会话窗口 | `frmP2PChat` | +| 联系人搜索框 | `skinAlphaTxt` | +| 消息展示区 | `rtbRecvMessage` | +| 发送编辑框 | `rtbSendMessage` | +| 发送按钮 | `btnSend` | +| 文件按钮 | `btnSendFile` | +| 离线提示 | `offlineSendBlocker` | + +当传入 `-ProbeClick` 时，脚本会调用当前分支已有的 WinHelper UIA 动作，把文本写入 `rtbSendMessage` 并点击 `btnSend`。按钮点击只写本地 marker 文件，字段固定为： + +- `sent_real_message=false` +- `uploaded_real_file=false` + +因此这个动作只验证 UIA 识别、定位、写入、点击链路，不触发真实消息发送。 + +## 自动化测试 + +新增测试： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\test-open-offline-chat-window-probe.ps1 +``` + +测试断言： + +- 根窗口 `root_automation_id=frmP2PChat` +- 找到 `rtbSendMessage` +- 找到 `btnSend` +- 找到 `btnSendFile` +- `ProbeClick` 后写入 marker +- marker 文本与 UIA 写入文本一致 +- `sent_real_message=false` +- `uploaded_real_file=false` + +## 真实 `frmP2PChat` 破解进度 + +已继续尝试直接构造真实类型： + +```text +IMPP.Client.Business.ChatManager.SingleChat.frmP2PChat +``` + +构造函数签名： + +```text +frmP2PChat(com.vision.smack.Chat chat, string pluginInfo, string extendTabJson) +``` + +已补过的离线依赖包括： + +- `IMPPManager.Instance.MessageCenter` +- `IMPPManager.Instance.LogonUser` +- `IMPPManager.Instance.UserInfo` +- `SmarkManager.Connection` +- `SmarkManager.RosterManager` +- `SmarkManager.PresenceManager` +- `SmarkManager.P2PChatManager` +- `ConfigSystemManager.config` +- `P2PChat.OtherJid` + +仍未成功打开真实窗体。当前阻塞不再是脚本能力，而是构造函数内部把窗体初始化、登录态、连接态、名册状态、消息中心事件、配置状态混在一起。IL 证据显示构造函数中直接访问： + +- `IMPPManager.Instance.Connection.add_OnConnectError` +- `IMPPManager.Instance.Connection.add_OnReConnectOk` +- `P2PChat.OtherJid.getBareJid` +- `IMPPManager.Instance.UserInfo.get_Jid` +- `UCChatSendMessageBox.GetChatRichTextBox` +- `UCChatSendMessageBox.GetReceiptCheckBox` +- `IMPPManager.Instance.GetMessageCenter().add_OnTcpMessageArrived` +- `IMPPManager.Instance.RosterManager.UpdateStrangerStatus` +- `Config.BaseConfig.VisualPhoneEnable` + +这说明真实 `frmP2PChat` 不是独立窗口类；它要求完整登录运行态。没有服务端/登录态时继续硬构造，投入会越来越像“重建一个假客户端运行时”。 + +## 当前业务结论 + +从“验证 RPA 能否识别、定位、点击会话窗口”的业务目标看，本轮已经可验证： + +- 会话窗口可打开。 +- 核心 AutomationId 可识别。 +- 文本可写入。 +- 按钮可点击。 +- 点击结果可留本地审计 marker。 +- 全程不发真实消息、不上传真实文件。 + +从“真实 iSphere 会话窗口”的目标看，仍需要在线登录环境或继续补完整客户端运行态。 + +## 下一步建议 + +1. 当前分支先使用 `open-offline-chat-window-probe.ps1` 验证 RPA 识别/定位/点击效果。 +2. 在线环境拿到真实登录后的 `frmP2PChat` HWND 后，用同一套 WinHelper 选择器直接验证真实窗口。 +3. 如果必须离线打开真实窗体，再继续走“补完整 IMPPManager/SmarkManager/Config/BaseConfig/MessageCenter 运行态”的破解路线。 diff --git a/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md b/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md index 05d4825..38cbb2a 100644 --- a/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md +++ b/docs/superpowers/plans/2026-07-11-a-route-rpa-send.md @@ -93,3 +93,23 @@ - [x] Run `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-win-helper.ps1`. - [x] Run `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-go-mcp.ps1`. - [x] Commit and push branch `codex/a-route-rpa-send`. + +## Task 6: Open conversation window probe for A-route RPA + +**Files:** +- Create: `scripts/open-offline-chat-window-probe.ps1` +- Create: `scripts/test-open-offline-chat-window-probe.ps1` +- Create: `docs/source-discovery/2026-07-11-a-route-open-conversation-window.md` + +**Purpose:** local environment cannot log in, but RPA still needs a stable +conversation-window surface to prove identify/locate/click behavior. This task +adds a no-network WinForms probe using the real selector names +`frmP2PChat`, `rtbRecvMessage`, `rtbSendMessage`, `btnSend`, and +`btnSendFile`. + +- [x] Write failing test proving the open-conversation probe script is required. +- [x] Implement the offline chat-window probe. +- [x] Verify UIA classification finds send editor, send button, and file button. +- [x] Verify `-ProbeClick` writes text and clicks `btnSend`. +- [x] Verify marker says `sent_real_message=false` and `uploaded_real_file=false`. +- [x] Document why direct offline construction of the real `frmP2PChat` is still blocked by login/runtime singletons. diff --git a/scripts/open-offline-chat-window-probe.ps1 b/scripts/open-offline-chat-window-probe.ps1 new file mode 100644 index 0000000..3fd67dd --- /dev/null +++ b/scripts/open-offline-chat-window-probe.ps1 @@ -0,0 +1,371 @@ +param( + [string]$HelperExe = "runs/win-helper/ISphereWinHelper.exe", + [int]$WaitSeconds = 10, + [int]$KeepOpenSeconds = 0, + [int]$X = 160, + [int]$Y = 120, + [int]$Width = 760, + [int]$Height = 560, + [ValidateSet("Visible", "Offscreen", "Minimized")] + [string]$WindowMode = "Visible", + [switch]$ProbeClick, + [switch]$NoScreenshot, + [switch]$NoActivate, + [switch]$TopMost +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$helperPath = Join-Path $repo $HelperExe +$runOut = Join-Path $repo "runs\offline-chat-window" +New-Item -ItemType Directory -Force -Path $runOut | Out-Null + +function Resolve-RepoPath([string]$PathValue) { + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return $PathValue + } + return (Join-Path $repo $PathValue) +} + +function Invoke-HelperJson([string]$Op, [hashtable]$OpArgs, [string]$RequestId, [int]$TimeoutMs = 5000) { + $request = @{ + protocol = "isphere.helper.v1" + request_id = $RequestId + op = $Op + timeout_ms = $TimeoutMs + args = $OpArgs + } + $json = $request | ConvertTo-Json -Depth 16 -Compress + $output = $json | & $script:HelperPath --json + if ($LASTEXITCODE -ne 0) { + throw "helper exited with code $LASTEXITCODE for op $Op. Output: $output" + } + try { + return $output | ConvertFrom-Json + } + catch { + throw "helper output was not JSON for op $Op`: $output" + } +} + +function Add-NativeWindowApi { + Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class ISphereOfflineChatProbeWindowApi { + [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); + [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr hWndInsertAfter, int X, int Y, int cx, int cy, uint uFlags); +} +"@ -ErrorAction SilentlyContinue +} + +function Set-ProbeWindowPlacement([string]$HwndText) { + $hwndValue = [Convert]::ToInt64(($HwndText -replace "^0x", ""), 16) + $hwnd = [IntPtr]::new($hwndValue) + $insertAfter = if ($TopMost) { [IntPtr]::new(-1) } else { [IntPtr]::Zero } + $flags = 0x0040 + if ($NoActivate) { + $flags = $flags -bor 0x0010 + } + + if ($WindowMode -eq "Minimized") { + [ISphereOfflineChatProbeWindowApi]::ShowWindow($hwnd, 6) | Out-Null + return + } + + $effectiveX = $X + $effectiveY = $Y + if ($WindowMode -eq "Offscreen") { + $effectiveX = -32000 + $effectiveY = -32000 + } + + [ISphereOfflineChatProbeWindowApi]::ShowWindow($hwnd, 9) | Out-Null + [ISphereOfflineChatProbeWindowApi]::SetWindowPos($hwnd, $insertAfter, $effectiveX, $effectiveY, $Width, $Height, [uint32]$flags) | Out-Null + if (-not $NoActivate) { + [ISphereOfflineChatProbeWindowApi]::SetForegroundWindow($hwnd) | Out-Null + } +} + +function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY, [int]$CaptureWidth, [int]$CaptureHeight) { + Add-Type -AssemblyName System.Drawing + $bitmap = New-Object System.Drawing.Bitmap $CaptureWidth, $CaptureHeight + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + try { + $graphics.CopyFromScreen(($ScreenX - 20), ($ScreenY - 20), 0, 0, $bitmap.Size) + $bitmap.Save($OutputPath, [System.Drawing.Imaging.ImageFormat]::Png) + } + finally { + $graphics.Dispose() + $bitmap.Dispose() + } +} + +function New-Sha256Hex([string]$Value) { + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $bytes = [System.Text.Encoding]::UTF8.GetBytes($Value) + return -join ($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") }) + } + finally { + $sha.Dispose() + } +} + +$script:HelperPath = Resolve-RepoPath $HelperExe +if (-not (Test-Path -LiteralPath $script:HelperPath)) { + throw "WinHelper not found: $script:HelperPath. Run scripts\build-win-helper.ps1 first." +} + +$timestamp = (Get-Date).ToString("yyyyMMdd-HHmmss") +$windowTitle = "iSphere Offline Chat Window Probe " + ([guid]::NewGuid().ToString("N").Substring(0, 8)) +$launcherScript = Join-Path $runOut ("offline-chat-window-probe-$timestamp.ps1") +$markerPath = Join-Path $runOut ("offline-chat-window-probe-marker-$timestamp.json") +$statePath = Join-Path $runOut ("offline-chat-window-probe-state-$timestamp.json") +$dumpFile = Join-Path $runOut ("uia-dump-offline-chat-window-probe-$timestamp.json") +$screenshotFile = Join-Path $runOut ("offline-chat-window-probe-$timestamp.png") + +$escapedTitle = $windowTitle.Replace("'", "''") +$escapedMarker = $markerPath.Replace("'", "''") +$escapedState = $statePath.Replace("'", "''") + +$formScript = @" +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing +`$form = New-Object System.Windows.Forms.Form +`$form.Name = 'frmP2PChat' +`$form.Text = '$escapedTitle' +`$form.StartPosition = 'Manual' +`$form.Location = New-Object System.Drawing.Point(160, 120) +`$form.Size = New-Object System.Drawing.Size(760, 560) +`$form.KeyPreview = `$true + +`$layout = New-Object System.Windows.Forms.TableLayoutPanel +`$layout.Name = 'chatRootLayout' +`$layout.Dock = 'Fill' +`$layout.RowCount = 5 +`$layout.ColumnCount = 1 +`$layout.RowStyles.Add((New-Object System.Windows.Forms.RowStyle([System.Windows.Forms.SizeType]::Absolute, 38))) | Out-Null +`$layout.RowStyles.Add((New-Object System.Windows.Forms.RowStyle([System.Windows.Forms.SizeType]::Percent, 55))) | Out-Null +`$layout.RowStyles.Add((New-Object System.Windows.Forms.RowStyle([System.Windows.Forms.SizeType]::Absolute, 42))) | Out-Null +`$layout.RowStyles.Add((New-Object System.Windows.Forms.RowStyle([System.Windows.Forms.SizeType]::Percent, 45))) | Out-Null +`$layout.RowStyles.Add((New-Object System.Windows.Forms.RowStyle([System.Windows.Forms.SizeType]::Absolute, 48))) | Out-Null +`$form.Controls.Add(`$layout) + +`$search = New-Object System.Windows.Forms.TextBox +`$search.Name = 'skinAlphaTxt' +`$search.Text = 'search contact placeholder' +`$search.Dock = 'Fill' +`$layout.Controls.Add(`$search, 0, 0) + +`$recv = New-Object System.Windows.Forms.RichTextBox +`$recv.Name = 'rtbRecvMessage' +`$recv.ReadOnly = `$true +`$recv.Text = 'Offline chat window probe. This mimics frmP2PChat for UIA identify, locate, and click verification only.' +`$recv.Dock = 'Fill' +`$layout.Controls.Add(`$recv, 0, 1) + +`$toolbar = New-Object System.Windows.Forms.FlowLayoutPanel +`$toolbar.Name = 'chatToolbar' +`$toolbar.Dock = 'Fill' +`$toolbar.FlowDirection = 'LeftToRight' +`$layout.Controls.Add(`$toolbar, 0, 2) + +`$file = New-Object System.Windows.Forms.Button +`$file.Name = 'btnSendFile' +`$file.Text = 'Send File' +`$file.Width = 92 +`$file.Height = 30 +`$file.Add_Click({ + `$payload = [ordered]@{ + clicked = `$true + action = 'file_button_probe_only' + sent_real_message = `$false + uploaded_real_file = `$false + at = (Get-Date).ToString('o') + } | ConvertTo-Json -Compress + Set-Content -LiteralPath '$escapedMarker' -Value `$payload -Encoding UTF8 +}) +`$toolbar.Controls.Add(`$file) + +`$offline = New-Object System.Windows.Forms.Label +`$offline.Name = 'offlineSendBlocker' +`$offline.Text = 'Offline probe only: no server connection and no real message is sent.' +`$offline.AutoSize = `$true +`$offline.Padding = New-Object System.Windows.Forms.Padding(8) +`$toolbar.Controls.Add(`$offline) + +`$send = New-Object System.Windows.Forms.RichTextBox +`$send.Name = 'rtbSendMessage' +`$send.Text = '' +`$send.Dock = 'Fill' +`$layout.Controls.Add(`$send, 0, 3) + +`$buttonPanel = New-Object System.Windows.Forms.FlowLayoutPanel +`$buttonPanel.Name = 'sendButtonPanel' +`$buttonPanel.Dock = 'Fill' +`$buttonPanel.FlowDirection = 'RightToLeft' +`$layout.Controls.Add(`$buttonPanel, 0, 4) + +`$sendButton = New-Object System.Windows.Forms.Button +`$sendButton.Name = 'btnSend' +`$sendButton.Text = 'Send(S)' +`$sendButton.Width = 92 +`$sendButton.Height = 32 +`$sendButton.Add_Click({ + `$payload = [ordered]@{ + clicked = `$true + action = 'send_button_probe_only' + text = `$send.Text + text_length = `$send.Text.Length + sent_real_message = `$false + uploaded_real_file = `$false + at = (Get-Date).ToString('o') + } | ConvertTo-Json -Compress + Set-Content -LiteralPath '$escapedMarker' -Value `$payload -Encoding UTF8 +}) +`$buttonPanel.Controls.Add(`$sendButton) + +`$form.Add_Shown({ + `$state = [ordered]@{ + ok = `$true + window_kind = 'offline_chat_window_probe' + automation_id = `$form.Name + title = `$form.Text + process_id = [System.Diagnostics.Process]::GetCurrentProcess().Id + marker_path = '$escapedMarker' + at = (Get-Date).ToString('o') + } | ConvertTo-Json -Compress + Set-Content -LiteralPath '$escapedState' -Value `$state -Encoding UTF8 +}) + +[void]`$form.ShowDialog() +"@ + +Set-Content -LiteralPath $launcherScript -Value $formScript -Encoding UTF8 + +$process = Start-Process powershell -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcherScript) -PassThru +$candidate = $null +$scan = $null +$deadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) +do { + Start-Sleep -Milliseconds 300 + $scan = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $true } -RequestId "offline-chat-probe-scan" + if ($scan.ok -and $scan.data.windows) { + $candidate = $scan.data.windows | Where-Object { + $_.pid -eq $process.Id -and $_.title -eq $windowTitle + } | Select-Object -First 1 + } +} while (-not $candidate -and (Get-Date) -lt $deadline) + +if (-not $candidate) { + if ($process -and -not $process.HasExited) { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + } + throw "offline chat probe window not found. Last scan: $($scan | ConvertTo-Json -Depth 12 -Compress)" +} + +Add-NativeWindowApi +Set-ProbeWindowPlacement -HwndText $candidate.hwnd +Start-Sleep -Milliseconds 500 + +$dump = Invoke-HelperJson -Op "dump_uia" -OpArgs @{ + hwnd = $candidate.hwnd + max_depth = 8 + max_children = 200 + include_text = $true +} -RequestId "offline-chat-probe-dump-uia" +$dump | ConvertTo-Json -Depth 40 | Set-Content -LiteralPath $dumpFile -Encoding UTF8 + +$classify = Invoke-HelperJson -Op "probe_send_uia_controls" -OpArgs @{ + hwnd = $candidate.hwnd + max_depth = 8 + max_children = 200 +} -RequestId "offline-chat-probe-classify-uia" + +$probeClickedUi = $false +$probeTypedText = $false +$probeMarkerFound = $false +$probeMarkerTextMatches = $false +$probeMarker = $null +$probeContent = "" +$probeAction = $null + +if ($ProbeClick) { + $probeContent = "Codex offline chat probe " + ([guid]::NewGuid().ToString("N").Substring(0, 8)) + $probeHash = New-Sha256Hex $probeContent + $probeAction = Invoke-HelperJson -Op "uia_send_message" -OpArgs @{ + hwnd = $candidate.hwnd + send_editor_automation_id = "rtbSendMessage" + send_button_automation_id = "btnSend" + target_ref = "offline-chat-window-probe" + content_text = $probeContent + content_sha256 = $probeHash + } -RequestId "offline-chat-probe-click" + $probeClickedUi = [bool]($probeAction.ok -and $probeAction.data.clicked_ui) + $probeTypedText = [bool]($probeAction.ok -and $probeAction.data.typed_text) + + for ($i = 0; $i -lt 30 -and -not (Test-Path -LiteralPath $markerPath); $i++) { + Start-Sleep -Milliseconds 100 + } + $probeMarkerFound = Test-Path -LiteralPath $markerPath + if ($probeMarkerFound) { + $probeMarker = Get-Content -LiteralPath $markerPath -Raw | ConvertFrom-Json + $probeMarkerTextMatches = [string]$probeMarker.text -eq $probeContent + } +} + +$screenshotPathForOutput = $null +if (-not $NoScreenshot -and $WindowMode -eq "Visible") { + Save-WindowScreenshot -OutputPath $screenshotFile -ScreenX $X -ScreenY $Y -CaptureWidth ($Width + 40) -CaptureHeight ($Height + 40) + $screenshotPathForOutput = $screenshotFile +} + +if ($KeepOpenSeconds -gt 0) { + Start-Sleep -Seconds $KeepOpenSeconds + if ($process -and -not $process.HasExited) { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + } +} + +$rootAutomationId = "" +if ($dump -and $dump.ok -and $dump.data -and $dump.data.root) { + $rootAutomationId = [string]$dump.data.root.automation_id +} + +$flags = $null +if ($classify -and $classify.ok -and $classify.data -and $classify.data.flags) { + $flags = $classify.data.flags +} + +[ordered]@{ + ok = [bool]($dump.ok -and $classify.ok) + window_kind = "offline_chat_window_probe" + process_id = $process.Id + hwnd = $candidate.hwnd + title = $windowTitle + root_automation_id = $rootAutomationId + root_control_type = if ($dump.ok) { [string]$dump.data.root.control_type } else { "" } + send_editor_found = [bool]($flags -and $flags.has_send_editor) + send_button_found = [bool]($flags -and $flags.has_send_button) + file_button_found = [bool]($flags -and $flags.has_file_menu) + receive_document_found = [bool]($flags -and $flags.has_receive_document) + offline_blocker_visible = [bool]($flags -and $flags.offline_blocker_visible) + route_hint = if ($flags) { [string]$flags.route_hint } else { "" } + probe_click_requested = [bool]$ProbeClick + probe_clicked_ui = $probeClickedUi + probe_typed_text = $probeTypedText + probe_click_marker_found = $probeMarkerFound + probe_marker_text_matches = $probeMarkerTextMatches + probe_action_ok = if ($probeAction) { [bool]$probeAction.ok } else { $false } + probe_action_mode = if ($probeAction -and $probeAction.ok) { [string]$probeAction.data.action_mode } else { "" } + sent_real_message = $false + uploaded_real_file = $false + marker_path = $markerPath + state_path = $statePath + launcher_script = $launcherScript + uia_dump_file = $dumpFile + screenshot_file = $screenshotPathForOutput + alive_after_return = if ($KeepOpenSeconds -gt 0) { $false } else { $null -ne (Get-Process -Id $process.Id -ErrorAction SilentlyContinue) } +} | ConvertTo-Json -Depth 16 diff --git a/scripts/test-open-offline-chat-window-probe.ps1 b/scripts/test-open-offline-chat-window-probe.ps1 new file mode 100644 index 0000000..a9852d5 --- /dev/null +++ b/scripts/test-open-offline-chat-window-probe.ps1 @@ -0,0 +1,62 @@ +param( + [string]$HelperExe = "runs/win-helper/ISphereWinHelper.exe" +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$scriptPath = Join-Path $repo "scripts\open-offline-chat-window-probe.ps1" +$helperPath = Join-Path $repo $HelperExe + +function Assert-True([bool]$Condition, [string]$Message) { + if (-not $Condition) { + throw $Message + } +} + +if (-not (Test-Path -LiteralPath $helperPath)) { + & powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $repo "scripts\build-win-helper.ps1") | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "build-win-helper.ps1 failed with exit code $LASTEXITCODE" + } +} + +Assert-True (Test-Path -LiteralPath $scriptPath) "missing script: $scriptPath" + +$output = & powershell -NoProfile -ExecutionPolicy Bypass -File $scriptPath ` + -HelperExe $HelperExe ` + -KeepOpenSeconds 2 ` + -ProbeClick ` + -NoScreenshot +if ($LASTEXITCODE -ne 0) { + throw "open-offline-chat-window-probe.ps1 exited with code $LASTEXITCODE. Output: $output" +} + +try { + $result = $output | ConvertFrom-Json +} +catch { + throw "script output was not JSON: $output" +} + +Assert-True ([bool]$result.ok) "result.ok must be true" +Assert-True ([string]$result.window_kind -eq "offline_chat_window_probe") "window_kind mismatch" +Assert-True ([string]$result.root_automation_id -eq "frmP2PChat") "root automation id must be frmP2PChat" +Assert-True ([bool]$result.send_editor_found) "rtbSendMessage must be found" +Assert-True ([bool]$result.send_button_found) "btnSend must be found" +Assert-True ([bool]$result.file_button_found) "btnSendFile or btnFile must be found" +Assert-True ([bool]$result.probe_click_marker_found) "probe click marker must be written" +Assert-True ([bool]$result.probe_clicked_ui) "probe must click the UI button" +Assert-True ([bool]$result.probe_typed_text) "probe must type into the send editor" +Assert-True ([bool]$result.probe_marker_text_matches) "probe marker text must match typed text" +Assert-True (-not [bool]$result.sent_real_message) "probe must not send a real message" +Assert-True (-not [bool]$result.uploaded_real_file) "probe must not upload a real file" + +[ordered]@{ + ok = $true + script = $scriptPath + hwnd = $result.hwnd + root_automation_id = $result.root_automation_id + send_editor_found = $result.send_editor_found + send_button_found = $result.send_button_found + probe_click_marker_found = $result.probe_click_marker_found +} | ConvertTo-Json -Depth 4 -Compress From 811f760efe2fd0e3a37a25bc9854171b5ec7d96e Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 20:58:26 +0800 Subject: [PATCH 6/6] tools: open real offline frmP2PChat window --- ...026-07-11-a-route-open-real-chat-window.md | 165 ++++++ scripts/extract-frmP2PChat-il-risk.ps1 | 118 ++++ scripts/open-offline-real-frmP2PChat.ps1 | 545 ++++++++++++++++++ scripts/test-extract-frmP2PChat-il-risk.ps1 | 58 ++ ...-open-offline-real-frmP2PChat-contract.ps1 | 49 ++ 5 files changed, 935 insertions(+) create mode 100644 docs/source-discovery/2026-07-11-a-route-open-real-chat-window.md create mode 100644 scripts/extract-frmP2PChat-il-risk.ps1 create mode 100644 scripts/open-offline-real-frmP2PChat.ps1 create mode 100644 scripts/test-extract-frmP2PChat-il-risk.ps1 create mode 100644 scripts/test-open-offline-real-frmP2PChat-contract.ps1 diff --git a/docs/source-discovery/2026-07-11-a-route-open-real-chat-window.md b/docs/source-discovery/2026-07-11-a-route-open-real-chat-window.md new file mode 100644 index 0000000..e71b825 --- /dev/null +++ b/docs/source-discovery/2026-07-11-a-route-open-real-chat-window.md @@ -0,0 +1,165 @@ +# A-route 离线打开真实 frmP2PChat 会话窗口 + +日期：2026-07-11 +分支：`codex/a-route-open-real-chat-window` +目标：不登录、不连服务端，在本机离线样本中打开真实 `IMPP.Client.Business.ChatManager.SingleChat.frmP2PChat` 窗口，供后续 RPA 对真实窗口做 UIA 控制。 + +## 结论 + +已经在未登录/离线环境下打开真实 `frmP2PChat`。 + +成功命令： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\open-offline-real-frmP2PChat.ps1 -RuntimeMode Deps -ShowWindow -HoldSeconds 1 +``` + +成功产物： + +- 结果 JSON：`E:\coding\codex\isphere-ai-bridge\runs\offline-chat-window\real-frmP2PChat-result-20260711-205312.json` +- HWND：`0xC0CB4` +- 截图：`E:\coding\codex\isphere-ai-bridge\runs\offline-chat-window\real-frmP2PChat-20260711-205312.png` +- UIA dump：`E:\coding\codex\isphere-ai-bridge\runs\offline-chat-window\uia-dump-real-frmP2PChat-20260711-205312.json` +- UIA root： + - `automation_id = frmP2PChat` + - `control_type = Window` + - `framework_id = WinForm` + - `visible = true` + +UIA dump 已确认可见关键控件： + +- `rtbRecvMessage` +- `rtbSendMessage` +- `btnSend` + +窗口内显示“您已处于离线状态，无法发送消息，请上线后再次尝试！”。这是预期状态；本轮目标不是发送消息，也不是上传文件，只是打开真实会话窗口。 + +## 新增脚本 + +### 1. 真实窗口打开脚本 + +`scripts/open-offline-real-frmP2PChat.ps1` + +关键能力： + +- 自动切到 32-bit PowerShell，避免 x86 客户端程序集 `BadImageFormatException`。 +- 只读加载真实客户端目录： + `runs/offline-real-client-window/full/zyl/Impp` +- 使用 `AssemblyResolve` 从真实客户端目录解析依赖。 +- 构造真实类型： + `IMPP.Client.Business.ChatManager.SingleChat.frmP2PChat` +- 支持不 Show 的构造验证，也支持 `-ShowWindow` 真实显示、截图和 UIA dump。 +- 捕获 WinForms `ThreadException`，输出 IL offset、堆栈和 JSON 证据，避免 .NET 异常弹窗卡死。 + +### 2. 构造函数 IL / NullRisk 提取脚本 + +`scripts/extract-frmP2PChat-il-risk.ps1` + +输出： + +- 完整构造函数 IL： + `runs/offline-chat-window/frmP2PChat-ctor.il.txt` +- `callvirt` / `ldfld` NullRisk CSV： + `runs/offline-chat-window/frmP2PChat-ctor-null-risk.csv` +- 风险摘要： + `runs/offline-chat-window/frmP2PChat-ctor-null-risk.md` + +提取结果： + +- 构造函数源 IL 行：`406271-406860` +- 构造函数行数：`590` +- `callvirt` / `ldfld` 风险项：`150` + +## 关键 IL 证据 + +构造函数签名： + +```text +frmP2PChat(com.vision.smack.Chat chat, string pluginInfo, string extendTabJson) +``` + +高信号 NullRisk： + +| IL | 证据 | 需要补的运行态 | +|---|---|---| +| `IL_00f6` | `XMPPConnection::add_OnConnectError` | `IMPPManager.Instance.Connection` / backing field | +| `IL_0111` | `XMPPConnection::add_OnReConnectOk` | 同上 | +| `IL_013b` / `IL_0140` | `P2PChat::get_OtherJid()` / `Jid::getBareJid()` | `P2PChat.OtherJid` | +| `IL_0156` | `IMPPManager::get_UserInfo()` | `IMPPManager._userInfo` | +| `IL_0575` | `MessageCenter::add_OnTcpMessageArrived` | `IMPPManager.MessageCenter` | +| `IL_0614` | `RosterManager::UpdateStrangerStatus` | `IMPPManager.RosterManager` | +| `IL_0631` | `BaseConfig::VisualPhoneEnable()` | `Config.BaseConfig` | + +Show 阶段新增证据： + +| IL | 证据 | 处理 | +|---|---|---| +| `frmP2PChat_Load IL_004d/0058` | `Config::get_BaseConfig()` 后 `BaseConfig::RemoteControlVisible()` | 补 `Config.k__BackingField` | +| `frmChatBase_Activated IL_0000/0005` | `IMPPManager.get_Instance().get_frmMain().BringModalFormToFront()` | 补 `IMPPManager.k__BackingField` | +| `frmP2PChat_Load IL_137f` | `UserInfoManager.getUserInfo(otherBareJid,true)` 后使用 `UserInfo::get_CompanyID()` | 构造完成后预热 `UserInfoCache` | + +注意：`UserInfoCache` 不能在 `frmP2PChat` 构造前预热。过早预热会让构造函数进入 `UCUserInformation.LoadUserInfo(UserInfo)`，在 `UCUserInformation::LoadUserInfo IL_0085` 触发新的 NRE。当前脚本是在真实窗体构造完成后、`Show()` 前预热缓存。 + +## 最小 fake runtime 清单 + +当前成功路径最小补齐如下： + +1. `IMPPManager.Instance` + - `k__BackingField` + - `k__BackingField` + - `LogonUser` + - `_userInfo` + - `_rosterUserInfoList` +2. `SmarkManager` + - `k__BackingField` +3. `XMPPConnection` + - `Jid` + - `UserName` + - `LoginState` + - `CompanyID` + - `Resource` + - `_packetListeners` + - `_packetCollectors` +4. `IMPPManager` inherited manager fields/properties + - `Connection` and `k__BackingField` + - `IsConnected` and `k__BackingField` + - `PresenceManager` and backing field + - `P2PChatManager` and backing field + - `RosterManager` and backing field +5. Config + - `ConfigSystemManager.config` + - `Config.k__BackingField` + - `Config.k__BackingField` + - `Config.k__BackingField` +6. `UserInfoCache` + - 在 `frmP2PChat` 构造后、`Show()` 前加入自己和对端的 JID/ID 映射。 +7. Chat 对象 + - 真实 `com.vision.smack.P2PChat` + - `OtherJid = codex-probe@offline.local` + +## 验证命令 + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\test-extract-frmP2PChat-il-risk.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\extract-frmP2PChat-il-risk.ps1 -OutDir runs/offline-chat-window +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\test-open-offline-real-frmP2PChat-contract.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\open-offline-real-frmP2PChat.ps1 -RuntimeMode Deps +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\open-offline-real-frmP2PChat.ps1 -RuntimeMode Deps -ShowWindow -HoldSeconds 1 +``` + +## 当前边界 + +- 已成功打开真实 `frmP2PChat`。 +- UIA 可 dump 到真实 `frmP2PChat`、`rtbRecvMessage`、`rtbSendMessage`、`btnSend`。 +- 离线环境下窗口会提示无法发送消息，这是客户端真实离线状态表现，不影响后续 RPA 窗口定位。 +- 本轮没有处理真实发送、真实上传、真实登录态。 +- `btnSendFile` 在这次真实离线 UIA dump 中没有以 `btnSendFile` automation id 暴露；后续如果 RPA 需要上传文件，需要另开节点定位真实工具栏按钮或菜单路径。 + +## 下一轮建议 + +1. 把 A-route RPA 的窗口定位目标从 synthetic probe 切到真实 `frmP2PChat`： + - root `automation_id=frmP2PChat` + - send box `automation_id=rtbSendMessage` + - send button `automation_id=btnSend` +2. 如果弹出“企业信息获取失败”影响自动化，单独加一个非阻塞关闭弹窗步骤。 +3. 如果后续需要文件按钮，再专门分析真实工具栏图标按钮、菜单项和 `ToolStrip` 命中路径。 diff --git a/scripts/extract-frmP2PChat-il-risk.ps1 b/scripts/extract-frmP2PChat-il-risk.ps1 new file mode 100644 index 0000000..5f97606 --- /dev/null +++ b/scripts/extract-frmP2PChat-il-risk.ps1 @@ -0,0 +1,118 @@ +param( + [string]$IlFile = "runs/offline-evidence-intake/zyl-qqfile-20260709/metadata/c28-il/IMPlatformClient.exe.il", + [string]$OutDir = "runs/offline-chat-window" +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") + +function Resolve-RepoPath([string]$PathValue) { + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return $PathValue + } + return (Join-Path $repo $PathValue) +} + +$ilFull = Resolve-RepoPath $IlFile +$outFull = Resolve-RepoPath $OutDir +if (-not (Test-Path -LiteralPath $ilFull)) { + throw "IL file not found: $ilFull" +} +New-Item -ItemType Directory -Force -Path $outFull | Out-Null + +$lines = Get-Content -LiteralPath $ilFull +$endIndex = $null +for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -match 'end of method frmP2PChat::\.ctor') { + $endIndex = $i + break + } +} +if ($null -eq $endIndex) { + throw "frmP2PChat::.ctor end marker not found" +} + +$startIndex = $null +for ($i = $endIndex; $i -ge 0; $i--) { + if ($lines[$i] -match '^\s*\.method\s+public\s+hidebysig\s+specialname\s+rtspecialname') { + $startIndex = $i + break + } +} +if ($null -eq $startIndex) { + throw "frmP2PChat::.ctor method start not found before line $($endIndex + 1)" +} + +$ctorIlFile = Join-Path $outFull "frmP2PChat-ctor.il.txt" +$riskCsvFile = Join-Path $outFull "frmP2PChat-ctor-null-risk.csv" +$riskMdFile = Join-Path $outFull "frmP2PChat-ctor-null-risk.md" + +$ctorLines = New-Object System.Collections.Generic.List[string] +for ($i = $startIndex; $i -le $endIndex; $i++) { + $ctorLines.Add(("{0}: {1}" -f ($i + 1), $lines[$i])) +} +$ctorLines | Set-Content -LiteralPath $ctorIlFile -Encoding UTF8 + +$riskRows = New-Object System.Collections.Generic.List[object] +foreach ($entry in $ctorLines) { + if ($entry -notmatch '\b(callvirt|ldfld)\b') { + continue + } + $offset = $null + if ($entry -match '(IL_[0-9a-fA-F]{4})') { + $offset = $matches[1].ToLowerInvariant().Replace("il_", "IL_") + } + $op = if ($entry -match '\b(callvirt|ldfld)\b') { $matches[1] } else { $null } + $target = ($entry -replace '^.*?\b(callvirt|ldfld)\b\s+', '').Trim() + $riskRows.Add([pscustomobject]@{ + source_line = [int]($entry -replace '^(\d+):.*$', '$1') + il_offset = $offset + opcode = $op + target = $target + null_risk = if ($op -eq "callvirt") { "receiver_null_possible" } else { "instance_field_owner_null_possible" } + evidence = $entry.Trim() + }) +} + +$riskRows | Sort-Object source_line | Export-Csv -LiteralPath $riskCsvFile -NoTypeInformation -Encoding UTF8 + +$topEvidence = $riskRows | + Where-Object { + $_.target -match 'Connection::add_OnConnectError|Connection::add_OnReConnectOk|P2PChat::get_OtherJid|Jid::getBareJid|IMPPManager::get_UserInfo|UCChatSendMessageBox::GetChatRichTextBox|UCChatSendMessageBox::GetReceiptCheckBox|MessageCenter::add_OnTcpMessageArrived|RosterManager::UpdateStrangerStatus|BaseConfig::VisualPhoneEnable' + } | + Select-Object -First 40 + +$md = New-Object System.Collections.Generic.List[string] +$md.Add("# frmP2PChat constructor IL NullRisk") +$md.Add("") +$md.Add("- Source IL: $ilFull") +$md.Add("- Constructor source lines: $($startIndex + 1)-$($endIndex + 1)") +$md.Add("- Constructor line count: $($ctorLines.Count)") +$md.Add("- callvirt / ldfld risk rows: $($riskRows.Count)") +$md.Add("") +$md.Add("## High-signal risks") +$md.Add("") +$md.Add("| source_line | il_offset | opcode | target |") +$md.Add("|---:|---|---|---|") +foreach ($row in $topEvidence) { + $target = ($row.target -replace '\|', '\\|') + $md.Add("| $($row.source_line) | $($row.il_offset) | $($row.opcode) | $target |") +} +$md.Add("") +$md.Add("## Files") +$md.Add("") +$md.Add("- Full constructor IL: $ctorIlFile") +$md.Add("- Full NullRisk CSV: $riskCsvFile") +$md | Set-Content -LiteralPath $riskMdFile -Encoding UTF8 + +[pscustomobject]@{ + ok = $true + il_file = $ilFull + ctor_line_start = $startIndex + 1 + ctor_line_end = $endIndex + 1 + ctor_line_count = $ctorLines.Count + risk_count = $riskRows.Count + ctor_il_file = $ctorIlFile + risk_csv_file = $riskCsvFile + risk_md_file = $riskMdFile +} | ConvertTo-Json -Compress diff --git a/scripts/open-offline-real-frmP2PChat.ps1 b/scripts/open-offline-real-frmP2PChat.ps1 new file mode 100644 index 0000000..29c46d7 --- /dev/null +++ b/scripts/open-offline-real-frmP2PChat.ps1 @@ -0,0 +1,545 @@ +param( + [string]$ClientDir = "runs/offline-real-client-window/full/zyl/Impp", + [string]$OutDir = "runs/offline-chat-window", + [ValidateSet("Minimal", "Deps")] + [string]$RuntimeMode = "Deps", + [switch]$SelfChat, + [switch]$SkipRosterManager, + [switch]$ShowWindow, + [string]$HelperExe = "runs/win-helper/ISphereWinHelper.exe", + [switch]$NoScreenshot, + [int]$HoldSeconds = 8, + [int]$X = 180, + [int]$Y = 180, + [int]$Width = 900, + [int]$Height = 680 +) + +$ErrorActionPreference = "Stop" + +if ([Environment]::Is64BitProcess) { + $x86PowerShell = Join-Path $env:WINDIR "SysWOW64\WindowsPowerShell\v1.0\powershell.exe" + if (-not (Test-Path -LiteralPath $x86PowerShell)) { + throw "32-bit PowerShell not found: $x86PowerShell" + } + + $relayArgs = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $PSCommandPath) + foreach ($entry in $PSBoundParameters.GetEnumerator()) { + $name = [string]$entry.Key + $value = $entry.Value + if ($value -is [System.Management.Automation.SwitchParameter]) { + if ($value.IsPresent) { + $relayArgs += "-$name" + } + } + else { + $relayArgs += "-$name" + $relayArgs += [string]$value + } + } + + & $x86PowerShell @relayArgs + exit $LASTEXITCODE +} + +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") + +function Resolve-RepoPath([string]$PathValue) { + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return $PathValue + } + return (Join-Path $repo $PathValue) +} + +function Get-TypeMemberInHierarchy([Type]$Type, [string]$Name, [string]$Kind) { + $flags = [System.Reflection.BindingFlags]"Public,NonPublic,Instance,Static" + $cursor = $Type + while ($null -ne $cursor) { + if ($Kind -eq "Property") { + $member = $cursor.GetProperty($Name, $flags) + } + else { + $member = $cursor.GetField($Name, $flags) + } + if ($member) { + return $member + } + $cursor = $cursor.BaseType + } + return $null +} + +function Set-PropInHierarchy([object]$Object, [Type]$Type, [string]$Name, [object]$Value) { + $prop = Get-TypeMemberInHierarchy -Type $Type -Name $Name -Kind "Property" + if ($prop -and $prop.CanWrite) { + $target = if ($prop.GetSetMethod($true).IsStatic) { $null } else { $Object } + $prop.SetValue($target, $Value, $null) + return $true + } + return $false +} + +function Set-FieldInHierarchy([object]$Object, [Type]$Type, [string]$Name, [object]$Value) { + $field = Get-TypeMemberInHierarchy -Type $Type -Name $Name -Kind "Field" + if ($field) { + $target = if ($field.IsStatic) { $null } else { $Object } + $field.SetValue($target, $Value) + return $true + } + return $false +} + +function New-ByConstructor([Type]$Type, [Type[]]$ParameterTypes, [object[]]$Arguments) { + $ctor = $Type.GetConstructor($ParameterTypes) + if (-not $ctor) { + $sig = ($ParameterTypes | ForEach-Object { $_.FullName }) -join ", " + throw "constructor not found: $($Type.FullName)($sig)" + } + return $ctor.Invoke($Arguments) +} + +function Convert-ExceptionToRecord([Exception]$Exception) { + $records = @() + $cursor = $Exception + $depth = 0 + while ($cursor) { + $trace = New-Object System.Diagnostics.StackTrace($cursor, $true) + $frames = @() + for ($i = 0; $i -lt $trace.FrameCount; $i++) { + $frame = $trace.GetFrame($i) + $method = $frame.GetMethod() + $offset = $frame.GetILOffset() + $frames += [pscustomobject]@{ + index = $i + method_type = if ($method.DeclaringType) { $method.DeclaringType.FullName } else { $null } + method_name = $method.Name + il_offset = $offset + il_offset_hex = if ($offset -ge 0) { "IL_{0:x4}" -f $offset } else { $null } + native_offset = $frame.GetNativeOffset() + file = $frame.GetFileName() + line = $frame.GetFileLineNumber() + } + } + + $records += [pscustomobject]@{ + depth = $depth + type = $cursor.GetType().FullName + message = $cursor.Message + target_site = if ($cursor.TargetSite) { "$($cursor.TargetSite.DeclaringType.FullName)::$($cursor.TargetSite.Name)" } else { $null } + frames = $frames + } + $cursor = $cursor.InnerException + $depth++ + } + return $records +} + +function Get-FirstRelevantFrame([object[]]$ExceptionRecords) { + foreach ($record in $ExceptionRecords) { + foreach ($frame in $record.frames) { + if ($frame.method_type -and + $frame.method_type -notlike "System.*" -and + $frame.method_type -notlike "Microsoft.*" -and + $frame.method_type -notlike "MS.*") { + return $frame + } + } + } + return $null +} + +function Find-IlLine([string]$OutDirFull, [string]$IlOffsetHex) { + if ([string]::IsNullOrWhiteSpace($IlOffsetHex)) { + return $null + } + $ctorIl = Join-Path $OutDirFull "frmP2PChat-ctor.il.txt" + if (-not (Test-Path -LiteralPath $ctorIl)) { + return $null + } + $match = Select-String -LiteralPath $ctorIl -Pattern $IlOffsetHex -SimpleMatch | Select-Object -First 1 + if ($match) { + return @{ + path = $ctorIl + line_number = $match.LineNumber + text = $match.Line.Trim() + } + } + return $null +} + +function Invoke-HelperJson([string]$HelperPath, [string]$Op, [hashtable]$OpArgs, [string]$RequestId) { + $request = @{ + protocol = "isphere.helper.v1" + request_id = $RequestId + op = $Op + timeout_ms = 5000 + args = $OpArgs + } + $json = $request | ConvertTo-Json -Depth 16 -Compress + $output = $json | & $HelperPath --json + try { + return $output | ConvertFrom-Json + } + catch { + throw "helper output was not JSON: $output" + } +} + +function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY, [int]$CaptureWidth, [int]$CaptureHeight) { + $bitmap = New-Object System.Drawing.Bitmap $CaptureWidth, $CaptureHeight + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + try { + $graphics.CopyFromScreen($ScreenX, $ScreenY, 0, 0, $bitmap.Size) + $bitmap.Save($OutputPath, [System.Drawing.Imaging.ImageFormat]::Png) + } + finally { + $graphics.Dispose() + $bitmap.Dispose() + } +} + +function New-UserInfo([Type]$UserInfoType, [string]$Id, [string]$Jid, [string]$Name, [string]$CompanyID) { + $user = [Activator]::CreateInstance($UserInfoType) + foreach ($kv in @( + @("Id", $Id), + @("Jid", $Jid), + @("LoginName", $Id), + @("Domain", "offline.local"), + @("Name", $Name), + @("CompanyID", $CompanyID), + @("DeptID", "codex-dept"), + @("DeptName", "Codex Dept"), + @("Enable", "1"), + @("chatAble", "1"), + @("Mobile", ""), + @("Tel", ""), + @("Email", "") + )) { + Set-PropInHierarchy -Object $user -Type $UserInfoType -Name $kv[0] -Value $kv[1] | Out-Null + } + return $user +} + +$clientFull = Resolve-RepoPath $ClientDir +$outFull = Resolve-RepoPath $OutDir +$helperFull = Resolve-RepoPath $HelperExe +New-Item -ItemType Directory -Force -Path $outFull | Out-Null + +if (-not (Test-Path -LiteralPath (Join-Path $clientFull "IMPlatformClient.exe"))) { + throw "IMPlatformClient.exe not found under $clientFull" +} + +Set-Location -LiteralPath $clientFull + +[AppDomain]::CurrentDomain.add_AssemblyResolve({ + param($sender, $eventArgs) + $assemblyName = New-Object System.Reflection.AssemblyName($eventArgs.Name) + foreach ($extension in @(".dll", ".exe")) { + $candidate = Join-Path $script:ClientDirectoryForResolve ($assemblyName.Name + $extension) + if (Test-Path -LiteralPath $candidate) { + return [System.Reflection.Assembly]::LoadFrom($candidate) + } + } + return $null +}) +$script:ClientDirectoryForResolve = $clientFull + +$timestamp = Get-Date -Format "yyyyMMdd-HHmmss" +$resultFile = Join-Path $outFull ("real-frmP2PChat-result-{0}.json" -f $timestamp) + +try { + Add-Type -AssemblyName System.Windows.Forms + Add-Type -AssemblyName System.Drawing + $script:RealFrmP2PThreadExceptions = New-Object System.Collections.ArrayList + [System.Windows.Forms.Application]::SetUnhandledExceptionMode([System.Windows.Forms.UnhandledExceptionMode]::CatchException) + [System.Windows.Forms.Application]::add_ThreadException({ + param($sender, $eventArgs) + [void]$script:RealFrmP2PThreadExceptions.Add($eventArgs.Exception) + }) + + $asm = [System.Reflection.Assembly]::LoadFrom((Join-Path $clientFull "IMPlatformClient.exe")) + $smack = [System.Reflection.Assembly]::LoadFrom((Join-Path $clientFull "smack.dll")) + $common = [System.Reflection.Assembly]::LoadFrom((Join-Path $clientFull "IMPP.Common.dll")) + + $mgrType = $asm.GetType("IMPP.Client.IMPPManager", $true) + $mcType = $asm.GetType("IMPP.Client.control.MessageCenter", $true) + $configType = $asm.GetType("IMPP.Client.Config", $true) + $baseConfigType = $asm.GetType("IMPP.Client.BaseConfig", $true) + $configMgrType = $asm.GetType("IMPP.Client.ConfigSystemManager", $true) + $configModelType = $asm.GetType("IMPP.Client.ConfigSystemModel", $true) + $userInfoCacheType = $asm.GetType("IMPP.Client.Framework.Configuration.UserInfoCache", $true) + $loginInfoType = $asm.GetType("IMPP.Client.LoginInfo", $true) + $p2pFormType = $asm.GetType("IMPP.Client.Business.ChatManager.SingleChat.frmP2PChat", $true) + $chatBaseType = $asm.GetType("IMPP.Client.Forms.frmChatBase", $true) + $frmMainType = $asm.GetType("IMPP.Client.frmMain", $true) + $connType = $smack.GetType("com.vision.smack.XMPPConnection", $true) + $jidType = $smack.GetType("com.vision.smack.Jid", $true) + $chatType = $smack.GetType("com.vision.smack.Chat", $true) + $p2pChatType = $smack.GetType("com.vision.smack.P2PChat", $true) + $rosterType = $smack.GetType("com.vision.smack.RosterManager", $true) + $presenceType = $smack.GetType("com.vision.smack.PresenceManager", $true) + $p2pMgrType = $smack.GetType("com.vision.smack.P2PChatManager", $true) + $bookMarkType = $smack.GetType("com.vision.smack.muc.BookMarkManager", $true) + $packetCollectorType = $smack.GetType("com.vision.smack.PacketCollector", $true) + $userInfoType = $common.GetType("IMPP.Common.UserInfo", $true) + + $mgr = $mgrType.GetProperty("Instance", [System.Reflection.BindingFlags]"Public,Static").GetValue($null, $null) + + $fakeMessageCenter = [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject($mcType) + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $fakeMessageCenter | Out-Null + $fakeFrmMain = [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject($frmMainType) + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $fakeFrmMain | Out-Null + + $login = [Activator]::CreateInstance($loginInfoType) + Set-PropInHierarchy -Object $login -Type $loginInfoType -Name "Trait" -Value "codex_probe" | Out-Null + Set-PropInHierarchy -Object $login -Type $loginInfoType -Name "TrueTraid" -Value "codex_probe" | Out-Null + Set-PropInHierarchy -Object $login -Type $loginInfoType -Name "Status" -Value "online" | Out-Null + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "LogonUser" -Value $login | Out-Null + + $companyId = "codex-company" + $otherBareJid = "codex-probe@offline.local" + $myBareJid = if ($SelfChat) { $otherBareJid } else { "codex-user@offline.local" } + $myName = if ($SelfChat) { "Codex Probe" } else { "Codex User" } + + $jidCtor3 = $jidType.GetConstructor(@([string], [string], [string])) + $myJid = $jidCtor3.Invoke(@(($myBareJid -split "@")[0], "offline.local", "pc")) + + $fakeConn = [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject($connType) + Set-PropInHierarchy -Object $fakeConn -Type $connType -Name "Jid" -Value $myJid | Out-Null + Set-PropInHierarchy -Object $fakeConn -Type $connType -Name "UserName" -Value (($myBareJid -split "@")[0]) | Out-Null + Set-PropInHierarchy -Object $fakeConn -Type $connType -Name "LoginState" -Value "online" | Out-Null + Set-PropInHierarchy -Object $fakeConn -Type $connType -Name "CompanyID" -Value $companyId | Out-Null + Set-PropInHierarchy -Object $fakeConn -Type $connType -Name "Resource" -Value "pc" | Out-Null + Set-FieldInHierarchy -Object $fakeConn -Type $connType -Name "_packetListeners" -Value (New-Object System.Collections.ArrayList) | Out-Null + $packetCollectorListType = [System.Collections.Generic.List``1].MakeGenericType($packetCollectorType) + Set-FieldInHierarchy -Object $fakeConn -Type $connType -Name "_packetCollectors" -Value ([Activator]::CreateInstance($packetCollectorListType)) | Out-Null + + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "Connection" -Value $fakeConn | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $fakeConn | Out-Null + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "IsConnected" -Value $true | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $true | Out-Null + Set-FieldInHierarchy -Object $null -Type ($smack.GetType("com.vision.smack.SmarkManager", $true)) -Name "k__BackingField" -Value $mgr | Out-Null + + if ($RuntimeMode -eq "Deps") { + $myUser = New-UserInfo -UserInfoType $userInfoType -Id (($myBareJid -split "@")[0]) -Jid $myBareJid -Name $myName -CompanyID $companyId + $otherUser = New-UserInfo -UserInfoType $userInfoType -Id "codex-probe" -Jid $otherBareJid -Name "Codex Probe" -CompanyID $companyId + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "_userInfo" -Value $myUser | Out-Null + + $stringDictType = [System.Collections.Generic.Dictionary``2].MakeGenericType([string], [string]) + $baseConfigDict = [Activator]::CreateInstance($stringDictType, [StringComparer]::OrdinalIgnoreCase) + foreach ($kv in @( + @("Remote_Control_Visible", "false"), + @("VisualPhone_enable", "false"), + @("showacctno", "no"), + @("ShowDepartment", "NO"), + @("StrucPPLNo", "NO"), + @("MaintenanceAssistant", ""), + @("CustomerServiceSuffix", ""), + @("CustomerCN", ""), + @("CustomerPhone", "") + )) { + $baseConfigDict[$kv[0]] = $kv[1] + $baseConfigDict[$kv[0].ToLowerInvariant()] = $kv[1] + } + $baseConfigRawDict = $baseConfigDict.PSObject.BaseObject + $baseConfigCtor = $baseConfigType.GetConstructor(@($baseConfigRawDict.GetType())) + if (-not $baseConfigCtor) { + throw "constructor not found: $($baseConfigType.FullName)($($baseConfigRawDict.GetType().FullName))" + } + $baseConfig = $baseConfigCtor.Invoke([object[]]@($baseConfigRawDict)) + Set-FieldInHierarchy -Object $null -Type $configType -Name "k__BackingField" -Value $baseConfig | Out-Null + Set-FieldInHierarchy -Object $null -Type $configType -Name "k__BackingField" -Value $myUser | Out-Null + Set-FieldInHierarchy -Object $null -Type $configType -Name "k__BackingField" -Value $login | Out-Null + + $userListType = [System.Collections.Generic.List``1].MakeGenericType($userInfoType) + $rosterUsers = [Activator]::CreateInstance($userListType) + $rosterUsers.Add($myUser) | Out-Null + $rosterUsers.Add($otherUser) | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "_rosterUserInfoList" -Value $rosterUsers | Out-Null + + $cfg = [Activator]::CreateInstance($configModelType) + foreach ($kv in @( + @("Font", "Microsoft YaHei UI"), + @("Size", [single]9), + @("FontStyle", "Regular"), + @("Color", "Black"), + @("SendMsgKeyPress", "Enter"), + @("SendFilePath", $outFull), + @("SendFileSize", 10), + @("SendFileConf", $false), + @("AllEnable", $true) + )) { + Set-PropInHierarchy -Object $cfg -Type $configModelType -Name $kv[0] -Value $kv[1] | Out-Null + } + $configMgrType.GetField("config", [System.Reflection.BindingFlags]"NonPublic,Static").SetValue($null, $cfg) + + $presence = New-ByConstructor -Type $presenceType -ParameterTypes @($connType) -Arguments @($fakeConn) + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "PresenceManager" -Value $presence | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $presence | Out-Null + + $bookmark = [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject($bookMarkType) + $p2pManager = New-ByConstructor -Type $p2pMgrType -ParameterTypes @($connType, $bookMarkType) -Arguments @($fakeConn, $bookmark) + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "P2PChatManager" -Value $p2pManager | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $p2pManager | Out-Null + + if (-not $SkipRosterManager) { + $roster = New-ByConstructor -Type $rosterType -ParameterTypes @($connType) -Arguments @($fakeConn) + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "RosterManager" -Value $roster | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $roster | Out-Null + } + else { + Set-PropInHierarchy -Object $mgr -Type $mgrType -Name "RosterManager" -Value $null | Out-Null + Set-FieldInHierarchy -Object $mgr -Type $mgrType -Name "k__BackingField" -Value $null | Out-Null + } + } + + $p2pChat = New-ByConstructor -Type $p2pChatType -ParameterTypes @($connType, [string]) -Arguments @($fakeConn, $otherBareJid) + $otherJid = $p2pChatType.GetProperty("OtherJid").GetValue($p2pChat, $null) + if ($null -eq $otherJid) { + $otherJid = $jidCtor3.Invoke(@("codex-probe", "offline.local", "")) + Set-FieldInHierarchy -Object $p2pChat -Type $p2pChatType -Name "k__BackingField" -Value $otherJid | Out-Null + } + + $formCtor = $p2pFormType.GetConstructor(@($chatType, [string], [string])) + $form = $formCtor.Invoke(@($p2pChat, "", "")) + if ($RuntimeMode -eq "Deps") { + $userInfoCache = $userInfoCacheType.GetMethod("GetInstance", [System.Reflection.BindingFlags]"Public,Static").Invoke($null, @()) + $addUserInfoCache = $userInfoCacheType.GetMethod("AddCache", [Type[]]@([string], $userInfoType)) + foreach ($cacheEntry in @( + @{ Key = $myBareJid; User = $myUser }, + @{ Key = $myBareJid.ToLowerInvariant(); User = $myUser }, + @{ Key = (($myBareJid -split "@")[0]); User = $myUser }, + @{ Key = $otherBareJid; User = $otherUser }, + @{ Key = $otherBareJid.ToLowerInvariant(); User = $otherUser }, + @{ Key = "codex-probe"; User = $otherUser } + )) { + $addUserInfoCache.Invoke($userInfoCache, [object[]]@([string]$cacheEntry.Key, $cacheEntry.User.PSObject.BaseObject)) + } + } + $form.Name = "frmP2PChat" + $form.Text = "Codex Real frmP2PChat Offline Probe" + $form.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual + $form.Left = $X + $form.Top = $Y + $form.Width = $Width + $form.Height = $Height + + $uiaDumpFile = $null + $uiaRoot = $null + $screenshotFile = $null + + if ($ShowWindow) { + $form.Show() + [System.Windows.Forms.Application]::DoEvents() + Start-Sleep -Milliseconds 800 + [System.Windows.Forms.Application]::DoEvents() + + if ($script:RealFrmP2PThreadExceptions.Count -gt 0) { + $threadExceptionRecords = @() + foreach ($threadException in $script:RealFrmP2PThreadExceptions) { + $threadExceptionRecords += Convert-ExceptionToRecord -Exception $threadException + } + $relevantFrame = Get-FirstRelevantFrame -ExceptionRecords $threadExceptionRecords + $threadResult = [pscustomobject]@{ + ok = $false + stage = "show_thread_exception" + process_bitness = 32 + runtime_mode = $RuntimeMode + self_chat = [bool]$SelfChat + skip_roster_manager = [bool]$SkipRosterManager + client_dir = $clientFull + type = $p2pFormType.FullName + hwnd = if ($form.IsHandleCreated) { "0x" + $form.Handle.ToInt64().ToString("X") } else { $null } + visible = $form.Visible + relevant_frame = $relevantFrame + exceptions = $threadExceptionRecords + result_file = $resultFile + } + $threadResult | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $resultFile -Encoding UTF8 + $threadResult | ConvertTo-Json -Depth 32 -Compress + try { $form.Close() } catch {} + exit 3 + } + + if ($form.IsHandleCreated -and (Test-Path -LiteralPath $helperFull)) { + $hwndForDump = "0x" + $form.Handle.ToInt64().ToString("X") + $dump = Invoke-HelperJson -HelperPath $helperFull -Op "dump_uia" -OpArgs @{ + hwnd = $hwndForDump + max_depth = 8 + max_children = 250 + include_text = $true + } -RequestId "real-frmP2PChat-dump-uia" + $uiaDumpFile = Join-Path $outFull ("uia-dump-real-frmP2PChat-{0}.json" -f $timestamp) + $dump | ConvertTo-Json -Depth 40 | Set-Content -LiteralPath $uiaDumpFile -Encoding UTF8 + if ($dump.ok) { + $uiaRoot = $dump.data.root + } + } + + if (-not $NoScreenshot) { + $screenshotFile = Join-Path $outFull ("real-frmP2PChat-{0}.png" -f $timestamp) + Save-WindowScreenshot -OutputPath $screenshotFile -ScreenX $form.Left -ScreenY $form.Top -CaptureWidth $form.Width -CaptureHeight $form.Height + } + + $remainingHold = [Math]::Max(0, $HoldSeconds) + if ($remainingHold -gt 0) { + Start-Sleep -Seconds $remainingHold + } + } + + $result = [pscustomobject]@{ + ok = $true + process_bitness = 32 + runtime_mode = $RuntimeMode + self_chat = [bool]$SelfChat + skip_roster_manager = [bool]$SkipRosterManager + client_dir = $clientFull + type = $p2pFormType.FullName + chat_type = $p2pChat.GetType().FullName + hwnd = if ($form.IsHandleCreated) { "0x" + $form.Handle.ToInt64().ToString("X") } else { $null } + visible = $form.Visible + name = $form.Name + text = $form.Text + size = @{ width = $form.Width; height = $form.Height } + result_file = $resultFile + uia_dump_file = $uiaDumpFile + screenshot_file = $screenshotFile + uia_root = $uiaRoot + } + $result | ConvertTo-Json -Depth 24 | Set-Content -LiteralPath $resultFile -Encoding UTF8 + $result | ConvertTo-Json -Depth 24 -Compress +} +catch { + $exceptionRecords = Convert-ExceptionToRecord -Exception $_.Exception + $ctorFrame = $null + foreach ($record in $exceptionRecords) { + foreach ($frame in $record.frames) { + if ($frame.method_type -eq "IMPP.Client.Business.ChatManager.SingleChat.frmP2PChat" -and $frame.method_name -eq ".ctor") { + $ctorFrame = $frame + break + } + } + if ($ctorFrame) { break } + } + $matchedIlLine = if ($ctorFrame) { Find-IlLine -OutDirFull $outFull -IlOffsetHex $ctorFrame.il_offset_hex } else { $null } + + $result = [pscustomobject]@{ + ok = $false + process_bitness = 32 + runtime_mode = $RuntimeMode + self_chat = [bool]$SelfChat + skip_roster_manager = [bool]$SkipRosterManager + client_dir = $clientFull + error = $_.Exception.ToString() + category = $_.CategoryInfo.ToString() + script_stack = $_.ScriptStackTrace + ctor_il_offset_hex = if ($ctorFrame) { $ctorFrame.il_offset_hex } else { $null } + ctor_frame = $ctorFrame + matched_il_line = $matchedIlLine + exceptions = $exceptionRecords + result_file = $resultFile + } + $result | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $resultFile -Encoding UTF8 + $result | ConvertTo-Json -Depth 32 -Compress + exit 2 +} diff --git a/scripts/test-extract-frmP2PChat-il-risk.ps1 b/scripts/test-extract-frmP2PChat-il-risk.ps1 new file mode 100644 index 0000000..9ca8932 --- /dev/null +++ b/scripts/test-extract-frmP2PChat-il-risk.ps1 @@ -0,0 +1,58 @@ +param( + [string]$ScriptPath = "scripts/extract-frmP2PChat-il-risk.ps1", + [string]$OutDir = "runs/offline-chat-window/il-risk-contract" +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$target = if ([System.IO.Path]::IsPathRooted($ScriptPath)) { + $ScriptPath +} +else { + Join-Path $repo $ScriptPath +} + +if (-not (Test-Path -LiteralPath $target)) { + throw "script not found: $target" +} + +$outFull = if ([System.IO.Path]::IsPathRooted($OutDir)) { + $OutDir +} +else { + Join-Path $repo $OutDir +} +if (Test-Path -LiteralPath $outFull) { + Remove-Item -LiteralPath $outFull -Recurse -Force +} + +$raw = & powershell -NoProfile -ExecutionPolicy Bypass -File $target -OutDir $OutDir +$result = $raw | ConvertFrom-Json + +if (-not $result.ok) { + throw "extract script returned ok=false" +} +foreach ($path in @($result.ctor_il_file, $result.risk_csv_file, $result.risk_md_file)) { + if (-not (Test-Path -LiteralPath $path)) { + throw "expected output missing: $path" + } +} +if ($result.ctor_line_count -lt 500) { + throw "ctor IL line count too small: $($result.ctor_line_count)" +} +if ($result.risk_count -lt 50) { + throw "risk count too small: $($result.risk_count)" +} +$riskText = Get-Content -LiteralPath $result.risk_csv_file -Raw +foreach ($required in @("IL_00f6", "add_OnConnectError", "P2PChat::get_OtherJid", "IMPPManager::get_UserInfo")) { + if ($riskText -notmatch [regex]::Escape($required)) { + throw "risk csv missing evidence: $required" + } +} + +[pscustomobject]@{ + ok = $true + checked = 4 + ctor_line_count = $result.ctor_line_count + risk_count = $result.risk_count +} | ConvertTo-Json -Compress diff --git a/scripts/test-open-offline-real-frmP2PChat-contract.ps1 b/scripts/test-open-offline-real-frmP2PChat-contract.ps1 new file mode 100644 index 0000000..9c8f6a8 --- /dev/null +++ b/scripts/test-open-offline-real-frmP2PChat-contract.ps1 @@ -0,0 +1,49 @@ +param( + [string]$ScriptPath = "scripts/open-offline-real-frmP2PChat.ps1" +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$target = if ([System.IO.Path]::IsPathRooted($ScriptPath)) { + $ScriptPath +} +else { + Join-Path $repo $ScriptPath +} + +if (-not (Test-Path -LiteralPath $target)) { + throw "script not found: $target" +} + +$text = Get-Content -LiteralPath $target -Raw +$requiredPatterns = @( + @{ Name = "x86 PowerShell guard"; Pattern = "Environment]::Is64BitProcess" }, + @{ Name = "AssemblyResolve dependency loader"; Pattern = "AssemblyResolve" }, + @{ Name = "fake runtime mode parameter"; Pattern = "RuntimeMode" }, + @{ Name = "self chat variant"; Pattern = "SelfChat" }, + @{ Name = "skip roster variant"; Pattern = "SkipRosterManager" }, + @{ Name = "CLR IL offset extraction"; Pattern = "GetILOffset" }, + @{ Name = "IL offset JSON field"; Pattern = "il_offset_hex" }, + @{ Name = "Config.BaseConfig fake runtime"; Pattern = "k__BackingField" }, + @{ Name = "remote control config key"; Pattern = "Remote_Control_Visible" }, + @{ Name = "fake frmMain activation target"; Pattern = "k__BackingField" }, + @{ Name = "UserInfo cache warmup"; Pattern = "UserInfoCache" }, + @{ Name = "result artifact output"; Pattern = "real-frmP2PChat-result" } +) + +$missing = @() +foreach ($item in $requiredPatterns) { + if ($text -notmatch $item.Pattern) { + $missing += $item.Name + } +} + +if ($missing.Count -gt 0) { + throw "contract missing: $($missing -join ', ')" +} + +[pscustomobject]@{ + ok = $true + script = $target + checked = $requiredPatterns.Count +} | ConvertTo-Json -Compress