From c2a1ca560f6dc0faab91bd9d27f2302a32e2e517 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Sat, 11 Jul 2026 20:04:34 +0800 Subject: [PATCH] tools: add offline rpa window watchdog --- .../2026-07-11-a-route-user-silent-rpa.md | 46 +++++++ scripts/open-offline-real-client-window.ps1 | 124 +++++++++++++++++- ...en-offline-real-client-window-watchdog.ps1 | 45 +++++++ 3 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 scripts/test-open-offline-real-client-window-watchdog.ps1 diff --git a/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md b/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md index 04eefa6..14f0fa6 100644 --- a/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md +++ b/docs/source-discovery/2026-07-11-a-route-user-silent-rpa.md @@ -31,6 +31,52 @@ powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-b - 不生成屏幕截图，避免因为窗口在屏幕外得到无意义图片。 - 继续输出 UIA dump，供 RPA 选择器验证。 +## 抗干扰能力 + +新增守护模式： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -WindowMode Visible -WatchSeconds 30 -PollIntervalMs 500 +``` + +也可和无感模式一起使用： + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File E:\coding\codex\isphere-ai-bridge\scripts\open-offline-real-client-window.ps1 -UserSilent -WatchSeconds 30 -PollIntervalMs 500 +``` + +当前守护逻辑： + +- 用户手动移动主窗口：检测到位置/大小偏离后自动移回目标位置。 +- 用户手动移动提示窗：检测到位置/大小偏离后自动移回目标位置。 +- 用户关闭主窗口/杀掉客户端进程：检测到主窗口消失后自动重新启动离线客户端。 +- 每次守护输出： + - `watch_iteration_count` + - `repair_count` + - `relaunch_count` + - `recovery_actions` + +本轮实际验证： + +1. 模拟移动窗口到 `760,260,310x610`： + - 守护输出 `repair_count=1` + - `recovery_actions=["repair_window_placement"]` + - 最终窗口恢复到 `120,120,270x570` + +2. 模拟关闭客户端进程： + - 被关闭 PID：`11592` + - 守护输出 `relaunch_count=1` + - 新启动 PID：`10504` + - `recovery_actions=["relaunch_missing_main_window","repair_after_relaunch"]` + - 最终 UIA 根控件仍为 `frmLogin` + +抗干扰强度结论： + +- 对“用户移动窗口”：强。 +- 对“用户关闭窗口/进程”：中强，可自动重启。 +- 对“用户在关键发送瞬间抢焦点/操作同一窗口”：当前还不是强，需要动作级锁定和发送前后校验。 +- 对“应用崩溃/服务端不可用”：只能自动拉起窗口；业务登录/聊天能力仍受真实服务端影响。 + 本轮验证结果： - 新启动进程：`IMPlatformClient.exe` diff --git a/scripts/open-offline-real-client-window.ps1 b/scripts/open-offline-real-client-window.ps1 index 3677a6e..8fb5c67 100644 --- a/scripts/open-offline-real-client-window.ps1 +++ b/scripts/open-offline-real-client-window.ps1 @@ -9,6 +9,8 @@ param( [int]$Y = 120, [int]$Width = 270, [int]$Height = 570, + [int]$WatchSeconds = 0, + [int]$PollIntervalMs = 500, [ValidateSet("Visible", "Offscreen", "Minimized")] [string]$WindowMode = "Visible", [switch]$SkipExtract, @@ -107,6 +109,19 @@ public static class ISphereOfflineWindowApi { "@ -ErrorAction SilentlyContinue } +function Start-OfflineClient { + if ($script:UserSilentMode -or $script:WindowModeValue -ne "Visible" -or $script:NoActivateMode) { + return Start-Process -FilePath $script:ClientExePath -WorkingDirectory $script:ClientDirPath -WindowStyle Minimized -PassThru + } + return Start-Process -FilePath $script:ClientExePath -WorkingDirectory $script:ClientDirPath -PassThru +} + +function Relaunch-OfflineClient { + Stop-ExistingOfflineCopies -RootPath $script:ExtractRootPath + Start-Sleep -Milliseconds 500 + return Start-OfflineClient +} + function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$MainWidth, [int]$MainHeight, [bool]$MakeTopMost, [bool]$Activate, [string]$Mode) { $hwndTop = [IntPtr]::Zero $topMostHandle = [IntPtr]::new(-1) @@ -157,6 +172,32 @@ function Move-ClientWindows([object[]]$Windows, [int]$MainX, [int]$MainY, [int]$ } } +function Get-ClientWindowsForProcess([object]$Process) { + $scanResult = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-watch-scan" + if (-not $scanResult.ok -or -not $scanResult.data.windows) { + return @() + } + return @($scanResult.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $Process) -or $_.pid -eq $Process.Id) + }) +} + +function Test-WindowNear([object]$Window, [int]$ExpectedX, [int]$ExpectedY, [int]$ExpectedWidth, [int]$ExpectedHeight) { + if (-not $Window -or -not $Window.bounds) { + return $false + } + $tolerance = 3 + return ([Math]::Abs([int]$Window.bounds.x - $ExpectedX) -le $tolerance) -and + ([Math]::Abs([int]$Window.bounds.y - $ExpectedY) -le $tolerance) -and + ([Math]::Abs([int]$Window.bounds.width - $ExpectedWidth) -le $tolerance) -and + ([Math]::Abs([int]$Window.bounds.height - $ExpectedHeight) -le $tolerance) +} + +function Repair-ClientWindowPlacement([object[]]$Windows) { + Move-ClientWindows -Windows $Windows -MainX $script:EffectiveX -MainY $script:EffectiveY -MainWidth $script:DesiredWidth -MainHeight $script:DesiredHeight -MakeTopMost ([bool]$script:TopMostMode) -Activate (-not [bool]$script:NoActivateMode) -Mode $script:WindowModeValue | Out-Null +} + function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY, [int]$CaptureWidth, [int]$CaptureHeight) { Add-Type -AssemblyName System.Drawing $bitmap = New-Object System.Drawing.Bitmap $CaptureWidth, $CaptureHeight @@ -174,6 +215,7 @@ function Save-WindowScreenshot([string]$OutputPath, [int]$ScreenX, [int]$ScreenY $archiveFull = Resolve-RepoPath $ArchivePath $extractFull = Resolve-RepoPath $ExtractDir $script:HelperPath = Resolve-RepoPath $HelperExe +$script:ExtractRootPath = $extractFull $runOut = Join-Path $repo "runs\offline-real-client-window" New-Item -ItemType Directory -Force -Path $runOut | Out-Null @@ -185,6 +227,16 @@ $clientDir = Join-Path $extractFull ("zyl\" + $ClientRoot) $clientExe = Join-Path $clientDir "IMPlatformClient.exe" $clientConfig = Join-Path $clientDir "IMPlatformClient.exe.config" $requiredDependency = Join-Path $clientDir "Utilities.Lib.Base.dll" +$script:ClientDirPath = $clientDir +$script:ClientExePath = $clientExe +$script:UserSilentMode = [bool]$UserSilent +$script:NoActivateMode = [bool]$NoActivate +$script:WindowModeValue = $WindowMode +$script:EffectiveX = $effectiveX +$script:EffectiveY = $effectiveY +$script:DesiredWidth = $Width +$script:DesiredHeight = $Height +$script:TopMostMode = [bool]$TopMost if (-not $SkipExtract -and (-not (Test-Path -LiteralPath $clientExe) -or -not (Test-Path -LiteralPath $clientConfig) -or -not (Test-Path -LiteralPath $requiredDependency))) { if (-not (Test-Path -LiteralPath $archiveFull)) { @@ -211,12 +263,7 @@ if (-not $KeepExisting) { $process = $null if (-not $NoLaunch) { - if ($UserSilent -or $WindowMode -ne "Visible" -or $NoActivate) { - $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -WindowStyle Minimized -PassThru - } - else { - $process = Start-Process -FilePath $clientExe -WorkingDirectory $clientDir -PassThru - } + $process = Start-OfflineClient } $deadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) @@ -276,6 +323,65 @@ if ($latePromptWindows.Count -gt 0) { }) } +$watchIterationCount = 0 +$repairCount = 0 +$relaunchCount = 0 +$recoveryActions = @() + +if ($WatchSeconds -gt 0) { + $safePollIntervalMs = [Math]::Max(100, $PollIntervalMs) + $watchDeadline = (Get-Date).AddSeconds($WatchSeconds) + while ((Get-Date) -lt $watchDeadline) { + Start-Sleep -Milliseconds $safePollIntervalMs + $watchIterationCount++ + $watchedWindows = Get-ClientWindowsForProcess -Process $process + $watchedMain = @($watchedWindows | Where-Object { [string]$_.title -ne $InfoPromptTitle }) | Select-Object -First 1 + $watchedPrompt = @($watchedWindows | Where-Object { [string]$_.title -eq $InfoPromptTitle }) | Select-Object -First 1 + + if (-not $watchedMain) { + if (-not $NoLaunch) { + $process = Relaunch-OfflineClient + $relaunchCount++ + $recoveryActions += "relaunch_missing_main_window" + + $relaunchDeadline = (Get-Date).AddSeconds([Math]::Max(1, $WaitSeconds)) + do { + Start-Sleep -Milliseconds $safePollIntervalMs + $watchedWindows = Get-ClientWindowsForProcess -Process $process + $watchedMain = @($watchedWindows | Where-Object { [string]$_.title -ne $InfoPromptTitle }) | Select-Object -First 1 + } while (-not $watchedMain -and (Get-Date) -lt $relaunchDeadline) + + if ($watchedWindows.Count -gt 0) { + Repair-ClientWindowPlacement -Windows $watchedWindows + $repairCount++ + $recoveryActions += "repair_after_relaunch" + } + } + continue + } + + $expectedPromptX = if ($WindowMode -eq "Offscreen") { $effectiveX } else { $effectiveX + $Width + 40 } + $expectedPromptY = if ($WindowMode -eq "Offscreen") { $effectiveY + $Height + 40 } else { $effectiveY + 40 } + + $needsRepair = -not (Test-WindowNear -Window $watchedMain -ExpectedX $effectiveX -ExpectedY $effectiveY -ExpectedWidth $Width -ExpectedHeight $Height) + if ($watchedPrompt) { + $needsRepair = $needsRepair -or -not (Test-WindowNear -Window $watchedPrompt -ExpectedX $expectedPromptX -ExpectedY $expectedPromptY -ExpectedWidth 300 -ExpectedHeight 190) + } + + if ($needsRepair) { + Repair-ClientWindowPlacement -Windows $watchedWindows + $repairCount++ + $recoveryActions += "repair_window_placement" + } + } + + $scanAfterMove = Invoke-HelperJson -Op "scan_windows" -OpArgs @{ include_all_visible = $false } -RequestId "offline-real-client-final-scan-after-watch" + $windowsAfterMove = @($scanAfterMove.data.windows | Where-Object { + $_.process_name -eq "IMPlatformClient" -and + ((-not $process) -or $_.pid -eq $process.Id) + }) +} + $mainWindowMatches = @($windowsAfterMove | Where-Object { [string]$_.title -ne $InfoPromptTitle }) $promptWindowMatches = @($windowsAfterMove | Where-Object { [string]$_.title -eq $InfoPromptTitle }) $mainWindow = if ($mainWindowMatches.Count -gt 0) { $mainWindowMatches[0] } else { $null } @@ -306,6 +412,12 @@ if (-not $NoScreenshot -and $WindowMode -eq "Visible") { user_silent = [bool]$UserSilent window_mode = $WindowMode no_activate = [bool]$NoActivate + watch_seconds = $WatchSeconds + poll_interval_ms = $PollIntervalMs + watch_iteration_count = $watchIterationCount + relaunch_count = $relaunchCount + repair_count = $repairCount + recovery_actions = $recoveryActions effective_bounds = @{ x = $effectiveX y = $effectiveY diff --git a/scripts/test-open-offline-real-client-window-watchdog.ps1 b/scripts/test-open-offline-real-client-window-watchdog.ps1 new file mode 100644 index 0000000..7ccb481 --- /dev/null +++ b/scripts/test-open-offline-real-client-window-watchdog.ps1 @@ -0,0 +1,45 @@ +param( + [string]$ScriptPath = "scripts/open-offline-real-client-window.ps1" +) + +$ErrorActionPreference = "Stop" +$repo = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..") +$target = if ([System.IO.Path]::IsPathRooted($ScriptPath)) { + $ScriptPath +} +else { + Join-Path $repo $ScriptPath +} + +if (-not (Test-Path -LiteralPath $target)) { + throw "script not found: $target" +} + +$text = Get-Content -LiteralPath $target -Raw + +$requiredPatterns = @( + @{ Name = "WatchSeconds parameter"; Pattern = '\[int\]\$WatchSeconds' }, + @{ Name = "PollIntervalMs parameter"; Pattern = '\[int\]\$PollIntervalMs' }, + @{ Name = "watch loop output"; Pattern = 'watch_iteration_count' }, + @{ Name = "relaunch counter output"; Pattern = 'relaunch_count' }, + @{ Name = "move repair counter output"; Pattern = 'repair_count' }, + @{ Name = "closed window recovery"; Pattern = 'Relaunch-OfflineClient' }, + @{ Name = "moved window recovery"; Pattern = 'Repair-ClientWindowPlacement' } +) + +$missing = @() +foreach ($item in $requiredPatterns) { + if ($text -notmatch $item.Pattern) { + $missing += $item.Name + } +} + +if ($missing.Count -gt 0) { + throw "watchdog contract missing: $($missing -join ', ')" +} + +[pscustomobject]@{ + ok = $true + script = $target + checked = $requiredPatterns.Count +} | ConvertTo-Json -Compress