From 10e7a616c60fb2285d7f9388251d6277b425b515 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Fri, 10 Jul 2026 03:13:57 +0800 Subject: [PATCH] test: add msglib mcp enrichment smoke --- docs/current-status-card.md | 5 +- docs/msglib-read-sidecar-contract.md | 1 + .../capability-source-matrix.md | 8 +- ...tage-c-log-backed-receive-messages-loop.md | 70 ++++++- scripts/verify-msglib-mcp-enrichment.ps1 | 173 ++++++++++++++++++ 5 files changed, 243 insertions(+), 14 deletions(-) create mode 100644 scripts/verify-msglib-mcp-enrichment.ps1 diff --git a/docs/current-status-card.md b/docs/current-status-card.md index f887548..b1c0f4a 100644 --- a/docs/current-status-card.md +++ b/docs/current-status-card.md @@ -56,7 +56,7 @@ N13/N14/N15 are pre-business validation results. They can help identify UI eleme ## Current loop -Current loop: `Stage C / Loop C31 - optional real MsgLib MCP enrichment smoke`. +Current loop: `Stage C / Loop C32 - receive message display-name enrichment`. Plan file: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md`. @@ -99,10 +99,11 @@ Current loop output so far: 28. C28: added `internal/msglib` Go client wrapper for the sidecar protocol with env config, fake-sidecar unit tests, timeout handling, structured sidecar errors, `SelfCheck`, and `DisplaySources`; no MCP tool wiring or message/display row reads yet. 29. C29: added bounded `display_entities` support across the x86 sidecar, Go client, and verification script; evidence-backed verification returned sanitized contact/group counts and source-table names only, with no message bodies, file paths, raw rows, sends, writes, hooks, or injection. 30. C30: wired `DisplayEntities` into `isphere_search_contacts` and `isphere_search_groups` as an optional env-configured enrichment source with fake-source tests; default MCP verification now clears MsgLib env to keep the standard smoke deterministic. +31. C31: added optional `scripts/verify-msglib-mcp-enrichment.ps1`; real copied-DB MCP smoke returned sanitized contact/group counts and MsgLib source refs without printing names, JIDs, message bodies, file paths, or raw rows. ## Next business mainline -1. Run Stage C Loop C31: add an optional copied-DB MCP enrichment smoke/hardening path that proves contact/group enrichment through the real sidecar while printing only sanitized counts/source metadata by default. +1. Run Stage C Loop C32: reuse the optional MsgLib display-entity source to enrich `isphere_receive_messages` sender/conversation display fields without changing message text/body handling. 2. Leave `isphere_send_message` blocked until bridge/API, UIA action-chain, or protocol connector evidence is validated. 3. Leave actual file download/cache mapping as a later connector node until a message-to-cache hash or client download source is validated. 4. Keep UIA selector/report work as fallback support, not as the primary roadmap. diff --git a/docs/msglib-read-sidecar-contract.md b/docs/msglib-read-sidecar-contract.md index f60868e..e7be403 100644 --- a/docs/msglib-read-sidecar-contract.md +++ b/docs/msglib-read-sidecar-contract.md @@ -184,6 +184,7 @@ Committed C29 implementation: - Go wrapper: `DisplayEntities(ctx, DisplayEntitiesOptions)`. - Verification script calls `display_entities` only when copied DB env paths are supplied, and prints only sanitized `entity_count` plus `source_tables` summaries. - Evidence-backed C29 verification against the copied DB returned contact/group entity counts and source-table names without printing entity values. +- C31 adds optional `scripts/verify-msglib-mcp-enrichment.ps1` to prove MCP-level contact/group enrichment through the real sidecar while printing only counts and `msglib:` refs. ## Verification diff --git a/docs/source-discovery/capability-source-matrix.md b/docs/source-discovery/capability-source-matrix.md index ec7d3be..136f404 100644 --- a/docs/source-discovery/capability-source-matrix.md +++ b/docs/source-discovery/capability-source-matrix.md @@ -17,9 +17,9 @@ Schema notes: `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md` | MCP tool | Primary source | Fallback source | Evidence file | Decision status | Next implementation slice | | --- | --- | --- | --- | --- | --- | -| `isphere_receive_messages` | decrypted `PacketReader.ProcessPacket` XMPP `` stanzas via `internal/isphere.EncryptedPacketLogSource`; configured by `ISPHERE_PACKET_LOG_FILE` or `ISPHERE_PACKET_LOG_DIR` | decrypted `Smark.SendReceive` transport stanzas; decrypted `SaveToDB` `Chat_OnMessageArrived` traces; copied `MsgLib.db` through a future x86 read-only sidecar/helper | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#field-mapping-evidence`; `docs/source-discovery/2026-07-10-dotnet-wrapper-static-analysis.md` | C20 supports and validates the current safe receive-message contract args; C30 wires bounded contact/group display entities into search enrichment; DB-backed message listing is still not implemented | Defer DB-backed message listing; run optional real MsgLib MCP enrichment smoke | -| `isphere_search_contacts` | bare sender/receiver JIDs extracted from normalized log-backed messages loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `TD_Roster`, `TD_CustomEffigy`, `tblRecent`, `tblChatLevel`, `tblPersonMsg`; decrypted roster/contact stanzas from `Smark.SendReceive`; UIA helper source if display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md` | C30 wires optional MsgLib display-entity enrichment into `isphere_search_contacts`; standard verification keeps MsgLib env cleared by default | C31 optional copied-DB MCP enrichment smoke with sanitized output | -| `isphere_search_groups` | decrypted `PacketReader.ProcessPacket` `type="groupchat"` stanzas and conference/MUC JIDs loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `tblMsgGroupPersonMsg`, `TD_WorkGroupAuth`, `tblRecent`; UIA helper source if group display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md`; C9 ignored-log scan: PacketReader 86 groupchat/86 conference hits; Smark 24 groupchat/658 conference/631 muc hits | C30 wires optional MsgLib display-entity enrichment into `isphere_search_groups`; standard verification keeps MsgLib env cleared by default | C31 optional copied-DB MCP enrichment smoke with sanitized output | +| `isphere_receive_messages` | decrypted `PacketReader.ProcessPacket` XMPP `` stanzas via `internal/isphere.EncryptedPacketLogSource`; configured by `ISPHERE_PACKET_LOG_FILE` or `ISPHERE_PACKET_LOG_DIR` | decrypted `Smark.SendReceive` transport stanzas; decrypted `SaveToDB` `Chat_OnMessageArrived` traces; copied `MsgLib.db` through a future x86 read-only sidecar/helper | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#field-mapping-evidence`; `docs/source-discovery/2026-07-10-dotnet-wrapper-static-analysis.md` | C20 supports and validates the current safe receive-message contract args; C31 proves real copied-DB MCP contact/group enrichment with sanitized output; DB-backed message listing is still not implemented | C32 enrich receive-message sender/conversation display fields from MsgLib metadata | +| `isphere_search_contacts` | bare sender/receiver JIDs extracted from normalized log-backed messages loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `TD_Roster`, `TD_CustomEffigy`, `tblRecent`, `tblChatLevel`, `tblPersonMsg`; decrypted roster/contact stanzas from `Smark.SendReceive`; UIA helper source if display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md` | C31 verifies optional copied-DB MCP contact enrichment with sanitized output | C32 reuse display metadata for receive-message display fields | +| `isphere_search_groups` | decrypted `PacketReader.ProcessPacket` `type="groupchat"` stanzas and conference/MUC JIDs loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `tblMsgGroupPersonMsg`, `TD_WorkGroupAuth`, `tblRecent`; UIA helper source if group display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md`; C9 ignored-log scan: PacketReader 86 groupchat/86 conference hits; Smark 24 groupchat/658 conference/631 muc hits | C31 verifies optional copied-DB MCP group enrichment with sanitized output | C32 reuse display metadata for receive-message conversation display fields | | `isphere_receive_files` | decrypted `PacketReader.ProcessPacket` file-transfer message stanzas via `internal/isphere.ListFilesFromMessages` and `isphere_receive_files` list mode; configured PacketReader file or directory source; `zyl\importal\` cache entries remain unlinked | decrypted `Smark.SendReceive` and `SaveToDB` traces for file-reference reconciliation; future UIA/client connector for download | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#c12-file-transfer-evidence-precheck` | C22 verifies safe file-list contract args; no log-to-cache/download mapping yet | defer download mapping; precheck contact/group display-name evidence | | `isphere_send_message` | none validated yet | existing bridge/API/local service preferred; UIA write connector only after internal-sandbox action evidence; network/protocol connector only after protocol discovery | `docs/source-discovery/2026-07-10-send-message-source-precheck.md` | C15 blocked: contract exists, but no committed bridge/API/local service, helper write op, UIA action chain, or protocol connector is validated | do not implement write behavior; harden existing read source configuration | | `isphere_send_file` | none validated yet | depends on send-message connector plus outbound file/upload evidence | `docs/source-discovery/2026-07-10-send-message-source-precheck.md` | blocked behind `isphere_send_message` connector selection and file upload/source policy | defer until send-message connector is validated | @@ -39,4 +39,4 @@ Start Stage C with a narrow log-backed `isphere_receive_messages` source abstrac ## Deferred source work -`MsgLib.db` has a validated copied read-only open path through the bundled 32-bit `System.Data.SQLite.dll` and password `123`. C27 adds `MsgLibReadSidecar` as the bounded x86 .NET reader boundary, C28 adds the Go `internal/msglib` process client, C29 adds bounded `display_entities` extraction, and C30 wires it as optional contact/group MCP enrichment. The next step is an optional copied-DB MCP smoke with sanitized output. +`MsgLib.db` has a validated copied read-only open path through the bundled 32-bit `System.Data.SQLite.dll` and password `123`. C27 adds `MsgLibReadSidecar` as the bounded x86 .NET reader boundary, C28 adds the Go `internal/msglib` process client, C29 adds bounded `display_entities` extraction, C30 wires it as optional contact/group MCP enrichment, and C31 proves real copied-DB MCP enrichment with sanitized output. The next step is receive-message display-name enrichment from the same metadata source. diff --git a/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md b/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md index ffe3b0b..8cde52e 100644 --- a/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md +++ b/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md @@ -2366,10 +2366,11 @@ Optional MsgLib MCP enrichment result: **Goal:** Add an optional verification path that proves `isphere_search_contacts` and `isphere_search_groups` can call the real copied-DB MsgLib sidecar through MCP, while printing only sanitized counts/source metadata by default. -**Planned files:** -- Modify: `scripts/verify-go-mcp.ps1` or add a focused optional verification script. +**Files:** +- Create: `scripts/verify-msglib-mcp-enrichment.ps1` - Modify: `docs/current-status-card.md` - Modify: `docs/source-discovery/capability-source-matrix.md` +- Modify: `docs/msglib-read-sidecar-contract.md` - Modify: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md` **Planned behavior:** @@ -2377,17 +2378,70 @@ Optional MsgLib MCP enrichment result: - Optional verification is gated by explicit env variables and copied DB paths. - Output only counts, source names, and booleans; do not print display names, JIDs, message bodies, file paths, or raw rows. -- [ ] **Step 1: Add optional real-MsgLib MCP smoke** +- [x] **Step 1: Add optional real-MsgLib MCP smoke** -Implement opt-in smoke with sanitized summary only. +Created `scripts/verify-msglib-mcp-enrichment.ps1`. It builds the sidecar, starts an in-memory MCP server with MsgLib env, calls `isphere_search_contacts` and `isphere_search_groups`, and outputs only counts/source refs. -- [ ] **Step 2: Run default and optional verification** +- [x] **Step 2: Run default and optional verification** -Run default full verification and optional copied-DB smoke if paths are available. +Default verification remains `scripts/verify-go-mcp.ps1` with MsgLib env cleared. Optional copied-DB smoke passed with sanitized output: `contact_count=5`, `contact_sources=["msglib:TD_CustomEffigy"]`, `group_count=3`, `group_sources=["msglib:TD_WorkGroupAuth"]`, `printed_entity_values=false`, `message_bodies_returned=false`, and `raw_rows_returned=false`. -- [ ] **Step 3: Update docs and commit** +- [x] **Step 3: Update docs and commit** -Record sanitized evidence and next loop. +Run before commit: + +```powershell +git diff --check +go test ./... +go build ./cmd/isphere-mcp +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\build-msglib-sidecar.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-msglib-sidecar.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-go-mcp.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-msglib-mcp-enrichment.ps1 # with copied DB env paths +``` + +Final verification passed: `git diff --check`, `go test ./...`, explicit `go build ./cmd/isphere-mcp`, default sidecar smoke, evidence-backed sidecar smoke, deterministic `scripts/verify-go-mcp.ps1`, and optional copied-DB `scripts/verify-msglib-mcp-enrichment.ps1` with sanitized output. + +--- + +## Loop C31 Result + +Optional real MsgLib MCP enrichment smoke result: + +- Added `scripts/verify-msglib-mcp-enrichment.ps1`. +- Proved MCP-level contact/group enrichment through the real copied DB sidecar. +- Sanitized output contains counts and `msglib:` refs only. +- No entity values, message bodies, file paths, or raw rows are printed by the smoke. + +--- + +## Loop C32: Receive-message display-name enrichment + +**Goal:** Reuse the optional MsgLib display-entity source to enrich `isphere_receive_messages` sender/conversation display fields, without changing message text/body handling or adding send/download/write behavior. + +**Planned files:** +- Modify: `internal/tools/isphere_read.go` +- Modify: `internal/tools/isphere_read_test.go` +- Modify: `internal/mcpserver/server.go` +- Modify: `scripts/verify-go-mcp.ps1` or optional enrichment smoke as needed +- Modify: docs/status/matrix/plan + +**Planned behavior:** +- Default behavior remains unchanged without MsgLib env. +- With injected display metadata, fill safe display fields such as `sender_name` and `conversation.display_name` when a matching contact/group entity exists. +- Do not alter `text`, `content_text`, attachments, raw refs, file paths, or message bodies. + +- [ ] **Step 1: Write failing receive-message display-name tests** + +Use fake display metadata to prove sender/conversation display enrichment while preserving existing message fields. + +- [ ] **Step 2: Implement optional display-name enrichment** + +Wire display metadata into receive message map output only. + +- [ ] **Step 3: Verify/docs/commit** + +Run the standard full gate and update status. --- diff --git a/scripts/verify-msglib-mcp-enrichment.ps1 b/scripts/verify-msglib-mcp-enrichment.ps1 new file mode 100644 index 0000000..80e86ae --- /dev/null +++ b/scripts/verify-msglib-mcp-enrichment.ps1 @@ -0,0 +1,173 @@ +param( + [string]$SidecarExe = "runs/msglib-sidecar/MsgLibReadSidecar.exe", + [string]$SQLiteDllPath = $env:ISPHERE_MSGLIB_SQLITE_DLL, + [string]$DbPath = $env:ISPHERE_MSGLIB_DB, + [switch]$SkipBuild +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$repo = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path +$sidecarPath = Join-Path $repo $SidecarExe +$harnessDir = Join-Path $repo ("runs\msglib-mcp-enrichment-harness-" + [guid]::NewGuid().ToString("N")) +$harnessMain = Join-Path $harnessDir "main.go" + +function Assert-True([bool]$Condition, [string]$Message) { + if (-not $Condition) { throw $Message } +} + +if (-not $SQLiteDllPath -or -not (Test-Path -LiteralPath $SQLiteDllPath)) { + throw "SQLite DLL not found; set ISPHERE_MSGLIB_SQLITE_DLL or pass -SQLiteDllPath" +} +if (-not $DbPath -or -not (Test-Path -LiteralPath $DbPath)) { + throw "MsgLib DB not found; set ISPHERE_MSGLIB_DB or pass -DbPath" +} + +try { + Push-Location $repo + try { + if (-not $SkipBuild) { + & powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $repo "scripts\build-msglib-sidecar.ps1") | Out-Host + if ($LASTEXITCODE -ne 0) { throw "build-msglib-sidecar.ps1 failed with exit code $LASTEXITCODE" } + } + Assert-True (Test-Path -LiteralPath $sidecarPath) "sidecar not found: $sidecarPath" + + New-Item -ItemType Directory -Force -Path $harnessDir | Out-Null + @' +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "isphere-ai-bridge/internal/mcpserver" +) + +type summary struct { + Count int `json:"count"` + Sources []string `json:"sources"` +} + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + serverTransport, clientTransport := mcp.NewInMemoryTransports() + server, err := mcpserver.NewServerFromEnv() + if err != nil { fail("server_config", err) } + serverErr := make(chan error, 1) + go func() { serverErr <- server.Run(ctx, serverTransport) }() + + client := mcp.NewClient(&mcp.Implementation{Name: "verify-msglib-mcp-enrichment", Version: "0.1.0"}, nil) + session, err := client.Connect(ctx, clientTransport, nil) + if err != nil { fail("connect", err) } + defer func() { + _ = session.Close() + cancel() + select { + case <-serverErr: + case <-time.After(2*time.Second): + fail("server_shutdown", fmt.Errorf("server did not stop")) + } + }() + + contacts := callSearch(ctx, session, "isphere_search_contacts", "contacts") + groups := callSearch(ctx, session, "isphere_search_groups", "groups") + if contacts.Count == 0 { fail("contacts", fmt.Errorf("no msglib_readonly contacts returned")) } + if groups.Count == 0 { fail("groups", fmt.Errorf("no msglib_readonly groups returned")) } + + out := map[string]any{ + "ok": true, + "verification": "msglib_mcp_enrichment", + "contact_count": contacts.Count, + "contact_sources": contacts.Sources, + "group_count": groups.Count, + "group_sources": groups.Sources, + "printed_entity_values": false, + "message_bodies_returned": false, + "raw_rows_returned": false, + } + encoded, _ := json.Marshal(out) + fmt.Println(string(encoded)) +} + +func callSearch(ctx context.Context, session *mcp.ClientSession, tool string, field string) summary { + result, err := session.CallTool(ctx, &mcp.CallToolParams{Name: tool, Arguments: map[string]any{ + "query": "", + "source_preference": "auto", + "cursor": "", + "limit": 5, + }}) + if err != nil { fail(tool, err) } + if result.IsError { fail(tool, fmt.Errorf("tool returned isError=true")) } + payload, _ := json.Marshal(result.StructuredContent) + var decoded map[string]any + if err := json.Unmarshal(payload, &decoded); err != nil { fail(tool+"_decode", err) } + values, ok := decoded[field].([]any) + if !ok { fail(tool+"_shape", fmt.Errorf("%s missing or not array", field)) } + sourceSet := map[string]bool{} + count := 0 + for _, value := range values { + item, ok := value.(map[string]any) + if !ok { continue } + source, _ := item["source"].(string) + if source != "msglib_readonly" { continue } + rawRef, _ := item["raw_ref"].(string) + if rawRef != "" { sourceSet[rawRef] = true } + count++ + } + sources := make([]string, 0, len(sourceSet)) + for source := range sourceSet { sources = append(sources, source) } + sort.Strings(sources) + return summary{Count: count, Sources: sources} +} + +func fail(step string, err error) { + encoded, _ := json.Marshal(map[string]any{"ok": false, "step": step, "error": err.Error()}) + fmt.Fprintln(os.Stderr, string(encoded)) + os.Exit(1) +} +'@ | Set-Content -LiteralPath $harnessMain -Encoding UTF8 + + $oldSidecar = $env:ISPHERE_MSGLIB_SIDECAR_EXE + $oldDll = $env:ISPHERE_MSGLIB_SQLITE_DLL + $oldDb = $env:ISPHERE_MSGLIB_DB + $env:ISPHERE_MSGLIB_SIDECAR_EXE = (Resolve-Path -LiteralPath $sidecarPath).Path + $env:ISPHERE_MSGLIB_SQLITE_DLL = (Resolve-Path -LiteralPath $SQLiteDllPath).Path + $env:ISPHERE_MSGLIB_DB = (Resolve-Path -LiteralPath $DbPath).Path + try { + Push-Location $harnessDir + try { + $output = & go run . + if ($LASTEXITCODE -ne 0) { throw "go run MCP enrichment harness failed with exit code $LASTEXITCODE" } + $output | ForEach-Object { Write-Host $_ } + $lastLine = @($output | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) | Select-Object -Last 1 + $json = $lastLine | ConvertFrom-Json + } + finally { Pop-Location } + } + finally { + $env:ISPHERE_MSGLIB_SIDECAR_EXE = $oldSidecar + $env:ISPHERE_MSGLIB_SQLITE_DLL = $oldDll + $env:ISPHERE_MSGLIB_DB = $oldDb + } + + Assert-True ($json.ok -eq $true) "harness did not return ok=true" + Assert-True ($json.printed_entity_values -eq $false) "harness must not print entity values" + Assert-True ($json.contact_count -gt 0) "contact_count should be > 0" + Assert-True ($json.group_count -gt 0) "group_count should be > 0" + } + finally { Pop-Location } +} +finally { + if (Test-Path -LiteralPath $harnessDir) { + Remove-Item -LiteralPath $harnessDir -Recurse -Force + } +}