From 8be4a6a6c5b31c57e1441f22e9bcfb183e8d74b7 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Fri, 10 Jul 2026 03:02:16 +0800 Subject: [PATCH] feat: add msglib display entity extraction --- docs/current-status-card.md | 9 +- docs/msglib-read-sidecar-contract.md | 49 ++++- .../capability-source-matrix.md | 8 +- ...tage-c-log-backed-receive-messages-loop.md | 90 +++++++-- internal/msglib/sidecar_client.go | 56 ++++++ internal/msglib/sidecar_client_test.go | 63 ++++++ native/MsgLibReadSidecar/Program.cs | 180 +++++++++++++++++- scripts/verify-msglib-sidecar.ps1 | 30 ++- 8 files changed, 459 insertions(+), 26 deletions(-) diff --git a/docs/current-status-card.md b/docs/current-status-card.md index 93f58b2..9d1caf1 100644 --- a/docs/current-status-card.md +++ b/docs/current-status-card.md @@ -27,9 +27,9 @@ Remote base before local roadmap commits: `b2d839e Merge branch 'codex/n15-repor - `scripts/verify-win-helper.ps1` and `scripts/verify-go-mcp.ps1` provide repeatable local checks for the helper, eight-tool MCP surface, default empty-source receive/contact/group/file calls, synthetic configured-file receive/contact/group/file smoke, and synthetic configured-directory receive/contact/group/file smoke. - `isphere_receive_messages` now returns the contract-facing `ok`, `conversation`, `messages`, `next_cursor`, and `audit` envelope while retaining legacy parser-native message fields for compatibility. - `isphere_receive_messages` accepts the read-contract argument set for the current local-readonly path: `conversation_id`, `query`, `since`, `limit`, `include_attachment_metadata`, `source_preference`, `preview`, and empty `cursor`. -- `isphere_search_contacts` and `isphere_search_groups` accept and validate the safe search-contract args for the current local-readonly path, but display names/member data remain JID-derived/unknown until the x86 `MsgLibReadSidecar` is extended for bounded display-entity reads and integrated. +- `isphere_search_contacts` and `isphere_search_groups` accept and validate the safe search-contract args for the current local-readonly path, but display names/member data are not yet enriched from `MsgLib.db` until C30 wires the C29 bounded display-entity reader into the MCP search tools. - `isphere_receive_files` list mode accepts and validates the safe file-list contract args for the current local-readonly path while keeping download blocked. -- `native/MsgLibReadSidecar` provides a bounded x86 .NET read-only boundary for `MsgLib.db` schema/display-source checks, and `internal/msglib` now wraps it from Go; it is not yet wired into MCP tools and does not return display row values yet. +- `native/MsgLibReadSidecar` provides a bounded x86 .NET read-only boundary for `MsgLib.db` schema/display-source checks and bounded contact/group display-entity reads; `internal/msglib` wraps it from Go, but MCP tools are not wired to it yet. - N12-pre and N12R documents define safe offline evidence intake and internal-sandbox live UIA capture procedures. - N13/N14/N15 produced selector catalog/report validation infrastructure, but those are supporting validation assets only. @@ -56,7 +56,7 @@ N13/N14/N15 are pre-business validation results. They can help identify UI eleme ## Current loop -Current loop: `Stage C / Loop C29 - bounded MsgLib display entity extraction`. +Current loop: `Stage C / Loop C30 - integrate MsgLib display entities into contact/group search`. Plan file: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md`. @@ -97,10 +97,11 @@ Current loop output so far: 26. C26: copied `MsgLib.db` schema extraction succeeded through a 32-bit .NET `System.Data.SQLite` read-only probe with password `123`; all three copied DB candidates opened, with confirmed contact/group/message/file schema tables. 27. C27: added `native/MsgLibReadSidecar` with protocol `isphere.msglib.v1`, x86 build/verify scripts, and safe `self_check`, `schema_summary`, and `display_sources` operations; evidence-backed smoke returned 8 display-source candidates without row/message values. 28. C28: added `internal/msglib` Go client wrapper for the sidecar protocol with env config, fake-sidecar unit tests, timeout handling, structured sidecar errors, `SelfCheck`, and `DisplaySources`; no MCP tool wiring or message/display row reads yet. +29. C29: added bounded `display_entities` support across the x86 sidecar, Go client, and verification script; evidence-backed verification returned sanitized contact/group counts and source-table names only, with no message bodies, file paths, raw rows, sends, writes, hooks, or injection. ## Next business mainline -1. Run Stage C Loop C29: extend the sidecar/client contract for bounded contact/group display-entity extraction only, with tests first; do not expose message bodies, file paths, downloads, sends, writes, hooks, injection, or raw row dumps. +1. Run Stage C Loop C30: wire C29 `DisplayEntities` into `isphere_search_contacts` and `isphere_search_groups` as an env-configured optional enrichment source, with fake-sidecar tests first; keep message bodies, file paths, downloads, sends, writes, hooks, injection, and raw row dumps out of scope. 2. Leave `isphere_send_message` blocked until bridge/API, UIA action-chain, or protocol connector evidence is validated. 3. Leave actual file download/cache mapping as a later connector node until a message-to-cache hash or client download source is validated. 4. Keep UIA selector/report work as fallback support, not as the primary roadmap. diff --git a/docs/msglib-read-sidecar-contract.md b/docs/msglib-read-sidecar-contract.md index 9636129..f60868e 100644 --- a/docs/msglib-read-sidecar-contract.md +++ b/docs/msglib-read-sidecar-contract.md @@ -110,12 +110,48 @@ Initial display sources: - `received_file_metadata` from `TD_ReceiveFileRecord`. - `sent_file_metadata` from `TD_SendFileRecord`. +### `display_entities` + +Same safe provider/DB validation as `display_sources`, but returns bounded normalized contact/group display metadata. + +Required args: + +- `sqlite_dll_path` +- `db_path` +- `entity_type`: `contacts` or `groups` + +Optional args: + +- `password`: default `123`. +- `query`: optional string matched against allowlisted jid/name columns. +- `limit`: bounded to `1..100`; default `25`. + +Returned data adds: + +- `metadata_only=true` +- `read_only=true` +- `message_body_values_returned=false` +- `raw_rows_returned=false` +- `entities[]` entries with: + - `entity_type` + - `source_table` + - `jid` + - `display_name` + - `confidence` + - `matched_columns` + +Initial allowlisted identity sources: + +- Contacts: `TD_Roster(JId,Nick)`, `TD_CustomEffigy(PersonJid,PersonName)`, `tblRecent(PersonId,PersonName)`, `tblChatLevel(PersonId,PersonName)`, selected `tblPersonMsg` id/name pairs only. +- Groups: `TD_WorkGroupAuth(GroupJID,GroupName)`, `TD_WorkGroupAuth(ChildGroupJid,GroupName)`, selected `tblMsgGroupPersonMsg` group/session id/name pairs only. + ## Security and scope rules - The sidecar must open DB files with read-only connection settings. - `db_path` must point to a file named `MsgLib.db`. -- The first contract returns schema, columns, counts, and display-source availability only. -- It does not return message bodies, contact row values, file paths, or raw row sets. +- The schema/display-source operations return schema, columns, counts, and display-source availability only. +- `display_entities` may return bounded contact/group `jid` and `display_name` metadata only. +- It does not return message bodies, message text, file paths, attachment contents, or raw row sets. - It does not implement send, file download, mark-read, login, hooks, injection, or DB writes. - The Go MCP layer remains the coordinator; this process is a bounded local reader. @@ -142,6 +178,13 @@ Committed C28 implementation: - Unit tests use a fake sidecar process; normal tests do not require real `System.Data.SQLite.dll` or copied `MsgLib.db`. - C28 does not register a new MCP tool and does not return message bodies, file paths, raw rows, contact row values, or group row values. +Committed C29 implementation: + +- Sidecar op: `display_entities`. +- Go wrapper: `DisplayEntities(ctx, DisplayEntitiesOptions)`. +- Verification script calls `display_entities` only when copied DB env paths are supplied, and prints only sanitized `entity_count` plus `source_tables` summaries. +- Evidence-backed C29 verification against the copied DB returned contact/group entity counts and source-table names without printing entity values. + ## Verification Default verification builds and checks `self_check` only: @@ -150,7 +193,7 @@ Default verification builds and checks `self_check` only: powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-msglib-sidecar.ps1 ``` -Evidence-backed provider verification is opt-in through environment variables and should use copied DB files only: +Evidence-backed provider verification is opt-in through environment variables and should use copied DB files only. The script also verifies `display_sources` and sanitized `display_entities` summaries when these variables are present: ```powershell $env:ISPHERE_MSGLIB_SQLITE_DLL = "" diff --git a/docs/source-discovery/capability-source-matrix.md b/docs/source-discovery/capability-source-matrix.md index 1ece41e..cd7ef28 100644 --- a/docs/source-discovery/capability-source-matrix.md +++ b/docs/source-discovery/capability-source-matrix.md @@ -17,9 +17,9 @@ Schema notes: `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md` | MCP tool | Primary source | Fallback source | Evidence file | Decision status | Next implementation slice | | --- | --- | --- | --- | --- | --- | -| `isphere_receive_messages` | decrypted `PacketReader.ProcessPacket` XMPP `` stanzas via `internal/isphere.EncryptedPacketLogSource`; configured by `ISPHERE_PACKET_LOG_FILE` or `ISPHERE_PACKET_LOG_DIR` | decrypted `Smark.SendReceive` transport stanzas; decrypted `SaveToDB` `Chat_OnMessageArrived` traces; copied `MsgLib.db` through a future x86 read-only sidecar/helper | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#field-mapping-evidence`; `docs/source-discovery/2026-07-10-dotnet-wrapper-static-analysis.md` | C20 supports and validates the current safe receive-message contract args; C28 adds the Go `internal/msglib` sidecar wrapper for schema/display-source operations; DB-backed message listing is still not implemented | Defer DB-backed message listing; first run C29 bounded contact/group display-entity extraction | -| `isphere_search_contacts` | bare sender/receiver JIDs extracted from normalized log-backed messages loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `TD_Roster`, `TD_CustomEffigy`, `tblRecent`, `tblChatLevel`, `tblPersonMsg`; decrypted roster/contact stanzas from `Smark.SendReceive`; UIA helper source if display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md` | C28 adds the Go sidecar client wrapper; the sidecar still needs a bounded display-entity operation before DB-backed display names can be exposed | C29 bounded contact display-entity extraction | -| `isphere_search_groups` | decrypted `PacketReader.ProcessPacket` `type="groupchat"` stanzas and conference/MUC JIDs loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `tblMsgGroupPersonMsg`, `TD_WorkGroupAuth`, `tblRecent`; UIA helper source if group display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md`; C9 ignored-log scan: PacketReader 86 groupchat/86 conference hits; Smark 24 groupchat/658 conference/631 muc hits | C28 adds the Go sidecar client wrapper; the sidecar still needs a bounded display-entity operation before DB-backed group/member names can be exposed | C29 bounded group display-entity extraction | +| `isphere_receive_messages` | decrypted `PacketReader.ProcessPacket` XMPP `` stanzas via `internal/isphere.EncryptedPacketLogSource`; configured by `ISPHERE_PACKET_LOG_FILE` or `ISPHERE_PACKET_LOG_DIR` | decrypted `Smark.SendReceive` transport stanzas; decrypted `SaveToDB` `Chat_OnMessageArrived` traces; copied `MsgLib.db` through a future x86 read-only sidecar/helper | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#field-mapping-evidence`; `docs/source-discovery/2026-07-10-dotnet-wrapper-static-analysis.md` | C20 supports and validates the current safe receive-message contract args; C29 adds bounded contact/group display-entity extraction; DB-backed message listing is still not implemented | Defer DB-backed message listing; first run C30 contact/group search enrichment | +| `isphere_search_contacts` | bare sender/receiver JIDs extracted from normalized log-backed messages loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `TD_Roster`, `TD_CustomEffigy`, `tblRecent`, `tblChatLevel`, `tblPersonMsg`; decrypted roster/contact stanzas from `Smark.SendReceive`; UIA helper source if display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md` | C29 verifies bounded contact display-entity extraction through the x86 sidecar and Go client; MCP search is not enriched yet | C30 wire optional MsgLib enrichment into `isphere_search_contacts` | +| `isphere_search_groups` | decrypted `PacketReader.ProcessPacket` `type="groupchat"` stanzas and conference/MUC JIDs loaded from configured PacketReader file or directory source | validated copied `MsgLib.db` tables: `tblMsgGroupPersonMsg`, `TD_WorkGroupAuth`, `tblRecent`; UIA helper source if group display names are still missing | `docs/source-discovery/2026-07-10-msglib-readonly-schema-extraction.md`; C9 ignored-log scan: PacketReader 86 groupchat/86 conference hits; Smark 24 groupchat/658 conference/631 muc hits | C29 verifies bounded group display-entity extraction through the x86 sidecar and Go client; MCP search is not enriched yet | C30 wire optional MsgLib enrichment into `isphere_search_groups` | | `isphere_receive_files` | decrypted `PacketReader.ProcessPacket` file-transfer message stanzas via `internal/isphere.ListFilesFromMessages` and `isphere_receive_files` list mode; configured PacketReader file or directory source; `zyl\importal\` cache entries remain unlinked | decrypted `Smark.SendReceive` and `SaveToDB` traces for file-reference reconciliation; future UIA/client connector for download | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#c12-file-transfer-evidence-precheck` | C22 verifies safe file-list contract args; no log-to-cache/download mapping yet | defer download mapping; precheck contact/group display-name evidence | | `isphere_send_message` | none validated yet | existing bridge/API/local service preferred; UIA write connector only after internal-sandbox action evidence; network/protocol connector only after protocol discovery | `docs/source-discovery/2026-07-10-send-message-source-precheck.md` | C15 blocked: contract exists, but no committed bridge/API/local service, helper write op, UIA action chain, or protocol connector is validated | do not implement write behavior; harden existing read source configuration | | `isphere_send_file` | none validated yet | depends on send-message connector plus outbound file/upload evidence | `docs/source-discovery/2026-07-10-send-message-source-precheck.md` | blocked behind `isphere_send_message` connector selection and file upload/source policy | defer until send-message connector is validated | @@ -39,4 +39,4 @@ Start Stage C with a narrow log-backed `isphere_receive_messages` source abstrac ## Deferred source work -`MsgLib.db` has a validated copied read-only open path through the bundled 32-bit `System.Data.SQLite.dll` and password `123`. C27 adds `MsgLibReadSidecar` as the bounded x86 .NET reader boundary, and C28 adds the Go `internal/msglib` process client. The next step is a bounded display-entity operation before contact/group display-name integration. +`MsgLib.db` has a validated copied read-only open path through the bundled 32-bit `System.Data.SQLite.dll` and password `123`. C27 adds `MsgLibReadSidecar` as the bounded x86 .NET reader boundary, C28 adds the Go `internal/msglib` process client, and C29 adds bounded `display_entities` extraction. The next step is optional contact/group MCP enrichment from that source. diff --git a/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md b/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md index 794e4e7..3286c1a 100644 --- a/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md +++ b/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md @@ -2238,7 +2238,7 @@ Go sidecar client result: **Goal:** Extend the sidecar/client boundary from schema/display-source availability to bounded contact/group display-entity metadata, so a later loop can enrich `isphere_search_contacts` and `isphere_search_groups` without returning raw rows, message bodies, file paths, downloads, sends, writes, hooks, injection, or DB mutations. -**Planned files:** +**Files:** - Modify: `native/MsgLibReadSidecar/Program.cs` - Modify: `docs/msglib-read-sidecar-contract.md` - Modify: `internal/msglib/sidecar_client.go` @@ -2249,25 +2249,91 @@ Go sidecar client result: - Modify: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md` **Planned behavior:** -- Add a sidecar operation tentatively named `display_entities`. +- Add a sidecar operation named `display_entities`. - Request args: `sqlite_dll_path`, `db_path`, optional `password`, `entity_type` (`contacts` or `groups`), optional `query`, and bounded `limit`. -- Response data: normalized metadata only, such as `entity_type`, `source_table`, `jid`, `display_name`, `confidence`, and `matched_columns`. -- Never return message bodies, file paths, raw SQL row maps, attachment contents, send/write capabilities, or unbounded result sets. -- Use tests first. Normal Go tests should still use fake sidecar data; real copied DB verification stays opt-in through script/env paths. +- Response data: normalized metadata only: `entity_type`, `source_table`, `jid`, `display_name`, `confidence`, and `matched_columns`. +- Never return message bodies, message text, file paths, raw SQL row maps, attachment contents, send/write capabilities, or unbounded result sets. +- Use tests first. Normal Go tests use fake sidecar data; real copied DB verification stays opt-in through script/env paths and prints only sanitized summaries. -- [ ] **Step 1: Write failing display-entity contract tests** +- [x] **Step 1: Write failing display-entity contract tests** -Add sidecar/client tests for bounded contact/group entity decoding, query/limit validation, and rejection of unsupported entity types. +Extended `internal/msglib/sidecar_client_test.go` with fake-sidecar tests for `DisplayEntities`, bounded args, decoded entity metadata, and unsupported entity type rejection. Verified RED with missing `DisplayEntities` / `DisplayEntitiesOptions` symbols. -- [ ] **Step 2: Implement sidecar `display_entities` operation** +- [x] **Step 2: Implement sidecar `display_entities` operation** -Implement allowlisted table/column reads only for contact/group identity metadata, with parameterized query/limit and no raw row dumps. +Implemented Go `DisplayEntities` wrapper and C# sidecar `display_entities` op. The sidecar uses allowlisted table/column pairs only, parameterized `query`, limit clamped to `1..100`, read-only SQLite connection settings, and normalized entity output. -- [ ] **Step 3: Extend Go client and verification script** +- [x] **Step 3: Extend Go client and verification script** -Add Go wrapper types/methods and optional evidence-backed verification that reports only counts/source-table names, not personal values. +Updated `scripts/verify-msglib-sidecar.ps1` to call `display_entities` when copied DB env paths are supplied. The verification output includes only `entity_count` and `source_tables`, not actual entity values. -- [ ] **Step 4: Update docs, run full verification, and commit** +- [x] **Step 4: Update docs, run full verification, and commit** + +Run before commit: + +```powershell +git diff --check +go test ./... +go build ./cmd/isphere-mcp +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\build-msglib-sidecar.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-msglib-sidecar.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-msglib-sidecar.ps1 # with copied DB env paths +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-go-mcp.ps1 +``` + +Final verification passed: `git diff --check`, `go test ./...`, explicit `go build ./cmd/isphere-mcp`, default sidecar smoke, evidence-backed sidecar smoke with sanitized `display_entity_summaries`, and `scripts/verify-go-mcp.ps1`. + +Verification evidence from the copied DB smoke was sanitized: `contacts` returned count/source-table summary and `groups` returned count/source-table summary without printing names, JIDs, message bodies, file paths, or raw rows. + +--- + +## Loop C29 Result + +Bounded display entity extraction result: + +- Added sidecar op `display_entities`. +- Added Go client method `DisplayEntities(ctx, DisplayEntitiesOptions)` plus `DisplayEntity` metadata decoding. +- Evidence-backed sidecar verification against the copied DB returned sanitized summaries: contacts count from `TD_CustomEffigy`; groups count from `TD_WorkGroupAuth`. +- `isphere_search_contacts` and `isphere_search_groups` are still not wired to this source. + +--- + +## Loop C30: Integrate MsgLib display entities into contact/group search + +**Goal:** Wire the C29 bounded `DisplayEntities` source into `isphere_search_contacts` and `isphere_search_groups` as an optional env-configured enrichment path, while preserving the current log-backed fallback and keeping message bodies, file paths, downloads, sends, writes, hooks, injection, DB mutation, and raw row dumps out of scope. + +**Planned files:** +- Modify: `internal/tools/isphere_contacts.go` +- Modify: `internal/tools/isphere_contacts_test.go` +- Modify: `internal/tools/isphere_groups.go` +- Modify: `internal/tools/isphere_groups_test.go` +- Modify: `internal/mcpserver/server.go` +- Modify: `internal/mcpserver/server_test.go` +- Modify: `scripts/verify-go-mcp.ps1` +- Modify: `docs/current-status-card.md` +- Modify: `docs/source-discovery/capability-source-matrix.md` +- Modify: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md` + +**Planned behavior:** +- Add an injectable display-entity source interface around `internal/msglib.Client`. +- Keep default MCP behavior unchanged when `ISPHERE_MSGLIB_SIDECAR_EXE`, `ISPHERE_MSGLIB_SQLITE_DLL`, or `ISPHERE_MSGLIB_DB` are absent. +- When all env vars are present, enrich contact/group search results from `DisplayEntities` with bounded `query` and `limit`. +- Normal tests use fake display-entity source data; verification script should include a synthetic/fake enrichment smoke before any real copied DB smoke. +- Do not expose a new MCP tool in this loop. + +- [ ] **Step 1: Write failing contact/group enrichment tests** + +Add tests proving contacts/groups can consume injected display entities and still pass existing log-backed/default behavior. + +- [ ] **Step 2: Implement optional display-entity source wiring** + +Implement source interface and env-configured `internal/msglib` client construction without changing default empty-source behavior. + +- [ ] **Step 3: Update verification/docs** + +Extend `scripts/verify-go-mcp.ps1` with synthetic/fake enrichment smoke and update status/matrix/plan. + +- [ ] **Step 4: Full verification and commit** Run: diff --git a/internal/msglib/sidecar_client.go b/internal/msglib/sidecar_client.go index db7ab82..6337e0c 100644 --- a/internal/msglib/sidecar_client.go +++ b/internal/msglib/sidecar_client.go @@ -16,6 +16,12 @@ import ( const ( Protocol = "isphere.msglib.v1" DefaultPassword = "123" + + EntityTypeContacts = "contacts" + EntityTypeGroups = "groups" + + DefaultDisplayEntityLimit = 25 + MaxDisplayEntityLimit = 100 ) var requestCounter uint64 @@ -81,6 +87,21 @@ type DisplaySource struct { PresentColumns []string `json:"present_columns"` } +type DisplayEntitiesOptions struct { + EntityType string + Query string + Limit int +} + +type DisplayEntity struct { + EntityType string `json:"entity_type"` + SourceTable string `json:"source_table"` + JID string `json:"jid"` + DisplayName string `json:"display_name"` + Confidence float64 `json:"confidence"` + MatchedColumns []string `json:"matched_columns"` +} + type SidecarError struct { Code string `json:"code"` Message string `json:"message"` @@ -118,10 +139,45 @@ func (c *Client) DisplaySources(ctx context.Context) ([]DisplaySource, error) { return out.DisplaySources, nil } +func (c *Client) DisplayEntities(ctx context.Context, opts DisplayEntitiesOptions) ([]DisplayEntity, error) { + entityType := strings.TrimSpace(opts.EntityType) + if entityType != EntityTypeContacts && entityType != EntityTypeGroups { + return nil, fmt.Errorf("entity_type must be %q or %q", EntityTypeContacts, EntityTypeGroups) + } + limit := opts.Limit + if limit <= 0 { + limit = DefaultDisplayEntityLimit + } + if limit > MaxDisplayEntityLimit { + limit = MaxDisplayEntityLimit + } + + args := map[string]any{ + "sqlite_dll_path": c.config.SQLiteDLLPath, + "db_path": c.config.DBPath, + "password": c.config.Password, + "entity_type": entityType, + "limit": limit, + } + if strings.TrimSpace(opts.Query) != "" { + args["query"] = strings.TrimSpace(opts.Query) + } + + var out displayEntitiesData + if err := c.call(ctx, "display_entities", args, &out); err != nil { + return nil, err + } + return out.Entities, nil +} + type displaySourcesData struct { DisplaySources []DisplaySource `json:"display_sources"` } +type displayEntitiesData struct { + Entities []DisplayEntity `json:"entities"` +} + type requestEnvelope struct { Protocol string `json:"protocol"` RequestID string `json:"request_id"` diff --git a/internal/msglib/sidecar_client_test.go b/internal/msglib/sidecar_client_test.go index 4eb2463..c87e32a 100644 --- a/internal/msglib/sidecar_client_test.go +++ b/internal/msglib/sidecar_client_test.go @@ -55,6 +55,44 @@ func TestDisplaySourcesPassesConfiguredPathsAndDecodesMetadata(t *testing.T) { } } +func TestDisplayEntitiesPassesBoundedArgsAndDecodesMetadata(t *testing.T) { + cfg := Config{ + SQLiteDLLPath: `C:\fixture\System.Data.SQLite.dll`, + DBPath: `C:\fixture\MsgLib.db`, + Password: "123", + } + client := newFakeClient(t, "display_entities_ok", cfg) + + got, err := client.DisplayEntities(context.Background(), DisplayEntitiesOptions{EntityType: "contacts", Query: "alice", Limit: 2}) + if err != nil { + t.Fatalf("DisplayEntities returned error: %v", err) + } + if len(got) != 2 { + t.Fatalf("len(DisplayEntities) = %d, want 2", len(got)) + } + if got[0].EntityType != "contacts" || got[0].SourceTable != "TD_Roster" || got[0].JID != "alice@example" || got[0].DisplayName != "Alice" { + t.Fatalf("unexpected first entity: %+v", got[0]) + } + if got[0].Confidence <= 0 || strings.Join(got[0].MatchedColumns, ",") != "JId,Nick" { + t.Fatalf("unexpected first entity metadata: %+v", got[0]) + } + if got[1].SourceTable != "tblRecent" || got[1].JID != "recent@example" || got[1].DisplayName != "Recent Alice" { + t.Fatalf("unexpected second entity: %+v", got[1]) + } +} + +func TestDisplayEntitiesRejectsUnsupportedEntityType(t *testing.T) { + client := newFakeClient(t, "display_entities_ok", Config{}) + + _, err := client.DisplayEntities(context.Background(), DisplayEntitiesOptions{EntityType: "messages", Limit: 5}) + if err == nil { + t.Fatal("DisplayEntities returned nil error, want unsupported entity_type error") + } + if !strings.Contains(err.Error(), "entity_type") { + t.Fatalf("DisplayEntities error = %v, want entity_type context", err) + } +} + func TestClientReturnsStructuredSidecarError(t *testing.T) { client := newFakeClient(t, "sidecar_error", Config{}) @@ -202,6 +240,31 @@ func fakeSidecarMain() int { "error": nil, "warnings": []any{}, }) + case "display_entities": + args, _ := req["args"].(map[string]any) + if args["sqlite_dll_path"] != `C:\fixture\System.Data.SQLite.dll` || args["db_path"] != `C:\fixture\MsgLib.db` || args["password"] != "123" || args["entity_type"] != "contacts" || args["query"] != "alice" || args["limit"] != float64(2) { + fmt.Fprintf(os.Stderr, "bad display_entities args: %#v\n", args) + return 2 + } + return writeFakeResponse(map[string]any{ + "protocol": Protocol, + "request_id": requestID, + "op": op, + "ok": true, + "data": map[string]any{ + "metadata_only": true, + "read_only": true, + "entity_type": "contacts", + "limit": 2, + "result_count": 2, + "entities": []any{ + map[string]any{"entity_type": "contacts", "source_table": "TD_Roster", "jid": "alice@example", "display_name": "Alice", "confidence": 0.95, "matched_columns": []any{"JId", "Nick"}}, + map[string]any{"entity_type": "contacts", "source_table": "tblRecent", "jid": "recent@example", "display_name": "Recent Alice", "confidence": 0.70, "matched_columns": []any{"PersonId", "PersonName"}}, + }, + }, + "error": nil, + "warnings": []any{}, + }) default: fmt.Fprintf(os.Stderr, "unexpected op: %s\n", op) return 2 diff --git a/native/MsgLibReadSidecar/Program.cs b/native/MsgLibReadSidecar/Program.cs index 0867a41..06d6b84 100644 --- a/native/MsgLibReadSidecar/Program.cs +++ b/native/MsgLibReadSidecar/Program.cs @@ -8,7 +8,7 @@ namespace MsgLibReadSidecar { internal static class Program { - private const string Version = "0.1.0"; + private const string Version = "0.2.0"; private const string DefaultPassword = "123"; private static readonly HashSet DefaultTargetTables = new HashSet(StringComparer.OrdinalIgnoreCase) @@ -46,6 +46,9 @@ namespace MsgLibReadSidecar case "display_sources": response = SidecarProtocol.Success(requestId, op, SchemaSummary(opArgs, true)); break; + case "display_entities": + response = SidecarProtocol.Success(requestId, op, DisplayEntities(opArgs)); + break; default: response = SidecarProtocol.Failure(requestId, op, "UNSUPPORTED_OP", "unsupported op: " + op); break; @@ -71,6 +74,8 @@ namespace MsgLibReadSidecar { "runtime", ".NET Framework " + Environment.Version }, { "process_bits", IntPtr.Size * 8 }, { "requires_process_bits", 32 }, + { "read_only", true }, + { "mutates_db", false }, { "db_mutation_allowed", false }, { "message_body_reads_allowed", false } }; @@ -118,6 +123,52 @@ namespace MsgLibReadSidecar return result; } + + private static Dictionary DisplayEntities(Dictionary args) + { + string sqliteDllPath = SidecarProtocol.GetString(args, "sqlite_dll_path", ""); + string dbPath = SidecarProtocol.GetString(args, "db_path", ""); + string password = SidecarProtocol.GetString(args, "password", DefaultPassword); + string entityType = SidecarProtocol.GetString(args, "entity_type", "").Trim().ToLowerInvariant(); + string query = SidecarProtocol.GetString(args, "query", "").Trim(); + int limit = Clamp(SidecarProtocol.GetInt(args, "limit", 25), 1, 100); + + if (!string.Equals(entityType, "contacts", StringComparison.OrdinalIgnoreCase) && + !string.Equals(entityType, "groups", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("entity_type must be contacts or groups"); + } + + ValidateInputs(sqliteDllPath, dbPath); + + var result = new Dictionary + { + { "metadata_only", true }, + { "read_only", true }, + { "message_body_values_returned", false }, + { "raw_rows_returned", false }, + { "entity_type", entityType }, + { "limit", limit }, + { "provider", "System.Data.SQLite" }, + { "process_bits", IntPtr.Size * 8 } + }; + + using (DbConnection connection = OpenSqliteConnection(sqliteDllPath, dbPath, password)) + { + connection.Open(); + TryExecuteNonQuery(connection, "PRAGMA query_only=ON"); + List> tables = ReadMasterTables(connection, 256); + HashSet entityTables = BuildDisplayEntityTableSet(entityType); + List> columns = ReadColumns(connection, tables, entityTables); + Dictionary> columnMap = BuildColumnMap(columns); + List> entities = ReadDisplayEntities(connection, entityType, query, limit, columnMap); + result["entities"] = entities; + result["result_count"] = entities.Count; + } + + return result; + } + private static void ValidateInputs(string sqliteDllPath, string dbPath) { if (string.IsNullOrWhiteSpace(sqliteDllPath) || !File.Exists(sqliteDllPath)) @@ -269,6 +320,133 @@ namespace MsgLibReadSidecar return sources; } + + private sealed class DisplayEntitySpec + { + public string EntityType; + public string SourceTable; + public string JidColumn; + public string NameColumn; + public double Confidence; + } + + private static HashSet BuildDisplayEntityTableSet(string entityType) + { + var tables = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (DisplayEntitySpec spec in BuildDisplayEntitySpecs(entityType)) + { + tables.Add(spec.SourceTable); + } + return tables; + } + + private static Dictionary> BuildColumnMap(List> columns) + { + var columnMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (Dictionary column in columns) + { + string table = Convert.ToString(column["table"]); + string name = Convert.ToString(column["name"]); + if (!columnMap.ContainsKey(table)) columnMap[table] = new HashSet(StringComparer.OrdinalIgnoreCase); + columnMap[table].Add(name); + } + return columnMap; + } + + private static List BuildDisplayEntitySpecs(string entityType) + { + var specs = new List(); + if (string.Equals(entityType, "contacts", StringComparison.OrdinalIgnoreCase)) + { + AddEntitySpec(specs, entityType, "TD_Roster", "JId", "Nick", 0.95); + AddEntitySpec(specs, entityType, "TD_CustomEffigy", "PersonJid", "PersonName", 0.90); + AddEntitySpec(specs, entityType, "tblRecent", "PersonId", "PersonName", 0.70); + AddEntitySpec(specs, entityType, "tblChatLevel", "PersonId", "PersonName", 0.70); + AddEntitySpec(specs, entityType, "tblPersonMsg", "SndPersonId", "SndPerson", 0.55); + AddEntitySpec(specs, entityType, "tblPersonMsg", "RecvPersonId", "RecvPerson", 0.55); + AddEntitySpec(specs, entityType, "tblPersonMsg", "ObjPersonId", "ObjPerson", 0.55); + } + else if (string.Equals(entityType, "groups", StringComparison.OrdinalIgnoreCase)) + { + AddEntitySpec(specs, entityType, "TD_WorkGroupAuth", "GroupJID", "GroupName", 0.90); + AddEntitySpec(specs, entityType, "TD_WorkGroupAuth", "ChildGroupJid", "GroupName", 0.80); + AddEntitySpec(specs, entityType, "tblMsgGroupPersonMsg", "MsgGroupId", "MsgGroupName", 0.70); + AddEntitySpec(specs, entityType, "tblMsgGroupPersonMsg", "SessionPersonId", "SessionPersonName", 0.60); + } + return specs; + } + + private static void AddEntitySpec(List specs, string entityType, string table, string jidColumn, string nameColumn, double confidence) + { + specs.Add(new DisplayEntitySpec + { + EntityType = entityType, + SourceTable = table, + JidColumn = jidColumn, + NameColumn = nameColumn, + Confidence = confidence + }); + } + + private static List> ReadDisplayEntities(DbConnection connection, string entityType, string query, int limit, Dictionary> columnMap) + { + var rows = new List>(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (DisplayEntitySpec spec in BuildDisplayEntitySpecs(entityType)) + { + if (rows.Count >= limit) break; + if (!columnMap.ContainsKey(spec.SourceTable)) continue; + if (!columnMap[spec.SourceTable].Contains(spec.JidColumn)) continue; + if (!columnMap[spec.SourceTable].Contains(spec.NameColumn)) continue; + ReadDisplayEntitiesForSpec(connection, spec, query, limit - rows.Count, rows, seen); + } + return rows; + } + + private static void ReadDisplayEntitiesForSpec(DbConnection connection, DisplayEntitySpec spec, string query, int limit, List> rows, HashSet seen) + { + if (limit <= 0) return; + using (DbCommand command = connection.CreateCommand()) + { + string jid = QuoteIdent(spec.JidColumn); + string name = QuoteIdent(spec.NameColumn); + command.CommandText = + "select " + jid + " as jid, " + name + " as display_name from " + QuoteIdent(spec.SourceTable) + + " where (coalesce(" + jid + ", '') <> '' or coalesce(" + name + ", '') <> '')"; + if (!string.IsNullOrWhiteSpace(query)) + { + command.CommandText += " and (" + jid + " like @query or " + name + " like @query)"; + DbParameter queryParam = command.CreateParameter(); + queryParam.ParameterName = "@query"; + queryParam.Value = "%" + query + "%"; + command.Parameters.Add(queryParam); + } + command.CommandText += " limit " + Convert.ToString(limit); + + using (DbDataReader reader = command.ExecuteReader()) + { + while (reader.Read() && rows.Count < limit) + { + string entityJid = Convert.ToString(reader["jid"]); + string displayName = Convert.ToString(reader["display_name"]); + if (string.IsNullOrWhiteSpace(entityJid) && string.IsNullOrWhiteSpace(displayName)) continue; + string key = spec.EntityType + "|" + entityJid + "|" + displayName + "|" + spec.SourceTable; + if (seen.Contains(key)) continue; + seen.Add(key); + rows.Add(new Dictionary + { + { "entity_type", spec.EntityType }, + { "source_table", spec.SourceTable }, + { "jid", entityJid }, + { "display_name", displayName }, + { "confidence", spec.Confidence }, + { "matched_columns", new string[] { spec.JidColumn, spec.NameColumn } } + }); + } + } + } + } + private static void AddSource(List> sources, HashSet tableSet, Dictionary> columnMap, string sourceName, string table, string[] columns) { var presentColumns = new List(); diff --git a/scripts/verify-msglib-sidecar.ps1 b/scripts/verify-msglib-sidecar.ps1 index 26457be..85f7397 100644 --- a/scripts/verify-msglib-sidecar.ps1 +++ b/scripts/verify-msglib-sidecar.ps1 @@ -46,6 +46,7 @@ if (-not $self.ok -or $self.data.sidecar_name -ne "MsgLibReadSidecar" -or $self. $providerChecked = $false $displaySources = $null +$displayEntitySummaries = @() if ($SQLiteDllPath -and $DbPath) { if (-not (Test-Path -LiteralPath $SQLiteDllPath)) { throw "SQLite DLL not found: $SQLiteDllPath" @@ -66,7 +67,31 @@ if ($SQLiteDllPath -and $DbPath) { } } if (-not $displaySources.ok -or -not $displaySources.data.metadata_only -or $displaySources.data.message_body_values_returned) { - throw "display_sources failed: $($displaySources | ConvertTo-Json -Depth 12 -Compress)" + throw "display_sources failed" + } + + foreach ($entityType in @("contacts", "groups")) { + $displayEntities = Invoke-SidecarJson @{ + protocol = "isphere.msglib.v1" + request_id = "verify-display-entities-$entityType" + op = "display_entities" + args = @{ + sqlite_dll_path = $SQLiteDllPath + db_path = $DbPath + password = "123" + entity_type = $entityType + limit = 5 + } + } + if (-not $displayEntities.ok -or -not $displayEntities.data.metadata_only -or -not $displayEntities.data.read_only -or $displayEntities.data.message_body_values_returned -or $displayEntities.data.raw_rows_returned) { + throw "display_entities failed for $entityType" + } + $sourceTables = @($displayEntities.data.entities | ForEach-Object { $_.source_table } | Sort-Object -Unique) + $displayEntitySummaries += [ordered]@{ + entity_type = $entityType + entity_count = @($displayEntities.data.entities).Count + source_tables = $sourceTables + } } $providerChecked = $true } @@ -78,4 +103,5 @@ if ($SQLiteDllPath -and $DbPath) { process_bits = $self.data.process_bits provider_checked = $providerChecked display_source_count = if ($displaySources) { $displaySources.data.display_sources.Count } else { 0 } -} | ConvertTo-Json -Depth 6 -Compress + display_entity_summaries = $displayEntitySummaries +} | ConvertTo-Json -Depth 8 -Compress