From fe242145bbe92cc3fe9c8458d7c6cfaf2c1e5c51 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Fri, 10 Jul 2026 00:45:08 +0800 Subject: [PATCH] feat: extract files from message logs --- docs/current-status-card.md | 7 +- .../capability-source-matrix.md | 2 +- ...tage-c-log-backed-receive-messages-loop.md | 74 ++++++++- internal/isphere/files.go | 142 ++++++++++++++++++ internal/isphere/files_test.go | 31 ++++ 5 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 internal/isphere/files.go create mode 100644 internal/isphere/files_test.go diff --git a/docs/current-status-card.md b/docs/current-status-card.md index 21d3bb6..216f5c3 100644 --- a/docs/current-status-card.md +++ b/docs/current-status-card.md @@ -50,7 +50,7 @@ N13/N14/N15 are pre-business validation results. They can help identify UI eleme ## Current loop -Current loop: `Stage C / Loop C13 - log-backed file metadata extraction`. +Current loop: `Stage C / Loop C14 - register isphere_receive_files list mode`. Plan file: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md`. @@ -75,10 +75,11 @@ Current loop output so far: 10. C10: log-backed group candidate extraction from groupchat/conference JIDs. 11. C11: `isphere_search_groups` MCP registration and seven-tool verification. 12. C12: file evidence precheck confirmed enough PacketReader/Smark evidence for file metadata listing, but not enough for real cache/download mapping. +13. C13: `internal/isphere.ListFilesFromMessages` extracts synthetic/redacted file metadata candidates from normalized messages; MCP registration is next. ## Next business mainline -1. Run Stage C Loop C13: implement log-backed file metadata extraction from normalized messages. -2. Register `isphere_receive_files` list mode after C13 source behavior passes; leave actual download/cache mapping as a later connector node. +1. Run Stage C Loop C14: register `isphere_receive_files` list mode on top of `ListFilesFromMessages`. +2. Leave actual download/cache mapping as a later connector node until a message-to-cache hash or client download source is validated. 3. Add `isphere_send_message` and `isphere_send_file` after source discovery and connector selection. 4. Keep UIA selector/report work as fallback support, not as the primary roadmap. diff --git a/docs/source-discovery/capability-source-matrix.md b/docs/source-discovery/capability-source-matrix.md index e31b992..f4b40a5 100644 --- a/docs/source-discovery/capability-source-matrix.md +++ b/docs/source-discovery/capability-source-matrix.md @@ -20,7 +20,7 @@ Schema notes: `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md` | `isphere_receive_messages` | decrypted `PacketReader.ProcessPacket` XMPP `` stanzas via `internal/isphere.EncryptedPacketLogSource` | decrypted `Smark.SendReceive` transport stanzas; decrypted `SaveToDB` `Chat_OnMessageArrived` traces; wrapped `MsgLib.db` after DB wrapper is solved | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#field-mapping-evidence` | C5 env loader wired through ISPHERE_PACKET_LOG_FILE; default source remains empty when unset | fixture-backed configured-source operator smoke | | `isphere_search_contacts` | bare sender/receiver JIDs extracted from normalized log-backed messages | decrypted roster/contact stanzas from `Smark.SendReceive`; `MsgLib.db` contact/conversation tables after DB wrapper is solved; UIA helper source if display names are missing | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#log-decryption-result` | C8 MCP tool registered and verify-go-mcp covers default/configured contact smoke; JID-only display/account fallback | enrich display names from roster/db/uia evidence | | `isphere_search_groups` | decrypted `PacketReader.ProcessPacket` `type="groupchat"` stanzas and conference/MUC JIDs | `MsgLib.db` group/conversation tables after DB wrapper is solved; UIA helper source if group display names are missing | C9 ignored-log scan: PacketReader 86 groupchat/86 conference hits; Smark 24 groupchat/658 conference/631 muc hits | C11 MCP tool registered and verify-go-mcp covers default/configured group smoke; JID-only display fallback | enrich group display/member data from roster/db/uia evidence | -| `isphere_receive_files` | decrypted `PacketReader.ProcessPacket` file-transfer message stanzas for metadata listing; `zyl\importal\` cache entries remain unlinked | decrypted `Smark.SendReceive` and `SaveToDB` traces for file-reference reconciliation; future UIA/client connector for download | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#c12-file-transfer-evidence-precheck` | C12 evidence confirmed metadata-list source: PacketReader has 107 parsed file candidates; no log-to-cache/download mapping yet | extract file metadata candidates from normalized messages | +| `isphere_receive_files` | decrypted `PacketReader.ProcessPacket` file-transfer message stanzas for metadata listing via `internal/isphere.ListFilesFromMessages`; `zyl\importal\` cache entries remain unlinked | decrypted `Smark.SendReceive` and `SaveToDB` traces for file-reference reconciliation; future UIA/client connector for download | `docs/source-discovery/2026-07-09-n12-pre-zyl-schema-notes.md#c12-file-transfer-evidence-precheck` | C13 source behavior implemented for metadata listing; no log-to-cache/download mapping yet | register `isphere_receive_files` MCP list mode | ## Stage C entry rule diff --git a/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md b/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md index b5465b3..b1a5205 100644 --- a/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md +++ b/docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md @@ -1277,7 +1277,7 @@ Evidence precheck result: - Query supports `ConversationID`, `MessageID`, `FileID`, `NameContains`, and `Limit`; default/zero limit returns all candidates. - This loop does not register the MCP tool yet. Registration/list-mode smoke becomes C14 after source behavior passes tests. -- [ ] **Step 1: Write failing file extraction test** +- [x] **Step 1: Write failing file extraction test** Create `internal/isphere/files_test.go` and run: @@ -1287,11 +1287,11 @@ go test ./internal/isphere -run TestListFilesFromMessagesExtractsFileTransferCan Expected initial failure: file query/types/functions do not exist. -- [ ] **Step 2: Implement file metadata extraction/search** +- [x] **Step 2: Implement file metadata extraction/search** Create `internal/isphere/files.go` with minimal pure Go logic over normalized messages and synthetic/redacted fixtures. -- [ ] **Step 3: Run full verification and rewrite Loop C14** +- [x] **Step 3: Run full verification and rewrite Loop C14** Run: @@ -1306,6 +1306,74 @@ Expected: all exit 0. Remove any generated root `isphere-mcp.exe` after build. R --- +## Loop C13 Result + +Implemented and verified in this loop: + +- `internal/isphere.ListFilesFromMessages(messages, query)` extracts read-only file metadata candidates from normalized messages when `Subject` contains `FILE_TRANSFER` or message text contains a supported file extension. +- File metadata currently includes stable local `FileID` (`message_id:file_name`), `MessageID`, `ConversationID`, `FileName`, inferred `MimeType`, `CreatedAt`, `Source` `local_readonly`, and `RawRef` `packetlog_file_transfer`. +- `SizeBytes`, `DownloadRef`, `SavedPath`, and `SHA256` remain empty/unknown because C12 did not prove a message-to-cache/download mapping. +- Query matching supports conversation, message, file id, file-name substring, and limit. Tests use only synthetic/redacted filenames. +- Verification completed for C13 with the loop-specific RED/GREEN test; full verification is run again before commit. + +--- + +## Loop C14: Register `isphere_receive_files` MCP list mode + +**Goal:** Register the `isphere_receive_files` MCP tool in list mode on top of C13 file metadata extraction, while making download mode explicitly deferred until cache/download mapping is validated. + +**Planned files:** +- Create: `internal/tools/isphere_files.go` +- Create: `internal/tools/isphere_files_test.go` +- Modify: `internal/mcpserver/server.go` +- Modify: `internal/mcpserver/server_test.go` +- Modify: `scripts/verify-go-mcp.ps1` +- Modify: `docs/go-mcp-runbook.md` +- Modify: `docs/current-status-card.md` +- Modify: `docs/source-discovery/capability-source-matrix.md` +- Modify: `docs/superpowers/plans/2026-07-09-stage-c-log-backed-receive-messages-loop.md` + +**Planned interface:** +- Tool name: `isphere_receive_files`. +- Input: `conversation_id`, `message_id`, `file_id`, `name_contains`, `mode`, `limit`. +- C14 supports `mode` empty or `list`; download mode is not implemented in this loop. +- Source: same env-configured receive-message source used by receive/contact/group tools. +- Output shape: `ok`, `mode`, `files`, `next_cursor`, and `audit`, aligned with `docs/mcp-core-tools-contract.md`. +- Default empty source returns `ok: true`, `mode: "list"`, `files: []`. + +- [ ] **Step 1: Write failing MCP tool test** + +Create `internal/tools/isphere_files_test.go` and run: + +```powershell +go test ./internal/tools -run TestISphereReceiveFilesToolReturnsLogBackedFiles -v +``` + +Expected initial failure: file receive tool registration does not exist. + +- [ ] **Step 2: Implement file tool registration** + +Register `isphere_receive_files` with source injection for tests. Use `ListFilesFromMessages` and return a contract-shaped map with minimal audit metadata. + +- [ ] **Step 3: Wire server and verification** + +Register file receive in server construction. Update server tool-count tests and `scripts/verify-go-mcp.ps1` to expect eight tools and call `isphere_receive_files` against default empty source and synthetic configured file-transfer fixture. + +- [ ] **Step 4: Run full verification and commit** + +Run: + +```powershell +git diff --check +go test ./... +go build ./cmd/isphere-mcp +powershell -NoProfile -ExecutionPolicy Bypass -File scripts/verify-go-mcp.ps1 +``` + +Expected: all exit 0. Remove any generated root `isphere-mcp.exe` after build. Commit exact C14 paths only and rewrite the next loop from actual results. + +--- + ## Self-Review - Spec coverage: the plan follows the user's cyclic requirement: plan first, implement one loop at a time, update the next loop after each result, and ask only on blockers. diff --git a/internal/isphere/files.go b/internal/isphere/files.go new file mode 100644 index 0000000..c02eda8 --- /dev/null +++ b/internal/isphere/files.go @@ -0,0 +1,142 @@ +package isphere + +import ( + "path/filepath" + "regexp" + "sort" + "strings" +) + +type File struct { + FileID string + MessageID string + ConversationID string + FileName string + SizeBytes *int64 + MimeType string + CreatedAt string + DownloadRef string + SavedPath string + SHA256 string + Source string + RawRef string +} + +type ListFilesQuery struct { + ConversationID string + MessageID string + FileID string + NameContains string + Limit int +} + +type ListFilesResult struct { + Files []File +} + +var fileNamePattern = regexp.MustCompile(`(?i)[^\s<>:"|?*]+\.(?:docx?|xlsx?|pptx?|pdf|zip|rar|png|jpe?g|txt|csv)\b`) + +func ListFilesFromMessages(messages []Message, query ListFilesQuery) ListFilesResult { + files := make([]File, 0) + for _, message := range messages { + if !messageLooksFileBacked(message) { + continue + } + fileName := extractMessageFileName(message.Text) + if fileName == "" { + continue + } + file := File{ + FileID: fileID(message.ID, fileName), + MessageID: message.ID, + ConversationID: message.ConversationID, + FileName: fileName, + MimeType: mimeTypeForFileName(fileName), + CreatedAt: message.Timestamp, + Source: "local_readonly", + RawRef: "packetlog_file_transfer", + } + if fileMatchesQuery(file, query) { + files = append(files, file) + } + } + + sort.Slice(files, func(i, j int) bool { + if files[i].FileID == files[j].FileID { + return files[i].MessageID < files[j].MessageID + } + return files[i].FileID < files[j].FileID + }) + + limit := query.Limit + if limit <= 0 || limit > len(files) { + limit = len(files) + } + return ListFilesResult{Files: files[:limit]} +} + +func messageLooksFileBacked(message Message) bool { + return strings.Contains(strings.ToUpper(message.Subject), "FILE_TRANSFER") || extractMessageFileName(message.Text) != "" +} + +func extractMessageFileName(text string) string { + match := fileNamePattern.FindString(strings.TrimSpace(text)) + return strings.Trim(match, " .,;，。；") +} + +func fileID(messageID string, fileName string) string { + if messageID == "" { + return fileName + } + return messageID + ":" + fileName +} + +func fileMatchesQuery(file File, query ListFilesQuery) bool { + if query.ConversationID != "" && file.ConversationID != query.ConversationID { + return false + } + if query.MessageID != "" && file.MessageID != query.MessageID { + return false + } + if query.FileID != "" && file.FileID != query.FileID { + return false + } + nameQuery := strings.ToLower(strings.TrimSpace(query.NameContains)) + if nameQuery != "" && !strings.Contains(strings.ToLower(file.FileName), nameQuery) { + return false + } + return true +} + +func mimeTypeForFileName(name string) string { + switch strings.ToLower(filepath.Ext(name)) { + case ".doc": + return "application/msword" + case ".docx": + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + case ".xls": + return "application/vnd.ms-excel" + case ".xlsx": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + case ".ppt": + return "application/vnd.ms-powerpoint" + case ".pptx": + return "application/vnd.openxmlformats-officedocument.presentationml.presentation" + case ".pdf": + return "application/pdf" + case ".zip": + return "application/zip" + case ".rar": + return "application/vnd.rar" + case ".png": + return "image/png" + case ".jpg", ".jpeg": + return "image/jpeg" + case ".txt": + return "text/plain" + case ".csv": + return "text/csv" + default: + return "" + } +} diff --git a/internal/isphere/files_test.go b/internal/isphere/files_test.go new file mode 100644 index 0000000..fd3c4f5 --- /dev/null +++ b/internal/isphere/files_test.go @@ -0,0 +1,31 @@ +package isphere + +import ( + "reflect" + "testing" +) + +func TestListFilesFromMessagesExtractsFileTransferCandidates(t *testing.T) { + messages := []Message{ + {ID: "msg-1", ConversationID: "alice@imopenfire1-lanzhou|bob@imopenfire1-lanzhou", Text: "redacted-report.docx", Timestamp: "1783423807000", Subject: "FILE_TRANSFER_CANCEL"}, + {ID: "msg-2", ConversationID: "alice@imopenfire1-lanzhou|bob@imopenfire1-lanzhou", Text: "plain chat", Timestamp: "1783423808000"}, + {ID: "msg-3", ConversationID: "room@conference.imopenfire1-lanzhou", Text: "redacted-screenshot.png", Timestamp: "1783423809000"}, + } + + got := ListFilesFromMessages(messages, ListFilesQuery{NameContains: "report", Limit: 10}) + want := ListFilesResult{Files: []File{ + { + FileID: "msg-1:redacted-report.docx", + MessageID: "msg-1", + ConversationID: "alice@imopenfire1-lanzhou|bob@imopenfire1-lanzhou", + FileName: "redacted-report.docx", + MimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + CreatedAt: "1783423807000", + Source: "local_readonly", + RawRef: "packetlog_file_transfer", + }, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ListFilesFromMessages() = %#v, want %#v", got, want) + } +}