From 2a7087218b73259f63f97c6896676f03422f5361 Mon Sep 17 00:00:00 2001 From: zhaoyilun Date: Fri, 10 Jul 2026 20:28:13 +0800 Subject: [PATCH] test: add send audit idempotency diagnostics --- docs/current-status-card.md | 15 +- docs/go-mcp-runbook.md | 10 ++ ...07-10-r6f-r14-continuous-execution-plan.md | 16 +- internal/tools/isphere_send_message_test.go | 41 +++++ .../test-verify-send-audit-idempotency.ps1 | 49 ++++++ scripts/verify-send-audit-idempotency.ps1 | 155 ++++++++++++++++++ 6 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 scripts/test-verify-send-audit-idempotency.ps1 create mode 100644 scripts/verify-send-audit-idempotency.ps1 diff --git a/docs/current-status-card.md b/docs/current-status-card.md index 84ee325..cb2371d 100644 --- a/docs/current-status-card.md +++ b/docs/current-status-card.md @@ -58,7 +58,7 @@ N13/N14/N15 are pre-business validation results. They can help identify UI eleme ## Current loop -Current loop: `R6f fake/sandbox send connector contract complete; next R6g send audit and idempotency replay diagnostics`. +Current loop: `R6g send audit and idempotency replay diagnostics complete; next R6h central production gate policy for send message`. Active continuous execution plan: `docs/superpowers/plans/2026-07-10-r6f-r14-continuous-execution-plan.md`. @@ -239,5 +239,14 @@ Continuous execution plan R6f-R14 is approved and execution has started: - Plan file: `docs/superpowers/plans/2026-07-10-r6f-r14-continuous-execution-plan.md`. - Scope: 15 ordered rounds covering send-message connector/gate hardening, send-file preview/idempotency/package, receive-file download preview, receive-message reconciliation, end-to-end business smoke, and release-candidate report. - Execution rule after approval: one round at a time, status-card update, verification, commit, push, then continue automatically until an evidence-only blocker requires user action. -- Last completed round: R6f fake/sandbox send connector contract. -- Next active round: R6g send audit and idempotency replay diagnostics. +- Last completed round: R6g send audit and idempotency replay diagnostics. +- Next active round: R6h central production gate policy for send message. + +R6g send audit and idempotency replay diagnostics is complete: + +- Added `scripts\verify-send-audit-idempotency.ps1`. +- Added `scripts\test-verify-send-audit-idempotency.ps1`; the first run failed before the verifier existed, then passed after implementation. +- Diagnostic output reports sanitized `audit_records`, `idempotency_records`, `duplicate_detected`, `conflict_detected`, `raw_body_present`, and `raw_idempotency_key_present`. +- Added `TestISphereSendMessageAuditRedactionIsStable` proving file-backed audit JSONL does not store raw message body or raw idempotency key. +- Standard verification still reports 9 tools and `production_send_enabled=false`. +- Next node: R6h central production gate policy for send message. diff --git a/docs/go-mcp-runbook.md b/docs/go-mcp-runbook.md index 64cd165..741b5ae 100644 --- a/docs/go-mcp-runbook.md +++ b/docs/go-mcp-runbook.md @@ -275,6 +275,16 @@ $env:ISPHERE_SEND_IDEMPOTENCY_PATH = "E:\coding\codex\isphere-ai-bridge\runs\sen If `ISPHERE_SEND_IDEMPOTENCY_PATH` is set, the send preview path records hashed idempotency state. Reusing the same idempotency key for the same target/content sets `duplicate_detected=true`; reusing it for different content or target returns blocked metadata with `conflict_detected=true`. Raw message body and raw idempotency key are not stored in the JSONL state. +R6g adds a local diagnostic for replaying the redacted audit/idempotency files without opening iSphere: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\verify-send-audit-idempotency.ps1 ` + -AuditPath $env:ISPHERE_SEND_AUDIT_PATH ` + -IdempotencyPath $env:ISPHERE_SEND_IDEMPOTENCY_PATH +``` + +The diagnostic prints sanitized counts and booleans: `audit_records`, `idempotency_records`, `duplicate_detected`, `conflict_detected`, `raw_body_present`, and `raw_idempotency_key_present`. It exits non-zero if a raw message body field or raw idempotency-key field is found. + ## 10. Current tool surface and next-stage route The current runbook verifies nine tools: diff --git a/docs/superpowers/plans/2026-07-10-r6f-r14-continuous-execution-plan.md b/docs/superpowers/plans/2026-07-10-r6f-r14-continuous-execution-plan.md index cd6e0de..ad7e5bb 100644 --- a/docs/superpowers/plans/2026-07-10-r6f-r14-continuous-execution-plan.md +++ b/docs/superpowers/plans/2026-07-10-r6f-r14-continuous-execution-plan.md @@ -170,7 +170,7 @@ git push gitea main - Consumes: `ISPHERE_SEND_AUDIT_PATH`, `ISPHERE_SEND_IDEMPOTENCY_PATH`. - Produces script output keys: `audit_records`, `idempotency_records`, `duplicate_detected`, `conflict_detected`, `raw_body_present`, `raw_idempotency_key_present`. -- [ ] **Step 1: Write fixture test script** +- [x] **Step 1: Write fixture test script** `scripts/test-verify-send-audit-idempotency.ps1` creates temporary JSONL audit/idempotency files with one normal record, one duplicate, and one conflict. @@ -180,7 +180,7 @@ powershell -NoProfile -ExecutionPolicy Bypass -File scripts\test-verify-send-aud Expected before implementation: FAIL because `verify-send-audit-idempotency.ps1` does not exist. -- [ ] **Step 2: Implement diagnostic script** +- [x] **Step 2: Implement diagnostic script** `scripts/verify-send-audit-idempotency.ps1` accepts: @@ -193,11 +193,11 @@ param( It prints sanitized JSON with counts and booleans only; it exits non-zero if raw `content_text` or raw `idempotency_key` appears. -- [ ] **Step 3: Add tool-level test for audit redaction regression** +- [x] **Step 3: Add tool-level test for audit redaction regression** Add `TestISphereSendMessageAuditRedactionIsStable` asserting JSONL audit files do not contain the submitted message body or raw idempotency key. -- [ ] **Step 4: Verify and commit** +- [x] **Step 4: Verify and commit** ```powershell git diff --check @@ -213,6 +213,14 @@ git push gitea main **Acceptance gate:** Diagnostics can prove audit/idempotency shape without opening iSphere. +**R6g Result:** + +- Added `scripts/test-verify-send-audit-idempotency.ps1` and confirmed the red test failed before the verifier existed. +- Added `scripts/verify-send-audit-idempotency.ps1` to summarize redacted audit/idempotency JSONL files with sanitized counts and booleans. +- Diagnostic detects duplicate same-key/same-target/content replay, conflict same-key/different-content replay, and raw field leaks. +- Added `TestISphereSendMessageAuditRedactionIsStable` to prove file-backed audit JSONL does not store raw message body or raw idempotency key. +- No iSphere login, real send, helper action, hook, upload, or network replay was introduced. + --- ### R6h: Central production gate policy for send message diff --git a/internal/tools/isphere_send_message_test.go b/internal/tools/isphere_send_message_test.go index d422f6c..296ff49 100644 --- a/internal/tools/isphere_send_message_test.go +++ b/internal/tools/isphere_send_message_test.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "os" "strings" "testing" @@ -315,6 +316,46 @@ func TestISphereSendMessageRejectsContentHashMismatch(t *testing.T) { } } +func TestISphereSendMessageAuditRedactionIsStable(t *testing.T) { + auditPath := t.TempDir() + "\\send-audit.jsonl" + audit := fileSendMessageAuditSink{path: auditPath} + session, cleanup := connectToolsTestSession(t, func(server *mcp.Server) { + RegisterISphereSendMessageToolWithState(server, audit, newFakeSendMessageIdempotencyStore()) + }) + defer cleanup() + + content := "raw body must not be stored in audit" + idempotencyKey := "raw-idempotency-key-must-not-be-stored" + contentHash := sha256HexForSendMessageTest(content) + callResult, err := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: ToolNameSendMessage, + Arguments: map[string]any{ + "target_type": "direct", + "target_id": "alice@imopenfire1-lanzhou", + "content_text": content, + "content_sha256": contentHash, + "idempotency_key": idempotencyKey, + "execution_mode": "preview", + }, + }) + if err != nil { + t.Fatalf("call %s: %v", ToolNameSendMessage, err) + } + if callResult.IsError { + t.Fatalf("preview should not return error: %+v", callResult) + } + payload, err := os.ReadFile(auditPath) + if err != nil { + t.Fatalf("read audit file: %v", err) + } + if strings.Contains(string(payload), content) || strings.Contains(string(payload), idempotencyKey) { + t.Fatalf("audit file leaked raw content or idempotency key: %s", payload) + } + if !strings.Contains(string(payload), contentHash) { + t.Fatalf("audit file missing content hash: %s", payload) + } +} + func TestISphereSendMessageDetectsDuplicateIdempotencyKey(t *testing.T) { audit := &fakeSendMessageAuditSink{} idempotency := newFakeSendMessageIdempotencyStore() diff --git a/scripts/test-verify-send-audit-idempotency.ps1 b/scripts/test-verify-send-audit-idempotency.ps1 new file mode 100644 index 0000000..65dd866 --- /dev/null +++ b/scripts/test-verify-send-audit-idempotency.ps1 @@ -0,0 +1,49 @@ +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$verifier = Join-Path $PSScriptRoot "verify-send-audit-idempotency.ps1" +$workDir = Join-Path ([System.IO.Path]::GetTempPath()) ("isphere-send-audit-idempotency-test-" + [System.Guid]::NewGuid().ToString("N")) +New-Item -ItemType Directory -Path $workDir | Out-Null + +try { + $auditPath = Join-Path $workDir "send-audit.jsonl" + $idempotencyPath = Join-Path $workDir "send-idempotency.jsonl" + + @( + @{ tool = "isphere_send_message"; target_ref = "contact:alice"; content_sha256 = "hash-a"; idempotency_key_sha256 = "key-a"; result = "planned" }, + @{ tool = "isphere_send_message"; target_ref = "contact:alice"; content_sha256 = "hash-a"; idempotency_key_sha256 = "key-a"; result = "planned" }, + @{ tool = "isphere_send_message"; target_ref = "contact:alice"; content_sha256 = "hash-b"; idempotency_key_sha256 = "key-a"; result = "blocked" } + ) | ForEach-Object { $_ | ConvertTo-Json -Compress } | Set-Content -Path $auditPath -Encoding UTF8 + + @( + @{ tool = "isphere_send_message"; target_ref = "contact:alice"; content_sha256 = "hash-a"; idempotency_key_sha256 = "key-a"; audit_ref = "send-message:first" } + ) | ForEach-Object { $_ | ConvertTo-Json -Compress } | Set-Content -Path $idempotencyPath -Encoding UTF8 + + $summary = & $verifier -AuditPath $auditPath -IdempotencyPath $idempotencyPath | ConvertFrom-Json + if (-not $summary.ok) { throw "expected ok summary, got: $($summary | ConvertTo-Json -Compress)" } + if ($summary.audit_records -ne 3) { throw "audit_records=$($summary.audit_records), want 3" } + if ($summary.idempotency_records -ne 1) { throw "idempotency_records=$($summary.idempotency_records), want 1" } + if (-not $summary.duplicate_detected) { throw "duplicate_detected should be true" } + if (-not $summary.conflict_detected) { throw "conflict_detected should be true" } + if ($summary.raw_body_present) { throw "raw_body_present should be false" } + if ($summary.raw_idempotency_key_present) { throw "raw_idempotency_key_present should be false" } + + $leakyAuditPath = Join-Path $workDir "leaky-audit.jsonl" + @{ tool = "isphere_send_message"; content_text = "raw body"; idempotency_key_sha256 = "key-b" } | ConvertTo-Json -Compress | Set-Content -Path $leakyAuditPath -Encoding UTF8 + $leakyOutput = & $verifier -AuditPath $leakyAuditPath -IdempotencyPath $idempotencyPath 2>&1 + if ($LASTEXITCODE -eq 0) { + throw "leaky audit unexpectedly passed: $leakyOutput" + } + + Write-Output (@{ + ok = $true + verification = "send_audit_idempotency_fixture" + repo_root = $repoRoot.Path + } | ConvertTo-Json -Compress) +} +finally { + if (Test-Path $workDir) { + Remove-Item -LiteralPath $workDir -Recurse -Force + } +} diff --git a/scripts/verify-send-audit-idempotency.ps1 b/scripts/verify-send-audit-idempotency.ps1 new file mode 100644 index 0000000..2724a95 --- /dev/null +++ b/scripts/verify-send-audit-idempotency.ps1 @@ -0,0 +1,155 @@ +param( + [Parameter(Mandatory = $true)][string]$AuditPath, + [Parameter(Mandatory = $true)][string]$IdempotencyPath +) + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" + +function Read-JsonLines { + param([Parameter(Mandatory = $true)][string]$Path) + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path)) { + return @() + } + $records = @() + $lineNo = 0 + foreach ($line in Get-Content -LiteralPath $Path) { + $lineNo++ + $trimmed = $line.Trim() + if ($trimmed.Length -eq 0) { + continue + } + try { + $records += ($trimmed | ConvertFrom-Json) + } + catch { + throw "failed to parse JSONL $Path line ${lineNo}: $($_.Exception.Message)" + } + } + return @($records) +} + +function Get-JsonPropertyNames { + param([AllowNull()]$Value) + $names = @() + if ($null -eq $Value) { + return $names + } + if ($Value -is [System.Array]) { + foreach ($item in $Value) { + $names += Get-JsonPropertyNames -Value $item + } + return $names + } + if ($Value -is [System.Collections.IDictionary]) { + foreach ($key in $Value.Keys) { + $names += [string]$key + $names += Get-JsonPropertyNames -Value $Value[$key] + } + return $names + } + if ($Value.PSObject -and $Value.PSObject.Properties) { + foreach ($property in $Value.PSObject.Properties) { + $names += $property.Name + $names += Get-JsonPropertyNames -Value $property.Value + } + } + return $names +} + +function Get-StringProperty { + param( + [Parameter(Mandatory = $true)]$Record, + [Parameter(Mandatory = $true)][string]$Name + ) + $property = $Record.PSObject.Properties[$Name] + if ($null -eq $property -or $null -eq $property.Value) { + return "" + } + return [string]$property.Value +} + +$auditRecords = @(Read-JsonLines -Path $AuditPath) +$idempotencyRecords = @(Read-JsonLines -Path $IdempotencyPath) +$allRecords = @($auditRecords + $idempotencyRecords) +$propertyNames = @() +foreach ($record in $allRecords) { + $propertyNames += Get-JsonPropertyNames -Value $record +} + +$rawBodyKeys = @("content_text", "ContentText", "message_body", "raw_body", "raw_content_text") +$rawIdempotencyKeys = @("idempotency_key", "IdempotencyKey", "raw_idempotency_key") +$rawBodyPresent = $false +$rawIdempotencyKeyPresent = $false +foreach ($name in $propertyNames) { + foreach ($rawName in $rawBodyKeys) { + if ([string]::Equals($name, $rawName, [System.StringComparison]::OrdinalIgnoreCase)) { + $rawBodyPresent = $true + } + } + foreach ($rawName in $rawIdempotencyKeys) { + if ([string]::Equals($name, $rawName, [System.StringComparison]::OrdinalIgnoreCase)) { + $rawIdempotencyKeyPresent = $true + } + } +} + +$byIdempotencyKey = @{} +foreach ($record in $auditRecords) { + $keyHash = Get-StringProperty -Record $record -Name "idempotency_key_sha256" + if ([string]::IsNullOrWhiteSpace($keyHash)) { + continue + } + $fingerprint = (Get-StringProperty -Record $record -Name "target_ref") + "|" + (Get-StringProperty -Record $record -Name "content_sha256") + if (-not $byIdempotencyKey.ContainsKey($keyHash)) { + $byIdempotencyKey[$keyHash] = @() + } + $byIdempotencyKey[$keyHash] = @($byIdempotencyKey[$keyHash] + $fingerprint) +} + +$duplicateDetected = $false +$conflictDetected = $false +foreach ($keyHash in $byIdempotencyKey.Keys) { + $fingerprints = @($byIdempotencyKey[$keyHash]) + if ($fingerprints.Count -lt 2) { + continue + } + $unique = @($fingerprints | Sort-Object -Unique) + if ($unique.Count -eq 1) { + $duplicateDetected = $true + } + else { + $conflictDetected = $true + $counts = @{} + foreach ($fingerprint in $fingerprints) { + if (-not $counts.ContainsKey($fingerprint)) { + $counts[$fingerprint] = 0 + } + $counts[$fingerprint]++ + } + foreach ($count in $counts.Values) { + if ($count -gt 1) { + $duplicateDetected = $true + } + } + } +} + +$ok = -not ($rawBodyPresent -or $rawIdempotencyKeyPresent) +$summary = [ordered]@{ + ok = $ok + verification = "send_audit_idempotency" + audit_records = $auditRecords.Count + idempotency_records = $idempotencyRecords.Count + duplicate_detected = $duplicateDetected + conflict_detected = $conflictDetected + raw_body_present = $rawBodyPresent + raw_idempotency_key_present = $rawIdempotencyKeyPresent + audit_path_configured = -not [string]::IsNullOrWhiteSpace($AuditPath) + idempotency_path_configured = -not [string]::IsNullOrWhiteSpace($IdempotencyPath) +} + +Write-Output ($summary | ConvertTo-Json -Compress) +if (-not $ok) { + exit 1 +}