param( [string]$ArchiveListPath = "", [string]$CacheRoot = "", [string]$SQLiteDllPath = $env:ISPHERE_MSGLIB_SQLITE_DLL, [string]$DbPath = $env:ISPHERE_MSGLIB_DB, [string]$Password = "123", [string]$FixtureRecordsPath = "", [string]$SanitizedOutputPath = "", [switch]$SkipDb ) $ErrorActionPreference = "Stop" Set-StrictMode -Version Latest $repo = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path function Add-Count([hashtable]$Map, [string]$Key, [int]$Increment = 1) { if ([string]::IsNullOrWhiteSpace($Key)) { return } if ($Map.ContainsKey($Key)) { $Map[$Key] = [int]$Map[$Key] + $Increment } else { $Map[$Key] = $Increment } } function Get-Sha256Text([string]$Value) { $sha = [System.Security.Cryptography.SHA256]::Create() try { $bytes = [System.Text.Encoding]::UTF8.GetBytes($Value) $hash = $sha.ComputeHash($bytes) return (($hash | ForEach-Object { $_.ToString("x2") }) -join "") } finally { $sha.Dispose() } } function Normalize-FileName([string]$Value) { if ([string]::IsNullOrWhiteSpace($Value)) { return "" } $leaf = ($Value -split '[\\/]')[-1] return $leaf.Trim().ToLowerInvariant() } function Normalize-Extension([string]$Value) { if ([string]::IsNullOrWhiteSpace($Value)) { return "" } $extension = $Value.Trim().ToLowerInvariant() if ($extension -match '^\.[a-z0-9]{1,8}$') { return $extension } return ".other" } function New-EmptyCacheSummary { return [ordered]@{ checked = $false candidate_count = 0 extension_counts = @{} hash_dir_length_counts = @{} hash_dir_value_hash_counts = @{} filename_hash_counts = @{} filename_size_hash_counts = @{} has_size_data = $false } } function Add-CacheCandidate($Summary, [string]$Name, [Nullable[Int64]]$Size, [string]$HashDir) { $normalizedName = Normalize-FileName $Name if ($normalizedName -eq "") { return } $extension = Normalize-Extension ([System.IO.Path]::GetExtension($normalizedName)) if ($extension -ne "") { Add-Count $Summary.extension_counts $extension } if (-not [string]::IsNullOrWhiteSpace($HashDir)) { Add-Count $Summary.hash_dir_length_counts ([string]$HashDir.Length) Add-Count $Summary.hash_dir_value_hash_counts (Get-Sha256Text $HashDir.ToLowerInvariant()) } Add-Count $Summary.filename_hash_counts (Get-Sha256Text $normalizedName) if ($null -ne $Size -and $Size.Value -ge 0) { $Summary["has_size_data"] = $true Add-Count $Summary.filename_size_hash_counts (Get-Sha256Text ($normalizedName + "|" + $Size.Value)) } $Summary["candidate_count"] = [int]$Summary["candidate_count"] + 1 } function Get-HashDirFromPathText([string]$Value) { $match = [regex]::Match($Value, '(?i)(?:^|[\\/])importal[\\/]([0-9a-f]{32,})(?:[\\/])') if ($match.Success) { return $match.Groups[1].Value.ToLowerInvariant() } return "" } function Read-CacheSummaryFromArchiveList([string]$Path) { if (-not (Test-Path -LiteralPath $Path)) { throw "ArchiveListPath not found" } $summary = New-EmptyCacheSummary $summary["checked"] = $true foreach ($line in Get-Content -LiteralPath $Path) { $text = [string]$line $hashDir = Get-HashDirFromPathText $text if ($hashDir -eq "") { continue } Add-CacheCandidate $summary $text $null $hashDir } return $summary } function Read-CacheSummaryFromRoot([string]$Path) { if (-not (Test-Path -LiteralPath $Path)) { throw "CacheRoot not found" } $summary = New-EmptyCacheSummary $summary["checked"] = $true foreach ($file in Get-ChildItem -LiteralPath $Path -File -Recurse) { $hashDir = Get-HashDirFromPathText $file.FullName if ($hashDir -eq "") { continue } Add-CacheCandidate $summary $file.Name ([Int64]$file.Length) $hashDir } return $summary } function ConvertTo-SafeFileRecord($Value) { $createdAt = $null if (-not [string]::IsNullOrWhiteSpace([string]$Value.created_at)) { [datetime]$parsed = [datetime]::MinValue if ([datetime]::TryParse([string]$Value.created_at, [ref]$parsed)) { $createdAt = $parsed.ToUniversalTime() } } return [pscustomobject]@{ normalized_name = Normalize-FileName ([string]$Value.file_name) size_bytes = if ($null -ne $Value.size_bytes) { [int64]$Value.size_bytes } else { -1 } source_id = ([string]$Value.source_id).Trim().ToLowerInvariant() created_at_utc = $createdAt } } function Read-FixtureFileRecords([string]$Path) { if (-not (Test-Path -LiteralPath $Path)) { throw "FixtureRecordsPath not found" } $json = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json $safeRecords = @() foreach ($record in $json) { $safeRecords += (ConvertTo-SafeFileRecord $record) } return $safeRecords } function Read-CacheCandidatesForScoring([string]$Path) { if (-not (Test-Path -LiteralPath $Path)) { throw "CacheRoot not found" } $candidates = @() foreach ($file in Get-ChildItem -LiteralPath $Path -File -Recurse) { $searchText = (($file.Name + " " + $file.DirectoryName) -replace '[\\/]', ' ').ToLowerInvariant() $candidates += [pscustomobject]@{ normalized_name = Normalize-FileName $file.Name size_bytes = [int64]$file.Length last_write_utc = $file.LastWriteTimeUtc search_text = $searchText } } return $candidates } function Test-CloseTimestamp($Record, $Candidate) { if ($null -eq $Record.created_at_utc -or $null -eq $Candidate.last_write_utc) { return $false } $delta = [Math]::Abs((New-TimeSpan -Start $Record.created_at_utc -End $Candidate.last_write_utc).TotalSeconds) return $delta -le 600 } function Get-ResolverScore($Record, $Candidate) { $nameMatches = ($Record.normalized_name -ne "" -and $Record.normalized_name -eq $Candidate.normalized_name) $sizeMatches = ($Record.size_bytes -ge 0 -and $Record.size_bytes -eq $Candidate.size_bytes) $sourceMatches = ($Record.source_id -ne "" -and $Candidate.search_text.Contains($Record.source_id)) if ($nameMatches -and $sizeMatches -and (Test-CloseTimestamp $Record $Candidate)) { return 100 } if ($sizeMatches -and $sourceMatches) { return 80 } if ($nameMatches) { return 40 } return 0 } function Invoke-ResolverScoring($Records, $Candidates) { $accepted = 0 $ambiguous = 0 $score100 = 0 $score80 = 0 $score40Only = 0 foreach ($record in @($Records)) { $scores = @() foreach ($candidate in @($Candidates)) { $score = Get-ResolverScore $record $candidate if ($score -gt 0) { $scores += $score } } if ($scores.Count -eq 0) { continue } $top = ($scores | Sort-Object -Descending | Select-Object -First 1) $topCount = @($scores | Where-Object { $_ -eq $top }).Count if ($top -ge 80 -and $topCount -eq 1) { $accepted++ if ($top -eq 100) { $score100++ } elseif ($top -eq 80) { $score80++ } } elseif ($top -ge 80) { $ambiguous++ } elseif ($top -eq 40) { $score40Only++ } } return [ordered]@{ records_checked = @($Records).Count cache_candidates_checked = @($Candidates).Count accepted_matches = $accepted ambiguous_matches = $ambiguous score_100_matches = $score100 score_80_matches = $score80 score_40_only_matches = $score40Only used_fixture_records = $true } } function Convert-CountObjectToHashtable($Value) { $map = @{} if ($null -eq $Value) { return $map } if ($Value -is [System.Collections.IDictionary]) { foreach ($key in $Value.Keys) { $map[[string]$key] = [int]$Value[$key] } return $map } foreach ($property in $Value.PSObject.Properties) { $map[$property.Name] = [int]$property.Value } return $map } function Count-Intersections([hashtable]$Left, [hashtable]$Right) { $count = 0 foreach ($key in $Left.Keys) { if ($Right.ContainsKey($key)) { $count += [Math]::Min([int]$Left[$key], [int]$Right[$key]) } } return $count } function Count-UniqueIntersections([hashtable]$Left, [hashtable]$Right) { $count = 0 foreach ($key in $Left.Keys) { if ($Right.ContainsKey($key) -and [int]$Left[$key] -eq 1 -and [int]$Right[$key] -eq 1) { $count++ } } return $count } function Invoke-ReceiveFileRecordProbe([string]$SQLiteDll, [string]$Database, [string]$DbPassword) { if (-not (Test-Path -LiteralPath $SQLiteDll)) { throw "SQLite DLL not found" } if (-not (Test-Path -LiteralPath $Database)) { throw "MsgLib DB not found" } $tempDir = Join-Path $repo ("runs\file-cache-mapping-db-probe-" + [guid]::NewGuid().ToString("N")) $source = Join-Path $tempDir "ReceiveFileRecordProbe.cs" $exe = Join-Path $tempDir "ReceiveFileRecordProbe.exe" $sqliteCopy = Join-Path $tempDir "System.Data.SQLite.dll" $csc = "$env:WINDIR\Microsoft.NET\Framework\v4.0.30319\csc.exe" if (-not (Test-Path -LiteralPath $csc)) { throw "32-bit .NET Framework csc.exe v4.0.30319 not found" } try { New-Item -ItemType Directory -Force -Path $tempDir | Out-Null Copy-Item -LiteralPath $SQLiteDll -Destination $sqliteCopy -Force @' using System; using System.Collections.Generic; using System.Data.Common; using System.IO; using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Web.Script.Serialization; internal static class ReceiveFileRecordProbe { private static int Main(string[] args) { try { string sqliteDllPath = args[0]; string dbPath = args[1]; string password = args.Length > 2 ? args[2] : "123"; AppDomain.CurrentDomain.AssemblyResolve += delegate(object sender, ResolveEventArgs eventArgs) { if (eventArgs.Name.StartsWith("System.Data.SQLite", StringComparison.OrdinalIgnoreCase)) { return Assembly.LoadFrom(sqliteDllPath); } return null; }; Assembly sqliteAssembly = Assembly.LoadFrom(sqliteDllPath); Type factoryType = sqliteAssembly.GetType("System.Data.SQLite.SQLiteFactory", true); FieldInfo instanceField = factoryType.GetField("Instance", BindingFlags.Public | BindingFlags.Static); DbProviderFactory factory = (DbProviderFactory)instanceField.GetValue(null); var result = NewResult(); using (DbConnection connection = factory.CreateConnection()) { connection.ConnectionString = "Data Source=" + dbPath + ";Version=3;Read Only=True;Password=" + password + ";"; connection.Open(); ExecuteNonQuery(connection, "PRAGMA query_only=ON"); if (!TableExists(connection, "TD_ReceiveFileRecord")) { PrintJson(result); return 0; } result["table_available"] = true; HashSet columns = ReadColumns(connection, "TD_ReceiveFileRecord"); string msgColumn = ExistingColumn(columns, "ReceiveMsgGuid"); string nameColumn = ExistingColumn(columns, "FileName"); string sizeColumn = ExistingColumn(columns, "FileSize", "Size"); string sourceColumn = ExistingColumn(columns, "SourceID", "FileGuid", "FileId"); if (nameColumn == "") { PrintJson(result); return 0; } string sql = "select " + SelectColumn(msgColumn, "receive_msg_guid") + ", " + SelectColumn(nameColumn, "file_name") + ", " + SelectColumn(sizeColumn, "size_bytes") + ", " + SelectColumn(sourceColumn, "source_id") + " from " + QuoteIdent("TD_ReceiveFileRecord") + " limit 100000"; using (DbCommand command = connection.CreateCommand()) { command.CommandText = sql; using (DbDataReader reader = command.ExecuteReader()) { while (reader.Read()) { result["record_count"] = (int)result["record_count"] + 1; string receiveMsgGuid = ReaderString(reader, "receive_msg_guid"); string fileName = SafeFileName(ReaderString(reader, "file_name")); long size = ReaderInt64(reader, "size_bytes"); string sourceId = ReaderString(reader, "source_id").Trim().ToLowerInvariant(); if (receiveMsgGuid.Trim() != "") result["receive_msg_guid_present_count"] = (int)result["receive_msg_guid_present_count"] + 1; if (fileName.Trim() != "") result["filename_present_count"] = (int)result["filename_present_count"] + 1; if (size > 0) result["size_present_count"] = (int)result["size_present_count"] + 1; if (sourceId != "") result["source_id_present_count"] = (int)result["source_id_present_count"] + 1; string normalizedName = fileName.Trim().ToLowerInvariant(); if (normalizedName != "") { AddCount((Dictionary)result["extension_counts"], SafeExtension(Path.GetExtension(normalizedName))); AddCount((Dictionary)result["filename_hash_counts"], Sha256(normalizedName)); if (size > 0) AddCount((Dictionary)result["filename_size_hash_counts"], Sha256(normalizedName + "|" + size.ToString())); } if (sourceId != "") { AddCount((Dictionary)result["source_id_hash_counts"], Sha256(sourceId)); if (IsHexLike(sourceId)) result["source_id_hash_shape_count"] = (int)result["source_id_hash_shape_count"] + 1; } } } } } PrintJson(result); return 0; } catch (Exception ex) { Console.Error.WriteLine(ex.GetType().Name + ": " + ex.Message); return 1; } } private static Dictionary NewResult() { return new Dictionary { { "table_available", false }, { "record_count", 0 }, { "filename_present_count", 0 }, { "size_present_count", 0 }, { "source_id_present_count", 0 }, { "receive_msg_guid_present_count", 0 }, { "source_id_hash_shape_count", 0 }, { "extension_counts", new Dictionary() }, { "filename_hash_counts", new Dictionary() }, { "filename_size_hash_counts", new Dictionary() }, { "source_id_hash_counts", new Dictionary() } }; } private static void PrintJson(Dictionary value) { Console.WriteLine(new JavaScriptSerializer().Serialize(value)); } private static void AddCount(Dictionary map, string key) { if (string.IsNullOrWhiteSpace(key)) return; if (map.ContainsKey(key)) map[key] = (int)map[key] + 1; else map[key] = 1; } private static bool TableExists(DbConnection connection, string table) { using (DbCommand command = connection.CreateCommand()) { command.CommandText = "select count(*) from sqlite_master where type='table' and name=@name"; DbParameter parameter = command.CreateParameter(); parameter.ParameterName = "@name"; parameter.Value = table; command.Parameters.Add(parameter); return Convert.ToInt32(command.ExecuteScalar()) > 0; } } private static HashSet ReadColumns(DbConnection connection, string table) { var columns = new HashSet(StringComparer.OrdinalIgnoreCase); using (DbCommand command = connection.CreateCommand()) { command.CommandText = "pragma table_info(" + QuoteIdent(table) + ")"; using (DbDataReader reader = command.ExecuteReader()) { while (reader.Read()) columns.Add(Convert.ToString(reader["name"])); } } return columns; } private static string ExistingColumn(HashSet columns, params string[] names) { foreach (string name in names) { if (!string.IsNullOrWhiteSpace(name) && columns.Contains(name)) return name; } return ""; } private static string SelectColumn(string column, string alias) { if (string.IsNullOrWhiteSpace(column)) return "'' as " + QuoteIdent(alias); return QuoteIdent(column) + " as " + QuoteIdent(alias); } private static string QuoteIdent(string value) { return "\"" + value.Replace("\"", "\"\"") + "\""; } private static string ReaderString(DbDataReader reader, string column) { object value = reader[column]; if (value == null || value == DBNull.Value) return ""; return Convert.ToString(value); } private static long ReaderInt64(DbDataReader reader, string column) { object value = reader[column]; if (value == null || value == DBNull.Value) return 0; long parsed; if (long.TryParse(Convert.ToString(value), out parsed)) return parsed; return 0; } private static string SafeFileName(string value) { if (string.IsNullOrWhiteSpace(value)) return ""; try { return Path.GetFileName(value); } catch { int slash = Math.Max(value.LastIndexOf('/'), value.LastIndexOf('\\')); if (slash >= 0 && slash + 1 < value.Length) return value.Substring(slash + 1); return value; } } private static string SafeExtension(string value) { if (string.IsNullOrWhiteSpace(value)) return ""; string extension = value.Trim().ToLowerInvariant(); if (extension.Length < 2 || extension.Length > 9) return ".other"; if (extension[0] != '.') return ".other"; for (int i = 1; i < extension.Length; i++) { char c = extension[i]; bool ok = (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); if (!ok) return ".other"; } return extension; } private static bool IsHexLike(string value) { if (value.Length < 16) return false; foreach (char c in value) { bool ok = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); if (!ok) return false; } return true; } private static string Sha256(string value) { using (SHA256 sha = SHA256.Create()) { byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(value)); var builder = new StringBuilder(hash.Length * 2); foreach (byte b in hash) builder.Append(b.ToString("x2")); return builder.ToString(); } } private static void ExecuteNonQuery(DbConnection connection, string sql) { using (DbCommand command = connection.CreateCommand()) { command.CommandText = sql; command.ExecuteNonQuery(); } } } '@ | Set-Content -LiteralPath $source -Encoding UTF8 & $csc ` /nologo ` /target:exe ` /platform:x86 ` /optimize+ ` /warn:0 ` /out:$exe ` /reference:System.dll ` /reference:System.Core.dll ` /reference:System.Data.dll ` /reference:System.Web.Extensions.dll ` /reference:$sqliteCopy ` $source | Out-Null if ($LASTEXITCODE -ne 0) { throw "ReceiveFileRecordProbe compilation failed" } @' '@ | Set-Content -LiteralPath ($exe + ".config") -Encoding UTF8 $output = & $exe $sqliteCopy $Database $DbPassword if ($LASTEXITCODE -ne 0) { throw "ReceiveFileRecordProbe failed" } return $output | ConvertFrom-Json } finally { if (Test-Path -LiteralPath $tempDir) { Remove-Item -LiteralPath $tempDir -Recurse -Force } } } $cacheSummary = New-EmptyCacheSummary if (-not [string]::IsNullOrWhiteSpace($ArchiveListPath)) { $cacheSummary = Read-CacheSummaryFromArchiveList $ArchiveListPath } elseif (-not [string]::IsNullOrWhiteSpace($CacheRoot)) { $cacheSummary = Read-CacheSummaryFromRoot $CacheRoot } $dbChecked = $false $dbProbe = [ordered]@{ table_available = $false record_count = 0 filename_present_count = 0 size_present_count = 0 source_id_present_count = 0 receive_msg_guid_present_count = 0 source_id_hash_shape_count = 0 extension_counts = @{} filename_hash_counts = @{} filename_size_hash_counts = @{} source_id_hash_counts = @{} } if (-not $SkipDb) { if ([string]::IsNullOrWhiteSpace($SQLiteDllPath) -or [string]::IsNullOrWhiteSpace($DbPath)) { throw "MsgLib DB probe requires SQLiteDllPath and DbPath, or pass -SkipDb" } $dbProbe = Invoke-ReceiveFileRecordProbe $SQLiteDllPath $DbPath $Password $dbChecked = $true } $dbFilenameHashes = Convert-CountObjectToHashtable $dbProbe.filename_hash_counts $dbFilenameSizeHashes = Convert-CountObjectToHashtable $dbProbe.filename_size_hash_counts $dbSourceIDHashes = Convert-CountObjectToHashtable $dbProbe.source_id_hash_counts $cacheFilenameHashes = [hashtable]$cacheSummary["filename_hash_counts"] $cacheFilenameSizeHashes = [hashtable]$cacheSummary["filename_size_hash_counts"] $cacheHashDirValueHashes = [hashtable]$cacheSummary["hash_dir_value_hash_counts"] $filenameOnlyMatches = Count-Intersections $dbFilenameHashes $cacheFilenameHashes $filenameSizeMatches = Count-Intersections $dbFilenameSizeHashes $cacheFilenameSizeHashes $uniqueFilenameSizeMatches = Count-UniqueIntersections $dbFilenameSizeHashes $cacheFilenameSizeHashes $sourceIDHashDirMatches = Count-Intersections $dbSourceIDHashes $cacheHashDirValueHashes $mappingValidated = ($uniqueFilenameSizeMatches -gt 0 -and [bool]$cacheSummary["has_size_data"]) $recommendedNextNode = "additional evidence request" if ($mappingValidated) { $recommendedNextNode = "R4 cache-file resolver candidate" } elseif ($filenameOnlyMatches -gt 0 -or $sourceIDHashDirMatches -gt 0) { $recommendedNextNode = "additional evidence request: collect file size or cache root metadata" } $resolverSummary = [ordered]@{ records_checked = [int]$dbProbe.record_count cache_candidates_checked = [int]$cacheSummary["candidate_count"] accepted_matches = 0 ambiguous_matches = 0 score_100_matches = 0 score_80_matches = 0 score_40_only_matches = $filenameOnlyMatches used_fixture_records = $false } if (-not [string]::IsNullOrWhiteSpace($FixtureRecordsPath)) { if ([string]::IsNullOrWhiteSpace($CacheRoot)) { throw "FixtureRecordsPath requires CacheRoot" } $fixtureRecords = Read-FixtureFileRecords $FixtureRecordsPath $cacheCandidates = Read-CacheCandidatesForScoring $CacheRoot $resolverSummary = Invoke-ResolverScoring $fixtureRecords $cacheCandidates $mappingValidated = ([int]$resolverSummary.accepted_matches -gt 0) if ($mappingValidated) { $recommendedNextNode = "R11 download preview resolver contract" } } $summary = [ordered]@{ ok = $true verification = "file_cache_mapping" db_checked = $dbChecked db_table_available = [bool]$dbProbe.table_available db_record_count = [int]$dbProbe.record_count db_filename_present_count = [int]$dbProbe.filename_present_count db_size_present_count = [int]$dbProbe.size_present_count db_source_id_present_count = [int]$dbProbe.source_id_present_count db_receive_msg_guid_present_count = [int]$dbProbe.receive_msg_guid_present_count db_source_id_hash_shape_count = [int]$dbProbe.source_id_hash_shape_count db_extension_counts = $dbProbe.extension_counts cache_checked = [bool]$cacheSummary["checked"] cache_candidate_count = [int]$cacheSummary["candidate_count"] cache_extension_counts = $cacheSummary["extension_counts"] hash_dir_length_counts = $cacheSummary["hash_dir_length_counts"] filename_only_match_count = $filenameOnlyMatches filename_size_match_count = $filenameSizeMatches unique_filename_size_match_count = $uniqueFilenameSizeMatches source_id_hash_dir_match_count = $sourceIDHashDirMatches records_checked = [int]$resolverSummary.records_checked cache_candidates_checked = [int]$resolverSummary.cache_candidates_checked accepted_matches = [int]$resolverSummary.accepted_matches ambiguous_matches = [int]$resolverSummary.ambiguous_matches score_100_matches = [int]$resolverSummary.score_100_matches score_80_matches = [int]$resolverSummary.score_80_matches score_40_only_matches = [int]$resolverSummary.score_40_only_matches used_fixture_records = [bool]$resolverSummary.used_fixture_records mapping_validated = $mappingValidated recommended_next_node = $recommendedNextNode raw_paths_printed = $false file_contents_read = $false file_paths_returned = $false raw_rows_returned = $false message_body_values_printed = $false } $json = $summary | ConvertTo-Json -Depth 8 -Compress if (-not [string]::IsNullOrWhiteSpace($SanitizedOutputPath)) { $parent = Split-Path -Parent $SanitizedOutputPath if (-not [string]::IsNullOrWhiteSpace($parent)) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } Set-Content -LiteralPath $SanitizedOutputPath -Value $json -Encoding UTF8 } $json