chore: initialize qiming workspace repository
This commit is contained in:
Codex
@@ -0,0 +1,845 @@
|
||||
/**
|
||||
* Extraction Queue Module
|
||||
*
|
||||
* Async queue for memory extraction with model config capture
|
||||
* IMPORTANT: Must capture full model config including API Key
|
||||
*
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import log from "electron-log";
|
||||
import type {
|
||||
ExtractionTask,
|
||||
ExtractedMemory,
|
||||
ModelConfig,
|
||||
DeduplicationConfig,
|
||||
} from "./types";
|
||||
import { MemoryExtractor } from "./MemoryExtractor";
|
||||
import { MemoryFileSync } from "./MemoryFileSync";
|
||||
import { MemoryDatabase } from "./MemoryDatabase";
|
||||
import { deduplicateMemories } from "./utils/deduplicator";
|
||||
import { callLlmApi } from "./utils/llmClient";
|
||||
|
||||
// ==================== Types ====================
|
||||
|
||||
interface QueueOptions {
|
||||
maxRetries: number;
|
||||
processingInterval: number; // ms
|
||||
maxQueueSize: number;
|
||||
}
|
||||
|
||||
interface ExtractionResult {
|
||||
taskId: string;
|
||||
success: boolean;
|
||||
memories?: ExtractedMemory[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ==================== ExtractionQueue Class ====================
|
||||
|
||||
export class ExtractionQueue extends EventEmitter {
|
||||
private queue: ExtractionTask[] = [];
|
||||
private processing: boolean = false;
|
||||
private paused: boolean = false;
|
||||
private extractor: MemoryExtractor;
|
||||
private fileSync: MemoryFileSync | null = null;
|
||||
private database: MemoryDatabase | null = null;
|
||||
private deduplicationConfig: DeduplicationConfig = {
|
||||
textSimilarityThreshold: 0.8,
|
||||
vectorSimilarityThreshold: 0.95,
|
||||
};
|
||||
private options: QueueOptions = {
|
||||
maxRetries: 2,
|
||||
processingInterval: 1000,
|
||||
maxQueueSize: 100,
|
||||
};
|
||||
private processTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(extractor: MemoryExtractor) {
|
||||
super();
|
||||
this.extractor = extractor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the queue
|
||||
*/
|
||||
init(
|
||||
fileSync: MemoryFileSync,
|
||||
options?: Partial<QueueOptions>,
|
||||
database?: MemoryDatabase,
|
||||
deduplicationConfig?: DeduplicationConfig,
|
||||
): void {
|
||||
this.fileSync = fileSync;
|
||||
this.database = database ?? null;
|
||||
this.options = { ...this.options, ...options };
|
||||
if (deduplicationConfig) {
|
||||
this.deduplicationConfig = deduplicationConfig;
|
||||
}
|
||||
log.info("[ExtractionQueue] Initialized");
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy the queue
|
||||
*/
|
||||
destroy(): void {
|
||||
this.stop();
|
||||
this.queue = [];
|
||||
log.info("[ExtractionQueue] Destroyed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Start processing queue
|
||||
*/
|
||||
start(): void {
|
||||
if (this.processTimer) return;
|
||||
|
||||
this.paused = false;
|
||||
this.processTimer = setInterval(() => {
|
||||
this.processNext();
|
||||
}, this.options.processingInterval);
|
||||
|
||||
log.info("[ExtractionQueue] Started");
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop processing queue
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.processTimer) {
|
||||
clearInterval(this.processTimer);
|
||||
this.processTimer = null;
|
||||
}
|
||||
this.paused = true;
|
||||
log.info("[ExtractionQueue] Stopped");
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause processing
|
||||
*/
|
||||
pause(): void {
|
||||
this.paused = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume processing
|
||||
*/
|
||||
resume(): void {
|
||||
this.paused = false;
|
||||
}
|
||||
|
||||
// ==================== Queue Operations ====================
|
||||
|
||||
/**
|
||||
* Add extraction task to queue
|
||||
*
|
||||
* IMPORTANT: modelConfig must include apiKey!
|
||||
* Electron client has no global API key storage,
|
||||
* so we must capture it at enqueue time.
|
||||
*/
|
||||
enqueue(
|
||||
sessionId: string,
|
||||
messageId: string,
|
||||
messages: Array<{ role: "user" | "assistant"; content: string }>,
|
||||
modelConfig: ModelConfig,
|
||||
segmentIndex?: number,
|
||||
startMsgIndex?: number,
|
||||
endMsgIndex?: number,
|
||||
): string {
|
||||
// Validate model config - API key is required!
|
||||
if (!modelConfig.apiKey) {
|
||||
const error = "API Key is required in modelConfig for extraction queue";
|
||||
log.error("[ExtractionQueue]", error);
|
||||
throw new Error(error);
|
||||
}
|
||||
|
||||
// Check queue size
|
||||
if (this.queue.length >= this.options.maxQueueSize) {
|
||||
log.warn("[ExtractionQueue] Queue is full, dropping oldest task");
|
||||
this.queue.shift();
|
||||
}
|
||||
|
||||
const task: ExtractionTask = {
|
||||
sessionId,
|
||||
messageId,
|
||||
messages,
|
||||
modelConfig, // Store full config including API key!
|
||||
timestamp: Date.now(),
|
||||
retryCount: 0,
|
||||
segmentIndex,
|
||||
startMsgIndex,
|
||||
endMsgIndex,
|
||||
};
|
||||
|
||||
const taskId = this.generateTaskId(task);
|
||||
this.queue.push(task);
|
||||
log.info("[ExtractionQueue] Task enqueued:", taskId);
|
||||
|
||||
this.emit("task:enqueued", task);
|
||||
|
||||
// Try to process immediately if not busy
|
||||
if (!this.processing && !this.paused) {
|
||||
this.processNext();
|
||||
}
|
||||
|
||||
return taskId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue length
|
||||
*/
|
||||
getLength(): number {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if queue is empty
|
||||
*/
|
||||
isEmpty(): boolean {
|
||||
return this.queue.length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all tasks
|
||||
*/
|
||||
clear(): void {
|
||||
this.queue = [];
|
||||
log.info("[ExtractionQueue] Queue cleared");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get pending tasks (without sensitive data)
|
||||
*/
|
||||
getPendingTasks(): Array<{
|
||||
taskId: string;
|
||||
sessionId: string;
|
||||
messageId: string;
|
||||
timestamp: number;
|
||||
retryCount: number;
|
||||
}> {
|
||||
return this.queue.map((task) => ({
|
||||
taskId: this.generateTaskId(task),
|
||||
sessionId: task.sessionId,
|
||||
messageId: task.messageId,
|
||||
timestamp: task.timestamp,
|
||||
retryCount: task.retryCount,
|
||||
}));
|
||||
}
|
||||
|
||||
// ==================== Processing ====================
|
||||
|
||||
/**
|
||||
* Process next task in queue
|
||||
*/
|
||||
private async processNext(): Promise<void> {
|
||||
if (this.processing || this.paused || this.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.processing = true;
|
||||
const task = this.queue.shift()!;
|
||||
const taskId = this.generateTaskId(task);
|
||||
|
||||
try {
|
||||
const result = await this.processTask(task);
|
||||
this.emit("task:completed", result);
|
||||
} catch (error) {
|
||||
log.error("[ExtractionQueue] Task processing failed:", error);
|
||||
this.emit("task:error", { taskId, error });
|
||||
} finally {
|
||||
this.processing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single extraction task
|
||||
*/
|
||||
private async processTask(task: ExtractionTask): Promise<ExtractionResult> {
|
||||
const taskId = this.generateTaskId(task);
|
||||
log.info("[ExtractionQueue] Processing task:", taskId);
|
||||
|
||||
// Update progress to 'processing' if tracking segment
|
||||
if (task.segmentIndex !== undefined && this.database) {
|
||||
this.database.updateExtractionProgress(
|
||||
task.sessionId,
|
||||
task.segmentIndex,
|
||||
{
|
||||
status: "processing",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// Extract memories using stored model config
|
||||
let memories = await this.extractMemories(task);
|
||||
|
||||
// Apply cross-segment deduplication
|
||||
if (memories.length > 0 && this.database) {
|
||||
const existingTexts = this.database.getRecentMemoryTexts(50);
|
||||
memories = deduplicateMemories(
|
||||
memories,
|
||||
existingTexts,
|
||||
this.deduplicationConfig,
|
||||
);
|
||||
}
|
||||
|
||||
if (memories.length > 0) {
|
||||
// Write to daily memory file
|
||||
await this.writeToDailyMemory(memories, task.sessionId);
|
||||
|
||||
log.info(
|
||||
"[ExtractionQueue] Extracted",
|
||||
memories.length,
|
||||
"memories from task:",
|
||||
taskId,
|
||||
);
|
||||
}
|
||||
|
||||
// Update progress to 'completed' if tracking segment
|
||||
if (task.segmentIndex !== undefined && this.database) {
|
||||
this.database.updateExtractionProgress(
|
||||
task.sessionId,
|
||||
task.segmentIndex,
|
||||
{
|
||||
status: "completed",
|
||||
memoriesExtracted: memories.length,
|
||||
completedAt: Date.now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
taskId,
|
||||
success: true,
|
||||
memories,
|
||||
};
|
||||
} catch (error) {
|
||||
// Retry logic
|
||||
if (task.retryCount < this.options.maxRetries) {
|
||||
task.retryCount++;
|
||||
this.queue.unshift(task); // Put back at front
|
||||
log.warn(
|
||||
`[ExtractionQueue] Retrying task (${task.retryCount}/${this.options.maxRetries}):`,
|
||||
taskId,
|
||||
);
|
||||
|
||||
return {
|
||||
taskId,
|
||||
success: false,
|
||||
error: String(error),
|
||||
};
|
||||
}
|
||||
|
||||
// Update progress to 'failed' if tracking segment
|
||||
if (task.segmentIndex !== undefined && this.database) {
|
||||
this.database.updateExtractionProgress(
|
||||
task.sessionId,
|
||||
task.segmentIndex,
|
||||
{
|
||||
status: "failed",
|
||||
errorMessage: String(error),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
log.error("[ExtractionQueue] Task failed after max retries:", taskId);
|
||||
return {
|
||||
taskId,
|
||||
success: false,
|
||||
error: String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract memories from task messages
|
||||
*
|
||||
* Pipeline:
|
||||
* 1. Regex + rules extraction (existing, no API call)
|
||||
* 2. LLM segmented extraction (new, API call)
|
||||
* 3. Merge + deduplicate results
|
||||
* 4. LLM validation for uncertain candidates (existing)
|
||||
*/
|
||||
private async extractMemories(
|
||||
task: ExtractionTask,
|
||||
): Promise<ExtractedMemory[]> {
|
||||
log.info("[ExtractionQueue] extractMemories: starting extraction for task");
|
||||
|
||||
// Step 1: Regex + rules extraction (no API call needed)
|
||||
const regexMemories = await this.extractor.extract(task.messages);
|
||||
log.info(
|
||||
"[ExtractionQueue] Step 1 - Regex extraction: " +
|
||||
regexMemories.length +
|
||||
" memories",
|
||||
);
|
||||
|
||||
// Step 2: LLM segmented extraction (requires API key)
|
||||
let llmMemories: ExtractedMemory[] = [];
|
||||
if (task.modelConfig.apiKey) {
|
||||
log.info(
|
||||
"[ExtractionQueue] Step 2 - Calling LLM for extraction (apiKey present)",
|
||||
);
|
||||
try {
|
||||
llmMemories = await this.callLlmForExtraction(task);
|
||||
log.info(
|
||||
"[ExtractionQueue] Step 2 - LLM extraction returned: " +
|
||||
llmMemories.length +
|
||||
" memories",
|
||||
);
|
||||
if (llmMemories.length > 0) {
|
||||
log.info(
|
||||
"[ExtractionQueue] LLM memories: " +
|
||||
llmMemories.map((m) => m.text.slice(0, 30)).join(", "),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
"[ExtractionQueue] LLM extraction failed, using regex results only:",
|
||||
error,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
log.info(
|
||||
"[ExtractionQueue] Step 2 - Skipping LLM extraction (no apiKey)",
|
||||
);
|
||||
}
|
||||
|
||||
// Step 3: Merge and deduplicate regex + LLM results
|
||||
let memories = this.mergeExtractionResults(regexMemories, llmMemories);
|
||||
log.info(
|
||||
"[ExtractionQueue] Step 3 - After merge: " +
|
||||
memories.length +
|
||||
" memories",
|
||||
);
|
||||
|
||||
// Step 4: LLM validation for medium-confidence candidates
|
||||
if (task.modelConfig.apiKey) {
|
||||
const needsValidation = memories.filter(
|
||||
(m) =>
|
||||
this.extractor.needsLlmValidation?.(m.confidence) ??
|
||||
this.defaultNeedsLlmValidation(m.confidence),
|
||||
);
|
||||
|
||||
if (needsValidation.length > 0) {
|
||||
log.info(
|
||||
"[ExtractionQueue] Step 4 - " +
|
||||
needsValidation.length +
|
||||
" memories need LLM validation",
|
||||
);
|
||||
try {
|
||||
const validatedMemories = await this.callLlmForValidation(
|
||||
task,
|
||||
needsValidation,
|
||||
);
|
||||
|
||||
// Merge validated results - keep high confidence, update validated ones
|
||||
memories = memories
|
||||
.map((m) => {
|
||||
const validated = validatedMemories.find(
|
||||
(v) => v.text === m.text,
|
||||
);
|
||||
return validated ?? m;
|
||||
})
|
||||
.filter((m) => m.confidence >= 0.5); // Filter out rejected ones
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
"[ExtractionQueue] LLM validation failed, using original results:",
|
||||
error,
|
||||
);
|
||||
// Continue with original extraction results
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[ExtractionQueue] extractMemories: final result = " +
|
||||
memories.length +
|
||||
" memories",
|
||||
);
|
||||
return memories;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call LLM for segmented extraction
|
||||
* Uses MemoryExtractor's buildExtractionPrompt/parseExtractionResponse
|
||||
*/
|
||||
private async callLlmForExtraction(
|
||||
task: ExtractionTask,
|
||||
): Promise<ExtractedMemory[]> {
|
||||
const { provider, model, apiKey, baseUrl, apiProtocol } = task.modelConfig;
|
||||
|
||||
if (!apiKey) {
|
||||
log.info(
|
||||
"[ExtractionQueue] callLlmForExtraction: no API key, returning empty",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[ExtractionQueue] callLlmForExtraction: provider=" +
|
||||
provider +
|
||||
", model=" +
|
||||
model +
|
||||
", apiProtocol=" +
|
||||
(apiProtocol ?? "auto"),
|
||||
);
|
||||
|
||||
// Get existing memories for deduplication context
|
||||
const existingMemories = this.database
|
||||
? this.database.getRecentMemoryTexts(30)
|
||||
: (this.fileSync?.readCoreMemory() ?? "")
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- "))
|
||||
.map((l) => l.slice(2));
|
||||
|
||||
log.info(
|
||||
"[ExtractionQueue] callLlmForExtraction: " +
|
||||
existingMemories.length +
|
||||
" existing memories for context",
|
||||
);
|
||||
|
||||
// Build segment metadata if available
|
||||
const segmentMeta =
|
||||
task.segmentIndex !== undefined
|
||||
? { index: task.segmentIndex, total: task.segmentIndex + 1 }
|
||||
: undefined;
|
||||
|
||||
// Build extraction prompt
|
||||
const prompt = this.extractor.buildExtractionPrompt(
|
||||
task.messages,
|
||||
segmentMeta,
|
||||
existingMemories,
|
||||
);
|
||||
|
||||
log.info(
|
||||
"[ExtractionQueue] callLlmForExtraction: prompt length=" + prompt.length,
|
||||
);
|
||||
|
||||
// Call LLM API
|
||||
const response = await this.callLlmApiInternal(
|
||||
provider,
|
||||
model,
|
||||
apiKey,
|
||||
baseUrl,
|
||||
apiProtocol,
|
||||
prompt,
|
||||
);
|
||||
log.info(
|
||||
"[ExtractionQueue] callLlmForExtraction: response length=" +
|
||||
response.length,
|
||||
);
|
||||
|
||||
// Parse response
|
||||
const memories = this.extractor.parseExtractionResponse(response);
|
||||
log.info(
|
||||
"[ExtractionQueue] callLlmForExtraction: parsed " +
|
||||
memories.length +
|
||||
" memories",
|
||||
);
|
||||
|
||||
return memories;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge regex and LLM extraction results, deduplicating by normalized text
|
||||
* When duplicates are found, keep the one with higher confidence
|
||||
*/
|
||||
private mergeExtractionResults(
|
||||
regexResults: ExtractedMemory[],
|
||||
llmResults: ExtractedMemory[],
|
||||
): ExtractedMemory[] {
|
||||
if (llmResults.length === 0) return regexResults;
|
||||
if (regexResults.length === 0) return llmResults;
|
||||
|
||||
const merged = new Map<string, ExtractedMemory>();
|
||||
|
||||
// Index regex results by normalized text
|
||||
for (const mem of regexResults) {
|
||||
const key = mem.text.toLowerCase().replace(/\s+/g, " ").trim();
|
||||
merged.set(key, mem);
|
||||
}
|
||||
|
||||
// Merge LLM results, preferring higher confidence
|
||||
for (const mem of llmResults) {
|
||||
const key = mem.text.toLowerCase().replace(/\s+/g, " ").trim();
|
||||
const existing = merged.get(key);
|
||||
if (!existing || mem.confidence > existing.confidence) {
|
||||
merged.set(key, mem);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(merged.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Default LLM validation check
|
||||
* Returns true if confidence is in the "uncertain" range (0.5-0.7)
|
||||
*/
|
||||
private defaultNeedsLlmValidation(confidence: number): boolean {
|
||||
return confidence >= 0.5 && confidence < 0.75;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call LLM for extraction validation
|
||||
* Based on Spec 5.4 LLM 提取 Prompt
|
||||
*/
|
||||
private async callLlmForValidation(
|
||||
task: ExtractionTask,
|
||||
candidates: ExtractedMemory[],
|
||||
): Promise<ExtractedMemory[]> {
|
||||
const { provider, model, apiKey, baseUrl, apiProtocol } = task.modelConfig;
|
||||
|
||||
if (!apiKey) {
|
||||
log.warn("[ExtractionQueue] No API key available for LLM validation");
|
||||
return candidates;
|
||||
}
|
||||
|
||||
// Build validation prompt
|
||||
const existingMemories = this.fileSync?.readCoreMemory() ?? "";
|
||||
const prompt =
|
||||
this.extractor.buildValidationPrompt?.(
|
||||
candidates.map((c) => c.text).join("\n"),
|
||||
existingMemories.split("\n").filter((l) => l.startsWith("- ")),
|
||||
) ?? this.buildDefaultValidationPrompt(candidates, existingMemories);
|
||||
|
||||
try {
|
||||
// Call LLM API based on provider
|
||||
const response = await this.callLlmApiInternal(
|
||||
provider,
|
||||
model,
|
||||
apiKey,
|
||||
baseUrl,
|
||||
apiProtocol,
|
||||
prompt,
|
||||
);
|
||||
const validation = this.parseValidationResponse(response);
|
||||
|
||||
if (!validation.accept) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
text: validation.mergedText ?? candidates[0].text,
|
||||
category: candidates[0].category,
|
||||
confidence: validation.confidence,
|
||||
isExplicit: false,
|
||||
},
|
||||
];
|
||||
} catch (error) {
|
||||
log.error("[ExtractionQueue] LLM validation call failed:", error);
|
||||
return candidates; // Fallback to original candidates
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build default validation prompt
|
||||
*/
|
||||
private buildDefaultValidationPrompt(
|
||||
candidates: ExtractedMemory[],
|
||||
existingMemories: string,
|
||||
): string {
|
||||
return `You are a memory validation assistant. Decide whether the candidate memory below is worth saving.
|
||||
|
||||
## Candidate memory
|
||||
${candidates.map((c) => c.text).join("\n")}
|
||||
|
||||
## Existing memories
|
||||
${existingMemories || "(none)"}
|
||||
|
||||
## Rules
|
||||
1. Does it conflict with or duplicate existing memories?
|
||||
2. Does it have long-term value?
|
||||
3. Is it personal information about the user rather than general knowledge?
|
||||
|
||||
## Output
|
||||
Return JSON:
|
||||
{
|
||||
"accept": true/false,
|
||||
"reason": "why accepted or rejected",
|
||||
"merged_text": "if merging is needed, the merged text",
|
||||
"confidence": 0.0-1.0
|
||||
}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse validation response from LLM
|
||||
*/
|
||||
private parseValidationResponse(response: string): {
|
||||
accept: boolean;
|
||||
reason: string;
|
||||
mergedText?: string;
|
||||
confidence: number;
|
||||
} {
|
||||
try {
|
||||
// Try to extract JSON from response
|
||||
const jsonMatch = response.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
return {
|
||||
accept: parsed.accept ?? false,
|
||||
reason: parsed.reason ?? "",
|
||||
mergedText: parsed.merged_text,
|
||||
confidence: parsed.confidence ?? 0.5,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
log.warn("[ExtractionQueue] Failed to parse validation response:", error);
|
||||
}
|
||||
|
||||
// Default to accepting with lower confidence
|
||||
return {
|
||||
accept: true,
|
||||
reason: "Failed to parse LLM response",
|
||||
confidence: 0.5,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Call LLM API (delegates to shared llmClient)
|
||||
*/
|
||||
private async callLlmApiInternal(
|
||||
provider: string,
|
||||
model: string,
|
||||
apiKey: string,
|
||||
baseUrl: string | undefined,
|
||||
apiProtocol: string | undefined,
|
||||
prompt: string,
|
||||
): Promise<string> {
|
||||
return callLlmApi(prompt, {
|
||||
provider,
|
||||
model,
|
||||
apiKey,
|
||||
baseUrl,
|
||||
apiProtocol,
|
||||
maxTokens: 500,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Write extracted memories to daily memory file
|
||||
*/
|
||||
private async writeToDailyMemory(
|
||||
memories: ExtractedMemory[],
|
||||
sessionId: string,
|
||||
): Promise<void> {
|
||||
if (!this.fileSync) {
|
||||
log.warn("[ExtractionQueue] FileSync not initialized");
|
||||
return;
|
||||
}
|
||||
|
||||
// Format memories as markdown
|
||||
const content = this.extractor.formatForDailyMemory(memories);
|
||||
|
||||
// Append to daily memory file
|
||||
this.fileSync.appendToDailyMemory(
|
||||
content,
|
||||
`Session (${sessionId.slice(0, 8)})`,
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== Breakpoint Recovery ====================
|
||||
|
||||
/**
|
||||
* Resume pending tasks from extraction_progress table
|
||||
* Called during initialization to recover from interruptions
|
||||
*/
|
||||
resumePendingTasks(
|
||||
transcriptReader: {
|
||||
readTranscriptRange: (
|
||||
sessionId: string,
|
||||
start: number,
|
||||
end: number,
|
||||
) => Array<{ role: "user" | "assistant"; content: string }>;
|
||||
},
|
||||
modelConfig: ModelConfig,
|
||||
): void {
|
||||
if (!this.database) {
|
||||
log.warn("[ExtractionQueue] Cannot resume: database not available");
|
||||
return;
|
||||
}
|
||||
|
||||
const pendingRecords = this.database.getPendingExtractionProgress();
|
||||
if (pendingRecords.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(
|
||||
`[ExtractionQueue] Resuming ${pendingRecords.length} pending extraction tasks`,
|
||||
);
|
||||
|
||||
for (const record of pendingRecords) {
|
||||
try {
|
||||
const messages = transcriptReader.readTranscriptRange(
|
||||
record.sessionId,
|
||||
record.startMsgIndex,
|
||||
record.endMsgIndex,
|
||||
);
|
||||
|
||||
if (messages.length === 0) {
|
||||
// Transcript no longer available
|
||||
this.database.updateExtractionProgress(
|
||||
record.sessionId,
|
||||
record.segmentIndex,
|
||||
{
|
||||
status: "failed",
|
||||
errorMessage: "transcript_expired",
|
||||
},
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
this.enqueue(
|
||||
record.sessionId,
|
||||
`resume-seg-${record.segmentIndex}`,
|
||||
messages,
|
||||
modelConfig,
|
||||
record.segmentIndex,
|
||||
record.startMsgIndex,
|
||||
record.endMsgIndex,
|
||||
);
|
||||
} catch (error) {
|
||||
log.error(
|
||||
`[ExtractionQueue] Failed to resume task for session ${record.sessionId}:`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Utility Methods ====================
|
||||
|
||||
/**
|
||||
* Generate unique task ID
|
||||
*/
|
||||
private generateTaskId(task: ExtractionTask): string {
|
||||
return `${task.sessionId}:${task.messageId}:${task.timestamp}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create extraction task with validation
|
||||
*
|
||||
* Helper to create a properly typed task
|
||||
*/
|
||||
static createTask(
|
||||
sessionId: string,
|
||||
messageId: string,
|
||||
messages: Array<{ role: "user" | "assistant"; content: string }>,
|
||||
modelConfig: ModelConfig,
|
||||
): ExtractionTask {
|
||||
if (!modelConfig.apiKey) {
|
||||
throw new Error("API Key is required for extraction task");
|
||||
}
|
||||
|
||||
return {
|
||||
sessionId,
|
||||
messageId,
|
||||
messages,
|
||||
modelConfig,
|
||||
timestamp: Date.now(),
|
||||
retryCount: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// We'll create the singleton after MemoryExtractor is defined
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,585 @@
|
||||
/**
|
||||
* Memory Extractor Module
|
||||
*
|
||||
* Extract memories from conversations using regex signals and rule-based scoring
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import log from "electron-log";
|
||||
import type {
|
||||
ExtractedMemory,
|
||||
SignalMatch,
|
||||
ValidationResult,
|
||||
ModelConfig,
|
||||
MemoryCategory,
|
||||
} from "./types";
|
||||
import {
|
||||
SCORE_PERSONAL_FACT,
|
||||
SCORE_APPROPRIATE_LENGTH,
|
||||
SCORE_CLEAR_PREFERENCE,
|
||||
SCORE_QUESTION,
|
||||
SCORE_TEMPORARY,
|
||||
SCORE_CODE,
|
||||
SCORE_SPECIFIC_TIME,
|
||||
SCORE_MIN_ACCEPT,
|
||||
SCORE_LLM_THRESHOLD_LOW,
|
||||
SCORE_LLM_THRESHOLD_HIGH,
|
||||
LLM_EXTRACTION_PROMPT,
|
||||
LLM_VALIDATION_PROMPT,
|
||||
} from "./constants";
|
||||
import {
|
||||
detectSignals,
|
||||
extractExplicitContent,
|
||||
hasExplicitCommand,
|
||||
hasImplicitSignals,
|
||||
isQuestion,
|
||||
isTemporary,
|
||||
isCode,
|
||||
countSignalStrength,
|
||||
} from "./utils/signals";
|
||||
import { calculateHash } from "./utils/hash";
|
||||
|
||||
// ==================== Types ====================
|
||||
|
||||
interface ExtractionOptions {
|
||||
explicitEnabled: boolean;
|
||||
implicitEnabled: boolean;
|
||||
guardLevel: "strict" | "standard" | "relaxed";
|
||||
}
|
||||
|
||||
interface ScoringResult {
|
||||
score: number;
|
||||
breakdown: {
|
||||
personalFact: number;
|
||||
appropriateLength: number;
|
||||
clearPreference: number;
|
||||
question: number;
|
||||
temporary: number;
|
||||
code: number;
|
||||
specificTime: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ==================== MemoryExtractor Class ====================
|
||||
|
||||
export class MemoryExtractor extends EventEmitter {
|
||||
private options: ExtractionOptions = {
|
||||
explicitEnabled: true,
|
||||
implicitEnabled: true,
|
||||
guardLevel: "standard",
|
||||
};
|
||||
|
||||
/**
|
||||
* Configure extractor
|
||||
*/
|
||||
configure(options: Partial<ExtractionOptions>): void {
|
||||
this.options = { ...this.options, ...options };
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract memories from conversation
|
||||
*/
|
||||
async extract(
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
options?: Partial<ExtractionOptions>,
|
||||
): Promise<ExtractedMemory[]> {
|
||||
const opts = { ...this.options, ...options };
|
||||
const results: ExtractedMemory[] = [];
|
||||
|
||||
for (const message of messages) {
|
||||
// Only process user messages for memory extraction
|
||||
if (message.role !== "user") continue;
|
||||
|
||||
const extracted = await this.extractFromMessage(message.content, opts);
|
||||
results.push(...extracted);
|
||||
}
|
||||
|
||||
// Deduplicate by text hash
|
||||
return this.deduplicate(results);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract memories from a single message
|
||||
*/
|
||||
private async extractFromMessage(
|
||||
text: string,
|
||||
options: ExtractionOptions,
|
||||
): Promise<ExtractedMemory[]> {
|
||||
const results: ExtractedMemory[] = [];
|
||||
|
||||
// Preprocess: extract user content from system prompts
|
||||
const cleanText = this.preprocessText(text);
|
||||
|
||||
// Detect signals
|
||||
const signals = detectSignals(cleanText);
|
||||
log.info(
|
||||
"[MemoryExtractor] extractFromMessage: signals=" +
|
||||
signals.length +
|
||||
', cleanText="' +
|
||||
cleanText.slice(0, 50) +
|
||||
'"',
|
||||
);
|
||||
|
||||
if (signals.length === 0) {
|
||||
return results;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[MemoryExtractor] Signal types: " +
|
||||
signals.map((s) => s.pattern).join(", "),
|
||||
);
|
||||
|
||||
// Process explicit commands
|
||||
if (options.explicitEnabled) {
|
||||
const explicitSignals = signals.filter((s) => s.type === "explicit");
|
||||
for (const signal of explicitSignals) {
|
||||
const memory = this.processExplicitSignal(signal, cleanText);
|
||||
if (memory) {
|
||||
results.push(memory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process implicit signals
|
||||
if (options.implicitEnabled) {
|
||||
const implicitSignals = signals.filter((s) => s.type === "implicit");
|
||||
if (implicitSignals.length > 0) {
|
||||
const memory = this.processImplicitSignals(
|
||||
implicitSignals,
|
||||
cleanText,
|
||||
options.guardLevel,
|
||||
);
|
||||
if (memory) {
|
||||
log.info(
|
||||
'[MemoryExtractor] Extracted implicit memory: "' +
|
||||
memory.text.slice(0, 50) +
|
||||
'"',
|
||||
);
|
||||
results.push(memory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process explicit signal (e.g., "记住: xxx")
|
||||
*/
|
||||
private processExplicitSignal(
|
||||
signal: SignalMatch,
|
||||
text: string,
|
||||
): ExtractedMemory | null {
|
||||
const explicitContent = extractExplicitContent(text);
|
||||
|
||||
if (!explicitContent || !explicitContent.content) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Skip "forget" commands for now (handled separately)
|
||||
if (explicitContent.command === "forget") {
|
||||
this.emit("forget:requested", explicitContent.content);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
text: explicitContent.content,
|
||||
category: this.inferCategory(explicitContent.content),
|
||||
confidence: 0.95, // High confidence for explicit commands
|
||||
isExplicit: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process implicit signals
|
||||
*/
|
||||
private processImplicitSignals(
|
||||
signals: SignalMatch[],
|
||||
text: string,
|
||||
guardLevel: "strict" | "standard" | "relaxed",
|
||||
): ExtractedMemory | null {
|
||||
// Score the candidate
|
||||
const scoring = this.scoreCandidate(text, signals);
|
||||
log.debug(
|
||||
"[MemoryExtractor] processImplicitSignals: score=",
|
||||
scoring.score,
|
||||
"breakdown=",
|
||||
scoring.breakdown,
|
||||
);
|
||||
|
||||
// Determine threshold based on guard level
|
||||
let threshold = SCORE_MIN_ACCEPT;
|
||||
if (guardLevel === "strict") {
|
||||
threshold = 0.7;
|
||||
} else if (guardLevel === "relaxed") {
|
||||
threshold = 0.5;
|
||||
}
|
||||
|
||||
// Check if score meets threshold
|
||||
if (scoring.score < threshold) {
|
||||
log.debug(
|
||||
"[MemoryExtractor] processImplicitSignals: score",
|
||||
scoring.score,
|
||||
"< threshold",
|
||||
threshold,
|
||||
"- rejected",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Extract the relevant text
|
||||
const extractedText = this.extractRelevantText(text, signals);
|
||||
|
||||
return {
|
||||
text: extractedText,
|
||||
category: this.inferCategoryFromSignals(signals),
|
||||
confidence: scoring.score,
|
||||
isExplicit: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a memory candidate
|
||||
*/
|
||||
scoreCandidate(text: string, signals: SignalMatch[]): ScoringResult {
|
||||
const breakdown = {
|
||||
personalFact: 0,
|
||||
appropriateLength: 0,
|
||||
clearPreference: 0,
|
||||
question: 0,
|
||||
temporary: 0,
|
||||
code: 0,
|
||||
specificTime: 0,
|
||||
};
|
||||
|
||||
// Positive scores
|
||||
if (
|
||||
signals.some((s) => s.pattern === "personal_info" || s.pattern === "fact")
|
||||
) {
|
||||
breakdown.personalFact = SCORE_PERSONAL_FACT;
|
||||
}
|
||||
|
||||
if (signals.some((s) => s.pattern === "preference")) {
|
||||
breakdown.clearPreference = SCORE_CLEAR_PREFERENCE;
|
||||
}
|
||||
|
||||
// Check length (10-200 chars is ideal)
|
||||
const cleanText = text.trim();
|
||||
if (cleanText.length >= 10 && cleanText.length <= 200) {
|
||||
breakdown.appropriateLength = SCORE_APPROPRIATE_LENGTH;
|
||||
}
|
||||
|
||||
// Negative scores
|
||||
if (isQuestion(cleanText)) {
|
||||
breakdown.question = SCORE_QUESTION;
|
||||
}
|
||||
|
||||
if (isTemporary(cleanText)) {
|
||||
breakdown.temporary = SCORE_TEMPORARY;
|
||||
}
|
||||
|
||||
if (isCode(cleanText)) {
|
||||
breakdown.code = SCORE_CODE;
|
||||
}
|
||||
|
||||
// Check for specific time references
|
||||
if (/\d{1,2}:\d{2}|\d{4}年\d{1,2}月\d{1,2}日/.test(cleanText)) {
|
||||
breakdown.specificTime = SCORE_SPECIFIC_TIME;
|
||||
}
|
||||
|
||||
// Calculate total score
|
||||
let score = 0.5; // Base score
|
||||
score += breakdown.personalFact;
|
||||
score += breakdown.appropriateLength;
|
||||
score += breakdown.clearPreference;
|
||||
score += breakdown.question;
|
||||
score += breakdown.temporary;
|
||||
score += breakdown.code;
|
||||
score += breakdown.specificTime;
|
||||
|
||||
// Clamp to 0-1
|
||||
score = Math.max(0, Math.min(1, score));
|
||||
|
||||
return { score, breakdown };
|
||||
}
|
||||
|
||||
/**
|
||||
* Preprocess text for memory extraction
|
||||
* Currently returns the text as-is, waiting for new API field for pure user input
|
||||
*/
|
||||
private preprocessText(text: string): string {
|
||||
if (!text) return "";
|
||||
log.info("[MemoryExtractor] preprocessText: length=" + text.length);
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract relevant text from message based on signals
|
||||
*/
|
||||
private extractRelevantText(text: string, signals: SignalMatch[]): string {
|
||||
// For implicit signals, extract the sentence containing the signal
|
||||
const sentences = text.split(/[。！？\n]/).filter((s) => s.trim());
|
||||
|
||||
for (const sentence of sentences) {
|
||||
for (const signal of signals) {
|
||||
if (sentence.includes(signal.matchedText.replace(/[:：]\s*.*/, ""))) {
|
||||
// Clean up the extracted sentence
|
||||
return this.cleanExtractedText(sentence.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: return the whole text if it's not too long
|
||||
if (text.length <= 200) {
|
||||
return this.cleanExtractedText(text.trim());
|
||||
}
|
||||
|
||||
// Return first 200 chars
|
||||
return this.cleanExtractedText(text.trim().slice(0, 200));
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean extracted text by removing tone particles and irrelevant suffixes
|
||||
*/
|
||||
private cleanExtractedText(text: string): string {
|
||||
let cleaned = text;
|
||||
|
||||
// Remove common Chinese tone particles and suffixes that aren't part of the memory
|
||||
// e.g., "你记住下" should just be the preceding content
|
||||
cleaned = cleaned.replace(/[，,]?你?(?:记住|记得)[下吧了啊]?/g, "");
|
||||
|
||||
// Remove trailing punctuation that looks incomplete
|
||||
cleaned = cleaned.replace(/[，,;；\s]+$/g, "");
|
||||
|
||||
// Remove markdown headers
|
||||
cleaned = cleaned.replace(/#+\s*$/g, "");
|
||||
|
||||
return cleaned.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer memory category from text content
|
||||
*/
|
||||
private inferCategory(text: string): MemoryCategory {
|
||||
const lowerText = text.toLowerCase();
|
||||
|
||||
// Check for preference indicators
|
||||
if (/喜欢|偏好|习惯|倾向|prefer|like|usually/.test(lowerText)) {
|
||||
return "preference";
|
||||
}
|
||||
|
||||
// Check for decision indicators
|
||||
if (
|
||||
/决定|选择|采用|使用|方案|decide|choose|adopt|approach/.test(lowerText)
|
||||
) {
|
||||
return "decision";
|
||||
}
|
||||
|
||||
// Check for event indicators
|
||||
if (
|
||||
/昨天|今天|明天|上周|下周|yesterday|tomorrow|next|last/.test(lowerText)
|
||||
) {
|
||||
return "event";
|
||||
}
|
||||
|
||||
// Check for skill indicators
|
||||
if (/会|能|擅长|skill|can|able|expert/.test(lowerText)) {
|
||||
return "skill";
|
||||
}
|
||||
|
||||
// Default to fact
|
||||
return "fact";
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer category from signal types
|
||||
*/
|
||||
private inferCategoryFromSignals(signals: SignalMatch[]): MemoryCategory {
|
||||
if (signals.some((s) => s.pattern === "preference")) {
|
||||
return "preference";
|
||||
}
|
||||
|
||||
if (
|
||||
signals.some(
|
||||
(s) => s.pattern === "personal_info" || s.pattern === "ownership",
|
||||
)
|
||||
) {
|
||||
return "fact";
|
||||
}
|
||||
|
||||
return "fact";
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate extracted memories
|
||||
*/
|
||||
private deduplicate(memories: ExtractedMemory[]): ExtractedMemory[] {
|
||||
const seen = new Set<string>();
|
||||
const result: ExtractedMemory[] = [];
|
||||
|
||||
for (const memory of memories) {
|
||||
const hash = calculateHash(memory.text);
|
||||
if (!seen.has(hash)) {
|
||||
seen.add(hash);
|
||||
result.push(memory);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ==================== LLM Integration ====================
|
||||
|
||||
/**
|
||||
* Check if LLM validation is needed based on score
|
||||
*/
|
||||
needsLlmValidation(score: number): boolean {
|
||||
return (
|
||||
score >= SCORE_LLM_THRESHOLD_LOW && score <= SCORE_LLM_THRESHOLD_HIGH
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build LLM extraction prompt
|
||||
*
|
||||
* @param messages - Conversation messages to extract from
|
||||
* @param segmentMeta - Optional segment metadata for segmented extraction
|
||||
* @param existingMemories - Optional list of existing memories to avoid duplicates
|
||||
*/
|
||||
buildExtractionPrompt(
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
segmentMeta?: { index: number; total: number },
|
||||
existingMemories?: string[],
|
||||
): string {
|
||||
const conversationHistory = messages
|
||||
.map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content}`)
|
||||
.join("\n\n");
|
||||
|
||||
const segmentInfo = segmentMeta
|
||||
? `(segment ${segmentMeta.index + 1}/${segmentMeta.total})`
|
||||
: "";
|
||||
|
||||
const existingMems =
|
||||
existingMemories && existingMemories.length > 0
|
||||
? existingMemories.map((m) => `- ${m}`).join("\n")
|
||||
: "(none)";
|
||||
|
||||
return LLM_EXTRACTION_PROMPT.replace("{segment_info}", segmentInfo)
|
||||
.replace("{conversation_history}", conversationHistory)
|
||||
.replace("{existing_memories}", existingMems);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build LLM validation prompt
|
||||
*/
|
||||
buildValidationPrompt(
|
||||
candidateMemory: string,
|
||||
existingMemories: string[],
|
||||
): string {
|
||||
return LLM_VALIDATION_PROMPT.replace(
|
||||
"{candidate_memory}",
|
||||
candidateMemory,
|
||||
).replace("{existing_memories}", existingMemories.join("\n") || "(none)");
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse LLM extraction response
|
||||
*/
|
||||
parseExtractionResponse(response: string): ExtractedMemory[] {
|
||||
try {
|
||||
// Try to extract JSON array from response
|
||||
const jsonMatch = response.match(/\[[\s\S]*\]/);
|
||||
if (!jsonMatch) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return parsed
|
||||
.map((item) => ({
|
||||
text: item.text || "",
|
||||
category: item.category || "fact",
|
||||
confidence:
|
||||
typeof item.confidence === "number" ? item.confidence : 0.75,
|
||||
isExplicit: false,
|
||||
}))
|
||||
.filter((m) => m.text.length > 0);
|
||||
} catch (error) {
|
||||
log.warn("[MemoryExtractor] Failed to parse LLM response:", error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse LLM validation response
|
||||
*/
|
||||
parseValidationResponse(response: string): ValidationResult {
|
||||
try {
|
||||
// Try to extract JSON from response
|
||||
const jsonMatch = response.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) {
|
||||
return {
|
||||
accept: false,
|
||||
reason: "Failed to parse response",
|
||||
confidence: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
|
||||
return {
|
||||
accept: parsed.accept === true,
|
||||
reason: parsed.reason,
|
||||
mergedText: parsed.merged_text,
|
||||
confidence: parsed.accept ? 0.9 : 0.1,
|
||||
};
|
||||
} catch (error) {
|
||||
log.warn("[MemoryExtractor] Failed to parse validation response:", error);
|
||||
return { accept: false, reason: "Parse error", confidence: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Utility Methods ====================
|
||||
|
||||
/**
|
||||
* Quick check if message contains extractable content
|
||||
*/
|
||||
hasExtractableContent(
|
||||
text: string,
|
||||
options?: Partial<ExtractionOptions>,
|
||||
): boolean {
|
||||
const opts = { ...this.options, ...options };
|
||||
|
||||
if (opts.explicitEnabled && hasExplicitCommand(text)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (opts.implicitEnabled && hasImplicitSignals(text)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get signal count for message
|
||||
*/
|
||||
getSignalCount(text: string): number {
|
||||
return countSignalStrength(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format memories for daily memory file
|
||||
*/
|
||||
formatForDailyMemory(memories: ExtractedMemory[]): string {
|
||||
return memories.map((m) => `- ${m.text}`).join("\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton
|
||||
export const memoryExtractor = new MemoryExtractor();
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* 单元测试: MemoryFileSync — destroy 竞态修复 & processPendingSync 守卫
|
||||
*
|
||||
* 覆盖内容：
|
||||
* - destroy() 先设 initialized=false，再清理 timers 和 watcher
|
||||
* - processPendingSync() 在 initialized=false 时提前返回
|
||||
* - destroy() 后 debounce 定时器触发不会导致错误
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
|
||||
vi.mock("electron-log", () => ({
|
||||
default: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("chokidar", () => ({
|
||||
default: {
|
||||
watch: vi.fn(() => ({
|
||||
on: vi.fn().mockReturnThis(),
|
||||
close: vi.fn(),
|
||||
})),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("./utils/hash", () => ({
|
||||
calculateHash: vi.fn((s: string) => `hash_${s.length}`),
|
||||
}));
|
||||
|
||||
vi.mock("./utils/chunker", () => ({
|
||||
chunkMarkdown: vi.fn(() => []),
|
||||
compareChunks: vi.fn(() => ({ added: [], removed: [], unchanged: [] })),
|
||||
}));
|
||||
|
||||
import { MemoryFileSync } from "./MemoryFileSync";
|
||||
import * as fs from "fs";
|
||||
|
||||
describe("MemoryFileSync — destroy race condition", () => {
|
||||
let sync: MemoryFileSync;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
sync = new MemoryFileSync();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("destroy() sets initialized=false before clearing timers", () => {
|
||||
// Track the order of operations via spying
|
||||
const states: boolean[] = [];
|
||||
|
||||
// Init with minimal mocks
|
||||
(sync as any).workspaceDir = "/tmp/test";
|
||||
(sync as any).database = {
|
||||
setSyncState: vi.fn(),
|
||||
getFileHash: vi.fn(),
|
||||
getSyncState: vi.fn(),
|
||||
};
|
||||
(sync as any).initialized = true;
|
||||
|
||||
// Spy on clearAllTimers to record initialized state when it's called
|
||||
const origClearAllTimers = (sync as any).clearAllTimers.bind(sync);
|
||||
(sync as any).clearAllTimers = () => {
|
||||
states.push((sync as any).initialized);
|
||||
origClearAllTimers();
|
||||
};
|
||||
|
||||
sync.destroy();
|
||||
|
||||
// clearAllTimers should have been called when initialized was already false
|
||||
expect(states).toEqual([false]);
|
||||
expect(sync.isInitialized()).toBe(false);
|
||||
});
|
||||
|
||||
it("processPendingSync returns early when initialized=false", async () => {
|
||||
(sync as any).workspaceDir = "/tmp/test";
|
||||
(sync as any).database = {
|
||||
setSyncState: vi.fn(),
|
||||
deleteBySourcePath: vi.fn(),
|
||||
deleteFileHash: vi.fn(),
|
||||
};
|
||||
(sync as any).initialized = false;
|
||||
|
||||
// Set up a pending sync that would normally be processed
|
||||
(sync as any).pendingSyncs.set("/tmp/test/file.md", {
|
||||
filePath: "/tmp/test/file.md",
|
||||
eventType: "change",
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
// syncFile would throw if actually called on uninitialized state
|
||||
const syncFileSpy = vi
|
||||
.spyOn(sync as any, "syncFile")
|
||||
.mockRejectedValue(new Error("should not be called"));
|
||||
|
||||
await (sync as any).processPendingSync("/tmp/test/file.md");
|
||||
|
||||
// syncFile should NOT have been called
|
||||
expect(syncFileSpy).not.toHaveBeenCalled();
|
||||
// pending sync should still be in the map (not consumed)
|
||||
expect((sync as any).pendingSyncs.has("/tmp/test/file.md")).toBe(true);
|
||||
});
|
||||
|
||||
it("debounced sync after destroy does not process", async () => {
|
||||
(sync as any).workspaceDir = "/tmp/test";
|
||||
(sync as any).database = {
|
||||
setSyncState: vi.fn(),
|
||||
};
|
||||
(sync as any).initialized = true;
|
||||
(sync as any).debounceMs = 100;
|
||||
|
||||
// Trigger debounced sync
|
||||
(sync as any).debounceSync("/tmp/test/MEMORY.md", "change");
|
||||
|
||||
// Destroy before debounce fires
|
||||
sync.destroy();
|
||||
|
||||
// The debounce timer should have been cleared by destroy
|
||||
// Advancing time should not trigger any processing
|
||||
const syncFileSpy = vi.spyOn(sync as any, "syncFile").mockResolvedValue({});
|
||||
vi.advanceTimersByTime(200);
|
||||
|
||||
expect(syncFileSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,746 @@
|
||||
/**
|
||||
* Memory File Sync Module
|
||||
*
|
||||
* File watching and hash-based synchronization
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import * as path from "path";
|
||||
import * as fs from "fs";
|
||||
import { EventEmitter } from "events";
|
||||
import chokidar, { FSWatcher } from "chokidar";
|
||||
import log from "electron-log";
|
||||
import type { MemoryChunk, SyncResult, SyncState, MemorySource } from "./types";
|
||||
import { MemoryDatabase } from "./MemoryDatabase";
|
||||
import {
|
||||
CORE_MEMORY_FILE,
|
||||
DAILY_MEMORY_DIR,
|
||||
WATCH_DEBOUNCE_MS,
|
||||
DEFAULT_SYNC_STATE,
|
||||
} from "./constants";
|
||||
import { calculateHash } from "./utils/hash";
|
||||
import { chunkMarkdown, compareChunks } from "./utils/chunker";
|
||||
|
||||
// ==================== Types ====================
|
||||
|
||||
interface FileSyncOptions {
|
||||
workspaceDir: string;
|
||||
database: MemoryDatabase;
|
||||
debounceMs?: number;
|
||||
}
|
||||
|
||||
interface PendingSync {
|
||||
filePath: string;
|
||||
eventType: "add" | "change" | "unlink";
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
// ==================== MemoryFileSync Class ====================
|
||||
|
||||
export class MemoryFileSync extends EventEmitter {
|
||||
private workspaceDir: string = "";
|
||||
private database: MemoryDatabase | null = null;
|
||||
private watcher: FSWatcher | null = null;
|
||||
private debounceMs: number;
|
||||
private debounceTimers: Map<string, NodeJS.Timeout> = new Map();
|
||||
private pendingSyncs: Map<string, PendingSync> = new Map();
|
||||
private initialized: boolean = false;
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.debounceMs = WATCH_DEBOUNCE_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize file sync
|
||||
*/
|
||||
async init(options: FileSyncOptions): Promise<void> {
|
||||
this.workspaceDir = options.workspaceDir;
|
||||
this.database = options.database;
|
||||
this.debounceMs = options.debounceMs ?? WATCH_DEBOUNCE_MS;
|
||||
|
||||
// Ensure directories exist
|
||||
this.ensureDirectories();
|
||||
|
||||
// Start file watcher
|
||||
this.startWatcher();
|
||||
|
||||
this.initialized = true;
|
||||
log.info("[MemoryFileSync] Initialized for:", this.workspaceDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy file sync
|
||||
*/
|
||||
destroy(): void {
|
||||
this.initialized = false;
|
||||
this.clearAllTimers();
|
||||
this.stopWatcher();
|
||||
log.info("[MemoryFileSync] Destroyed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if initialized
|
||||
*/
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
// ==================== Directory Management ====================
|
||||
|
||||
/**
|
||||
* Ensure memory directories exist
|
||||
*/
|
||||
private ensureDirectories(): void {
|
||||
const coreFile = this.getCoreMemoryPath();
|
||||
const dailyDir = this.getDailyMemoryDir();
|
||||
|
||||
// Create daily memory directory
|
||||
if (!fs.existsSync(dailyDir)) {
|
||||
fs.mkdirSync(dailyDir, { recursive: true });
|
||||
log.info("[MemoryFileSync] Created daily memory directory:", dailyDir);
|
||||
}
|
||||
|
||||
// Create core memory file if not exists
|
||||
if (!fs.existsSync(coreFile)) {
|
||||
const defaultContent = this.getDefaultCoreMemoryContent();
|
||||
fs.writeFileSync(coreFile, defaultContent, "utf8");
|
||||
log.info("[MemoryFileSync] Created core memory file:", coreFile);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get default MEMORY.md content
|
||||
*/
|
||||
private getDefaultCoreMemoryContent(): string {
|
||||
return `# Long-term Memory
|
||||
|
||||
> This file is auto-generated by Qiming Agent. You may edit it directly.
|
||||
|
||||
## User profile
|
||||
|
||||
## Preferences
|
||||
|
||||
## Project-related
|
||||
|
||||
## Important decisions
|
||||
|
||||
---
|
||||
*Last updated: ${new Date().toISOString().split("T")[0]}*
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get core memory file path
|
||||
*/
|
||||
getCoreMemoryPath(): string {
|
||||
return path.join(this.workspaceDir, CORE_MEMORY_FILE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get daily memory directory
|
||||
*/
|
||||
getDailyMemoryDir(): string {
|
||||
return path.join(this.workspaceDir, DAILY_MEMORY_DIR);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get daily memory file path for a specific date
|
||||
*/
|
||||
getDailyMemoryPath(date?: Date): string {
|
||||
const d = date ?? new Date();
|
||||
const dateStr = d.toISOString().split("T")[0]; // YYYY-MM-DD
|
||||
return path.join(this.getDailyMemoryDir(), `${dateStr}.md`);
|
||||
}
|
||||
|
||||
// ==================== File Watching ====================
|
||||
|
||||
/**
|
||||
* Start file watcher
|
||||
*/
|
||||
private startWatcher(): void {
|
||||
if (this.watcher) {
|
||||
return;
|
||||
}
|
||||
|
||||
const watchPatterns = [
|
||||
this.getCoreMemoryPath(),
|
||||
path.join(this.getDailyMemoryDir(), "*.md"),
|
||||
];
|
||||
|
||||
this.watcher = chokidar.watch(watchPatterns, {
|
||||
ignored: /(^|[\/\\])\../, // Ignore dotfiles
|
||||
persistent: true,
|
||||
ignoreInitial: true, // Don't trigger on initial scan
|
||||
awaitWriteFinish: {
|
||||
stabilityThreshold: 500,
|
||||
pollInterval: 100,
|
||||
},
|
||||
});
|
||||
|
||||
this.watcher
|
||||
.on("add", (filePath) => this.handleFileEvent(filePath, "add"))
|
||||
.on("change", (filePath) => this.handleFileEvent(filePath, "change"))
|
||||
.on("unlink", (filePath) => this.handleFileEvent(filePath, "unlink"))
|
||||
.on("error", (error) => {
|
||||
log.error("[MemoryFileSync] Watcher error:", error);
|
||||
this.emit("error", error);
|
||||
});
|
||||
|
||||
log.info("[MemoryFileSync] Watcher started");
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop file watcher
|
||||
*/
|
||||
private stopWatcher(): void {
|
||||
if (this.watcher) {
|
||||
this.watcher.close();
|
||||
this.watcher = null;
|
||||
log.info("[MemoryFileSync] Watcher stopped");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file system event
|
||||
*/
|
||||
private handleFileEvent(
|
||||
filePath: string,
|
||||
eventType: "add" | "change" | "unlink",
|
||||
): void {
|
||||
log.debug(`[MemoryFileSync] File event: ${eventType} - ${filePath}`);
|
||||
|
||||
// Mark as dirty
|
||||
this.database?.setSyncState({ dirty: true });
|
||||
|
||||
// Debounce the sync
|
||||
this.debounceSync(filePath, eventType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Debounce sync operation
|
||||
*/
|
||||
private debounceSync(
|
||||
filePath: string,
|
||||
eventType: "add" | "change" | "unlink",
|
||||
): void {
|
||||
// Clear existing timer for this file
|
||||
const existingTimer = this.debounceTimers.get(filePath);
|
||||
if (existingTimer) {
|
||||
clearTimeout(existingTimer);
|
||||
}
|
||||
|
||||
// Store pending sync
|
||||
this.pendingSyncs.set(filePath, {
|
||||
filePath,
|
||||
eventType,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
// Set new timer
|
||||
const timer = setTimeout(() => {
|
||||
this.debounceTimers.delete(filePath);
|
||||
this.processPendingSync(filePath);
|
||||
}, this.debounceMs);
|
||||
|
||||
this.debounceTimers.set(filePath, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process pending sync for a file
|
||||
*/
|
||||
private async processPendingSync(filePath: string): Promise<void> {
|
||||
if (!this.initialized) return;
|
||||
|
||||
const pending = this.pendingSyncs.get(filePath);
|
||||
if (!pending) return;
|
||||
|
||||
this.pendingSyncs.delete(filePath);
|
||||
|
||||
try {
|
||||
await this.syncFile(filePath, pending.eventType);
|
||||
} catch (error) {
|
||||
log.error("[MemoryFileSync] Failed to sync file:", filePath, error);
|
||||
this.emit("sync:error", { filePath, error });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all timers
|
||||
*/
|
||||
private clearAllTimers(): void {
|
||||
for (const timer of this.debounceTimers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.debounceTimers.clear();
|
||||
this.pendingSyncs.clear();
|
||||
}
|
||||
|
||||
// ==================== Sync Operations ====================
|
||||
|
||||
/**
|
||||
* Sync on startup (full hash verification)
|
||||
*/
|
||||
async syncOnStartup(): Promise<SyncResult> {
|
||||
log.info("[MemoryFileSync] Starting startup sync...");
|
||||
|
||||
const result: SyncResult = {
|
||||
added: 0,
|
||||
removed: 0,
|
||||
unchanged: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
try {
|
||||
// Get all existing memory files
|
||||
const files = this.getAllMemoryFiles();
|
||||
const db = this.database!;
|
||||
|
||||
// Check each file
|
||||
for (const filePath of files) {
|
||||
try {
|
||||
const syncResult = await this.syncFileWithHashCheck(filePath);
|
||||
result.added += syncResult.added;
|
||||
result.removed += syncResult.removed;
|
||||
result.unchanged += syncResult.unchanged;
|
||||
} catch (error) {
|
||||
result.errors.push(`${filePath}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for deleted files
|
||||
const deletedResult = await this.checkDeletedFiles(files);
|
||||
result.removed += deletedResult;
|
||||
|
||||
// Update sync state
|
||||
db.setSyncState({
|
||||
dirty: false,
|
||||
syncing: false,
|
||||
lastSyncTime: Date.now(),
|
||||
});
|
||||
|
||||
log.info("[MemoryFileSync] Startup sync complete:", result);
|
||||
this.emit("sync:complete", result);
|
||||
} catch (error) {
|
||||
log.error("[MemoryFileSync] Startup sync failed:", error);
|
||||
result.errors.push(String(error));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a single file
|
||||
*/
|
||||
async syncFile(
|
||||
filePath: string,
|
||||
eventType: "add" | "change" | "unlink",
|
||||
): Promise<SyncResult> {
|
||||
const result: SyncResult = {
|
||||
added: 0,
|
||||
removed: 0,
|
||||
unchanged: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
const db = this.database!;
|
||||
const relativePath = path.relative(this.workspaceDir, filePath);
|
||||
|
||||
if (eventType === "unlink") {
|
||||
// File deleted
|
||||
const deleted = db.deleteBySourcePath(relativePath);
|
||||
db.deleteFileHash(relativePath);
|
||||
result.removed = deleted;
|
||||
} else {
|
||||
// File added or changed
|
||||
const syncResult = await this.syncFileWithHashCheck(filePath);
|
||||
result.added = syncResult.added;
|
||||
result.removed = syncResult.removed;
|
||||
result.unchanged = syncResult.unchanged;
|
||||
}
|
||||
|
||||
// Update sync state
|
||||
db.setSyncState({
|
||||
dirty: false,
|
||||
lastSyncTime: Date.now(),
|
||||
});
|
||||
|
||||
this.emit("sync:file", { filePath, result });
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync file with hash check
|
||||
*/
|
||||
private async syncFileWithHashCheck(filePath: string): Promise<SyncResult> {
|
||||
const result: SyncResult = {
|
||||
added: 0,
|
||||
removed: 0,
|
||||
unchanged: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
const db = this.database!;
|
||||
const relativePath = path.relative(this.workspaceDir, filePath);
|
||||
|
||||
// Check if file exists
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Read file content
|
||||
const content = fs.readFileSync(filePath, "utf8");
|
||||
const fileHash = calculateHash(content);
|
||||
const stats = fs.statSync(filePath);
|
||||
|
||||
// Check if file has changed
|
||||
const existingHash = db.getFileHash(relativePath);
|
||||
if (existingHash && existingHash.hash === fileHash) {
|
||||
result.unchanged = existingHash.chunkCount;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Determine source type
|
||||
const source: MemorySource =
|
||||
path.basename(filePath) === CORE_MEMORY_FILE ? "core" : "daily";
|
||||
|
||||
// Parse and chunk file
|
||||
const newChunks = chunkMarkdown(content, relativePath);
|
||||
|
||||
// Get existing chunks from database
|
||||
const existingMemories = db
|
||||
.getMemories({
|
||||
status: "active",
|
||||
source,
|
||||
})
|
||||
.filter((m) => m.sourcePath === relativePath);
|
||||
|
||||
const oldChunks: MemoryChunk[] = existingMemories.map((m) => ({
|
||||
text: m.text,
|
||||
hash: m.fingerprint,
|
||||
startLine: m.startLine!,
|
||||
endLine: m.endLine!,
|
||||
}));
|
||||
|
||||
// Compare chunks
|
||||
const { added, removed, unchanged } = compareChunks(oldChunks, newChunks);
|
||||
|
||||
// Delete removed chunks
|
||||
for (const removedHash of removed) {
|
||||
const memory = existingMemories.find(
|
||||
(m) => m.fingerprint === removedHash,
|
||||
);
|
||||
if (memory) {
|
||||
db.deleteMemory(memory.id);
|
||||
result.removed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Insert added chunks
|
||||
for (const chunk of added) {
|
||||
const entry = db.createEntryFromChunk(chunk, source, relativePath);
|
||||
db.insertMemory(entry);
|
||||
result.added++;
|
||||
}
|
||||
|
||||
result.unchanged = unchanged.length;
|
||||
|
||||
// Update file hash record
|
||||
db.setFileHash({
|
||||
path: relativePath,
|
||||
hash: fileHash,
|
||||
chunkCount: newChunks.length,
|
||||
lastModified: stats.mtimeMs,
|
||||
syncedAt: Date.now(),
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for files that have been deleted
|
||||
*/
|
||||
private async checkDeletedFiles(existingFiles: string[]): Promise<number> {
|
||||
const db = this.database!;
|
||||
const dbHashes = db.getAllFileHashes();
|
||||
let deleted = 0;
|
||||
|
||||
const existingRelativePaths = new Set(
|
||||
existingFiles.map((f) => path.relative(this.workspaceDir, f)),
|
||||
);
|
||||
|
||||
for (const record of dbHashes) {
|
||||
if (!existingRelativePaths.has(record.path)) {
|
||||
// File has been deleted
|
||||
const removed = db.deleteBySourcePath(record.path);
|
||||
db.deleteFileHash(record.path);
|
||||
deleted += removed;
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild entire index
|
||||
*/
|
||||
async rebuildIndex(): Promise<SyncResult> {
|
||||
log.info("[MemoryFileSync] Rebuilding index...");
|
||||
|
||||
const result: SyncResult = {
|
||||
added: 0,
|
||||
removed: 0,
|
||||
unchanged: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
const db = this.database!;
|
||||
|
||||
// Clear all existing data
|
||||
// Note: This is a destructive operation
|
||||
const dbInstance = db.getDb();
|
||||
if (dbInstance) {
|
||||
dbInstance.exec("DELETE FROM memories");
|
||||
dbInstance.exec("DELETE FROM file_hashes");
|
||||
}
|
||||
|
||||
// Re-sync all files
|
||||
const files = this.getAllMemoryFiles();
|
||||
for (const filePath of files) {
|
||||
try {
|
||||
const syncResult = await this.syncFileWithHashCheck(filePath);
|
||||
result.added += syncResult.added;
|
||||
result.removed += syncResult.removed;
|
||||
result.unchanged += syncResult.unchanged;
|
||||
} catch (error) {
|
||||
result.errors.push(`${filePath}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync state
|
||||
db.setSyncState({
|
||||
dirty: false,
|
||||
syncing: false,
|
||||
lastSyncTime: Date.now(),
|
||||
});
|
||||
|
||||
log.info("[MemoryFileSync] Index rebuild complete:", result);
|
||||
this.emit("rebuild:complete", result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ==================== File Operations ====================
|
||||
|
||||
/**
|
||||
* Get all memory files
|
||||
*/
|
||||
getAllMemoryFiles(): string[] {
|
||||
const files: string[] = [];
|
||||
|
||||
// Core memory file
|
||||
const corePath = this.getCoreMemoryPath();
|
||||
if (fs.existsSync(corePath)) {
|
||||
files.push(corePath);
|
||||
}
|
||||
|
||||
// Daily memory files
|
||||
const dailyDir = this.getDailyMemoryDir();
|
||||
if (fs.existsSync(dailyDir)) {
|
||||
const dailyFiles = fs
|
||||
.readdirSync(dailyDir)
|
||||
.filter((f) => f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
|
||||
.map((f) => path.join(dailyDir, f));
|
||||
files.push(...dailyFiles);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append to daily memory file and sync to database
|
||||
*/
|
||||
appendToDailyMemory(content: string, title?: string): string {
|
||||
const filePath = this.getDailyMemoryPath();
|
||||
const now = new Date();
|
||||
const timeStr = now.toTimeString().slice(0, 5); // HH:MM
|
||||
const sessionTitle = title ?? "Session";
|
||||
|
||||
// Build content to append
|
||||
const appendContent = `
|
||||
---
|
||||
|
||||
### ${timeStr} ${sessionTitle}
|
||||
${content}
|
||||
`;
|
||||
|
||||
// Create file if not exists
|
||||
if (!fs.existsSync(filePath)) {
|
||||
const dateStr = now.toISOString().split("T")[0];
|
||||
const header = `# ${dateStr}\n\n---\n`;
|
||||
fs.writeFileSync(filePath, header, "utf8");
|
||||
}
|
||||
|
||||
// Append content
|
||||
fs.appendFileSync(filePath, appendContent, "utf8");
|
||||
|
||||
log.info("[MemoryFileSync] Appended to daily memory:", filePath);
|
||||
|
||||
// Immediately sync this file to database for real-time retrieval
|
||||
this.syncFileImmediate(filePath, content, sessionTitle);
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Immediately sync appended content to database (without re-reading file)
|
||||
*/
|
||||
private syncFileImmediate(
|
||||
filePath: string,
|
||||
content: string,
|
||||
title: string,
|
||||
): void {
|
||||
if (!this.database) return;
|
||||
|
||||
try {
|
||||
// Parse the appended content into individual memories
|
||||
const lines = content
|
||||
.split("\n")
|
||||
.filter((line) => line.trim().startsWith("- "));
|
||||
|
||||
for (const line of lines) {
|
||||
const memoryText = line.replace(/^-\s*/, "").trim();
|
||||
if (!memoryText || memoryText.length < 2) continue;
|
||||
|
||||
// Generate memory entry
|
||||
const now = Date.now();
|
||||
const fingerprint = calculateHash(memoryText);
|
||||
const relativePath = path
|
||||
.relative(this.workspaceDir, filePath)
|
||||
.replace(/\\/g, "/");
|
||||
|
||||
const entry = {
|
||||
id: `mem_${fingerprint}`,
|
||||
text: memoryText,
|
||||
fingerprint,
|
||||
category: "fact" as const,
|
||||
confidence: 0.75,
|
||||
isExplicit: true,
|
||||
importance: 0.5,
|
||||
source: "daily" as const,
|
||||
sourcePath: relativePath,
|
||||
status: "active" as const,
|
||||
accessCount: 0,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
// Check if already exists
|
||||
const exists = this.database.existsByFingerprint(fingerprint);
|
||||
if (!exists) {
|
||||
this.database.insertMemory(entry);
|
||||
log.debug(
|
||||
"[MemoryFileSync] Inserted memory to database:",
|
||||
memoryText.slice(0, 50),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[MemoryFileSync] Synced %d memories to database from append",
|
||||
lines.length,
|
||||
);
|
||||
} catch (error) {
|
||||
log.error(
|
||||
"[MemoryFileSync] Failed to sync appended content to database:",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read core memory file
|
||||
*/
|
||||
readCoreMemory(): string {
|
||||
const filePath = this.getCoreMemoryPath();
|
||||
if (fs.existsSync(filePath)) {
|
||||
return fs.readFileSync(filePath, "utf8");
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Write core memory file
|
||||
*/
|
||||
writeCoreMemory(content: string): void {
|
||||
const filePath = this.getCoreMemoryPath();
|
||||
fs.writeFileSync(filePath, content, "utf8");
|
||||
log.info("[MemoryFileSync] Wrote core memory:", filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read daily memory files for recent days
|
||||
*/
|
||||
readRecentDailyMemories(days: number = 2): Map<string, string> {
|
||||
const result = new Map<string, string>();
|
||||
|
||||
for (let i = 0; i < days; i++) {
|
||||
const date = new Date();
|
||||
date.setDate(date.getDate() - i);
|
||||
const filePath = this.getDailyMemoryPath(date);
|
||||
|
||||
if (fs.existsSync(filePath)) {
|
||||
const content = fs.readFileSync(filePath, "utf8");
|
||||
const dateStr = date.toISOString().split("T")[0];
|
||||
result.set(dateStr, content);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete old daily memory files
|
||||
*/
|
||||
deleteOldDailyFiles(retentionDays: number): number {
|
||||
const dailyDir = this.getDailyMemoryDir();
|
||||
if (!fs.existsSync(dailyDir)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffStr = cutoffDate.toISOString().split("T")[0];
|
||||
|
||||
let deleted = 0;
|
||||
const files = fs
|
||||
.readdirSync(dailyDir)
|
||||
.filter((f) => f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}\.md$/.test(f));
|
||||
|
||||
for (const file of files) {
|
||||
const dateStr = file.replace(".md", "");
|
||||
if (dateStr < cutoffStr) {
|
||||
const filePath = path.join(dailyDir, file);
|
||||
fs.unlinkSync(filePath);
|
||||
|
||||
// Also delete from database
|
||||
this.database?.deleteBySourcePath(file);
|
||||
|
||||
deleted++;
|
||||
log.info("[MemoryFileSync] Deleted old daily file:", file);
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sync state
|
||||
*/
|
||||
getSyncState(): SyncState {
|
||||
return this.database?.getSyncState() ?? DEFAULT_SYNC_STATE;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton
|
||||
export const memoryFileSync = new MemoryFileSync();
|
||||
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* Memory Injector Module
|
||||
*
|
||||
* Inject retrieved memories into system prompt
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import log from "electron-log";
|
||||
import type { InjectionOptions, MemorySearchResult } from "./types";
|
||||
import { MemoryRetriever } from "./MemoryRetriever";
|
||||
|
||||
// ==================== Constants ====================
|
||||
|
||||
const DEFAULT_MAX_TOKENS = 2000;
|
||||
const MEMORY_START_MARKER = "<!-- MEMORY_CONTEXT_START -->";
|
||||
const MEMORY_END_MARKER = "<!-- MEMORY_CONTEXT_END -->";
|
||||
|
||||
// ==================== MemoryInjector Class ====================
|
||||
|
||||
export class MemoryInjector {
|
||||
private retriever: MemoryRetriever | null = null;
|
||||
private defaultMaxTokens: number = DEFAULT_MAX_TOKENS;
|
||||
|
||||
/**
|
||||
* Initialize injector
|
||||
*/
|
||||
init(retriever: MemoryRetriever): void {
|
||||
this.retriever = retriever;
|
||||
log.info("[MemoryInjector] Initialized");
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure injector
|
||||
*/
|
||||
configure(options: { maxTokens?: number }): void {
|
||||
if (options.maxTokens !== undefined) {
|
||||
this.defaultMaxTokens = options.maxTokens;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Context Building ====================
|
||||
|
||||
/**
|
||||
* Build memory context for injection
|
||||
*/
|
||||
async buildContext(
|
||||
query: string,
|
||||
options?: InjectionOptions,
|
||||
): Promise<string> {
|
||||
if (!this.retriever) {
|
||||
log.debug("[MemoryInjector] buildContext: no retriever");
|
||||
return "";
|
||||
}
|
||||
|
||||
const maxTokens = options?.maxTokens ?? this.defaultMaxTokens;
|
||||
const format = options?.format ?? "xml";
|
||||
const includeScores = options?.includeScores ?? false;
|
||||
|
||||
// Retrieve relevant memories
|
||||
log.debug(
|
||||
"[MemoryInjector] buildContext: searching for query=",
|
||||
query.slice(0, 100),
|
||||
);
|
||||
const results = await this.retriever.search(query);
|
||||
log.debug(
|
||||
"[MemoryInjector] buildContext: found",
|
||||
results.length,
|
||||
"results",
|
||||
);
|
||||
|
||||
if (results.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Truncate to max tokens
|
||||
const truncated = this.truncateToTokenLimit(results, maxTokens);
|
||||
|
||||
// Format based on requested format
|
||||
if (format === "xml") {
|
||||
return this.formatAsXml(truncated, includeScores);
|
||||
} else {
|
||||
return this.formatAsMarkdown(truncated, includeScores);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build injection context without query (get all recent)
|
||||
*/
|
||||
async buildRecentContext(options?: InjectionOptions): Promise<string> {
|
||||
if (!this.retriever) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Use a broad query to get recent memories
|
||||
const results = await this.retriever.search("", { limit: 10 });
|
||||
|
||||
if (results.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const maxTokens = options?.maxTokens ?? this.defaultMaxTokens;
|
||||
const format = options?.format ?? "xml";
|
||||
const includeScores = options?.includeScores ?? false;
|
||||
|
||||
const truncated = this.truncateToTokenLimit(results, maxTokens);
|
||||
|
||||
if (format === "xml") {
|
||||
return this.formatAsXml(truncated, includeScores);
|
||||
} else {
|
||||
return this.formatAsMarkdown(truncated, includeScores);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Injection Methods ====================
|
||||
|
||||
/**
|
||||
* Inject memories into system prompt
|
||||
*/
|
||||
async injectIntoPrompt(
|
||||
systemPrompt: string,
|
||||
query: string,
|
||||
options?: InjectionOptions,
|
||||
): Promise<string> {
|
||||
const memoryContext = await this.buildContext(query, options);
|
||||
|
||||
if (!memoryContext) {
|
||||
return systemPrompt;
|
||||
}
|
||||
|
||||
// Check if there's an existing memory section
|
||||
const startIndex = systemPrompt.indexOf(MEMORY_START_MARKER);
|
||||
const endIndex = systemPrompt.indexOf(MEMORY_END_MARKER);
|
||||
|
||||
if (startIndex !== -1 && endIndex !== -1 && endIndex > startIndex) {
|
||||
// Replace existing memory section
|
||||
const before = systemPrompt.slice(
|
||||
0,
|
||||
startIndex + MEMORY_START_MARKER.length,
|
||||
);
|
||||
const after = systemPrompt.slice(endIndex);
|
||||
|
||||
return `${before}\n${memoryContext}\n${after}`;
|
||||
}
|
||||
|
||||
// Check for placeholder
|
||||
if (systemPrompt.includes("{{MEMORY_CONTEXT}}")) {
|
||||
return systemPrompt.replace("{{MEMORY_CONTEXT}}", memoryContext);
|
||||
}
|
||||
|
||||
// Append to end of system prompt
|
||||
return `${systemPrompt}\n\n${MEMORY_START_MARKER}\n${memoryContext}\n${MEMORY_END_MARKER}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove memory section from prompt
|
||||
*/
|
||||
removeFromPrompt(systemPrompt: string): string {
|
||||
const startIndex = systemPrompt.indexOf(MEMORY_START_MARKER);
|
||||
const endIndex = systemPrompt.indexOf(MEMORY_END_MARKER);
|
||||
|
||||
if (startIndex === -1 || endIndex === -1 || endIndex <= startIndex) {
|
||||
return systemPrompt;
|
||||
}
|
||||
|
||||
return (
|
||||
systemPrompt.slice(0, startIndex) +
|
||||
systemPrompt.slice(endIndex + MEMORY_END_MARKER.length)
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== Formatting ====================
|
||||
|
||||
/**
|
||||
* Format memories as XML
|
||||
*/
|
||||
private formatAsXml(
|
||||
results: MemorySearchResult[],
|
||||
includeScores: boolean,
|
||||
): string {
|
||||
const header = `<memory_context>
|
||||
<!--
|
||||
Long-term memory context: historical memories relevant to this conversation.
|
||||
-->
|
||||
<memories>`;
|
||||
|
||||
const memories = results
|
||||
.map((r) => {
|
||||
if (includeScores) {
|
||||
return ` <memory score="${r.score.toFixed(2)}" category="${r.entry.category}" source="${r.entry.source}">
|
||||
${this.escapeXml(r.entry.text)}
|
||||
</memory>`;
|
||||
}
|
||||
return ` <memory category="${r.entry.category}">
|
||||
${this.escapeXml(r.entry.text)}
|
||||
</memory>`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
return `${header}
|
||||
${memories}
|
||||
</memories>
|
||||
</memory_context>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format memories as Markdown
|
||||
*/
|
||||
private formatAsMarkdown(
|
||||
results: MemorySearchResult[],
|
||||
includeScores: boolean,
|
||||
): string {
|
||||
const lines: string[] = [
|
||||
"## Related memories",
|
||||
"",
|
||||
"> Memories relevant to this conversation.",
|
||||
"",
|
||||
];
|
||||
|
||||
for (const r of results) {
|
||||
const prefix = `- **[${r.entry.category}]**`;
|
||||
if (includeScores) {
|
||||
lines.push(
|
||||
`${prefix} ${r.entry.text} *(relevance: ${(r.score * 100).toFixed(0)}%)*`,
|
||||
);
|
||||
} else {
|
||||
lines.push(`${prefix} ${r.entry.text}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape XML special characters
|
||||
*/
|
||||
private escapeXml(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
// ==================== Token Management ====================
|
||||
|
||||
/**
|
||||
* Truncate results to fit within token limit
|
||||
* Simple estimation: ~4 chars per token for mixed Chinese/English
|
||||
*/
|
||||
private truncateToTokenLimit(
|
||||
results: MemorySearchResult[],
|
||||
maxTokens: number,
|
||||
): MemorySearchResult[] {
|
||||
const maxChars = maxTokens * 4;
|
||||
let totalChars = 0;
|
||||
const truncated: MemorySearchResult[] = [];
|
||||
|
||||
for (const result of results) {
|
||||
const entryChars = result.entry.text.length;
|
||||
|
||||
if (totalChars + entryChars > maxChars) {
|
||||
break;
|
||||
}
|
||||
|
||||
truncated.push(result);
|
||||
totalChars += entryChars;
|
||||
}
|
||||
|
||||
return truncated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate token count for text
|
||||
*/
|
||||
estimateTokens(text: string): number {
|
||||
// Simple estimation: ~4 chars per token for mixed content
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
// ==================== Utility Methods ====================
|
||||
|
||||
/**
|
||||
* Get memory statistics for display
|
||||
*/
|
||||
getMemoryStats(results: MemorySearchResult[]): {
|
||||
count: number;
|
||||
avgScore: number;
|
||||
categories: Record<string, number>;
|
||||
sources: Record<string, number>;
|
||||
} {
|
||||
const stats = {
|
||||
count: results.length,
|
||||
avgScore: 0,
|
||||
categories: {} as Record<string, number>,
|
||||
sources: {} as Record<string, number>,
|
||||
};
|
||||
|
||||
if (results.length === 0) {
|
||||
return stats;
|
||||
}
|
||||
|
||||
let totalScore = 0;
|
||||
|
||||
for (const r of results) {
|
||||
totalScore += r.score;
|
||||
|
||||
const cat = r.entry.category;
|
||||
stats.categories[cat] = (stats.categories[cat] ?? 0) + 1;
|
||||
|
||||
const src = r.entry.source;
|
||||
stats.sources[src] = (stats.sources[src] ?? 0) + 1;
|
||||
}
|
||||
|
||||
stats.avgScore = totalScore / results.length;
|
||||
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton
|
||||
export const memoryInjector = new MemoryInjector();
|
||||
@@ -0,0 +1,368 @@
|
||||
/**
|
||||
* Memory Retriever Module
|
||||
*
|
||||
* Three-tier retrieval: sqlite-vec -> JS vector -> FTS5
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import log from "electron-log";
|
||||
import type {
|
||||
MemorySearchResult,
|
||||
HybridSearchOptions,
|
||||
MemorySource,
|
||||
} from "./types";
|
||||
import { MemoryDatabase } from "./MemoryDatabase";
|
||||
|
||||
// ==================== Types ====================
|
||||
|
||||
interface EmbeddingProvider {
|
||||
getEmbedding(text: string): Promise<Float32Array | null>;
|
||||
}
|
||||
|
||||
// ==================== MemoryRetriever Class ====================
|
||||
|
||||
export class MemoryRetriever {
|
||||
private database: MemoryDatabase | null = null;
|
||||
private embeddingProvider: EmbeddingProvider | null = null;
|
||||
private vectorWeight: number = 0.7;
|
||||
private ftsWeight: number = 0.3;
|
||||
private defaultLimit: number = 12;
|
||||
private defaultMinScore: number = 0.4;
|
||||
private dailyMemoryDays: number = 2;
|
||||
|
||||
/**
|
||||
* Initialize retriever
|
||||
*/
|
||||
init(database: MemoryDatabase): void {
|
||||
this.database = database;
|
||||
log.info("[MemoryRetriever] Initialized");
|
||||
}
|
||||
|
||||
/**
|
||||
* Set embedding provider
|
||||
*/
|
||||
setEmbeddingProvider(provider: EmbeddingProvider | null): void {
|
||||
this.embeddingProvider = provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure retrieval parameters
|
||||
*/
|
||||
configure(options: {
|
||||
vectorWeight?: number;
|
||||
ftsWeight?: number;
|
||||
limit?: number;
|
||||
minScore?: number;
|
||||
dailyMemoryDays?: number;
|
||||
}): void {
|
||||
if (options.vectorWeight !== undefined)
|
||||
this.vectorWeight = options.vectorWeight;
|
||||
if (options.ftsWeight !== undefined) this.ftsWeight = options.ftsWeight;
|
||||
if (options.limit !== undefined) this.defaultLimit = options.limit;
|
||||
if (options.minScore !== undefined) this.defaultMinScore = options.minScore;
|
||||
if (options.dailyMemoryDays !== undefined)
|
||||
this.dailyMemoryDays = options.dailyMemoryDays;
|
||||
}
|
||||
|
||||
// ==================== Search Methods ====================
|
||||
|
||||
/**
|
||||
* Search memories using hybrid retrieval
|
||||
*/
|
||||
async search(
|
||||
query: string,
|
||||
options?: HybridSearchOptions,
|
||||
): Promise<MemorySearchResult[]> {
|
||||
if (!this.database) {
|
||||
log.debug("[MemoryRetriever] search: no database");
|
||||
return [];
|
||||
}
|
||||
|
||||
// Pre-retrieval dirty check (Spec 4.2 场景 5)
|
||||
// If index is dirty, log a warning - caller should use ensureMemoryReadyForSession() for proper sync
|
||||
if (options?.checkDirty !== false) {
|
||||
const { dirty } = this.database.getSyncState();
|
||||
if (dirty) {
|
||||
log.debug(
|
||||
"[MemoryRetriever] Index is dirty, search results may be stale. " +
|
||||
"Call memory.ensureReady() before session start for proper sync.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const limit = options?.limit ?? this.defaultLimit;
|
||||
const minScore = options?.minScore ?? this.defaultMinScore;
|
||||
const vectorWeight = options?.vectorWeight ?? this.vectorWeight;
|
||||
const ftsWeight = options?.ftsWeight ?? this.ftsWeight;
|
||||
|
||||
// Layer 1: Always do FTS5 search
|
||||
log.debug("[MemoryRetriever] search: FTS query=", query.slice(0, 100));
|
||||
const ftsResults = this.database.searchFTS(query, limit * 2);
|
||||
log.debug(
|
||||
"[MemoryRetriever] search: FTS returned",
|
||||
ftsResults.length,
|
||||
"results",
|
||||
);
|
||||
|
||||
// Check if embedding is available
|
||||
const embeddingEnabled = this.embeddingProvider !== null;
|
||||
const vectorAvailable =
|
||||
this.database.isVectorAvailable() || embeddingEnabled;
|
||||
|
||||
if (!vectorAvailable || !embeddingEnabled) {
|
||||
// Return FTS results only
|
||||
return ftsResults.filter((r) => r.score >= minScore).slice(0, limit);
|
||||
}
|
||||
|
||||
// Get query embedding
|
||||
let queryEmbedding: Float32Array | null = null;
|
||||
try {
|
||||
queryEmbedding = await this.embeddingProvider!.getEmbedding(query);
|
||||
} catch (error) {
|
||||
log.warn("[MemoryRetriever] Failed to get query embedding:", error);
|
||||
return ftsResults.filter((r) => r.score >= minScore).slice(0, limit);
|
||||
}
|
||||
|
||||
if (!queryEmbedding) {
|
||||
return ftsResults.filter((r) => r.score >= minScore).slice(0, limit);
|
||||
}
|
||||
|
||||
// Layer 2: Vector search
|
||||
const vecResults = this.database.searchVector(queryEmbedding, limit * 2, 0);
|
||||
|
||||
// Layer 3: Hybrid merge
|
||||
return this.mergeResults(ftsResults, vecResults, {
|
||||
vectorWeight,
|
||||
ftsWeight,
|
||||
limit,
|
||||
minScore,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search using FTS5 only (no vector)
|
||||
*/
|
||||
searchFTS(query: string, limit?: number): MemorySearchResult[] {
|
||||
if (!this.database) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.database.searchFTS(query, limit ?? this.defaultLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search using vector similarity
|
||||
*/
|
||||
async searchVector(
|
||||
query: string,
|
||||
limit?: number,
|
||||
minScore?: number,
|
||||
): Promise<MemorySearchResult[]> {
|
||||
if (!this.database || !this.embeddingProvider) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const embedding = await this.embeddingProvider.getEmbedding(query);
|
||||
if (!embedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.database.searchVector(
|
||||
embedding,
|
||||
limit ?? this.defaultLimit,
|
||||
minScore ?? this.defaultMinScore,
|
||||
);
|
||||
} catch (error) {
|
||||
log.error("[MemoryRetriever] Vector search failed:", error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Hybrid Merge ====================
|
||||
|
||||
/**
|
||||
* Merge FTS and vector results using weighted combination
|
||||
*/
|
||||
private mergeResults(
|
||||
ftsResults: MemorySearchResult[],
|
||||
vecResults: MemorySearchResult[],
|
||||
options: {
|
||||
vectorWeight: number;
|
||||
ftsWeight: number;
|
||||
limit: number;
|
||||
minScore: number;
|
||||
},
|
||||
): MemorySearchResult[] {
|
||||
const { vectorWeight, ftsWeight, limit, minScore } = options;
|
||||
|
||||
// Normalize scores for each result set
|
||||
const normalizedFts = this.normalizeScores(ftsResults);
|
||||
const normalizedVec = this.normalizeScores(vecResults);
|
||||
|
||||
// Create map for combining scores
|
||||
const scoreMap = new Map<
|
||||
string,
|
||||
{
|
||||
entry: MemorySearchResult["entry"];
|
||||
ftsScore: number;
|
||||
vecScore: number;
|
||||
}
|
||||
>();
|
||||
|
||||
// Add FTS results
|
||||
for (const result of normalizedFts) {
|
||||
scoreMap.set(result.entry.id, {
|
||||
entry: result.entry,
|
||||
ftsScore: result.score,
|
||||
vecScore: 0,
|
||||
});
|
||||
}
|
||||
|
||||
// Add/update with vector results
|
||||
for (const result of normalizedVec) {
|
||||
const existing = scoreMap.get(result.entry.id);
|
||||
if (existing) {
|
||||
existing.vecScore = result.score;
|
||||
} else {
|
||||
scoreMap.set(result.entry.id, {
|
||||
entry: result.entry,
|
||||
ftsScore: 0,
|
||||
vecScore: result.score,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate final scores
|
||||
const merged: MemorySearchResult[] = [];
|
||||
|
||||
for (const [id, data] of scoreMap) {
|
||||
// Weighted combination
|
||||
const finalScore =
|
||||
vectorWeight * data.vecScore + ftsWeight * data.ftsScore;
|
||||
|
||||
if (finalScore >= minScore) {
|
||||
merged.push({
|
||||
entry: data.entry,
|
||||
score: finalScore,
|
||||
source:
|
||||
data.ftsScore > 0 && data.vecScore > 0
|
||||
? "hybrid"
|
||||
: data.vecScore > 0
|
||||
? "vector"
|
||||
: "fts",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score descending
|
||||
merged.sort((a, b) => b.score - a.score);
|
||||
|
||||
return merged.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize scores to 0-1 range using min-max normalization
|
||||
*/
|
||||
private normalizeScores(results: MemorySearchResult[]): MemorySearchResult[] {
|
||||
if (results.length === 0) {
|
||||
return results;
|
||||
}
|
||||
|
||||
const scores = results.map((r) => r.score);
|
||||
const min = Math.min(...scores);
|
||||
const max = Math.max(...scores);
|
||||
const range = max - min;
|
||||
|
||||
if (range === 0) {
|
||||
// All scores are the same
|
||||
return results.map((r) => ({ ...r, score: 1 }));
|
||||
}
|
||||
|
||||
return results.map((r) => ({
|
||||
...r,
|
||||
score: (r.score - min) / range,
|
||||
}));
|
||||
}
|
||||
|
||||
// ==================== Context Building ====================
|
||||
|
||||
/**
|
||||
* Get memory context for query
|
||||
*/
|
||||
async getContext(
|
||||
query: string,
|
||||
options?: HybridSearchOptions,
|
||||
): Promise<string> {
|
||||
const results = await this.search(query, options);
|
||||
|
||||
if (results.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Format as XML
|
||||
const memories = results
|
||||
.map(
|
||||
(r, i) =>
|
||||
` <memory score="${r.score.toFixed(2)}" source="${r.source}">
|
||||
${r.entry.text}
|
||||
</memory>`,
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
return `<memories>
|
||||
${memories}
|
||||
</memories>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get memory context formatted as markdown
|
||||
*/
|
||||
async getContextMarkdown(
|
||||
query: string,
|
||||
options?: HybridSearchOptions,
|
||||
): Promise<string> {
|
||||
const results = await this.search(query, options);
|
||||
|
||||
if (results.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return results
|
||||
.map((r) => `- ${r.entry.text} (relevance: ${r.score.toFixed(2)})`)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
// ==================== Filtering ====================
|
||||
|
||||
/**
|
||||
* Get recent daily memory sources for search scope
|
||||
*/
|
||||
getRecentDailySources(): string[] {
|
||||
const sources: string[] = [];
|
||||
const today = new Date();
|
||||
|
||||
for (let i = 0; i < this.dailyMemoryDays; i++) {
|
||||
const date = new Date(today);
|
||||
date.setDate(date.getDate() - i);
|
||||
const dateStr = date.toISOString().split("T")[0];
|
||||
sources.push(`memory/${dateStr}.md`);
|
||||
}
|
||||
|
||||
return sources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter results by source
|
||||
*/
|
||||
filterBySource(
|
||||
results: MemorySearchResult[],
|
||||
sources: MemorySource[],
|
||||
): MemorySearchResult[] {
|
||||
const sourceSet = new Set(sources);
|
||||
return results.filter((r) => sourceSet.has(r.entry.source));
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton
|
||||
export const memoryRetriever = new MemoryRetriever();
|
||||
@@ -0,0 +1,594 @@
|
||||
/**
|
||||
* Memory Scheduler Module
|
||||
*
|
||||
* Cron-based daily consolidation and cleanup tasks
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import * as cron from "node-cron";
|
||||
import log from "electron-log";
|
||||
import type {
|
||||
MemoryConfig,
|
||||
ConsolidationResult,
|
||||
CleanupResult,
|
||||
ModelConfig,
|
||||
} from "./types";
|
||||
import { MemoryFileSync } from "./MemoryFileSync";
|
||||
import { MemoryDatabase } from "./MemoryDatabase";
|
||||
import { TranscriptWriter } from "./TranscriptWriter";
|
||||
import { LLM_CONSOLIDATION_PROMPT } from "./constants";
|
||||
import { callLlmApi } from "./utils/llmClient";
|
||||
|
||||
// ==================== Types ====================
|
||||
|
||||
interface SchedulerConfig {
|
||||
consolidationCron: string;
|
||||
cleanupCron: string;
|
||||
consolidationEnabled: boolean;
|
||||
cleanupEnabled: boolean;
|
||||
dailyRetentionDays: number;
|
||||
transcriptRetentionDays: number;
|
||||
stalePendingHours: number;
|
||||
timezone: string;
|
||||
}
|
||||
|
||||
// ==================== MemoryScheduler Class ====================
|
||||
|
||||
export class MemoryScheduler extends EventEmitter {
|
||||
private fileSync: MemoryFileSync | null = null;
|
||||
private database: MemoryDatabase | null = null;
|
||||
private transcriptWriter: TranscriptWriter | null = null;
|
||||
private consolidationJob: cron.ScheduledTask | null = null;
|
||||
private cleanupJob: cron.ScheduledTask | null = null;
|
||||
private running: boolean = false;
|
||||
|
||||
/** Stored model config for cron-triggered consolidation */
|
||||
private storedModelConfig: ModelConfig | null = null;
|
||||
|
||||
/** Provider for active session IDs (to protect from cleanup) */
|
||||
private activeSessionProvider: (() => Set<string>) | null = null;
|
||||
|
||||
private config: SchedulerConfig = {
|
||||
consolidationCron: "0 0 * * *", // 00:00 daily
|
||||
cleanupCron: "0 1 * * *", // 01:00 daily
|
||||
consolidationEnabled: true,
|
||||
cleanupEnabled: true,
|
||||
dailyRetentionDays: 30,
|
||||
transcriptRetentionDays: 7,
|
||||
stalePendingHours: 24,
|
||||
timezone:
|
||||
Intl.DateTimeFormat().resolvedOptions().timeZone || "Asia/Shanghai",
|
||||
};
|
||||
|
||||
/**
|
||||
* Initialize scheduler
|
||||
*/
|
||||
init(
|
||||
fileSync: MemoryFileSync,
|
||||
database: MemoryDatabase,
|
||||
transcriptWriter?: TranscriptWriter,
|
||||
): void {
|
||||
this.fileSync = fileSync;
|
||||
this.database = database;
|
||||
this.transcriptWriter = transcriptWriter ?? null;
|
||||
log.info("[MemoryScheduler] Initialized");
|
||||
}
|
||||
|
||||
/**
|
||||
* Set model config for cron-triggered consolidation
|
||||
* This allows the cron job to use LLM consolidation instead of simple merge
|
||||
*/
|
||||
setModelConfig(config: ModelConfig): void {
|
||||
this.storedModelConfig = config;
|
||||
log.debug("[MemoryScheduler] Model config stored for cron consolidation");
|
||||
}
|
||||
|
||||
/**
|
||||
* Set active session provider for cleanup protection
|
||||
* Active sessions' transcript files will be skipped during cleanup
|
||||
*/
|
||||
setActiveSessionProvider(provider: () => Set<string>): void {
|
||||
this.activeSessionProvider = provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy scheduler
|
||||
*/
|
||||
destroy(): void {
|
||||
this.stop();
|
||||
this.fileSync = null;
|
||||
this.database = null;
|
||||
this.transcriptWriter = null;
|
||||
this.storedModelConfig = null;
|
||||
this.activeSessionProvider = null;
|
||||
log.info("[MemoryScheduler] Destroyed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure scheduler
|
||||
*/
|
||||
configure(
|
||||
config: Partial<SchedulerConfig> & { dailyRetentionDays?: number },
|
||||
): void {
|
||||
if (config.consolidationCron !== undefined) {
|
||||
this.config.consolidationCron = config.consolidationCron;
|
||||
}
|
||||
if (config.cleanupCron !== undefined) {
|
||||
this.config.cleanupCron = config.cleanupCron;
|
||||
}
|
||||
if (config.consolidationEnabled !== undefined) {
|
||||
this.config.consolidationEnabled = config.consolidationEnabled;
|
||||
}
|
||||
if (config.cleanupEnabled !== undefined) {
|
||||
this.config.cleanupEnabled = config.cleanupEnabled;
|
||||
}
|
||||
if (config.dailyRetentionDays !== undefined) {
|
||||
this.config.dailyRetentionDays = config.dailyRetentionDays;
|
||||
}
|
||||
if (config.transcriptRetentionDays !== undefined) {
|
||||
this.config.transcriptRetentionDays = config.transcriptRetentionDays;
|
||||
}
|
||||
if (config.stalePendingHours !== undefined) {
|
||||
this.config.stalePendingHours = config.stalePendingHours;
|
||||
}
|
||||
if (config.timezone !== undefined) {
|
||||
this.config.timezone = config.timezone;
|
||||
}
|
||||
|
||||
// Restart if running
|
||||
if (this.running) {
|
||||
this.stop();
|
||||
this.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start scheduled tasks
|
||||
*/
|
||||
start(): void {
|
||||
if (this.running) return;
|
||||
|
||||
// Consolidation job (00:00)
|
||||
if (this.config.consolidationEnabled) {
|
||||
try {
|
||||
this.consolidationJob = cron.schedule(
|
||||
this.config.consolidationCron,
|
||||
() => this.runConsolidation(this.storedModelConfig ?? undefined),
|
||||
{ timezone: this.config.timezone },
|
||||
);
|
||||
log.info(
|
||||
"[MemoryScheduler] Consolidation job scheduled:",
|
||||
this.config.consolidationCron,
|
||||
);
|
||||
} catch (error) {
|
||||
log.error("[MemoryScheduler] Failed to schedule consolidation:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup job (01:00)
|
||||
if (this.config.cleanupEnabled) {
|
||||
try {
|
||||
this.cleanupJob = cron.schedule(
|
||||
this.config.cleanupCron,
|
||||
() => this.runCleanup(),
|
||||
{ timezone: this.config.timezone },
|
||||
);
|
||||
log.info(
|
||||
"[MemoryScheduler] Cleanup job scheduled:",
|
||||
this.config.cleanupCron,
|
||||
);
|
||||
} catch (error) {
|
||||
log.error("[MemoryScheduler] Failed to schedule cleanup:", error);
|
||||
}
|
||||
}
|
||||
|
||||
this.running = true;
|
||||
log.info("[MemoryScheduler] Started");
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop scheduled tasks
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.consolidationJob) {
|
||||
this.consolidationJob.stop();
|
||||
this.consolidationJob = null;
|
||||
}
|
||||
|
||||
if (this.cleanupJob) {
|
||||
this.cleanupJob.stop();
|
||||
this.cleanupJob = null;
|
||||
}
|
||||
|
||||
this.running = false;
|
||||
log.info("[MemoryScheduler] Stopped");
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if scheduler is running
|
||||
*/
|
||||
isRunning(): boolean {
|
||||
return this.running;
|
||||
}
|
||||
|
||||
// ==================== Consolidation ====================
|
||||
|
||||
/**
|
||||
* Run consolidation task
|
||||
*
|
||||
* Merges recent daily memories into core MEMORY.md
|
||||
* Uses LLM consolidation if modelConfig is provided, otherwise falls back to simple merge
|
||||
*
|
||||
* @param modelConfig - Optional model config for LLM-based consolidation
|
||||
*/
|
||||
async runConsolidation(modelConfig?: {
|
||||
provider: string;
|
||||
model: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
apiProtocol?: string;
|
||||
}): Promise<ConsolidationResult> {
|
||||
log.info("[MemoryScheduler] Running consolidation...");
|
||||
|
||||
const result: ConsolidationResult = {
|
||||
success: false,
|
||||
memoriesProcessed: 0,
|
||||
memoriesAdded: 0,
|
||||
memoriesMerged: 0,
|
||||
};
|
||||
|
||||
if (!this.fileSync || !this.database) {
|
||||
result.error = "Scheduler not initialized";
|
||||
return result;
|
||||
}
|
||||
|
||||
try {
|
||||
// Read recent daily memories (last 2 days)
|
||||
const dailyMemories = this.fileSync.readRecentDailyMemories(2);
|
||||
|
||||
if (dailyMemories.size === 0) {
|
||||
log.info("[MemoryScheduler] No daily memories to consolidate");
|
||||
result.success = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Read current core memory
|
||||
const coreMemory = this.fileSync.readCoreMemory();
|
||||
|
||||
// Count memories to process
|
||||
for (const content of dailyMemories.values()) {
|
||||
const lines = content.split("\n").filter((l) => l.startsWith("- "));
|
||||
result.memoriesProcessed += lines.length;
|
||||
}
|
||||
|
||||
// Use LLM consolidation if model config is provided, otherwise use simple merge
|
||||
let consolidatedContent: string;
|
||||
if (modelConfig?.apiKey) {
|
||||
log.info(
|
||||
"[MemoryScheduler] Using LLM consolidation with",
|
||||
modelConfig.provider,
|
||||
);
|
||||
try {
|
||||
consolidatedContent = await this.llmConsolidation(
|
||||
dailyMemories,
|
||||
coreMemory,
|
||||
modelConfig,
|
||||
);
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
"[MemoryScheduler] LLM consolidation failed, falling back to simple merge:",
|
||||
error,
|
||||
);
|
||||
consolidatedContent = this.simpleConsolidation(
|
||||
dailyMemories,
|
||||
coreMemory,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
log.debug(
|
||||
"[MemoryScheduler] Using simple consolidation (no LLM config provided)",
|
||||
);
|
||||
consolidatedContent = this.simpleConsolidation(
|
||||
dailyMemories,
|
||||
coreMemory,
|
||||
);
|
||||
}
|
||||
|
||||
// Write updated core memory
|
||||
this.fileSync.writeCoreMemory(consolidatedContent);
|
||||
|
||||
// Update meta
|
||||
this.database.setMeta("consolidation_last_run", Date.now().toString());
|
||||
|
||||
result.success = true;
|
||||
result.memoriesAdded = result.memoriesProcessed; // Simplified
|
||||
|
||||
log.info("[MemoryScheduler] Consolidation complete:", result);
|
||||
this.emit("consolidation:complete", result);
|
||||
} catch (error) {
|
||||
result.error = String(error);
|
||||
log.error("[MemoryScheduler] Consolidation failed:", error);
|
||||
this.emit("consolidation:error", result);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* LLM-based consolidation
|
||||
* Uses LLM to intelligently merge and deduplicate memories
|
||||
*/
|
||||
private async llmConsolidation(
|
||||
dailyMemories: Map<string, string>,
|
||||
coreMemory: string,
|
||||
modelConfig: {
|
||||
provider: string;
|
||||
model: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
apiProtocol?: string;
|
||||
},
|
||||
): Promise<string> {
|
||||
const prompt = this.buildConsolidationPrompt(dailyMemories, coreMemory);
|
||||
|
||||
try {
|
||||
const response = await callLlmApi(prompt, {
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
apiKey: modelConfig.apiKey,
|
||||
baseUrl: modelConfig.baseUrl,
|
||||
apiProtocol: modelConfig.apiProtocol,
|
||||
maxTokens: 2000,
|
||||
});
|
||||
return this.parseConsolidationResponse(response, coreMemory);
|
||||
} catch (error) {
|
||||
log.error("[MemoryScheduler] LLM consolidation call failed:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse consolidation response from LLM
|
||||
* Extracts the new MEMORY.md content from the response
|
||||
*/
|
||||
private parseConsolidationResponse(
|
||||
response: string,
|
||||
originalCoreMemory: string,
|
||||
): string {
|
||||
// Try to extract content between markdown code blocks
|
||||
const codeBlockMatch = response.match(/```markdown\n?([\s\S]*?)\n?```/);
|
||||
if (codeBlockMatch) {
|
||||
return codeBlockMatch[1].trim();
|
||||
}
|
||||
|
||||
// Core MEMORY.md uses English title only (see MemoryFileSync default + LLM_CONSOLIDATION_PROMPT)
|
||||
const headerMatch = response.match(/(#\s*Long-term Memory[\s\S]*)/i);
|
||||
if (headerMatch) {
|
||||
return headerMatch[1].trim();
|
||||
}
|
||||
|
||||
// If response looks like valid markdown, use it directly
|
||||
if (response.includes("## ") && response.includes("- ")) {
|
||||
return response.trim();
|
||||
}
|
||||
|
||||
// Fallback to original
|
||||
log.warn(
|
||||
"[MemoryScheduler] Could not parse LLM consolidation response, keeping original",
|
||||
);
|
||||
return originalCoreMemory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build consolidation prompt for LLM
|
||||
*/
|
||||
buildConsolidationPrompt(
|
||||
dailyMemories: Map<string, string>,
|
||||
coreMemory: string,
|
||||
): string {
|
||||
const dailyContent = Array.from(dailyMemories.entries())
|
||||
.map(([date, content]) => `### ${date}\n${content}`)
|
||||
.join("\n\n");
|
||||
|
||||
return LLM_CONSOLIDATION_PROMPT.replace(
|
||||
"{daily_memories}",
|
||||
dailyContent,
|
||||
).replace("{core_memories}", coreMemory || "(none)");
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple consolidation without LLM
|
||||
*/
|
||||
private simpleConsolidation(
|
||||
dailyMemories: Map<string, string>,
|
||||
coreMemory: string,
|
||||
): string {
|
||||
// Extract new facts from daily memories
|
||||
const newFacts: string[] = [];
|
||||
|
||||
for (const [date, content] of dailyMemories.entries()) {
|
||||
const lines = content
|
||||
.split("\n")
|
||||
.filter((l) => l.trim().startsWith("- "))
|
||||
.map((l) => l.trim().slice(2));
|
||||
|
||||
for (const line of lines) {
|
||||
// Check if this fact already exists in core memory
|
||||
if (!coreMemory.includes(line)) {
|
||||
newFacts.push(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no new facts, return original
|
||||
if (newFacts.length === 0) {
|
||||
return coreMemory;
|
||||
}
|
||||
|
||||
// Categorize facts by keyword matching (section titles match English MEMORY.md)
|
||||
const categorized: Record<string, string[]> = {
|
||||
Preferences: [],
|
||||
"User profile": [],
|
||||
"Project-related": [],
|
||||
"Important decisions": [],
|
||||
};
|
||||
|
||||
for (const fact of newFacts) {
|
||||
const lowerFact = fact.toLowerCase();
|
||||
if (
|
||||
/喜欢|偏好|习惯|倾向|prefer|like|usually|favorite|favourite/.test(
|
||||
lowerFact,
|
||||
)
|
||||
) {
|
||||
categorized.Preferences.push(fact);
|
||||
} else if (
|
||||
/名字|职业|住在|年龄|邮箱|电话|name|work|live|age|email|phone|occupation/.test(
|
||||
lowerFact,
|
||||
)
|
||||
) {
|
||||
categorized["User profile"].push(fact);
|
||||
} else if (
|
||||
/项目|repo|仓库|project|codebase|代码库|技术栈|框架/.test(lowerFact)
|
||||
) {
|
||||
categorized["Project-related"].push(fact);
|
||||
} else {
|
||||
categorized["Important decisions"].push(fact);
|
||||
}
|
||||
}
|
||||
|
||||
const today = new Date().toISOString().split("T")[0];
|
||||
let updated = coreMemory;
|
||||
|
||||
// Append to each section that has new facts
|
||||
for (const [section, facts] of Object.entries(categorized)) {
|
||||
if (facts.length === 0) continue;
|
||||
|
||||
const newSection = facts.map((f) => `- ${f}`).join("\n");
|
||||
const sectionHeader = `## ${section}`;
|
||||
|
||||
if (updated.includes(sectionHeader)) {
|
||||
// Append to existing section (after the header line, handle both \n and \r\n)
|
||||
updated = updated.replace(
|
||||
new RegExp(
|
||||
`(${sectionHeader.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\r?\\n)`,
|
||||
),
|
||||
`$1\n### ${today}\n${newSection}\n\n`,
|
||||
);
|
||||
} else {
|
||||
updated += `\n\n${sectionHeader}\n\n### ${today}\n${newSection}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// Footer line is English-only (*Last updated: YYYY-MM-DD*), matching default MEMORY.md
|
||||
updated = updated.replace(
|
||||
/\*Last updated:\s*[^*]+\*/,
|
||||
`*Last updated: ${today}*`,
|
||||
);
|
||||
if (!/\*Last updated:\s*[^*]+\*/.test(updated)) {
|
||||
updated = `${updated.trimEnd()}\n\n---\n*Last updated: ${today}*\n`;
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
// ==================== Cleanup ====================
|
||||
|
||||
/**
|
||||
* Run cleanup task
|
||||
*
|
||||
* Three-part cleanup:
|
||||
* 1. Daily memory files older than dailyRetentionDays
|
||||
* 2. Transcript files older than transcriptRetentionDays
|
||||
* 3. Extraction progress records (completed/failed older than transcriptRetentionDays, stale pending)
|
||||
*/
|
||||
async runCleanup(): Promise<CleanupResult> {
|
||||
log.info("[MemoryScheduler] Running cleanup...");
|
||||
|
||||
const result: CleanupResult = {
|
||||
success: false,
|
||||
filesDeleted: 0,
|
||||
memoriesDeleted: 0,
|
||||
transcriptsDeleted: 0,
|
||||
progressRecordsCleaned: 0,
|
||||
};
|
||||
|
||||
if (!this.fileSync || !this.database) {
|
||||
result.error = "Scheduler not initialized";
|
||||
return result;
|
||||
}
|
||||
|
||||
try {
|
||||
// Cleanup 1: Delete old daily memory files
|
||||
const filesDeleted = this.fileSync.deleteOldDailyFiles(
|
||||
this.config.dailyRetentionDays,
|
||||
);
|
||||
result.filesDeleted = filesDeleted;
|
||||
|
||||
// Cleanup 2: Delete old transcript files (skip active sessions)
|
||||
if (this.transcriptWriter) {
|
||||
const activeSessionIds = this.activeSessionProvider?.() ?? undefined;
|
||||
const transcriptsDeleted = this.transcriptWriter.cleanupOldTranscripts(
|
||||
this.config.transcriptRetentionDays,
|
||||
activeSessionIds,
|
||||
);
|
||||
result.transcriptsDeleted = transcriptsDeleted;
|
||||
}
|
||||
|
||||
// Cleanup 3: Cleanup extraction progress records
|
||||
const progressCleanup = this.database.cleanupExtractionProgress(
|
||||
this.config.transcriptRetentionDays,
|
||||
this.config.stalePendingHours,
|
||||
);
|
||||
result.progressRecordsCleaned =
|
||||
progressCleanup.deleted + progressCleanup.markedFailed;
|
||||
|
||||
// Update meta
|
||||
this.database.setMeta("cleanup_last_run", Date.now().toString());
|
||||
|
||||
result.success = true;
|
||||
|
||||
log.info("[MemoryScheduler] Cleanup complete:", result);
|
||||
this.emit("cleanup:complete", result);
|
||||
} catch (error) {
|
||||
result.error = String(error);
|
||||
log.error("[MemoryScheduler] Cleanup failed:", error);
|
||||
this.emit("cleanup:error", result);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ==================== Status ====================
|
||||
|
||||
/**
|
||||
* Get scheduler status
|
||||
*/
|
||||
getStatus(): {
|
||||
running: boolean;
|
||||
consolidationEnabled: boolean;
|
||||
cleanupEnabled: boolean;
|
||||
lastConsolidation: number;
|
||||
lastCleanup: number;
|
||||
} {
|
||||
const lastConsolidation = parseInt(
|
||||
this.database?.getMeta("consolidation_last_run") ?? "0",
|
||||
10,
|
||||
);
|
||||
const lastCleanup = parseInt(
|
||||
this.database?.getMeta("cleanup_last_run") ?? "0",
|
||||
10,
|
||||
);
|
||||
|
||||
return {
|
||||
running: this.running,
|
||||
consolidationEnabled: this.config.consolidationEnabled,
|
||||
cleanupEnabled: this.config.cleanupEnabled,
|
||||
lastConsolidation,
|
||||
lastCleanup,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton
|
||||
export const memoryScheduler = new MemoryScheduler();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,361 @@
|
||||
/**
|
||||
* Transcript Writer Module
|
||||
*
|
||||
* Manages JSONL session transcript files for memory extraction.
|
||||
* Each session's messages are appended to a JSONL file in memory/transcripts/.
|
||||
*
|
||||
* Based on specs/long-memory/long-memory.md Section 5.2
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import log from 'electron-log';
|
||||
import type { TranscriptEntry } from './types';
|
||||
import { TRANSCRIPT_DIR } from './constants';
|
||||
|
||||
// ==================== Sensitive Data Patterns ====================
|
||||
|
||||
const SENSITIVE_PATTERNS = [
|
||||
// API keys
|
||||
/(?:api[_-]?key|apikey|secret[_-]?key)\s*[:=]\s*['"]?[\w\-]{20,}['"]?/gi,
|
||||
// Bearer tokens
|
||||
/Bearer\s+[\w\-\.]{20,}/gi,
|
||||
// Generic tokens
|
||||
/(?:token|authorization)\s*[:=]\s*['"]?[\w\-\.]{20,}['"]?/gi,
|
||||
// AWS keys
|
||||
/AKIA[\w]{16}/g,
|
||||
// Base64-encoded credentials
|
||||
/(?:password|passwd|pwd)\s*[:=]\s*['"]?[^\s'"]{8,}['"]?/gi,
|
||||
];
|
||||
|
||||
const SENSITIVE_REPLACEMENT = '[REDACTED]';
|
||||
|
||||
// ==================== TranscriptWriter Class ====================
|
||||
|
||||
export class TranscriptWriter {
|
||||
private workspaceDir: string = '';
|
||||
private transcriptsDir: string = '';
|
||||
private initialized: boolean = false;
|
||||
|
||||
/** In-memory cache of message counts per session */
|
||||
private messageCounts = new Map<string, number>();
|
||||
|
||||
/**
|
||||
* Initialize transcript writer
|
||||
*/
|
||||
init(workspaceDir: string): void {
|
||||
this.workspaceDir = workspaceDir;
|
||||
// Store transcripts in memory/transcripts/ directory
|
||||
// TRANSCRIPT_DIR is already a full relative path (memory/transcripts)
|
||||
this.transcriptsDir = path.join(workspaceDir, TRANSCRIPT_DIR);
|
||||
|
||||
// Ensure transcripts directory exists
|
||||
if (!fs.existsSync(this.transcriptsDir)) {
|
||||
fs.mkdirSync(this.transcriptsDir, { recursive: true });
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
log.info('[TranscriptWriter] Initialized at:', this.transcriptsDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy transcript writer
|
||||
*/
|
||||
destroy(): void {
|
||||
this.initialized = false;
|
||||
this.workspaceDir = '';
|
||||
this.transcriptsDir = '';
|
||||
this.messageCounts.clear();
|
||||
}
|
||||
|
||||
// ==================== Write Operations ====================
|
||||
|
||||
/**
|
||||
* Append a message to session transcript (JSONL)
|
||||
*
|
||||
* Writes are append-only and atomic (single line).
|
||||
* Content is sanitized to remove sensitive data before writing.
|
||||
*/
|
||||
appendMessage(
|
||||
sessionId: string,
|
||||
role: 'user' | 'assistant',
|
||||
content: string,
|
||||
msgId: string
|
||||
): void {
|
||||
if (!this.initialized) {
|
||||
log.warn('[TranscriptWriter] Not initialized, skipping append');
|
||||
return;
|
||||
}
|
||||
|
||||
const sanitizedContent = this.sanitizeContent(content);
|
||||
|
||||
const entry: TranscriptEntry = {
|
||||
ts: Date.now(),
|
||||
role,
|
||||
content: sanitizedContent,
|
||||
msgId,
|
||||
};
|
||||
|
||||
const line = JSON.stringify(entry) + '\n';
|
||||
const filePath = this.getTranscriptPath(sessionId);
|
||||
|
||||
try {
|
||||
fs.appendFileSync(filePath, line, 'utf-8');
|
||||
// Update message count cache (recover from file on first append after restart)
|
||||
if (!this.messageCounts.has(sessionId)) {
|
||||
// Cold start: count existing lines in the file to initialize cache correctly
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
this.messageCounts.set(sessionId, content.split('\n').filter(l => l.trim().length > 0).length);
|
||||
} catch {
|
||||
this.messageCounts.set(sessionId, 1);
|
||||
}
|
||||
} else {
|
||||
this.messageCounts.set(sessionId, this.messageCounts.get(sessionId)! + 1);
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to append message:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Read Operations ====================
|
||||
|
||||
/**
|
||||
* Read complete transcript for a session
|
||||
*/
|
||||
readTranscript(sessionId: string): TranscriptEntry[] {
|
||||
const filePath = this.getTranscriptPath(sessionId);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
return this.parseJsonl(content);
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to read transcript:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a range of messages from transcript
|
||||
*
|
||||
* Optimized: skips JSON.parse for lines outside the range,
|
||||
* and stops reading once endIndex is reached.
|
||||
*
|
||||
* @param sessionId - Session ID
|
||||
* @param startIndex - Start index (inclusive, 0-based)
|
||||
* @param endIndex - End index (exclusive)
|
||||
*/
|
||||
readTranscriptRange(
|
||||
sessionId: string,
|
||||
startIndex: number,
|
||||
endIndex: number
|
||||
): TranscriptEntry[] {
|
||||
const filePath = this.getTranscriptPath(sessionId);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
const lines = content.split('\n');
|
||||
const entries: TranscriptEntry[] = [];
|
||||
let lineIndex = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
|
||||
if (lineIndex >= endIndex) break;
|
||||
|
||||
if (lineIndex >= startIndex) {
|
||||
try {
|
||||
entries.push(JSON.parse(trimmed) as TranscriptEntry);
|
||||
} catch {
|
||||
log.warn('[TranscriptWriter] Skipping malformed JSONL line in range read');
|
||||
}
|
||||
}
|
||||
|
||||
lineIndex++;
|
||||
}
|
||||
|
||||
return entries;
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to read transcript range:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count messages in transcript
|
||||
* Uses in-memory cache when available, falls back to file read on cold start
|
||||
*/
|
||||
countMessages(sessionId: string): number {
|
||||
// Fast path: return cached count
|
||||
const cached = this.messageCounts.get(sessionId);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Cold start: read file and cache the count
|
||||
const filePath = this.getTranscriptPath(sessionId);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
// Count non-empty lines
|
||||
const count = content.split('\n').filter(line => line.trim().length > 0).length;
|
||||
this.messageCounts.set(sessionId, count);
|
||||
return count;
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to count messages:', error);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Path Operations ====================
|
||||
|
||||
/**
|
||||
* Get transcript file path for a session
|
||||
*/
|
||||
getTranscriptPath(sessionId: string): string {
|
||||
// Sanitize sessionId to prevent path traversal
|
||||
const safeId = sessionId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return path.join(this.transcriptsDir, `${safeId}.jsonl`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if transcript exists for a session
|
||||
*/
|
||||
hasTranscript(sessionId: string): boolean {
|
||||
return fs.existsSync(this.getTranscriptPath(sessionId));
|
||||
}
|
||||
|
||||
// ==================== Cleanup Operations ====================
|
||||
|
||||
/**
|
||||
* Delete transcript for a session
|
||||
*/
|
||||
deleteTranscript(sessionId: string): void {
|
||||
const filePath = this.getTranscriptPath(sessionId);
|
||||
|
||||
if (fs.existsSync(filePath)) {
|
||||
try {
|
||||
fs.unlinkSync(filePath);
|
||||
log.debug('[TranscriptWriter] Deleted transcript:', sessionId);
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to delete transcript:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup old transcript files
|
||||
*
|
||||
* Deletes transcript files whose mtime is older than retentionDays.
|
||||
* Skips files belonging to active sessions.
|
||||
* Returns the number of files deleted.
|
||||
*/
|
||||
cleanupOldTranscripts(retentionDays: number, activeSessionIds?: Set<string>): number {
|
||||
if (!this.initialized || !fs.existsSync(this.transcriptsDir)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
|
||||
let deleted = 0;
|
||||
|
||||
// Build a set of sanitized active session IDs for comparison with filenames
|
||||
let sanitizedActiveIds: Set<string> | undefined;
|
||||
if (activeSessionIds && activeSessionIds.size > 0) {
|
||||
sanitizedActiveIds = new Set(
|
||||
Array.from(activeSessionIds).map(id => id.replace(/[^a-zA-Z0-9_-]/g, '_'))
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const files = fs.readdirSync(this.transcriptsDir);
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.jsonl')) continue;
|
||||
|
||||
// Extract sessionId from filename (remove .jsonl extension)
|
||||
const sessionId = file.slice(0, -6);
|
||||
|
||||
// Skip active sessions (compare with sanitized IDs to match filename)
|
||||
if (sanitizedActiveIds && sanitizedActiveIds.has(sessionId)) {
|
||||
log.debug('[TranscriptWriter] Skipping active session transcript:', file);
|
||||
continue;
|
||||
}
|
||||
|
||||
const filePath = path.join(this.transcriptsDir, file);
|
||||
|
||||
try {
|
||||
const stats = fs.statSync(filePath);
|
||||
|
||||
if (stats.mtimeMs < cutoff) {
|
||||
fs.unlinkSync(filePath);
|
||||
deleted++;
|
||||
log.debug('[TranscriptWriter] Cleaned up old transcript:', file);
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to check/delete file:', file, error);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('[TranscriptWriter] Failed to cleanup transcripts:', error);
|
||||
}
|
||||
|
||||
if (deleted > 0) {
|
||||
log.info(`[TranscriptWriter] Cleaned up ${deleted} old transcript files`);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// ==================== Private Helpers ====================
|
||||
|
||||
/**
|
||||
* Sanitize content by removing sensitive data
|
||||
*/
|
||||
private sanitizeContent(content: string): string {
|
||||
let sanitized = content;
|
||||
|
||||
for (const pattern of SENSITIVE_PATTERNS) {
|
||||
sanitized = sanitized.replace(pattern, SENSITIVE_REPLACEMENT);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse JSONL content into TranscriptEntry array
|
||||
*/
|
||||
private parseJsonl(content: string): TranscriptEntry[] {
|
||||
const entries: TranscriptEntry[] = [];
|
||||
const lines = content.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
|
||||
try {
|
||||
const entry = JSON.parse(trimmed) as TranscriptEntry;
|
||||
entries.push(entry);
|
||||
} catch (error) {
|
||||
log.warn('[TranscriptWriter] Skipping malformed JSONL line');
|
||||
}
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton
|
||||
export const transcriptWriter = new TranscriptWriter();
|
||||
@@ -0,0 +1,256 @@
|
||||
/**
|
||||
* Long-Term Memory Constants
|
||||
*
|
||||
* Default values and constants based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
import * as path from "path";
|
||||
import type { MemoryConfig, SyncState } from "./types";
|
||||
|
||||
// ==================== File Names ====================
|
||||
|
||||
export const CORE_MEMORY_FILE = "MEMORY.md";
|
||||
|
||||
// Memory directory structure:
|
||||
// memory/
|
||||
// ├── index.sqlite (database)
|
||||
// ├── transcripts/ (session transcripts)
|
||||
// └── daily/ (daily memory files)
|
||||
export const MEMORY_ROOT_DIR = "memory";
|
||||
export const DAILY_MEMORY_DIR = path.join("memory", "daily"); // Daily memory files: memory/daily/
|
||||
export const MEMORY_DB_DIR = "memory"; // Database: memory/index.sqlite
|
||||
export const MEMORY_DB_FILE = "index.sqlite";
|
||||
export const TRANSCRIPT_DIR = path.join("memory", "transcripts"); // Transcripts: memory/transcripts/
|
||||
|
||||
// ==================== Default Configuration ====================
|
||||
|
||||
export const DEFAULT_CONFIG: MemoryConfig = {
|
||||
enabled: true,
|
||||
|
||||
extraction: {
|
||||
enabled: true,
|
||||
implicitEnabled: true,
|
||||
explicitEnabled: true,
|
||||
guardLevel: "standard",
|
||||
trigger: {
|
||||
onEveryTurn: true,
|
||||
onSegmentFull: true,
|
||||
onSessionEnd: true,
|
||||
onIdleTimeout: true, // Extract after idle timeout
|
||||
idleTimeoutMs: 60000, // 60 seconds idle timeout
|
||||
},
|
||||
llm: {
|
||||
maxTokensPerExtract: 800,
|
||||
temperature: 0.3,
|
||||
maxRetries: 2,
|
||||
},
|
||||
},
|
||||
|
||||
storage: {
|
||||
workspacePath: "",
|
||||
dailyRetentionDays: 30,
|
||||
maxMemories: 500,
|
||||
},
|
||||
|
||||
embedding: {
|
||||
enabled: false,
|
||||
backend: "auto",
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
dimensions: 1536,
|
||||
cacheMaxEntries: 10000,
|
||||
},
|
||||
|
||||
retrieval: {
|
||||
vectorWeight: 0.7,
|
||||
ftsWeight: 0.3,
|
||||
limit: 12,
|
||||
minScore: 0.4,
|
||||
dailyMemoryDays: 2,
|
||||
},
|
||||
|
||||
scheduler: {
|
||||
consolidationCron: "0 0 * * *", // 00:00 daily
|
||||
cleanupCron: "0 1 * * *", // 01:00 daily
|
||||
consolidationEnabled: true,
|
||||
cleanupEnabled: true,
|
||||
},
|
||||
|
||||
segmentation: {
|
||||
segmentSize: 5, // Changed from 5 to 1 for testing - triggers extraction after each message
|
||||
segmentOverlap: 0, // Changed from 2 to 0 for testing
|
||||
maxSegmentTokens: 4000,
|
||||
maxContentPerMessage: 1500,
|
||||
},
|
||||
|
||||
transcript: {
|
||||
enabled: true,
|
||||
retentionDays: 7,
|
||||
},
|
||||
|
||||
deduplication: {
|
||||
textSimilarityThreshold: 0.8,
|
||||
vectorSimilarityThreshold: 0.95,
|
||||
},
|
||||
};
|
||||
|
||||
// ==================== Sync State Defaults ====================
|
||||
|
||||
export const DEFAULT_SYNC_STATE: SyncState = {
|
||||
dirty: false,
|
||||
syncing: false,
|
||||
lastSyncTime: 0,
|
||||
syncVersion: 0,
|
||||
};
|
||||
|
||||
// ==================== Chunking Constants ====================
|
||||
|
||||
export const CHUNK_MAX_CHARS = 1000;
|
||||
export const CHUNK_OVERLAP_CHARS = 100;
|
||||
|
||||
// ==================== File Watching Constants ====================
|
||||
|
||||
export const WATCH_DEBOUNCE_MS = 1500; // 1.5 seconds
|
||||
|
||||
// ==================== Vector Loading Constants ====================
|
||||
|
||||
export const VECTOR_LOAD_TIMEOUT_MS = 30000; // 30 seconds
|
||||
|
||||
// ==================== Session Sync Constants ====================
|
||||
|
||||
export const SESSION_SYNC_WAIT_TIMEOUT_MS = 5000; // 5 seconds max wait
|
||||
|
||||
// ==================== Scoring Constants ====================
|
||||
|
||||
// Positive scoring rules
|
||||
export const SCORE_PERSONAL_FACT = 0.3; // Contains personal fact
|
||||
export const SCORE_APPROPRIATE_LENGTH = 0.1; // Length 10-200 chars
|
||||
export const SCORE_CLEAR_PREFERENCE = 0.2; // Clear preference expression
|
||||
|
||||
// Negative scoring rules
|
||||
export const SCORE_QUESTION = -0.2; // Is a question
|
||||
export const SCORE_TEMPORARY = -0.2; // Temporary information
|
||||
export const SCORE_CODE = -0.3; // Pure code/instruction
|
||||
export const SCORE_SPECIFIC_TIME = -0.1; // Too specific time point
|
||||
|
||||
// Validation thresholds
|
||||
export const SCORE_MIN_ACCEPT = 0.6; // Minimum score to accept
|
||||
export const SCORE_LLM_THRESHOLD_LOW = 0.5; // Below this: reject
|
||||
export const SCORE_LLM_THRESHOLD_HIGH = 0.7; // Above this: accept without LLM
|
||||
|
||||
// ==================== Meta Keys ====================
|
||||
|
||||
export const META_KEYS = {
|
||||
SCHEMA_VERSION: "schema_version",
|
||||
EMBEDDING_ENABLED: "embedding_enabled",
|
||||
EMBEDDING_MODEL: "embedding_model",
|
||||
EMBEDDING_DIMS: "embedding_dims",
|
||||
EMBEDDING_PROVIDER: "embedding_provider",
|
||||
VECTOR_AVAILABLE: "vector_available",
|
||||
DIRTY: "dirty",
|
||||
SYNCING: "syncing",
|
||||
SYNC_VERSION: "sync_version",
|
||||
LAST_SYNC_TIME: "last_sync_time",
|
||||
CONSOLIDATION_LAST_RUN: "consolidation_last_run",
|
||||
CLEANUP_LAST_RUN: "cleanup_last_run",
|
||||
} as const;
|
||||
|
||||
// ==================== Schema Version ====================
|
||||
|
||||
export const SCHEMA_VERSION = "2";
|
||||
|
||||
// ==================== Memory Status ====================
|
||||
|
||||
export const MEMORY_STATUS = {
|
||||
ACTIVE: "active",
|
||||
ARCHIVED: "archived",
|
||||
DELETED: "deleted",
|
||||
} as const;
|
||||
|
||||
// ==================== Memory Categories ====================
|
||||
|
||||
export const MEMORY_CATEGORIES = {
|
||||
FACT: "fact",
|
||||
PREFERENCE: "preference",
|
||||
EVENT: "event",
|
||||
SKILL: "skill",
|
||||
DECISION: "decision",
|
||||
} as const;
|
||||
|
||||
// ==================== Memory Sources ====================
|
||||
|
||||
export const MEMORY_SOURCES = {
|
||||
CORE: "core",
|
||||
DAILY: "daily",
|
||||
} as const;
|
||||
|
||||
// ==================== LLM Prompts ====================
|
||||
|
||||
export const LLM_EXTRACTION_PROMPT = `You are a memory extraction assistant. From the conversation excerpt below, extract information worth remembering long-term.
|
||||
|
||||
## Conversation excerpt {segment_info}
|
||||
{conversation_history}
|
||||
|
||||
## Known memories (avoid duplicates)
|
||||
{existing_memories}
|
||||
|
||||
## Extraction rules
|
||||
1. Only extract personal facts, preferences, habits, and important decisions about the user.
|
||||
2. Ignore:
|
||||
- Ephemeral state (e.g. "I'm tired today")
|
||||
- Pure questions or instructions
|
||||
- Information that clearly duplicates known memories
|
||||
- The code itself (but preferences or decisions about code may be extracted)
|
||||
3. Each memory should be a standalone, complete statement.
|
||||
|
||||
## Output format
|
||||
Return a JSON array; each item has:
|
||||
{
|
||||
"text": "memory text",
|
||||
"category": "fact|preference|event|skill|decision",
|
||||
"confidence": 0.0-1.0
|
||||
}
|
||||
|
||||
If nothing is worth remembering, return an empty array [].`;
|
||||
|
||||
export const LLM_VALIDATION_PROMPT = `You are a memory validation assistant. Decide whether the candidate memory below is worth saving.
|
||||
|
||||
## Candidate memory
|
||||
{candidate_memory}
|
||||
|
||||
## Existing memories
|
||||
{existing_memories}
|
||||
|
||||
## Rules
|
||||
1. Does it conflict with or duplicate existing memories?
|
||||
2. Does it have long-term value?
|
||||
3. Is it personal information about the user rather than general knowledge?
|
||||
|
||||
## Output
|
||||
{
|
||||
"accept": true/false,
|
||||
"reason": "why accepted or rejected",
|
||||
"merged_text": "if merging is needed, the merged text"
|
||||
}`;
|
||||
|
||||
export const LLM_CONSOLIDATION_PROMPT = `You are a memory consolidation assistant. Merge recent daily memories into the core long-term memory.
|
||||
|
||||
## Daily memories (last ~2 days)
|
||||
{daily_memories}
|
||||
|
||||
## Current core memory
|
||||
{core_memories}
|
||||
|
||||
## Rules
|
||||
1. Pull important facts from the daily memories.
|
||||
2. Merge with core memory and deduplicate.
|
||||
3. Preserve structure and Markdown formatting.
|
||||
4. Organize under: User profile, Preferences, Project-related, Important decisions.
|
||||
|
||||
## Output
|
||||
Return the full updated MEMORY.md content in Markdown.`;
|
||||
|
||||
// ==================== Cleanup Constants ====================
|
||||
|
||||
export const DEFAULT_TRANSCRIPT_RETENTION_DAYS = 7;
|
||||
export const DEFAULT_STALE_PENDING_HOURS = 24;
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Memory Service Module
|
||||
*
|
||||
* Long-term memory system for Electron client
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
// ==================== Types ====================
|
||||
export * from './types';
|
||||
|
||||
// ==================== Constants ====================
|
||||
export * from './constants';
|
||||
|
||||
// ==================== Core Modules ====================
|
||||
export { MemoryDatabase, memoryDatabase } from './MemoryDatabase';
|
||||
export { MemoryFileSync, memoryFileSync } from './MemoryFileSync';
|
||||
export { MemoryExtractor, memoryExtractor } from './MemoryExtractor';
|
||||
export { ExtractionQueue } from './ExtractionQueue';
|
||||
export { MemoryRetriever, memoryRetriever } from './MemoryRetriever';
|
||||
export { MemoryInjector, memoryInjector } from './MemoryInjector';
|
||||
export { MemoryScheduler, memoryScheduler } from './MemoryScheduler';
|
||||
|
||||
// ==================== Main Service ====================
|
||||
export { MemoryService, memoryService } from './MemoryService';
|
||||
|
||||
// ==================== Utilities ====================
|
||||
export * from './utils/hash';
|
||||
export * from './utils/vector';
|
||||
export * from './utils/chunker';
|
||||
export * from './utils/signals';
|
||||
@@ -0,0 +1,344 @@
|
||||
/**
|
||||
* Long-Term Memory Type Definitions
|
||||
*
|
||||
* Based on specs/long-memory/long-memory.md
|
||||
*/
|
||||
|
||||
// ==================== Configuration Types ====================
|
||||
|
||||
export type GuardLevel = 'strict' | 'standard' | 'relaxed';
|
||||
export type EmbeddingBackend = 'auto' | 'sqlite-vec' | 'js' | 'none';
|
||||
export type EmbeddingProvider = 'openai' | 'ollama' | 'custom';
|
||||
export type MemoryCategory = 'fact' | 'preference' | 'event' | 'skill' | 'decision';
|
||||
export type MemorySource = 'core' | 'daily';
|
||||
export type MemoryStatus = 'active' | 'archived' | 'deleted';
|
||||
|
||||
export interface ExtractionTriggerConfig {
|
||||
onEveryTurn: boolean; // Check after every conversation turn
|
||||
onSegmentFull: boolean; // Extract when segment is full (default: true)
|
||||
onSessionEnd: boolean; // Extract on session end
|
||||
onIdleTimeout: boolean; // Extract after idle timeout (default: true)
|
||||
idleTimeoutMs: number; // Idle timeout in milliseconds (default: 60000)
|
||||
}
|
||||
|
||||
export interface ExtractionLlmConfig {
|
||||
maxTokensPerExtract: number; // Max tokens per extraction (default: 500)
|
||||
temperature: number; // LLM temperature (default: 0.3)
|
||||
maxRetries: number; // Max retry attempts (default: 2)
|
||||
}
|
||||
|
||||
export interface ExtractionConfig {
|
||||
enabled: boolean;
|
||||
implicitEnabled: boolean; // Enable implicit signal detection
|
||||
explicitEnabled: boolean; // Enable explicit command detection
|
||||
guardLevel: GuardLevel;
|
||||
trigger: ExtractionTriggerConfig;
|
||||
llm: ExtractionLlmConfig;
|
||||
}
|
||||
|
||||
export interface StorageConfig {
|
||||
workspacePath: string;
|
||||
dailyRetentionDays: number; // Days to keep daily memory files (default: 30)
|
||||
maxMemories: number; // Max memory entries (default: 500)
|
||||
}
|
||||
|
||||
export interface EmbeddingConfig {
|
||||
enabled: boolean;
|
||||
backend: EmbeddingBackend;
|
||||
provider: EmbeddingProvider;
|
||||
model: string; // e.g., 'text-embedding-3-small'
|
||||
dimensions: number; // Vector dimensions (default: 1536)
|
||||
apiKey?: string; // API key for embedding provider
|
||||
baseUrl?: string; // Custom API endpoint
|
||||
cacheMaxEntries: number; // Max cache entries (default: 10000)
|
||||
}
|
||||
|
||||
export interface RetrievalConfig {
|
||||
vectorWeight: number; // Vector search weight (default: 0.7)
|
||||
ftsWeight: number; // FTS search weight (default: 0.3)
|
||||
limit: number; // Max results (default: 12)
|
||||
minScore: number; // Minimum score threshold (default: 0.4)
|
||||
dailyMemoryDays: number; // Search recent N days of daily memories (default: 2)
|
||||
}
|
||||
|
||||
export interface SchedulerConfig {
|
||||
consolidationCron: string; // Consolidation task cron (default: '0 0 * * *')
|
||||
cleanupCron: string; // Cleanup task cron (default: '0 1 * * *')
|
||||
consolidationEnabled: boolean;
|
||||
cleanupEnabled: boolean;
|
||||
}
|
||||
|
||||
export interface SegmentationConfig {
|
||||
segmentSize: number; // Messages per segment (default: 5)
|
||||
segmentOverlap: number; // Overlap messages between segments (default: 2)
|
||||
maxSegmentTokens: number; // Max tokens per segment (default: 4000)
|
||||
maxContentPerMessage: number; // Max chars per message before truncation (default: 1500)
|
||||
}
|
||||
|
||||
export interface TranscriptConfig {
|
||||
enabled: boolean;
|
||||
retentionDays: number; // Days to keep transcript files (default: 7)
|
||||
}
|
||||
|
||||
export interface DeduplicationConfig {
|
||||
textSimilarityThreshold: number; // Jaccard similarity threshold (default: 0.8)
|
||||
vectorSimilarityThreshold: number; // Cosine similarity threshold (default: 0.95)
|
||||
}
|
||||
|
||||
export interface MemoryConfig {
|
||||
enabled: boolean;
|
||||
extraction: ExtractionConfig;
|
||||
storage: StorageConfig;
|
||||
embedding: EmbeddingConfig;
|
||||
retrieval: RetrievalConfig;
|
||||
scheduler: SchedulerConfig;
|
||||
segmentation: SegmentationConfig;
|
||||
transcript: TranscriptConfig;
|
||||
deduplication: DeduplicationConfig;
|
||||
}
|
||||
|
||||
// ==================== Memory Entry Types ====================
|
||||
|
||||
export interface MemoryEntry {
|
||||
id: string;
|
||||
text: string;
|
||||
fingerprint: string; // SHA256 hash for deduplication
|
||||
category: MemoryCategory;
|
||||
confidence: number; // 0-1 extraction confidence
|
||||
isExplicit: boolean; // Explicit user command
|
||||
importance: number; // 0-1 user-defined importance
|
||||
source: MemorySource; // 'core' (MEMORY.md) or 'daily' (memory/*.md)
|
||||
sourcePath: string; // Relative file path
|
||||
startLine?: number; // Markdown start line (1-indexed)
|
||||
endLine?: number; // Markdown end line (1-indexed)
|
||||
embedding?: Float32Array; // Vector embedding
|
||||
embeddingModel?: string;
|
||||
embeddingDims?: number;
|
||||
status: MemoryStatus;
|
||||
accessCount: number;
|
||||
lastAccessedAt?: number;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
// Database representation (BLOB fields as Buffer)
|
||||
export interface MemoryEntryRow {
|
||||
id: string;
|
||||
text: string;
|
||||
fingerprint: string;
|
||||
category: string;
|
||||
confidence: number;
|
||||
is_explicit: number;
|
||||
importance: number;
|
||||
source: string;
|
||||
source_path: string;
|
||||
start_line: number | null;
|
||||
end_line: number | null;
|
||||
embedding: Buffer | null;
|
||||
embedding_model: string | null;
|
||||
embedding_dims: number | null;
|
||||
status: string;
|
||||
access_count: number;
|
||||
last_accessed_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
// ==================== Extraction Types ====================
|
||||
|
||||
export interface ModelConfig {
|
||||
provider: string;
|
||||
model: string;
|
||||
apiKey: string; // Required! Electron has no global storage
|
||||
baseUrl?: string;
|
||||
apiProtocol?: string; // 'anthropic' or 'openai' - API protocol to use
|
||||
}
|
||||
|
||||
export interface ExtractionTask {
|
||||
sessionId: string;
|
||||
messageId: string;
|
||||
messages: Array<{
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
}>;
|
||||
modelConfig: ModelConfig; // Must capture full model config including API key
|
||||
timestamp: number;
|
||||
retryCount: number;
|
||||
segmentIndex?: number; // Segment index (0-based)
|
||||
startMsgIndex?: number; // Segment start message index
|
||||
endMsgIndex?: number; // Segment end message index (exclusive)
|
||||
}
|
||||
|
||||
export interface ExtractedMemory {
|
||||
text: string;
|
||||
category: MemoryCategory;
|
||||
confidence: number;
|
||||
isExplicit: boolean;
|
||||
}
|
||||
|
||||
export interface SignalMatch {
|
||||
type: 'explicit' | 'implicit';
|
||||
pattern: string;
|
||||
matchedText: string;
|
||||
extractedText?: string; // For explicit commands
|
||||
}
|
||||
|
||||
export interface ValidationResult {
|
||||
accept: boolean;
|
||||
reason?: string;
|
||||
mergedText?: string; // If merging with existing memory
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
// ==================== Search Types ====================
|
||||
|
||||
export interface SearchOptions {
|
||||
limit?: number;
|
||||
minScore?: number;
|
||||
categories?: MemoryCategory[];
|
||||
sources?: MemorySource[];
|
||||
includeVector?: boolean;
|
||||
checkDirty?: boolean; // Whether to check dirty state before search (default: true)
|
||||
}
|
||||
|
||||
export interface MemorySearchResult {
|
||||
entry: MemoryEntry;
|
||||
score: number;
|
||||
source: 'vector' | 'fts' | 'hybrid';
|
||||
}
|
||||
|
||||
export interface HybridSearchOptions extends SearchOptions {
|
||||
vectorWeight?: number;
|
||||
ftsWeight?: number;
|
||||
}
|
||||
|
||||
// ==================== Injection Types ====================
|
||||
|
||||
export interface InjectionOptions {
|
||||
maxTokens?: number;
|
||||
format?: 'xml' | 'markdown';
|
||||
includeScores?: boolean;
|
||||
}
|
||||
|
||||
// ==================== Sync Types ====================
|
||||
|
||||
export interface SyncState {
|
||||
dirty: boolean; // Has pending file changes
|
||||
syncing: boolean; // Sync in progress
|
||||
lastSyncTime: number;
|
||||
syncVersion: number; // For concurrency control
|
||||
}
|
||||
|
||||
export interface FileHashRecord {
|
||||
path: string; // Relative file path
|
||||
hash: string; // SHA256 of file content
|
||||
chunkCount: number;
|
||||
lastModified: number;
|
||||
syncedAt: number;
|
||||
}
|
||||
|
||||
export interface MemoryChunk {
|
||||
text: string;
|
||||
hash: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
added: number;
|
||||
removed: number;
|
||||
unchanged: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
// ==================== Scheduler Types ====================
|
||||
|
||||
export interface ConsolidationResult {
|
||||
success: boolean;
|
||||
memoriesProcessed: number;
|
||||
memoriesAdded: number;
|
||||
memoriesMerged: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface CleanupResult {
|
||||
success: boolean;
|
||||
filesDeleted: number;
|
||||
memoriesDeleted: number;
|
||||
transcriptsDeleted: number;
|
||||
progressRecordsCleaned: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ==================== Status Types ====================
|
||||
|
||||
export interface MemoryServiceStatus {
|
||||
initialized: boolean;
|
||||
workspacePath: string | null;
|
||||
databasePath: string | null;
|
||||
vectorAvailable: boolean | null; // null = not tested
|
||||
totalMemories: number;
|
||||
activeMemories: number;
|
||||
syncState: SyncState;
|
||||
config: MemoryConfig;
|
||||
}
|
||||
|
||||
// ==================== Utility Types ====================
|
||||
|
||||
export type VectorAvailability = 'unknown' | 'available' | 'unavailable';
|
||||
|
||||
export interface EmbeddingCacheEntry {
|
||||
contentHash: string;
|
||||
embedding: Float32Array;
|
||||
model: string;
|
||||
dims: number;
|
||||
accessCount: number;
|
||||
createdAt: number;
|
||||
lastAccessedAt: number;
|
||||
}
|
||||
|
||||
// ==================== Transcript Types ====================
|
||||
|
||||
export interface TranscriptEntry {
|
||||
ts: number; // Unix millisecond timestamp
|
||||
role: 'user' | 'assistant';
|
||||
content: string; // Text content only (filtered tool_use/tool_result)
|
||||
msgId: string; // Unique message ID
|
||||
}
|
||||
|
||||
// ==================== Segmentation Types ====================
|
||||
|
||||
export interface Segment {
|
||||
index: number; // Segment index (0-based)
|
||||
messages: Array<{ role: 'user' | 'assistant'; content: string }>;
|
||||
startMsgIndex: number; // Start message index in transcript (inclusive)
|
||||
endMsgIndex: number; // End message index in transcript (exclusive)
|
||||
}
|
||||
|
||||
// ==================== Extraction Progress Types ====================
|
||||
|
||||
export type ExtractionProgressStatus = 'pending' | 'processing' | 'completed' | 'failed';
|
||||
|
||||
export interface ExtractionProgressRecord {
|
||||
sessionId: string;
|
||||
segmentIndex: number;
|
||||
startMsgIndex: number;
|
||||
endMsgIndex: number;
|
||||
status: ExtractionProgressStatus;
|
||||
memoriesExtracted: number;
|
||||
createdAt: number;
|
||||
completedAt: number | null;
|
||||
errorMessage: string | null;
|
||||
}
|
||||
|
||||
export interface ExtractionProgressRow {
|
||||
session_id: string;
|
||||
segment_index: number;
|
||||
start_msg_index: number;
|
||||
end_msg_index: number;
|
||||
status: string;
|
||||
memories_extracted: number;
|
||||
created_at: number;
|
||||
completed_at: number | null;
|
||||
error_message: string | null;
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
/**
|
||||
* Markdown Chunker
|
||||
*
|
||||
* Split Markdown files into chunks for indexing
|
||||
*/
|
||||
|
||||
import type { MemoryChunk } from '../types';
|
||||
import { calculateHash } from './hash';
|
||||
import { CHUNK_MAX_CHARS, CHUNK_OVERLAP_CHARS } from '../constants';
|
||||
|
||||
/**
|
||||
* Chunk options
|
||||
*/
|
||||
export interface ChunkOptions {
|
||||
maxChars?: number;
|
||||
overlapChars?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Markdown file into chunks
|
||||
* - Core memory (MEMORY.md): splits by ## headings (level 2)
|
||||
* - Daily memory (daily/*.md): splits by --- separator
|
||||
*/
|
||||
export function chunkMarkdown(
|
||||
content: string,
|
||||
sourcePath: string,
|
||||
options: ChunkOptions = {}
|
||||
): MemoryChunk[] {
|
||||
const maxChars = options.maxChars ?? CHUNK_MAX_CHARS;
|
||||
const overlapChars = options.overlapChars ?? CHUNK_OVERLAP_CHARS;
|
||||
|
||||
// Detect file type based on source path
|
||||
const isDailyMemory = sourcePath.includes('daily/');
|
||||
|
||||
if (isDailyMemory) {
|
||||
return chunkDailyMemory(content, maxChars);
|
||||
} else {
|
||||
return chunkCoreMemory(content, maxChars, overlapChars);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk core memory file (MEMORY.md)
|
||||
* Splits by ## headings (level 2)
|
||||
*/
|
||||
function chunkCoreMemory(
|
||||
content: string,
|
||||
maxChars: number,
|
||||
overlapChars: number
|
||||
): MemoryChunk[] {
|
||||
const chunks: MemoryChunk[] = [];
|
||||
const lines = content.split('\n');
|
||||
let currentChunk: string[] = [];
|
||||
let startLine = 1;
|
||||
let currentLine = 0;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
currentLine = i + 1; // 1-indexed
|
||||
|
||||
// Check for ## heading (section boundary)
|
||||
const isSectionHeading = /^##\s/.test(line);
|
||||
|
||||
if (isSectionHeading && currentChunk.length > 0) {
|
||||
// Save current chunk
|
||||
const chunkText = currentChunk.join('\n').trim();
|
||||
if (chunkText) {
|
||||
chunks.push(createChunk(chunkText, startLine, currentLine - 1));
|
||||
}
|
||||
|
||||
// Start new chunk
|
||||
currentChunk = [line];
|
||||
startLine = currentLine;
|
||||
} else {
|
||||
currentChunk.push(line);
|
||||
|
||||
// Check if chunk exceeds max size
|
||||
const chunkText = currentChunk.join('\n');
|
||||
if (chunkText.length > maxChars) {
|
||||
// Find a good break point
|
||||
const breakResult = findBreakPoint(currentChunk, maxChars, overlapChars);
|
||||
|
||||
if (breakResult) {
|
||||
// Save the chunk before break
|
||||
const chunkText = breakResult.before.join('\n').trim();
|
||||
if (chunkText) {
|
||||
chunks.push(createChunk(chunkText, startLine, startLine + breakResult.before.length - 1));
|
||||
}
|
||||
|
||||
// Continue with overlap
|
||||
currentChunk = breakResult.after;
|
||||
startLine = startLine + breakResult.before.length - breakResult.after.length;
|
||||
} else {
|
||||
// No good break point, just save what we have
|
||||
const chunkText = currentChunk.join('\n').trim();
|
||||
if (chunkText) {
|
||||
chunks.push(createChunk(chunkText, startLine, currentLine));
|
||||
}
|
||||
currentChunk = [];
|
||||
startLine = currentLine + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save remaining chunk
|
||||
if (currentChunk.length > 0) {
|
||||
const chunkText = currentChunk.join('\n').trim();
|
||||
if (chunkText) {
|
||||
chunks.push(createChunk(chunkText, startLine, currentLine));
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk daily memory file (daily/*.md)
|
||||
* Splits by --- separator
|
||||
*/
|
||||
function chunkDailyMemory(
|
||||
content: string,
|
||||
maxChars: number
|
||||
): MemoryChunk[] {
|
||||
const chunks: MemoryChunk[] = [];
|
||||
const lines = content.split('\n');
|
||||
let currentChunk: string[] = [];
|
||||
let startLine = 1;
|
||||
let currentLine = 0;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
currentLine = i + 1; // 1-indexed
|
||||
|
||||
// Check for --- separator (section boundary for daily memory)
|
||||
const isSeparator = /^---$/.test(line.trim());
|
||||
|
||||
if (isSeparator && currentChunk.length > 0) {
|
||||
// Save current chunk
|
||||
const chunkText = currentChunk.join('\n').trim();
|
||||
if (chunkText && chunkText.length >= 2) {
|
||||
chunks.push(createChunk(chunkText, startLine, currentLine - 1));
|
||||
}
|
||||
|
||||
// Start new chunk (skip the separator line itself)
|
||||
currentChunk = [];
|
||||
startLine = currentLine + 1;
|
||||
} else {
|
||||
currentChunk.push(line);
|
||||
|
||||
// Check if chunk exceeds max size (split by ### session heading as fallback)
|
||||
const chunkText = currentChunk.join('\n');
|
||||
if (chunkText.length > maxChars) {
|
||||
// Try to find a ### heading to split on
|
||||
const splitIndex = findSessionSplitPoint(currentChunk);
|
||||
if (splitIndex >= 0) {
|
||||
const before = currentChunk.slice(0, splitIndex);
|
||||
const beforeText = before.join('\n').trim();
|
||||
if (beforeText && beforeText.length >= 2) {
|
||||
chunks.push(createChunk(beforeText, startLine, startLine + before.length - 1));
|
||||
}
|
||||
currentChunk = currentChunk.slice(splitIndex);
|
||||
startLine = startLine + splitIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save remaining chunk
|
||||
if (currentChunk.length > 0) {
|
||||
const chunkText = currentChunk.join('\n').trim();
|
||||
if (chunkText && chunkText.length >= 2) {
|
||||
chunks.push(createChunk(chunkText, startLine, currentLine));
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a good split point in daily memory chunk (### session heading)
|
||||
*/
|
||||
function findSessionSplitPoint(lines: string[]): number {
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
if (/^###\s/.test(lines[i])) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a memory chunk object
|
||||
*/
|
||||
function createChunk(text: string, startLine: number, endLine: number): MemoryChunk {
|
||||
return {
|
||||
text,
|
||||
hash: calculateHash(text),
|
||||
startLine,
|
||||
endLine,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a good break point in a chunk
|
||||
*/
|
||||
function findBreakPoint(
|
||||
lines: string[],
|
||||
maxChars: number,
|
||||
overlapChars: number
|
||||
): { before: string[]; after: string[] } | null {
|
||||
// Try to find paragraph break (empty line)
|
||||
let accumulated = 0;
|
||||
let breakIndex = -1;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const lineLen = lines[i].length + 1; // +1 for newline
|
||||
|
||||
if (accumulated + lineLen > maxChars && breakIndex >= 0) {
|
||||
// Found a break point
|
||||
const before = lines.slice(0, breakIndex + 1);
|
||||
|
||||
// Calculate overlap
|
||||
let overlapAccum = 0;
|
||||
let overlapStart = before.length;
|
||||
for (let j = before.length - 1; j >= 0 && overlapAccum < overlapChars; j--) {
|
||||
overlapAccum += before[j].length + 1;
|
||||
overlapStart = j;
|
||||
}
|
||||
|
||||
const after = lines.slice(overlapStart);
|
||||
return { before, after };
|
||||
}
|
||||
|
||||
accumulated += lineLen;
|
||||
|
||||
// Mark paragraph breaks as potential break points
|
||||
if (lines[i].trim() === '' || lines[i].startsWith('###')) {
|
||||
breakIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse daily memory file to extract session blocks
|
||||
*/
|
||||
export function parseDailyMemoryFile(content: string): Array<{ time: string; content: string; startLine: number; endLine: number }> {
|
||||
const sessions: Array<{ time: string; content: string; startLine: number; endLine: number }> = [];
|
||||
const lines = content.split('\n');
|
||||
|
||||
let currentSession: { time: string; lines: string[]; startLine: number } | null = null;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const sessionMatch = line.match(/^###\s+(\d{1,2}:\d{2})\s+(.+)$/);
|
||||
|
||||
if (sessionMatch) {
|
||||
// Save previous session
|
||||
if (currentSession) {
|
||||
sessions.push({
|
||||
time: currentSession.time,
|
||||
content: currentSession.lines.join('\n').trim(),
|
||||
startLine: currentSession.startLine,
|
||||
endLine: i,
|
||||
});
|
||||
}
|
||||
|
||||
// Start new session
|
||||
currentSession = {
|
||||
time: `${sessionMatch[1]} ${sessionMatch[2]}`,
|
||||
lines: [],
|
||||
startLine: i + 1,
|
||||
};
|
||||
} else if (currentSession) {
|
||||
// Skip separator lines
|
||||
if (!line.match(/^---$/)) {
|
||||
currentSession.lines.push(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save last session
|
||||
if (currentSession) {
|
||||
sessions.push({
|
||||
time: currentSession.time,
|
||||
content: currentSession.lines.join('\n').trim(),
|
||||
startLine: currentSession.startLine,
|
||||
endLine: lines.length,
|
||||
});
|
||||
}
|
||||
|
||||
return sessions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare old and new chunks to find changes
|
||||
*/
|
||||
export function compareChunks(
|
||||
oldChunks: MemoryChunk[],
|
||||
newChunks: MemoryChunk[]
|
||||
): { added: MemoryChunk[]; removed: string[]; unchanged: MemoryChunk[] } {
|
||||
const oldHashes = new Set(oldChunks.map(c => c.hash));
|
||||
const newHashes = new Set(newChunks.map(c => c.hash));
|
||||
|
||||
const added = newChunks.filter(c => !oldHashes.has(c.hash));
|
||||
const removed = oldChunks.filter(c => !newHashes.has(c.hash)).map(c => c.hash);
|
||||
const unchanged = newChunks.filter(c => oldHashes.has(c.hash));
|
||||
|
||||
return { added, removed, unchanged };
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Deduplicator Utility
|
||||
*
|
||||
* Three-layer cross-segment deduplication for extracted memories:
|
||||
* 1. SHA256 fingerprint exact match
|
||||
* 2. Text similarity (Jaccard)
|
||||
* 3. Vector similarity (optional, cosine)
|
||||
*
|
||||
* Based on specs/long-memory/long-memory.md Section 5.8
|
||||
*/
|
||||
|
||||
import type { ExtractedMemory, DeduplicationConfig } from '../types';
|
||||
import { calculateHash } from './hash';
|
||||
|
||||
// ==================== Text Normalization ====================
|
||||
|
||||
/**
|
||||
* Normalize text for fingerprint comparison
|
||||
* - Lowercase
|
||||
* - Remove punctuation
|
||||
* - Collapse whitespace
|
||||
*/
|
||||
export function normalizeText(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}\s]/gu, '') // Remove non-letter/number/space (Unicode-aware)
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 fingerprint of normalized text
|
||||
*/
|
||||
export function calculateFingerprint(text: string): string {
|
||||
return calculateHash(normalizeText(text));
|
||||
}
|
||||
|
||||
// ==================== Jaccard Similarity ====================
|
||||
|
||||
/**
|
||||
* Tokenize text into word set (supports CJK characters)
|
||||
*/
|
||||
function tokenize(text: string): Set<string> {
|
||||
const normalized = normalizeText(text);
|
||||
// Split on spaces for alphabetic languages
|
||||
// For CJK, also create bigrams
|
||||
const words = normalized.split(/\s+/).filter(w => w.length > 0);
|
||||
const tokens = new Set(words);
|
||||
|
||||
// Add CJK bigrams for better Chinese matching
|
||||
const cjk = normalized.replace(/\s+/g, '');
|
||||
for (let i = 0; i < cjk.length - 1; i++) {
|
||||
tokens.add(cjk.slice(i, i + 2));
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate Jaccard similarity between two texts
|
||||
* Returns a value between 0 and 1
|
||||
*/
|
||||
export function jaccardSimilarity(textA: string, textB: string): number {
|
||||
const setA = tokenize(textA);
|
||||
const setB = tokenize(textB);
|
||||
|
||||
if (setA.size === 0 && setB.size === 0) return 1;
|
||||
if (setA.size === 0 || setB.size === 0) return 0;
|
||||
|
||||
let intersection = 0;
|
||||
for (const token of setA) {
|
||||
if (setB.has(token)) {
|
||||
intersection++;
|
||||
}
|
||||
}
|
||||
|
||||
const union = setA.size + setB.size - intersection;
|
||||
return union === 0 ? 0 : intersection / union;
|
||||
}
|
||||
|
||||
// ==================== Deduplication ====================
|
||||
|
||||
/**
|
||||
* Deduplicate extracted memories against existing texts
|
||||
*
|
||||
* Three-layer deduplication:
|
||||
* 1. Exact fingerprint match (SHA256 of normalized text)
|
||||
* 2. Text similarity (Jaccard > threshold)
|
||||
* 3. Vector similarity (optional, not implemented here — handled by caller if needed)
|
||||
*
|
||||
* For duplicates against existingTexts (from DB): always discard the candidate.
|
||||
* For duplicates within the current batch: keep the one with higher confidence.
|
||||
*
|
||||
* @param candidates - Newly extracted memory candidates
|
||||
* @param existingTexts - Texts of existing memories to check against
|
||||
* @param config - Deduplication config with thresholds
|
||||
* @returns Deduplicated candidates (duplicates removed)
|
||||
*/
|
||||
export function deduplicateMemories(
|
||||
candidates: ExtractedMemory[],
|
||||
existingTexts: string[],
|
||||
config: DeduplicationConfig
|
||||
): ExtractedMemory[] {
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
// Build fingerprint set from existing memories (DB source)
|
||||
const existingFingerprints = new Set(
|
||||
existingTexts.map(text => calculateFingerprint(text))
|
||||
);
|
||||
|
||||
// Track accepted candidates with their fingerprints for intra-batch dedup
|
||||
const acceptedCandidates: ExtractedMemory[] = [];
|
||||
const acceptedFingerprints = new Set<string>();
|
||||
// Map from fingerprint to index in acceptedCandidates for efficient lookup
|
||||
const fingerprintToIndex = new Map<string, number>();
|
||||
|
||||
for (const candidate of candidates) {
|
||||
// Layer 1: Exact fingerprint match against DB
|
||||
const fingerprint = calculateFingerprint(candidate.text);
|
||||
if (existingFingerprints.has(fingerprint)) {
|
||||
continue; // Skip exact duplicate of DB entry
|
||||
}
|
||||
|
||||
// Layer 2: Text similarity check against DB entries
|
||||
let isDuplicateOfExisting = false;
|
||||
for (const existingText of existingTexts) {
|
||||
const similarity = jaccardSimilarity(candidate.text, existingText);
|
||||
if (similarity > config.textSimilarityThreshold) {
|
||||
isDuplicateOfExisting = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (isDuplicateOfExisting) {
|
||||
continue; // Skip near-duplicate of DB entry
|
||||
}
|
||||
|
||||
// Layer 1b: Exact fingerprint match within batch
|
||||
if (acceptedFingerprints.has(fingerprint)) {
|
||||
// Find the existing batch entry and compare confidence
|
||||
const idx = fingerprintToIndex.get(fingerprint)!;
|
||||
if (candidate.confidence > acceptedCandidates[idx].confidence) {
|
||||
acceptedCandidates[idx] = candidate; // Replace with higher confidence
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Layer 2b: Text similarity check within batch (compare confidence)
|
||||
let batchDuplicateIdx = -1;
|
||||
for (let i = 0; i < acceptedCandidates.length; i++) {
|
||||
const similarity = jaccardSimilarity(candidate.text, acceptedCandidates[i].text);
|
||||
if (similarity > config.textSimilarityThreshold) {
|
||||
batchDuplicateIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (batchDuplicateIdx >= 0) {
|
||||
// Keep the one with higher confidence
|
||||
if (candidate.confidence > acceptedCandidates[batchDuplicateIdx].confidence) {
|
||||
acceptedCandidates[batchDuplicateIdx] = candidate;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Passed all checks — accept this candidate
|
||||
const idx = acceptedCandidates.length;
|
||||
acceptedCandidates.push(candidate);
|
||||
acceptedFingerprints.add(fingerprint);
|
||||
fingerprintToIndex.set(fingerprint, idx);
|
||||
}
|
||||
|
||||
return acceptedCandidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate within a batch of candidates (self-dedup)
|
||||
* Useful when processing multiple segments that may produce overlapping results
|
||||
*/
|
||||
export function deduplicateWithinBatch(
|
||||
candidates: ExtractedMemory[],
|
||||
config: DeduplicationConfig
|
||||
): ExtractedMemory[] {
|
||||
return deduplicateMemories(candidates, [], config);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Hash Utilities
|
||||
*
|
||||
* SHA256 hash calculation for memory deduplication and file change detection
|
||||
*/
|
||||
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash of text content
|
||||
*/
|
||||
export function calculateHash(text: string): string {
|
||||
return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash of buffer content
|
||||
*/
|
||||
export function calculateBufferHash(buffer: Buffer): string {
|
||||
return crypto.createHash('sha256').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate memory ID from content hash
|
||||
*/
|
||||
export function generateMemoryId(content: string): string {
|
||||
return `mem_${calculateHash(content)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if two contents have the same hash
|
||||
*/
|
||||
export function isContentChanged(oldContent: string, newContent: string): boolean {
|
||||
return calculateHash(oldContent) !== calculateHash(newContent);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Utility exports
|
||||
*/
|
||||
|
||||
export * from './hash';
|
||||
export * from './vector';
|
||||
export * from './chunker';
|
||||
export * from './signals';
|
||||
export * from './llmClient';
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Shared LLM API Client
|
||||
*
|
||||
* Unified LLM API client used by ExtractionQueue and MemoryScheduler.
|
||||
* Supports both Anthropic and OpenAI-compatible APIs.
|
||||
*/
|
||||
|
||||
import log from 'electron-log';
|
||||
|
||||
export interface LlmCallOptions {
|
||||
provider: string;
|
||||
model: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
maxTokens?: number;
|
||||
apiProtocol?: string; // 'anthropic' or 'openai' - explicitly specify API protocol
|
||||
}
|
||||
|
||||
/**
|
||||
* Call LLM API (supports Anthropic and OpenAI-compatible)
|
||||
*
|
||||
* @param prompt - The prompt to send
|
||||
* @param options - Provider, model, API key, and optional settings
|
||||
* @returns The text response from the LLM
|
||||
*/
|
||||
export async function callLlmApi(
|
||||
prompt: string,
|
||||
options: LlmCallOptions
|
||||
): Promise<string> {
|
||||
const { provider, model, apiKey, baseUrl, maxTokens = 800, apiProtocol } = options;
|
||||
|
||||
// Determine API protocol:
|
||||
// Only use apiProtocol field to determine the protocol
|
||||
// If apiProtocol is not provided, default to OpenAI-compatible format
|
||||
const isAnthropic = apiProtocol?.toLowerCase() === 'anthropic';
|
||||
log.info('[llmClient] apiProtocol=' + (apiProtocol ?? 'undefined') + ' -> isAnthropic=' + isAnthropic);
|
||||
|
||||
log.info('[llmClient] callLlmApi: provider=' + provider + ', model=' + model +
|
||||
', baseUrl=' + (baseUrl ?? 'default') + ', isAnthropic=' + isAnthropic);
|
||||
|
||||
try {
|
||||
if (isAnthropic) {
|
||||
const result = await callAnthropicApi(prompt, apiKey, baseUrl, model, maxTokens);
|
||||
log.info('[llmClient] Anthropic response length: ' + result.length);
|
||||
return result;
|
||||
} else {
|
||||
const result = await callOpenAiApi(prompt, apiKey, baseUrl, model, maxTokens);
|
||||
log.info('[llmClient] OpenAI response length: ' + result.length);
|
||||
return result;
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('[llmClient] API call failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Call Anthropic Messages API
|
||||
*/
|
||||
async function callAnthropicApi(
|
||||
prompt: string,
|
||||
apiKey: string,
|
||||
baseUrl: string | undefined,
|
||||
model: string,
|
||||
maxTokens: number
|
||||
): Promise<string> {
|
||||
// Build URL: if baseUrl doesn't end with /v1/messages, append it
|
||||
let url = baseUrl ?? 'https://api.anthropic.com/v1/messages';
|
||||
if (baseUrl && !baseUrl.endsWith('/v1/messages') && !baseUrl.endsWith('/v1/messages/')) {
|
||||
// Remove trailing slash and append /v1/messages
|
||||
url = baseUrl.replace(/\/$/, '') + '/v1/messages';
|
||||
}
|
||||
|
||||
const requestBody = {
|
||||
model: model || 'claude-3-5-sonnet-20241022',
|
||||
max_tokens: maxTokens,
|
||||
messages: [
|
||||
{ role: 'user', content: prompt }
|
||||
],
|
||||
};
|
||||
|
||||
log.info('[llmClient] Calling Anthropic API: ' + url);
|
||||
log.info('[llmClient] Request body: model=' + requestBody.model + ', max_tokens=' + requestBody.max_tokens + ', prompt_length=' + prompt.length);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
log.error('[llmClient] Anthropic API error: ' + response.status + ' - ' + errorText);
|
||||
throw new Error(`Anthropic API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as Record<string, unknown>;
|
||||
log.info('[llmClient] Anthropic response structure: ' + JSON.stringify(data).slice(0, 200));
|
||||
|
||||
// Handle proxy server error responses (HTTP 200 with error body)
|
||||
// e.g., {"code":500,"msg":"404 NOT_FOUND","success":false}
|
||||
if ('success' in data && data.success === false) {
|
||||
const errorMsg = (data.msg as string) || (data.code ? `Error code: ${data.code}` : 'Unknown proxy error');
|
||||
log.error('[llmClient] Proxy server error: ' + errorMsg);
|
||||
throw new Error(`Proxy server error: ${errorMsg}`);
|
||||
}
|
||||
if ('code' in data && typeof data.code === 'number' && data.code >= 400) {
|
||||
const errorMsg = (data.msg as string) || `Error code: ${data.code}`;
|
||||
log.error('[llmClient] API returned error code: ' + errorMsg);
|
||||
throw new Error(`API error: ${errorMsg}`);
|
||||
}
|
||||
|
||||
// Standard Anthropic response format
|
||||
const anthropicData = data as { content?: Array<{ text?: string }> };
|
||||
return anthropicData.content?.[0]?.text ?? '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Call OpenAI-compatible Chat Completions API
|
||||
*/
|
||||
async function callOpenAiApi(
|
||||
prompt: string,
|
||||
apiKey: string,
|
||||
baseUrl: string | undefined,
|
||||
model: string,
|
||||
maxTokens: number
|
||||
): Promise<string> {
|
||||
// Build URL: if baseUrl doesn't contain /chat/completions, append it
|
||||
// Note: Use /chat/completions (not /v1/chat/completions) because:
|
||||
// - For official OpenAI API, baseUrl should be provided as "https://api.openai.com/v1"
|
||||
// - For proxy servers, they typically expect path like "/api/proxy/model/chat/completions"
|
||||
let url = baseUrl ?? 'https://api.openai.com/v1/chat/completions';
|
||||
if (baseUrl && !baseUrl.includes('/chat/completions')) {
|
||||
// Remove trailing slash and append /chat/completions
|
||||
url = baseUrl.replace(/\/$/, '') + '/chat/completions';
|
||||
}
|
||||
|
||||
log.info('[llmClient] Calling OpenAI API: ' + url);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model || 'gpt-4o-mini',
|
||||
max_tokens: maxTokens,
|
||||
messages: [
|
||||
{ role: 'user', content: prompt }
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
log.error('[llmClient] OpenAI API error: ' + response.status + ' - ' + errorText);
|
||||
throw new Error(`OpenAI API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as Record<string, unknown>;
|
||||
log.info('[llmClient] OpenAI response structure: ' + JSON.stringify(data).slice(0, 200));
|
||||
|
||||
// Handle proxy server error responses (HTTP 200 with error body)
|
||||
// e.g., {"code":500,"msg":"404 NOT_FOUND","success":false}
|
||||
if ('success' in data && data.success === false) {
|
||||
const errorMsg = (data.msg as string) || (data.code ? `Error code: ${data.code}` : 'Unknown proxy error');
|
||||
log.error('[llmClient] Proxy server error: ' + errorMsg);
|
||||
throw new Error(`Proxy server error: ${errorMsg}`);
|
||||
}
|
||||
if ('code' in data && typeof data.code === 'number' && data.code >= 400) {
|
||||
const errorMsg = (data.msg as string) || `Error code: ${data.code}`;
|
||||
log.error('[llmClient] API returned error code: ' + errorMsg);
|
||||
throw new Error(`API error: ${errorMsg}`);
|
||||
}
|
||||
|
||||
// Standard OpenAI response format
|
||||
const openaiData = data as { choices?: Array<{ message?: { content?: string } }> };
|
||||
return openaiData.choices?.[0]?.message?.content ?? '';
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
/**
|
||||
* Segmenter Utility
|
||||
*
|
||||
* Builds overlapping segments from transcript entries for memory extraction.
|
||||
* Handles large message preprocessing (code block summarization, truncation).
|
||||
*
|
||||
* Based on specs/long-memory/long-memory.md Section 5.3
|
||||
*/
|
||||
|
||||
import type { TranscriptEntry, Segment, SegmentationConfig } from "../types";
|
||||
|
||||
// ==================== Token Estimation ====================
|
||||
|
||||
/**
|
||||
* Estimate token count for text
|
||||
*
|
||||
* Uses a simple heuristic:
|
||||
* - Chinese characters: ~1.5 tokens each
|
||||
* - English words: ~1.3 tokens per word
|
||||
* - Other characters: ~0.25 tokens each (4 chars = 1 token)
|
||||
*
|
||||
* This is an approximation; actual token count depends on the tokenizer.
|
||||
*/
|
||||
export function estimateTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
|
||||
// Count Chinese characters (CJK range)
|
||||
const chineseChars = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || [])
|
||||
.length;
|
||||
|
||||
// Count English words (approximate)
|
||||
const englishWords = (text.match(/[a-zA-Z]+/g) || []).length;
|
||||
|
||||
// Count other characters
|
||||
const otherChars =
|
||||
text.length -
|
||||
chineseChars -
|
||||
(text.match(/[a-zA-Z]+/g) || []).join("").length;
|
||||
|
||||
// Estimate tokens
|
||||
const tokens = Math.ceil(
|
||||
chineseChars * 1.5 + // Chinese chars
|
||||
englishWords * 1.3 + // English words
|
||||
otherChars * 0.25, // Other chars (punctuation, spaces, etc.)
|
||||
);
|
||||
|
||||
return Math.max(1, tokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate tokens for an array of messages
|
||||
*/
|
||||
export function estimateMessagesTokens(
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
): number {
|
||||
let total = 0;
|
||||
for (const msg of messages) {
|
||||
// Role prefix adds ~2 tokens per message
|
||||
total += 2 + estimateTokens(msg.content);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
// ==================== Message Preprocessing ====================
|
||||
|
||||
/**
|
||||
* Code block pattern: ```lang\n...code...\n```
|
||||
*/
|
||||
const CODE_BLOCK_PATTERN = /```(\w*)\n([\s\S]*?)```/g;
|
||||
|
||||
/**
|
||||
* JSON structured data pattern (NOT XML - XML tags are handled separately)
|
||||
* Matches large JSON objects or arrays that are unlikely to contain memory signals
|
||||
*/
|
||||
const JSON_DATA_PATTERN = /(?:^|\n)\s*[{\[][\s\S]{300,}[}\]]\s*(?:\n|$)/g;
|
||||
|
||||
/**
|
||||
* Preprocess a single message content for extraction
|
||||
*
|
||||
* - Replaces code blocks with summaries
|
||||
* - Truncates long pure text (keep head + tail)
|
||||
* - Replaces large JSON data with type markers
|
||||
* - Does NOT replace XML - that's handled by MemoryExtractor.preprocessText
|
||||
*/
|
||||
export function preprocessMessageContent(
|
||||
content: string,
|
||||
maxChars: number,
|
||||
): string {
|
||||
if (content.length <= maxChars) {
|
||||
return content;
|
||||
}
|
||||
|
||||
let processed = content;
|
||||
|
||||
// Replace code blocks with summaries
|
||||
processed = processed.replace(CODE_BLOCK_PATTERN, (_match, lang, code) => {
|
||||
const lineCount = code.split("\n").length;
|
||||
const language = lang || "unknown";
|
||||
return `[code: ${language}, ${lineCount} lines]`;
|
||||
});
|
||||
|
||||
// Replace large JSON data (but NOT XML - XML may contain user content)
|
||||
processed = processed.replace(JSON_DATA_PATTERN, (match) => {
|
||||
const trimmed = match.trim();
|
||||
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
||||
return "\n[structured data: JSON]\n";
|
||||
}
|
||||
return match;
|
||||
});
|
||||
|
||||
// If still too long after code/data replacement, truncate
|
||||
if (processed.length > maxChars) {
|
||||
const headSize = Math.floor(maxChars * 0.4);
|
||||
const tailSize = Math.floor(maxChars * 0.4);
|
||||
const omitted = processed.length - headSize - tailSize;
|
||||
|
||||
processed =
|
||||
processed.slice(0, headSize) +
|
||||
`\n[... ${omitted} characters omitted ...]\n` +
|
||||
processed.slice(-tailSize);
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate message content to fit within token limit
|
||||
* Returns truncated content and whether truncation occurred
|
||||
*/
|
||||
export function truncateToTokenLimit(
|
||||
content: string,
|
||||
maxTokens: number,
|
||||
): { content: string; truncated: boolean } {
|
||||
const currentTokens = estimateTokens(content);
|
||||
if (currentTokens <= maxTokens) {
|
||||
return { content, truncated: false };
|
||||
}
|
||||
|
||||
// Estimate chars needed (conservative: assume 3 chars per token)
|
||||
const targetChars = Math.floor(maxTokens * 3);
|
||||
|
||||
if (content.length <= targetChars) {
|
||||
return { content, truncated: false };
|
||||
}
|
||||
|
||||
// Keep head and tail
|
||||
const headSize = Math.floor(targetChars * 0.45);
|
||||
const tailSize = Math.floor(targetChars * 0.45);
|
||||
const omitted = content.length - headSize - tailSize;
|
||||
|
||||
const truncated =
|
||||
content.slice(0, headSize) +
|
||||
`\n[... ${omitted} characters omitted ...]\n` +
|
||||
content.slice(-tailSize);
|
||||
|
||||
return { content: truncated, truncated: true };
|
||||
}
|
||||
|
||||
// ==================== Segment Builder ====================
|
||||
|
||||
/**
|
||||
* Build overlapping segments from transcript entries
|
||||
*
|
||||
* Example with segmentSize=5, overlap=2:
|
||||
* Segment 0: [M0, M1, M2, M3, M4]
|
||||
* Segment 1: [M3, M4, M5, M6, M7]
|
||||
* Segment 2: [M6, M7, M8, M9, M10]
|
||||
* Segment 3: [M9, M10, M11] (tail, possibly shorter)
|
||||
*
|
||||
* Also respects maxSegmentTokens - if a segment exceeds token limit,
|
||||
* it will be dynamically reduced in size.
|
||||
*/
|
||||
export function buildSegments(
|
||||
entries: TranscriptEntry[],
|
||||
config: SegmentationConfig,
|
||||
): Segment[] {
|
||||
const {
|
||||
segmentSize,
|
||||
segmentOverlap,
|
||||
maxContentPerMessage,
|
||||
maxSegmentTokens,
|
||||
} = config;
|
||||
|
||||
if (entries.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const step = segmentSize - segmentOverlap;
|
||||
if (step <= 0) {
|
||||
throw new Error(
|
||||
`Invalid segmentation config: segmentSize (${segmentSize}) must be > segmentOverlap (${segmentOverlap})`,
|
||||
);
|
||||
}
|
||||
|
||||
const segments: Segment[] = [];
|
||||
let segmentIndex = 0;
|
||||
|
||||
for (let start = 0; start < entries.length; start += step) {
|
||||
const end = Math.min(start + segmentSize, entries.length);
|
||||
const segmentEntries = entries.slice(start, end);
|
||||
|
||||
// Preprocess messages
|
||||
let messages = segmentEntries.map((entry) => ({
|
||||
role: entry.role,
|
||||
content: preprocessMessageContent(entry.content, maxContentPerMessage),
|
||||
}));
|
||||
|
||||
// Check token limit and reduce if necessary
|
||||
let actualEnd = end;
|
||||
while (messages.length > 1) {
|
||||
const tokenCount = estimateMessagesTokens(messages);
|
||||
if (tokenCount <= maxSegmentTokens) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Remove last message to reduce token count
|
||||
messages.pop();
|
||||
actualEnd = start + messages.length;
|
||||
}
|
||||
|
||||
// If still over limit with single message, truncate that message
|
||||
if (messages.length === 1) {
|
||||
const singleMsg = messages[0];
|
||||
const tokenCount = estimateMessagesTokens([singleMsg]);
|
||||
if (tokenCount > maxSegmentTokens) {
|
||||
// Reserve tokens for role prefix (~2 tokens)
|
||||
const { content: truncated } = truncateToTokenLimit(
|
||||
singleMsg.content,
|
||||
maxSegmentTokens - 2,
|
||||
);
|
||||
messages[0] = { ...singleMsg, content: truncated };
|
||||
}
|
||||
}
|
||||
|
||||
segments.push({
|
||||
index: segmentIndex,
|
||||
messages,
|
||||
startMsgIndex: start,
|
||||
endMsgIndex: actualEnd,
|
||||
});
|
||||
|
||||
segmentIndex++;
|
||||
|
||||
// If we've reached the end, stop
|
||||
if (actualEnd >= entries.length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build segments from a specific start index
|
||||
* Used for session-end extraction to process only unprocessed messages
|
||||
*/
|
||||
export function buildSegmentsFromIndex(
|
||||
entries: TranscriptEntry[],
|
||||
startFromIndex: number,
|
||||
config: SegmentationConfig,
|
||||
): Segment[] {
|
||||
if (startFromIndex >= entries.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const remaining = entries.slice(startFromIndex);
|
||||
const segments = buildSegments(remaining, config);
|
||||
|
||||
// Adjust indices to be relative to the original transcript
|
||||
return segments.map((segment) => ({
|
||||
...segment,
|
||||
startMsgIndex: segment.startMsgIndex + startFromIndex,
|
||||
endMsgIndex: segment.endMsgIndex + startFromIndex,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Signal Detection Patterns
|
||||
*
|
||||
* Regex patterns for detecting memory extraction signals
|
||||
*/
|
||||
|
||||
import type { SignalMatch } from '../types';
|
||||
|
||||
// ==================== Explicit Command Patterns ====================
|
||||
|
||||
/**
|
||||
* Explicit memory commands
|
||||
* Matches: "记住: xxx", "记得xxx", "remember: xxx", "remember this: xxx"
|
||||
*/
|
||||
export const EXPLICIT_PATTERNS: Array<{ pattern: RegExp; command: string }> = [
|
||||
{
|
||||
// "记住/记得" + optional colon + content (e.g., "记得我的名字是小花花", "记住：我的名字是小花花")
|
||||
// Stop at common delimiters: punctuation, comma, newline, ## (markdown headers), or end of string
|
||||
pattern: /(?:记住|记得)(?:[:：]\s*)?(.+?)(?:[。！？，,\n#]|$)/gi,
|
||||
command: 'remember',
|
||||
},
|
||||
{
|
||||
// English: "remember: xxx", "remember this: xxx"
|
||||
pattern: /(?:remember(?:\s+this)?)[:：]\s*(.+?)(?:[.!?,\n#]|$)/gi,
|
||||
command: 'remember',
|
||||
},
|
||||
{
|
||||
pattern: /(?:删除记忆|忘掉|忘记)[:：]\s*(.+?)(?:[。！？，,\n#]|$)/gi,
|
||||
command: 'forget',
|
||||
},
|
||||
{
|
||||
pattern: /(?:forget)[:：]\s*(.+?)(?:[.!?,\n#]|$)/gi,
|
||||
command: 'forget',
|
||||
},
|
||||
];
|
||||
|
||||
// ==================== Implicit Signal Patterns ====================
|
||||
|
||||
/**
|
||||
* Personal information signals
|
||||
* Matches: "我叫xxx", "我是xxx", "我的名字是xxx"
|
||||
*/
|
||||
export const PERSONAL_INFO_PATTERNS: RegExp[] = [
|
||||
/(?:我叫|我是|我的名字(?:是|叫)?)\s*[:：]?\s*(\S+)/gi,
|
||||
/(?:I\s+am|my\s+name\s+is)\s+(\w+)/gi,
|
||||
];
|
||||
|
||||
/**
|
||||
* Preference expression signals
|
||||
* Matches: "我喜欢xxx", "我偏好xxx", "我习惯xxx"
|
||||
*/
|
||||
export const PREFERENCE_PATTERNS: RegExp[] = [
|
||||
/(?:我喜欢|我偏好|我习惯|我更倾向|我比较喜欢)\s*[:：]?\s*(.+?)(?:[。！？\n]|$)/gi,
|
||||
/(?:I\s+(?:like|prefer|usually|tend\s+to))\s+(.+?)(?:[.!?\n]|$)/gi,
|
||||
];
|
||||
|
||||
/**
|
||||
* Ownership signals
|
||||
* Matches: "我养了xxx", "我家有xxx", "我的xxx是"
|
||||
*/
|
||||
export const OWNERSHIP_PATTERNS: RegExp[] = [
|
||||
/(?:我养了|我家有|我的|我有一(?:只|个|台|辆))\s*(\S+)(?:是|叫)?\s*(\S*)/gi,
|
||||
/(?:I\s+have|my\s+|I\s+own)\s+(.+?)(?:[.!?\n]|$)/gi,
|
||||
];
|
||||
|
||||
/**
|
||||
* Fact statement signals
|
||||
* Matches: "我在xxx", "我住xxx", "我来自xxx"
|
||||
*/
|
||||
export const FACT_STATEMENT_PATTERNS: RegExp[] = [
|
||||
/(?:我在|我住|我来自|我工作是|我在.*工作)/gi,
|
||||
/(?:I\s+(?:live|work|am\s+from|am\s+based))\s+(?:in|at)\s+/gi,
|
||||
];
|
||||
|
||||
// ==================== Signal Detection Functions ====================
|
||||
|
||||
/**
|
||||
* Detect all signals in text
|
||||
*/
|
||||
export function detectSignals(text: string): SignalMatch[] {
|
||||
const matches: SignalMatch[] = [];
|
||||
|
||||
// Check explicit patterns
|
||||
for (const { pattern, command } of EXPLICIT_PATTERNS) {
|
||||
pattern.lastIndex = 0; // Reset regex state
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
matches.push({
|
||||
type: 'explicit',
|
||||
pattern: command,
|
||||
matchedText: match[0],
|
||||
extractedText: match[1]?.trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check implicit patterns - personal info
|
||||
for (const pattern of PERSONAL_INFO_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
matches.push({
|
||||
type: 'implicit',
|
||||
pattern: 'personal_info',
|
||||
matchedText: match[0],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check implicit patterns - preferences
|
||||
for (const pattern of PREFERENCE_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
matches.push({
|
||||
type: 'implicit',
|
||||
pattern: 'preference',
|
||||
matchedText: match[0],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check implicit patterns - ownership
|
||||
for (const pattern of OWNERSHIP_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
matches.push({
|
||||
type: 'implicit',
|
||||
pattern: 'ownership',
|
||||
matchedText: match[0],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check implicit patterns - fact statements
|
||||
for (const pattern of FACT_STATEMENT_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
matches.push({
|
||||
type: 'implicit',
|
||||
pattern: 'fact',
|
||||
matchedText: match[0],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text contains explicit memory command
|
||||
*/
|
||||
export function hasExplicitCommand(text: string): boolean {
|
||||
for (const { pattern } of EXPLICIT_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
if (pattern.test(text)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract explicit command content
|
||||
*/
|
||||
export function extractExplicitContent(text: string): { command: string; content: string } | null {
|
||||
for (const { pattern, command } of EXPLICIT_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
const match = pattern.exec(text);
|
||||
if (match && match[1]) {
|
||||
let content = match[1].trim();
|
||||
|
||||
// Skip if content is just a tone particle (下, 吧, 了, 啊, etc.)
|
||||
// These are common Chinese particles that indicate mood/tone, not actual content
|
||||
if (/^[下吧了啊呢吗呀哦]+$/.test(content)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip if content is too short (likely noise)
|
||||
if (content.length < 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
content,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text contains implicit signals
|
||||
*/
|
||||
export function hasImplicitSignals(text: string): boolean {
|
||||
const allImplicitPatterns = [
|
||||
...PERSONAL_INFO_PATTERNS,
|
||||
...PREFERENCE_PATTERNS,
|
||||
...OWNERSHIP_PATTERNS,
|
||||
...FACT_STATEMENT_PATTERNS,
|
||||
];
|
||||
|
||||
for (const pattern of allImplicitPatterns) {
|
||||
pattern.lastIndex = 0;
|
||||
if (pattern.test(text)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count signal strength (number of signal matches)
|
||||
*/
|
||||
export function countSignalStrength(text: string): number {
|
||||
return detectSignals(text).length;
|
||||
}
|
||||
|
||||
// ==================== Validation Patterns ====================
|
||||
|
||||
/**
|
||||
* Patterns that indicate temporary information
|
||||
*/
|
||||
export const TEMPORARY_PATTERNS: RegExp[] = [
|
||||
/(?:今天|昨天|明天|现在|刚才|待会)/,
|
||||
/(?:today|yesterday|tomorrow|now|just now|later)/i,
|
||||
];
|
||||
|
||||
/**
|
||||
* Patterns that indicate questions
|
||||
*/
|
||||
export const QUESTION_PATTERNS: RegExp[] = [
|
||||
/\?|？/,
|
||||
/^(?:什么|怎么|如何|为什么|谁|哪|是否|能不能|可以)/,
|
||||
/^(?:what|how|why|who|where|when|can|could|would|is|are|do|does)/i,
|
||||
];
|
||||
|
||||
/**
|
||||
* Check if text is a question
|
||||
*/
|
||||
export function isQuestion(text: string): boolean {
|
||||
for (const pattern of QUESTION_PATTERNS) {
|
||||
if (pattern.test(text.trim())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text contains temporary information
|
||||
*/
|
||||
export function isTemporary(text: string): boolean {
|
||||
for (const pattern of TEMPORARY_PATTERNS) {
|
||||
if (pattern.test(text)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text is primarily code
|
||||
*/
|
||||
export function isCode(text: string): boolean {
|
||||
const codeIndicators = [
|
||||
/^(?:function|const|let|var|class|import|export|if|for|while|return)/m,
|
||||
/(?:=>|\{|\}|\[|\]|;)$/,
|
||||
/```/,
|
||||
];
|
||||
|
||||
let codeScore = 0;
|
||||
for (const pattern of codeIndicators) {
|
||||
if (pattern.test(text)) {
|
||||
codeScore++;
|
||||
}
|
||||
}
|
||||
|
||||
// If more than half of code indicators match, likely code
|
||||
return codeScore > codeIndicators.length / 2;
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* Vector Utilities
|
||||
*
|
||||
* Pure JavaScript vector operations for fallback when sqlite-vec is unavailable
|
||||
*/
|
||||
|
||||
/**
|
||||
* Calculate cosine similarity between two vectors
|
||||
*/
|
||||
export function cosineSimilarity(a: number[] | Float32Array, b: number[] | Float32Array): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error(`Vector length mismatch: ${a.length} vs ${b.length}`);
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let magnitudeA = 0;
|
||||
let magnitudeB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
magnitudeA += a[i] * a[i];
|
||||
magnitudeB += b[i] * b[i];
|
||||
}
|
||||
|
||||
magnitudeA = Math.sqrt(magnitudeA);
|
||||
magnitudeB = Math.sqrt(magnitudeB);
|
||||
|
||||
if (magnitudeA === 0 || magnitudeB === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return dotProduct / (magnitudeA * magnitudeB);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate Euclidean distance between two vectors
|
||||
*/
|
||||
export function euclideanDistance(a: number[] | Float32Array, b: number[] | Float32Array): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error(`Vector length mismatch: ${a.length} vs ${b.length}`);
|
||||
}
|
||||
|
||||
let sum = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const diff = a[i] - b[i];
|
||||
sum += diff * diff;
|
||||
}
|
||||
|
||||
return Math.sqrt(sum);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a vector to unit length
|
||||
*/
|
||||
export function normalizeVector(vector: number[] | Float32Array): number[] {
|
||||
let magnitude = 0;
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
magnitude += vector[i] * vector[i];
|
||||
}
|
||||
magnitude = Math.sqrt(magnitude);
|
||||
|
||||
if (magnitude === 0) {
|
||||
return new Array(vector.length).fill(0);
|
||||
}
|
||||
|
||||
const normalized: number[] = [];
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
normalized.push(vector[i] / magnitude);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Float32Array to Buffer for SQLite BLOB storage
|
||||
*/
|
||||
export function float32ToBuffer(arr: Float32Array): Buffer {
|
||||
return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Buffer to Float32Array from SQLite BLOB storage
|
||||
*/
|
||||
export function bufferToFloat32(buf: Buffer): Float32Array {
|
||||
return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert number array to Float32Array
|
||||
*/
|
||||
export function arrayToFloat32(arr: number[]): Float32Array {
|
||||
return new Float32Array(arr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Float32Array to number array
|
||||
*/
|
||||
export function float32ToArray(arr: Float32Array): number[] {
|
||||
return Array.from(arr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find top-K nearest neighbors by cosine similarity
|
||||
*/
|
||||
export function findTopK(
|
||||
query: number[] | Float32Array,
|
||||
vectors: Array<{ id: string; vector: number[] | Float32Array }>,
|
||||
k: number,
|
||||
minScore: number = 0
|
||||
): Array<{ id: string; score: number }> {
|
||||
const scores: Array<{ id: string; score: number }> = [];
|
||||
|
||||
for (const { id, vector } of vectors) {
|
||||
const score = cosineSimilarity(query, vector);
|
||||
if (score >= minScore) {
|
||||
scores.push({ id, score });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score descending
|
||||
scores.sort((a, b) => b.score - a.score);
|
||||
|
||||
// Return top K
|
||||
return scores.slice(0, k);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch cosine similarity calculation
|
||||
*/
|
||||
export function batchCosineSimilarity(
|
||||
query: number[] | Float32Array,
|
||||
vectors: Array<number[] | Float32Array>
|
||||
): number[] {
|
||||
return vectors.map(v => cosineSimilarity(query, v));
|
||||
}
|
||||
Reference in New Issue
Block a user