// ═══════════════════════════════════════════════════════════════════════════ // Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence) // // 命名规范: // - 存储层用 L0/L1/L2/L3(StateAtom/Chunk/Event/Fact) // - 召回层用语义名称:anchor/evidence/event/constraint // // 架构: // 阶段 1: Query Build(确定性,无 LLM) // 阶段 2: Round 1 Dense Retrieval(L0 + L2) // 阶段 3: Query Refinement(用已命中记忆增强) // 阶段 4: Round 2 Dense Retrieval(L0 + L2) // 阶段 5: Lexical Retrieval + L0 Merge // 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine 预筛选 // 阶段 7: L1 配对组装(L0 → top-1 AI L1 + top-1 USER L1) // 阶段 8: Causation Trace // ═══════════════════════════════════════════════════════════════════════════ import { getAllEventVectors, getChunksByFloors, getMeta, getChunkVectorsByIds } from '../storage/chunk-store.js'; import { getAllStateVectors, getStateAtoms } from '../storage/state-store.js'; import { getEngineFingerprint, embed } from '../utils/embedder.js'; import { xbLog } from '../../../../core/debug-core.js'; import { getContext } from '../../../../../../../extensions.js'; import { buildQueryBundle, refineQueryBundle } from './query-builder.js'; import { getLexicalIndex, searchLexicalIndex } from './lexical-index.js'; import { rerankChunks } from '../llm/reranker.js'; import { createMetrics, calcSimilarityStats } from './metrics.js'; const MODULE_ID = 'recall'; // ═══════════════════════════════════════════════════════════════════════════ // 配置 // ═══════════════════════════════════════════════════════════════════════════ const CONFIG = { // 窗口 LAST_MESSAGES_K: 2, // Anchor (L0 StateAtoms) ANCHOR_MIN_SIMILARITY: 0.58, // Event (L2 Events) EVENT_CANDIDATE_MAX: 100, EVENT_SELECT_MAX: 50, EVENT_MIN_SIMILARITY: 0.55, EVENT_MMR_LAMBDA: 0.72, // W-RRF 融合(L0-only) RRF_K: 60, RRF_W_DENSE: 1.0, RRF_W_LEX: 0.9, FUSION_CAP: 100, // Rerank(L0-only) RERANK_TOP_N: 50, RERANK_MIN_SCORE: 0.15, // 因果链 CAUSAL_CHAIN_MAX_DEPTH: 10, CAUSAL_INJECT_MAX: 30, }; // ═══════════════════════════════════════════════════════════════════════════ // 工具函数 // ═══════════════════════════════════════════════════════════════════════════ /** * 计算余弦相似度 * @param {number[]} a * @param {number[]} b * @returns {number} */ function cosineSimilarity(a, b) { if (!a?.length || !b?.length || a.length !== b.length) return 0; let dot = 0, nA = 0, nB = 0; for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; nA += a[i] * a[i]; nB += b[i] * b[i]; } return nA && nB ? dot / (Math.sqrt(nA) * Math.sqrt(nB)) : 0; } /** * 标准化字符串 * @param {string} s * @returns {string} */ function normalize(s) { return String(s || '') .normalize('NFKC') .replace(/[\u200B-\u200D\uFEFF]/g, '') .trim() .toLowerCase(); } /** * 获取最近消息 * @param {object[]} chat * @param {number} count * @param {boolean} excludeLastAi * @returns {object[]} */ function getLastMessages(chat, count = 2, excludeLastAi = false) { if (!chat?.length) return []; let messages = [...chat]; if (excludeLastAi && messages.length > 0 && !messages[messages.length - 1]?.is_user) { messages = messages.slice(0, -1); } return messages.slice(-count); } // ═══════════════════════════════════════════════════════════════════════════ // MMR 选择算法 // ═══════════════════════════════════════════════════════════════════════════ /** * Maximal Marginal Relevance 选择 * @param {object[]} candidates * @param {number} k * @param {number} lambda * @param {Function} getVector * @param {Function} getScore * @returns {object[]} */ function mmrSelect(candidates, k, lambda, getVector, getScore) { const selected = []; const ids = new Set(); while (selected.length < k && candidates.length) { let best = null; let bestScore = -Infinity; for (const c of candidates) { if (ids.has(c._id)) continue; const rel = getScore(c); let div = 0; if (selected.length) { const vC = getVector(c); if (vC?.length) { for (const s of selected) { const sim = cosineSimilarity(vC, getVector(s)); if (sim > div) div = sim; } } } const score = lambda * rel - (1 - lambda) * div; if (score > bestScore) { bestScore = score; best = c; } } if (!best) break; selected.push(best); ids.add(best._id); } return selected; } // ═══════════════════════════════════════════════════════════════════════════ // [Anchors] L0 StateAtoms 检索 // ═══════════════════════════════════════════════════════════════════════════ /** * 检索语义锚点 * @param {number[]} queryVector * @param {object} vectorConfig * @param {object|null} metrics * @returns {Promise<{hits: object[], floors: Set}>} */ async function recallAnchors(queryVector, vectorConfig, metrics) { const { chatId } = getContext(); if (!chatId || !queryVector?.length) { return { hits: [], floors: new Set() }; } const meta = await getMeta(chatId); const fp = getEngineFingerprint(vectorConfig); if (meta.fingerprint && meta.fingerprint !== fp) { xbLog.warn(MODULE_ID, 'Anchor fingerprint 不匹配'); return { hits: [], floors: new Set() }; } const stateVectors = await getAllStateVectors(chatId); if (!stateVectors.length) { return { hits: [], floors: new Set() }; } const atomsList = getStateAtoms(); const atomMap = new Map(atomsList.map(a => [a.atomId, a])); const scored = stateVectors .map(sv => { const atom = atomMap.get(sv.atomId); if (!atom) return null; return { atomId: sv.atomId, floor: sv.floor, similarity: cosineSimilarity(queryVector, sv.vector), atom, }; }) .filter(Boolean) .filter(s => s.similarity >= CONFIG.ANCHOR_MIN_SIMILARITY) .sort((a, b) => b.similarity - a.similarity); const floors = new Set(scored.map(s => s.floor)); if (metrics) { metrics.anchor.matched = scored.length; metrics.anchor.floorsHit = floors.size; metrics.anchor.topHits = scored.slice(0, 5).map(s => ({ floor: s.floor, semantic: s.atom?.semantic?.slice(0, 50), similarity: Math.round(s.similarity * 1000) / 1000, })); } return { hits: scored, floors }; } // ═══════════════════════════════════════════════════════════════════════════ // [Events] L2 Events 检索 // ═══════════════════════════════════════════════════════════════════════════ /** * 检索事件 * @param {number[]} queryVector * @param {object[]} allEvents * @param {object} vectorConfig * @param {string[]} focusEntities * @param {object|null} metrics * @returns {Promise} */ async function recallEvents(queryVector, allEvents, vectorConfig, focusEntities, metrics) { const { chatId } = getContext(); if (!chatId || !queryVector?.length || !allEvents?.length) { return []; } const meta = await getMeta(chatId); const fp = getEngineFingerprint(vectorConfig); if (meta.fingerprint && meta.fingerprint !== fp) { xbLog.warn(MODULE_ID, 'Event fingerprint 不匹配'); return []; } const eventVectors = await getAllEventVectors(chatId); const vectorMap = new Map(eventVectors.map(v => [v.eventId, v.vector])); if (!vectorMap.size) { return []; } const focusSet = new Set((focusEntities || []).map(normalize)); const scored = allEvents.map(event => { const v = vectorMap.get(event.id); const baseSim = v ? cosineSimilarity(queryVector, v) : 0; const participants = (event.participants || []).map(p => normalize(p)); const hasEntityMatch = participants.some(p => focusSet.has(p)); return { _id: event.id, event, similarity: baseSim, _hasEntityMatch: hasEntityMatch, vector: v, }; }); if (metrics) { metrics.event.inStore = allEvents.length; } let candidates = scored .filter(s => s.similarity >= CONFIG.EVENT_MIN_SIMILARITY) .sort((a, b) => b.similarity - a.similarity) .slice(0, CONFIG.EVENT_CANDIDATE_MAX); if (metrics) { metrics.event.considered = candidates.length; } // 实体过滤 if (focusSet.size > 0) { const beforeFilter = candidates.length; candidates = candidates.filter(c => { if (c.similarity >= 0.85) return true; return c._hasEntityMatch; }); if (metrics) { metrics.event.entityFilter = { focusEntities: focusEntities || [], before: beforeFilter, after: candidates.length, filtered: beforeFilter - candidates.length, }; } } // MMR 选择 const selected = mmrSelect( candidates, CONFIG.EVENT_SELECT_MAX, CONFIG.EVENT_MMR_LAMBDA, c => c.vector, c => c.similarity ); let directCount = 0; let relatedCount = 0; const results = selected.map(s => { const recallType = s._hasEntityMatch ? 'DIRECT' : 'RELATED'; if (recallType === 'DIRECT') directCount++; else relatedCount++; return { event: s.event, similarity: s.similarity, _recallType: recallType, }; }); if (metrics) { metrics.event.selected = results.length; metrics.event.byRecallType = { direct: directCount, related: relatedCount, causal: 0, lexical: 0 }; metrics.event.similarityDistribution = calcSimilarityStats(results.map(r => r.similarity)); } return results; } // ═══════════════════════════════════════════════════════════════════════════ // [Causation] 因果链追溯 // ═══════════════════════════════════════════════════════════════════════════ /** * 构建事件索引 * @param {object[]} allEvents * @returns {Map} */ function buildEventIndex(allEvents) { const map = new Map(); for (const e of allEvents || []) { if (e?.id) map.set(e.id, e); } return map; } /** * 追溯因果链 * @param {object[]} eventHits * @param {Map} eventIndex * @param {number} maxDepth * @returns {{results: object[], maxDepth: number}} */ function traceCausation(eventHits, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MAX_DEPTH) { const out = new Map(); const idRe = /^evt-\d+$/; let maxActualDepth = 0; function visit(parentId, depth, chainFrom) { if (depth > maxDepth) return; if (!idRe.test(parentId)) return; const ev = eventIndex.get(parentId); if (!ev) return; if (depth > maxActualDepth) maxActualDepth = depth; const existed = out.get(parentId); if (!existed) { out.set(parentId, { event: ev, depth, chainFrom: [chainFrom] }); } else { if (depth < existed.depth) existed.depth = depth; if (!existed.chainFrom.includes(chainFrom)) existed.chainFrom.push(chainFrom); } for (const next of (ev.causedBy || [])) { visit(String(next || '').trim(), depth + 1, chainFrom); } } for (const r of eventHits || []) { const rid = r?.event?.id; if (!rid) continue; for (const cid of (r.event?.causedBy || [])) { visit(String(cid || '').trim(), 1, rid); } } const results = Array.from(out.values()) .sort((a, b) => { const refDiff = b.chainFrom.length - a.chainFrom.length; if (refDiff !== 0) return refDiff; return a.depth - b.depth; }) .slice(0, CONFIG.CAUSAL_INJECT_MAX); return { results, maxDepth: maxActualDepth }; } // ═══════════════════════════════════════════════════════════════════════════ // [W-RRF] 加权倒数排名融合(L0-only) // ═══════════════════════════════════════════════════════════════════════════ /** * @typedef {object} RankedItem * @property {string} id - 唯一标识符 * @property {number} score - 该路的原始分数 */ /** * W-RRF 融合两路 L0 候选(dense + lexical) * * @param {RankedItem[]} denseRank - Dense 路(cosine 降序) * @param {RankedItem[]} lexRank - Lexical 路(MiniSearch score 降序) * @param {number} cap - 输出上限 * @returns {{top: {id: string, fusionScore: number}[], totalUnique: number}} */ function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) { const k = CONFIG.RRF_K; const wD = CONFIG.RRF_W_DENSE; const wL = CONFIG.RRF_W_LEX; const buildRankMap = (ranked) => { const map = new Map(); for (let i = 0; i < ranked.length; i++) { const id = ranked[i].id; if (!map.has(id)) map.set(id, i); } return map; }; const denseMap = buildRankMap(denseRank || []); const lexMap = buildRankMap(lexRank || []); const allIds = new Set([ ...denseMap.keys(), ...lexMap.keys(), ]); const totalUnique = allIds.size; const scored = []; for (const id of allIds) { let score = 0; if (denseMap.has(id)) { score += wD / (k + denseMap.get(id)); } if (lexMap.has(id)) { score += wL / (k + lexMap.get(id)); } scored.push({ id, fusionScore: score }); } scored.sort((a, b) => b.fusionScore - a.fusionScore); return { top: scored.slice(0, cap), totalUnique, }; } // ═══════════════════════════════════════════════════════════════════════════ // [Stage 6] L0-only 融合 + Rerank ‖ 并发 L1 Cosine 预筛选 // ═══════════════════════════════════════════════════════════════════════════ /** * L0 融合 + rerank,并发拉取 L1 并 cosine 打分 * * @param {object[]} anchorHits - L0 dense 命中(Round 2) * @param {Set} anchorFloors - L0 命中楼层(含 lexical 扩展) * @param {number[]} queryVector - 查询向量(v1) * @param {string} rerankQuery - rerank 查询文本 * @param {object} lexicalResult - 词法检索结果 * @param {object} metrics * @returns {Promise<{l0Selected: object[], l1ByFloor: Map}>} */ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rerankQuery, lexicalResult, metrics) { const { chatId, chat } = getContext(); if (!chatId) return { l0Selected: [], l1ByFloor: new Map() }; const T_Start = performance.now(); // ───────────────────────────────────────────────────────────────── // 6a. 构建 L0 候选对象(用于 rerank) // // 重要:支持 lexical-only 的 L0(atom)进入候选池。 // 否则 hybrid 会退化为 dense-only:lexical 命中的 atom 若未被 dense 命中会被直接丢弃。 // ───────────────────────────────────────────────────────────────── const l0ObjectMap = new Map(); for (const a of (anchorHits || [])) { const id = `anchor-${a.atomId}`; l0ObjectMap.set(id, { id, atomId: a.atomId, floor: a.floor, similarity: a.similarity, atom: a.atom, text: a.atom?.semantic || '', }); } // lexical-only atoms:从全量 StateAtoms 补齐(similarity 记为 0,靠 lex rank 贡献 W-RRF) const lexAtomIds = lexicalResult?.atomIds || []; if (lexAtomIds.length > 0) { const atomsList = getStateAtoms(); const atomMap = new Map(atomsList.map(a => [a.atomId, a])); for (const atomId of lexAtomIds) { const id = `anchor-${atomId}`; if (l0ObjectMap.has(id)) continue; const atom = atomMap.get(atomId); if (!atom) continue; if (typeof atom.floor !== 'number' || atom.floor < 0) continue; l0ObjectMap.set(id, { id, atomId, floor: atom.floor, similarity: 0, atom, text: atom.semantic || '', }); } } // ───────────────────────────────────────────────────────────────── // 6b. 构建两路排名(L0-only) // ───────────────────────────────────────────────────────────────── // Dense 路:anchorHits 按 similarity 排序 const denseRank = (anchorHits || []) .map(a => ({ id: `anchor-${a.atomId}`, score: a.similarity })) .sort((a, b) => b.score - a.score); // Lexical 路:从 lexicalResult.atomIds 构建排名(允许 lexical-only) // atomIds 已按 MiniSearch score 排序(searchLexicalIndex 返回顺序);W-RRF 依赖 rank,score 为占位 const lexRank = (lexAtomIds || []) .map(atomId => ({ id: `anchor-${atomId}`, score: 1 })) .filter(item => l0ObjectMap.has(item.id)); // ───────────────────────────────────────────────────────────────── // 6c. W-RRF 融合(L0-only) // ───────────────────────────────────────────────────────────────── const T_Fusion_Start = performance.now(); const { top: fusionResult, totalUnique } = fuseL0Candidates(denseRank, lexRank, CONFIG.FUSION_CAP); const fusionTime = Math.round(performance.now() - T_Fusion_Start); // 构建 rerank 候选列表 const rerankCandidates = fusionResult .map(f => l0ObjectMap.get(f.id)) .filter(Boolean); if (metrics) { metrics.fusion.denseCount = denseRank.length; metrics.fusion.lexCount = lexRank.length; metrics.fusion.totalUnique = totalUnique; metrics.fusion.afterCap = rerankCandidates.length; metrics.fusion.time = fusionTime; metrics.evidence.l0Candidates = rerankCandidates.length; } if (rerankCandidates.length === 0) { if (metrics) { metrics.evidence.l0Selected = 0; metrics.evidence.l1Pulled = 0; metrics.evidence.l1Attached = 0; metrics.evidence.l1CosineTime = 0; metrics.evidence.rerankApplied = false; } return { l0Selected: [], l1ByFloor: new Map() }; } // ───────────────────────────────────────────────────────────────── // 6d. 收集所有候选 L0 的楼层(用于并发拉取 L1) // 包含 AI 楼层本身 + 上方 USER 楼层 // ───────────────────────────────────────────────────────────────── const candidateFloors = new Set(); for (const c of rerankCandidates) { candidateFloors.add(c.floor); // 上方 USER 楼层 const userFloor = c.floor - 1; if (userFloor >= 0 && chat?.[userFloor]?.is_user) { candidateFloors.add(userFloor); } } // ───────────────────────────────────────────────────────────────── // 6e. 并发:rerank L0 ‖ 拉取 L1 chunks + 向量 + cosine 打分 // ───────────────────────────────────────────────────────────────── const T_Rerank_Start = performance.now(); // 并发任务 1:rerank L0 const rerankPromise = rerankChunks(rerankQuery, rerankCandidates, { topN: CONFIG.RERANK_TOP_N, minScore: CONFIG.RERANK_MIN_SCORE, }); // 并发任务 2:拉取 L1 chunks + 向量 → cosine 打分 const l1Promise = pullAndScoreL1(chatId, Array.from(candidateFloors), queryVector, chat); // 等待两个任务完成 const [rerankedL0, l1ScoredByFloor] = await Promise.all([rerankPromise, l1Promise]); const rerankTime = Math.round(performance.now() - T_Rerank_Start); // ───────────────────────────────────────────────────────────────── // 6f. 记录 rerank metrics // ───────────────────────────────────────────────────────────────── if (metrics) { metrics.evidence.rerankApplied = true; metrics.evidence.beforeRerank = rerankCandidates.length; metrics.evidence.afterRerank = rerankedL0.length; metrics.evidence.l0Selected = rerankedL0.length; metrics.evidence.rerankTime = rerankTime; metrics.timing.evidenceRerank = rerankTime; const scores = rerankedL0.map(c => c._rerankScore || 0).filter(s => s > 0); if (scores.length > 0) { scores.sort((a, b) => a - b); metrics.evidence.rerankScores = { min: Number(scores[0].toFixed(3)), max: Number(scores[scores.length - 1].toFixed(3)), mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)), }; } } // ───────────────────────────────────────────────────────────────── // 6g. 构建最终 l0Selected + l1ByFloor // ───────────────────────────────────────────────────────────────── const l0Selected = rerankedL0.map(item => ({ id: item.id, atomId: item.atomId, floor: item.floor, similarity: item.similarity, rerankScore: item._rerankScore || 0, atom: item.atom, text: item.text, })); // 为每个选中的 L0 楼层组装 top-1 L1 配对 const selectedFloors = new Set(l0Selected.map(l => l.floor)); const l1ByFloor = new Map(); let contextPairsAdded = 0; for (const floor of selectedFloors) { const aiChunks = l1ScoredByFloor.get(floor) || []; const userFloor = floor - 1; const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user) ? (l1ScoredByFloor.get(userFloor) || []) : []; // top-1:取 cosine 最高的 const aiTop1 = aiChunks.length > 0 ? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best)) : null; const userTop1 = userChunks.length > 0 ? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best)) : null; // context pair = 上方 USER 楼层成功挂载(用于 metrics) if (userTop1) contextPairsAdded++; l1ByFloor.set(floor, { aiTop1, userTop1 }); } // ───────────────────────────────────────────────────────────────── // 6h. L1 metrics // ───────────────────────────────────────────────────────────────── if (metrics) { let totalPulled = 0; let totalAttached = 0; for (const [, scored] of l1ScoredByFloor) { totalPulled += scored.length; } for (const [, pair] of l1ByFloor) { if (pair.aiTop1) totalAttached++; if (pair.userTop1) totalAttached++; } metrics.evidence.l1Pulled = totalPulled; metrics.evidence.l1Attached = totalAttached; metrics.evidence.contextPairsAdded = contextPairsAdded; metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0; } const totalTime = Math.round(performance.now() - T_Start); if (metrics) { metrics.timing.evidenceRetrieval = Math.max(0, totalTime - fusionTime - rerankTime); } xbLog.info(MODULE_ID, `Evidence: ${anchorHits?.length || 0} L0 dense → fusion=${rerankCandidates.length} → rerank=${rerankedL0.length} → L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)` ); return { l0Selected, l1ByFloor }; } // ═══════════════════════════════════════════════════════════════════════════ // [L1] 拉取 + Cosine 打分(并发子任务) // ═══════════════════════════════════════════════════════════════════════════ /** * 从 IndexedDB 拉取指定楼层的 L1 chunks + 向量,用 queryVector cosine 打分 * * @param {string} chatId * @param {number[]} floors - 需要拉取的楼层列表 * @param {number[]} queryVector - 查询向量(v1) * @param {object[]} chat - 聊天消息数组 * @returns {Promise>} floor → scored chunks(带 _cosineScore) */ async function pullAndScoreL1(chatId, floors, queryVector, chat) { const T0 = performance.now(); /** @type {Map} */ const result = new Map(); if (!chatId || !floors?.length || !queryVector?.length) { result._cosineTime = 0; return result; } // 拉取 chunks let dbChunks = []; try { dbChunks = await getChunksByFloors(chatId, floors); } catch (e) { xbLog.warn(MODULE_ID, 'L1 chunks 拉取失败', e); result._cosineTime = Math.round(performance.now() - T0); return result; } if (!dbChunks.length) { result._cosineTime = Math.round(performance.now() - T0); return result; } // 拉取向量 const chunkIds = dbChunks.map(c => c.chunkId); let chunkVectors = []; try { chunkVectors = await getChunkVectorsByIds(chatId, chunkIds); } catch (e) { xbLog.warn(MODULE_ID, 'L1 向量拉取失败', e); result._cosineTime = Math.round(performance.now() - T0); return result; } const vectorMap = new Map(chunkVectors.map(v => [v.chunkId, v.vector])); // Cosine 打分 + 按楼层分组 for (const chunk of dbChunks) { const vec = vectorMap.get(chunk.chunkId); const cosineScore = vec?.length ? cosineSimilarity(queryVector, vec) : 0; const scored = { chunkId: chunk.chunkId, floor: chunk.floor, chunkIdx: chunk.chunkIdx, speaker: chunk.speaker, isUser: chunk.isUser, text: chunk.text, _cosineScore: cosineScore, }; if (!result.has(chunk.floor)) { result.set(chunk.floor, []); } result.get(chunk.floor).push(scored); } // 每楼层按 cosine 降序排序 for (const [, chunks] of result) { chunks.sort((a, b) => b._cosineScore - a._cosineScore); } result._cosineTime = Math.round(performance.now() - T0); xbLog.info(MODULE_ID, `L1 pull: ${floors.length} floors → ${dbChunks.length} chunks → scored (${result._cosineTime}ms)` ); return result; } // ═══════════════════════════════════════════════════════════════════════════ // 主函数 // ═══════════════════════════════════════════════════════════════════════════ /** * 执行记忆召回 * * @param {object[]} allEvents - 所有事件(L2) * @param {object} vectorConfig - 向量配置 * @param {object} options * @param {boolean} options.excludeLastAi * @param {string|null} options.pendingUserMessage * @returns {Promise} */ export async function recallMemory(allEvents, vectorConfig, options = {}) { const T0 = performance.now(); const { chat } = getContext(); const { pendingUserMessage = null, excludeLastAi = false } = options; const metrics = createMetrics(); if (!allEvents?.length) { metrics.anchor.needRecall = false; metrics.timing.total = Math.round(performance.now() - T0); return { events: [], l0Selected: [], l1ByFloor: new Map(), causalChain: [], focusEntities: [], elapsed: metrics.timing.total, logText: 'No events.', metrics, }; } metrics.anchor.needRecall = true; // ═══════════════════════════════════════════════════════════════════ // 阶段 1: Query Build // ═══════════════════════════════════════════════════════════════════ const T_Build_Start = performance.now(); const lastMessages = getLastMessages(chat, CONFIG.LAST_MESSAGES_K, excludeLastAi); const bundle = buildQueryBundle(lastMessages, pendingUserMessage); metrics.query.buildTime = Math.round(performance.now() - T_Build_Start); metrics.anchor.focusEntities = bundle.focusEntities; if (metrics.query?.lengths) { metrics.query.lengths.v0Chars = String(bundle.queryText_v0 || '').length; metrics.query.lengths.v1Chars = null; metrics.query.lengths.rerankChars = String(bundle.rerankQuery || bundle.queryText_v0 || '').length; } xbLog.info(MODULE_ID, `Query Build: focus=[${bundle.focusEntities.join(',')}] lexTerms=[${bundle.lexicalTerms.slice(0, 5).join(',')}]` ); // ═══════════════════════════════════════════════════════════════════ // 阶段 2: Round 1 Dense Retrieval // ═══════════════════════════════════════════════════════════════════ let queryVector_v0; try { const [vec] = await embed([bundle.queryText_v0], vectorConfig, { timeout: 10000 }); queryVector_v0 = vec; } catch (e1) { xbLog.warn(MODULE_ID, 'Round 1 向量化失败,500ms 后重试', e1); await new Promise(r => setTimeout(r, 500)); try { const [vec] = await embed([bundle.queryText_v0], vectorConfig, { timeout: 15000 }); queryVector_v0 = vec; } catch (e2) { xbLog.error(MODULE_ID, 'Round 1 向量化重试仍失败', e2); metrics.timing.total = Math.round(performance.now() - T0); return { events: [], l0Selected: [], l1ByFloor: new Map(), causalChain: [], focusEntities: bundle.focusEntities, elapsed: metrics.timing.total, logText: 'Embedding failed (round 1, after retry).', metrics, }; } } if (!queryVector_v0?.length) { metrics.timing.total = Math.round(performance.now() - T0); return { events: [], l0Selected: [], l1ByFloor: new Map(), causalChain: [], focusEntities: bundle.focusEntities, elapsed: metrics.timing.total, logText: 'Empty query vector (round 1).', metrics, }; } const T_R1_Anchor_Start = performance.now(); const { hits: anchorHits_v0 } = await recallAnchors(queryVector_v0, vectorConfig, null); const r1AnchorTime = Math.round(performance.now() - T_R1_Anchor_Start); const T_R1_Event_Start = performance.now(); const eventHits_v0 = await recallEvents(queryVector_v0, allEvents, vectorConfig, bundle.focusEntities, null); const r1EventTime = Math.round(performance.now() - T_R1_Event_Start); xbLog.info(MODULE_ID, `Round 1: anchors=${anchorHits_v0.length} events=${eventHits_v0.length} (anchor=${r1AnchorTime}ms event=${r1EventTime}ms)` ); // ═══════════════════════════════════════════════════════════════════ // 阶段 3: Query Refinement // ═══════════════════════════════════════════════════════════════════ const T_Refine_Start = performance.now(); refineQueryBundle(bundle, anchorHits_v0, eventHits_v0); metrics.query.refineTime = Math.round(performance.now() - T_Refine_Start); metrics.anchor.focusEntities = bundle.focusEntities; if (metrics.query?.lengths) { metrics.query.lengths.v1Chars = bundle.queryText_v1 == null ? null : String(bundle.queryText_v1).length; metrics.query.lengths.rerankChars = String(bundle.rerankQuery || bundle.queryText_v1 || bundle.queryText_v0 || '').length; } xbLog.info(MODULE_ID, `Refinement: focus=[${bundle.focusEntities.join(',')}] hasV1=${!!bundle.queryText_v1} (${metrics.query.refineTime}ms)` ); // ═══════════════════════════════════════════════════════════════════ // 阶段 4: Round 2 Dense Retrieval // ═══════════════════════════════════════════════════════════════════ const queryTextFinal = bundle.queryText_v1 || bundle.queryText_v0; let queryVector_v1; try { const [vec] = await embed([queryTextFinal], vectorConfig, { timeout: 10000 }); queryVector_v1 = vec; } catch (e) { xbLog.warn(MODULE_ID, 'Round 2 向量化失败,降级使用 Round 1 向量', e); queryVector_v1 = queryVector_v0; } const T_R2_Anchor_Start = performance.now(); const { hits: anchorHits, floors: anchorFloors_dense } = await recallAnchors(queryVector_v1, vectorConfig, metrics); metrics.timing.anchorSearch = Math.round(performance.now() - T_R2_Anchor_Start); const T_R2_Event_Start = performance.now(); let eventHits = await recallEvents(queryVector_v1, allEvents, vectorConfig, bundle.focusEntities, metrics); metrics.timing.eventRetrieval = Math.round(performance.now() - T_R2_Event_Start); xbLog.info(MODULE_ID, `Round 2: anchors=${anchorHits.length} floors=${anchorFloors_dense.size} events=${eventHits.length}` ); // ═══════════════════════════════════════════════════════════════════ // 阶段 5: Lexical Retrieval + L0 Merge // ═══════════════════════════════════════════════════════════════════ const T_Lex_Start = performance.now(); let lexicalResult = { atomIds: [], atomFloors: new Set(), chunkIds: [], chunkFloors: new Set(), eventIds: [], chunkScores: [], searchTime: 0, }; try { const index = await getLexicalIndex(); if (index) { lexicalResult = searchLexicalIndex(index, bundle.lexicalTerms); } } catch (e) { xbLog.warn(MODULE_ID, 'Lexical 检索失败', e); } const lexTime = Math.round(performance.now() - T_Lex_Start); if (metrics) { metrics.lexical.atomHits = lexicalResult.atomIds.length; metrics.lexical.chunkHits = lexicalResult.chunkIds.length; metrics.lexical.eventHits = lexicalResult.eventIds.length; metrics.lexical.searchTime = lexTime; metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10); } // 合并 L0 floors(dense + lexical) const anchorFloors = new Set(anchorFloors_dense); for (const f of lexicalResult.atomFloors) { anchorFloors.add(f); } // 合并 L2 events(lexical 命中但 dense 未命中的 events) const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean)); const eventIndex = buildEventIndex(allEvents); let lexicalEventCount = 0; for (const eid of lexicalResult.eventIds) { if (!existingEventIds.has(eid)) { const ev = eventIndex.get(eid); if (ev) { eventHits.push({ event: ev, similarity: 0, _recallType: 'LEXICAL', }); existingEventIds.add(eid); lexicalEventCount++; } } } if (metrics && lexicalEventCount > 0) { metrics.event.byRecallType.lexical = lexicalEventCount; metrics.event.selected += lexicalEventCount; } xbLog.info(MODULE_ID, `Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedFloors=${anchorFloors.size} mergedEvents=+${lexicalEventCount} (${lexTime}ms)` ); // ═══════════════════════════════════════════════════════════════════ // 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine // ═══════════════════════════════════════════════════════════════════ const { l0Selected, l1ByFloor } = await locateAndPullEvidence( anchorHits, anchorFloors, queryVector_v1, bundle.rerankQuery, lexicalResult, metrics ); // ═══════════════════════════════════════════════════════════════════ // 阶段 7: Causation Trace // ═══════════════════════════════════════════════════════════════════ const { results: causalMap, maxDepth: causalMaxDepth } = traceCausation(eventHits, eventIndex); const recalledIdSet = new Set(eventHits.map(x => x?.event?.id).filter(Boolean)); const causalChain = causalMap .filter(x => x?.event?.id && !recalledIdSet.has(x.event.id)) .map(x => ({ event: x.event, similarity: 0, _recallType: 'CAUSAL', _causalDepth: x.depth, chainFrom: x.chainFrom, })); if (metrics.event.byRecallType) { metrics.event.byRecallType.causal = causalChain.length; } metrics.event.causalChainDepth = causalMaxDepth; metrics.event.causalCount = causalChain.length; // ═══════════════════════════════════════════════════════════════════ // 完成 // ═══════════════════════════════════════════════════════════════════ metrics.timing.total = Math.round(performance.now() - T0); metrics.event.entityNames = bundle.focusEntities; metrics.event.entitiesUsed = bundle.focusEntities.length; console.group('%c[Recall v7]', 'color: #7c3aed; font-weight: bold'); console.log(`Total: ${metrics.timing.total}ms`); console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`); console.log(`Focus: [${bundle.focusEntities.join(', ')}]`); console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors.size} floors`); console.log(`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`); console.log(`Fusion (L0-only): dense=${metrics.fusion.denseCount} lex=${metrics.fusion.lexCount} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`); console.log(`L0 Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.l0Selected || 0} (${metrics.evidence.rerankTime || 0}ms)`); console.log(`L1 Pull: ${metrics.evidence.l1Pulled || 0} chunks → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`); console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`); console.groupEnd(); return { events: eventHits, causalChain, l0Selected, l1ByFloor, focusEntities: bundle.focusEntities, elapsed: metrics.timing.total, metrics, }; }