diff --git a/modules/story-summary/vector/retrieval/metrics.js b/modules/story-summary/vector/retrieval/metrics.js index 5c6bbd4..51162fb 100644 --- a/modules/story-summary/vector/retrieval/metrics.js +++ b/modules/story-summary/vector/retrieval/metrics.js @@ -48,17 +48,15 @@ export function createMetrics() { // L3 Evidence Assembly l3: { floorsFromL0: 0, - // 候选规模（rerank 前） + l1Total: 0, + l1AfterCoarse: 0, chunksInRange: 0, chunksInRangeByType: { l0Virtual: 0, l1Real: 0 }, - // 最终注入（rerank + sparse 后） chunksSelected: 0, chunksSelectedByType: { l0Virtual: 0, l1Real: 0 }, - // 上下文配对 contextPairsAdded: 0, tokens: 0, assemblyTime: 0, - // Rerank 相关 rerankApplied: false, beforeRerank: 0, afterRerank: 0, @@ -80,7 +78,6 @@ export function createMetrics() { breakdown: { constraints: 0, events: 0, - entities: 0, chunks: 0, recentOrphans: 0, arcs: 0, @@ -204,8 +201,15 @@ export function formatMetricsLog(metrics) { lines.push('[L3] Evidence Assembly'); lines.push(`├─ floors_from_l0: ${m.l3.floorsFromL0}`); - // 候选规模 - lines.push(`├─ chunks_in_range: ${m.l3.chunksInRange}`); + // L1 粗筛信息 + if (m.l3.l1Total > 0) { + lines.push(`├─ l1_coarse_filter:`); + lines.push(`│ ├─ total: ${m.l3.l1Total}`); + lines.push(`│ ├─ after: ${m.l3.l1AfterCoarse}`); + lines.push(`│ └─ filtered: ${m.l3.l1Total - m.l3.l1AfterCoarse}`); + } + + lines.push(`├─ chunks_merged: ${m.l3.chunksInRange}`); if (m.l3.chunksInRangeByType) { const cir = m.l3.chunksInRangeByType; lines.push(`│ ├─ l0_virtual: ${cir.l0Virtual || 0}`); @@ -226,7 +230,6 @@ export function formatMetricsLog(metrics) { lines.push(`├─ rerank_applied: false`); } - // 最终注入规模 lines.push(`├─ chunks_selected: ${m.l3.chunksSelected}`); if (m.l3.chunksSelectedByType) { const cs = m.l3.chunksSelectedByType; @@ -341,6 +344,14 @@ export function detectIssues(metrics) { issues.push('L0 atoms not matched - may need to generate anchors'); } + // L1 粗筛问题 + if (m.l3.l1Total > 0 && m.l3.l1AfterCoarse > 0) { + const coarseFilterRatio = 1 - (m.l3.l1AfterCoarse / m.l3.l1Total); + if (coarseFilterRatio > 0.9) { + issues.push(`Very high L1 coarse filter ratio (${(coarseFilterRatio * 100).toFixed(0)}%) - query may be too specific`); + } + } + // Rerank 相关问题 if (m.l3.rerankApplied) { if (m.l3.beforeRerank > 0 && m.l3.afterRerank > 0) { @@ -365,7 +376,7 @@ export function detectIssues(metrics) { } } - // 证据密度问题（基于 selected 的构成） + // 证据密度问题 if (m.l3.chunksSelected > 0 && m.l3.chunksSelectedByType) { const l1Real = m.l3.chunksSelectedByType.l1Real || 0; const density = l1Real / m.l3.chunksSelected; diff --git a/modules/story-summary/vector/retrieval/recall.js b/modules/story-summary/vector/retrieval/recall.js index c5c094d..9da6ea5 100644 --- a/modules/story-summary/vector/retrieval/recall.js +++ b/modules/story-summary/vector/retrieval/recall.js @@ -1,13 +1,8 @@ // ═══════════════════════════════════════════════════════════════════════════ -// Story Summary - Recall Engine (v3 - L0 作为 L3 索引 + Rerank 精排) -// -// 架构： -// - Query Expansion → L0（主索引）→ L3（按楼层拉取）→ Rerank（精排） -// - Query Expansion → L2（独立检索） -// - L0 和 L2 不在同一抽象层，分开处理 +// Story Summary - Recall Engine (v4 - L0 无上限 + L1 粗筛) // ═══════════════════════════════════════════════════════════════════════════ -import { getAllEventVectors, getChunksByFloors, getMeta } from '../storage/chunk-store.js'; +import { getAllEventVectors, getChunksByFloors, getMeta, getChunkVectorsByIds } from '../storage/chunk-store.js'; import { getAllStateVectors, getStateAtoms } from '../storage/state-store.js'; import { getEngineFingerprint, embed } from '../utils/embedder.js'; import { xbLog } from '../../../../core/debug-core.js'; @@ -27,9 +22,11 @@ const CONFIG = { // Query Expansion QUERY_EXPANSION_TIMEOUT: 6000, - // L0 配置 - L0_MAX_RESULTS: 30, - L0_MIN_SIMILARITY: 0.50, + // L0 配置 - 去掉硬上限，提高阈值 + L0_MIN_SIMILARITY: 0.58, + + // L1 粗筛配置 + L1_MAX_CANDIDATES: 100, // L2 配置 L2_CANDIDATE_MAX: 100, @@ -37,11 +34,8 @@ const CONFIG = { L2_MIN_SIMILARITY: 0.55, L2_MMR_LAMBDA: 0.72, - // L3 配置（从 L0 楼层拉取） - L3_MAX_CHUNKS_PER_FLOOR: 3, - L3_MAX_TOTAL_CHUNKS: 60, - // Rerank 配置 + RERANK_THRESHOLD: 80, RERANK_TOP_N: 50, RERANK_MIN_SCORE: 0.15, @@ -49,6 +43,8 @@ const CONFIG = { CAUSAL_CHAIN_MAX_DEPTH: 10, CAUSAL_INJECT_MAX: 30, }; + +// ═══════════════════════════════════════════════════════════════════════════ // 工具函数 // ═══════════════════════════════════════════════════════════════════════════ @@ -75,12 +71,6 @@ function cleanForRecall(text) { return filterText(text).replace(/\[tts:[^\]]*\]/gi, '').trim(); } -/** - * 从 focusEntities 中移除用户名 - * @param {Array} focusEntities - 焦点实体 - * @param {string} userName - 用户名 - * @returns {Array} 过滤后的实体 - */ function removeUserNameFromFocus(focusEntities, userName) { const u = normalize(userName); if (!u) return Array.isArray(focusEntities) ? focusEntities : []; @@ -91,28 +81,17 @@ function removeUserNameFromFocus(focusEntities, userName) { .filter(e => normalize(e) !== u); } -/** - * 构建用于 Rerank 的查询文本 - * 综合 Query Expansion 结果和最近对话 - * @param {object} expansion - Query Expansion 结果 - * @param {Array} lastMessages - 最近的消息 - * @param {string} pendingUserMessage - 待发送的用户消息 - * @returns {string} Rerank 用的查询文本 - */ function buildRerankQuery(expansion, lastMessages, pendingUserMessage) { const parts = []; - // 1. focus entities if (expansion?.focus?.length) { parts.push(expansion.focus.join(' ')); } - // 2. DSL queries（取前3个） if (expansion?.queries?.length) { parts.push(...expansion.queries.slice(0, 3)); } - // 3. 最近对话的关键内容 const recentTexts = (lastMessages || []) .slice(-2) .map(m => cleanForRecall(m.mes || '').slice(0, 150)) @@ -122,7 +101,6 @@ function buildRerankQuery(expansion, lastMessages, pendingUserMessage) { parts.push(...recentTexts); } - // 4. 待发送消息 if (pendingUserMessage) { parts.push(cleanForRecall(pendingUserMessage).slice(0, 200)); } @@ -134,15 +112,6 @@ function buildRerankQuery(expansion, lastMessages, pendingUserMessage) { // MMR 选择 // ═══════════════════════════════════════════════════════════════════════════ -/** - * MMR 多样性选择 - * @param {Array} candidates - 候选项 - * @param {number} k - 选择数量 - * @param {number} lambda - MMR 参数 - * @param {Function} getVector - 获取向量函数 - * @param {Function} getScore - 获取分数函数 - * @returns {Array} 选中的项 - */ function mmrSelect(candidates, k, lambda, getVector, getScore) { const selected = []; const ids = new Set(); @@ -183,23 +152,15 @@ function mmrSelect(candidates, k, lambda, getVector, getScore) { } // ═══════════════════════════════════════════════════════════════════════════ -// L0 检索：Query → L0 → 楼层集合 +// L0 检索：无上限，阈值过滤 // ═══════════════════════════════════════════════════════════════════════════ -/** - * L0 向量检索 - * @param {Array} queryVector - 查询向量 - * @param {object} vectorConfig - 向量配置 - * @param {object} metrics - 指标对象 - * @returns {Promise} {atoms, floors} - */ async function searchL0(queryVector, vectorConfig, metrics) { const { chatId } = getContext(); if (!chatId || !queryVector?.length) { return { atoms: [], floors: new Set() }; } - // 检查 fingerprint const meta = await getMeta(chatId); const fp = getEngineFingerprint(vectorConfig); if (meta.fingerprint && meta.fingerprint !== fp) { @@ -207,17 +168,15 @@ async function searchL0(queryVector, vectorConfig, metrics) { return { atoms: [], floors: new Set() }; } - // 获取向量 const stateVectors = await getAllStateVectors(chatId); if (!stateVectors.length) { return { atoms: [], floors: new Set() }; } - // 获取 atoms 元数据 const atomsList = getStateAtoms(); const atomMap = new Map(atomsList.map(a => [a.atomId, a])); - // 计算相似度 + // ★ 只按阈值过滤，不设硬上限 const scored = stateVectors .map(sv => { const atom = atomMap.get(sv.atomId); @@ -232,13 +191,10 @@ async function searchL0(queryVector, vectorConfig, metrics) { }) .filter(Boolean) .filter(s => s.similarity >= CONFIG.L0_MIN_SIMILARITY) - .sort((a, b) => b.similarity - a.similarity) - .slice(0, CONFIG.L0_MAX_RESULTS); + .sort((a, b) => b.similarity - a.similarity); - // 收集楼层 const floors = new Set(scored.map(s => s.floor)); - // 更新 metrics if (metrics) { metrics.l0.atomsMatched = scored.length; metrics.l0.floorsHit = floors.size; @@ -253,48 +209,9 @@ async function searchL0(queryVector, vectorConfig, metrics) { } // ═══════════════════════════════════════════════════════════════════════════ -// L3 拉取：L0 楼层 → Chunks（带 Rerank 精排） +// 统计 chunks 类型构成 // ═══════════════════════════════════════════════════════════════════════════ -/** - * 按楼层稀疏去重 - * 每楼层最多保留 limit 个 chunk，优先保留分数高的 - * @param {Array} chunks - chunk 列表（假设已按分数排序） - * @param {number} limit - 每楼层上限 - * @returns {Array} 去重后的 chunks - */ -function sparseByFloor(chunks, limit = 3) { - const byFloor = new Map(); - - for (const c of chunks) { - const arr = byFloor.get(c.floor) || []; - if (arr.length < limit) { - arr.push(c); - byFloor.set(c.floor, arr); - } - } - - const result = []; - const seen = new Set(); - - for (const c of chunks) { - if (!seen.has(c.chunkId)) { - const arr = byFloor.get(c.floor); - if (arr?.includes(c)) { - result.push(c); - seen.add(c.chunkId); - } - } - } - - return result; -} - -/** - * 统计 chunks 的类型构成 - * @param {Array} chunks - chunk 列表 - * @returns {object} {l0Virtual, l1Real} - */ function countChunksByType(chunks) { let l0Virtual = 0; let l1Real = 0; @@ -310,15 +227,11 @@ function countChunksByType(chunks) { return { l0Virtual, l1Real }; } -/** - * 从 L0 命中楼层拉取 chunks，并用 Reranker 精排 - * @param {Set} l0Floors - L0 命中的楼层 - * @param {Array} l0Atoms - L0 atoms（用于构建虚拟 chunks） - * @param {string} queryText - 查询文本（用于 rerank） - * @param {object} metrics - 指标对象 - * @returns {Promise} chunks 列表 - */ -async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) { +// ═══════════════════════════════════════════════════════════════════════════ +// L3 拉取 + L1 粗筛 + Rerank +// ═══════════════════════════════════════════════════════════════════════════ + +async function getChunksFromL0Floors(l0Floors, l0Atoms, queryVector, queryText, metrics) { const { chatId } = getContext(); if (!chatId || !l0Floors.size) { return []; @@ -326,15 +239,7 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) { const floorArray = Array.from(l0Floors); - // 从 DB 拉取 chunks - let dbChunks = []; - try { - dbChunks = await getChunksByFloors(chatId, floorArray); - } catch (e) { - xbLog.warn(MODULE_ID, '从 DB 拉取 chunks 失败', e); - } - - // 构建 L0 虚拟 chunks + // 1. 构建 L0 虚拟 chunks const l0VirtualChunks = (l0Atoms || []).map(a => ({ chunkId: `state-${a.atomId}`, floor: a.floor, @@ -347,40 +252,69 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) { _atom: a.atom, })); - // 合并所有 chunks - const allChunks = [...l0VirtualChunks, ...dbChunks.map(c => ({ - ...c, - isL0: false, - similarity: 0.5, - }))]; + // 2. 拉取 L1 chunks + let dbChunks = []; + try { + dbChunks = await getChunksByFloors(chatId, floorArray); + } catch (e) { + xbLog.warn(MODULE_ID, '从 DB 拉取 chunks 失败', e); + } - // ★ 更新 metrics - 候选规模（rerank 前） + // 3. ★ L1 向量粗筛 + let l1Filtered = []; + if (dbChunks.length > 0 && queryVector?.length) { + const chunkIds = dbChunks.map(c => c.chunkId); + let chunkVectors = []; + try { + chunkVectors = await getChunkVectorsByIds(chatId, chunkIds); + } catch (e) { + xbLog.warn(MODULE_ID, 'L1 向量获取失败', e); + } + + const vectorMap = new Map(chunkVectors.map(v => [v.chunkId, v.vector])); + + l1Filtered = dbChunks + .map(c => { + const vec = vectorMap.get(c.chunkId); + if (!vec?.length) return null; + + return { + ...c, + isL0: false, + similarity: cosineSimilarity(queryVector, vec), + }; + }) + .filter(Boolean) + .sort((a, b) => b.similarity - a.similarity) + .slice(0, CONFIG.L1_MAX_CANDIDATES); + } + + // 4. 合并 + const allChunks = [...l0VirtualChunks, ...l1Filtered]; + + // ★ 更新 metrics if (metrics) { metrics.l3.floorsFromL0 = floorArray.length; - metrics.l3.chunksInRange = allChunks.length; + metrics.l3.l1Total = dbChunks.length; + metrics.l3.l1AfterCoarse = l1Filtered.length; + metrics.l3.chunksInRange = l0VirtualChunks.length + l1Filtered.length; metrics.l3.chunksInRangeByType = { l0Virtual: l0VirtualChunks.length, - l1Real: dbChunks.length, + l1Real: l1Filtered.length, }; } - // 如果数量不超限，直接按楼层去重返回 - if (allChunks.length <= CONFIG.L3_MAX_TOTAL_CHUNKS) { - allChunks.sort((a, b) => (b.similarity || 0) - (a.similarity || 0)); - - const selected = sparseByFloor(allChunks, CONFIG.L3_MAX_CHUNKS_PER_FLOOR); - - // ★ 更新 metrics - 最终注入规模 + // 5. 是否需要 Rerank + if (allChunks.length <= CONFIG.RERANK_THRESHOLD) { if (metrics) { metrics.l3.rerankApplied = false; - metrics.l3.chunksSelected = selected.length; - metrics.l3.chunksSelectedByType = countChunksByType(selected); + metrics.l3.chunksSelected = allChunks.length; + metrics.l3.chunksSelectedByType = countChunksByType(allChunks); } - - return selected; + return allChunks; } - // ★ Reranker 精排 + // 6. Rerank 精排 const T_Rerank_Start = performance.now(); const reranked = await rerankChunks(queryText, allChunks, { @@ -390,21 +324,16 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) { const rerankTime = Math.round(performance.now() - T_Rerank_Start); - // 按楼层稀疏去重 - const selected = sparseByFloor(reranked, CONFIG.L3_MAX_CHUNKS_PER_FLOOR); - - // ★ 更新 metrics if (metrics) { metrics.l3.rerankApplied = true; metrics.l3.beforeRerank = allChunks.length; metrics.l3.afterRerank = reranked.length; - metrics.l3.chunksSelected = selected.length; - metrics.l3.chunksSelectedByType = countChunksByType(selected); + metrics.l3.chunksSelected = reranked.length; + metrics.l3.chunksSelectedByType = countChunksByType(reranked); metrics.l3.rerankTime = rerankTime; metrics.timing.l3Rerank = rerankTime; - // rerank 分数分布（基于 selected） - const scores = selected.map(c => c._rerankScore || 0).filter(s => s > 0); + const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0); if (scores.length > 0) { scores.sort((a, b) => a - b); metrics.l3.rerankScoreDistribution = { @@ -415,31 +344,21 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) { } } - xbLog.info(MODULE_ID, `L3 Rerank: ${allChunks.length} → ${reranked.length} → ${selected.length} (${rerankTime}ms)`); + xbLog.info(MODULE_ID, `L3: ${dbChunks.length} L1 → ${l1Filtered.length} 粗筛 → ${reranked.length} Rerank (${rerankTime}ms)`); - return selected; + return reranked; } // ═══════════════════════════════════════════════════════════════════════════ -// L2 检索：Query → Events（独立） +// L2 检索（保持不变） // ═══════════════════════════════════════════════════════════════════════════ -/** - * L2 事件向量检索 - * @param {Array} queryVector - 查询向量 - * @param {Array} allEvents - 所有事件 - * @param {object} vectorConfig - 向量配置 - * @param {Array} focusEntities - 焦点实体（用于实体过滤） - * @param {object} metrics - 指标对象 - * @returns {Promise} 事件列表 - */ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntities, metrics) { const { chatId } = getContext(); if (!chatId || !queryVector?.length || !allEvents?.length) { return []; } - // 检查 fingerprint const meta = await getMeta(chatId); const fp = getEngineFingerprint(vectorConfig); if (meta.fingerprint && meta.fingerprint !== fp) { @@ -447,7 +366,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie return []; } - // 获取事件向量 const eventVectors = await getAllEventVectors(chatId); const vectorMap = new Map(eventVectors.map(v => [v.eventId, v.vector])); @@ -455,19 +373,15 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie return []; } - // 实体匹配集合 const focusSet = new Set((focusEntities || []).map(normalize)); - // 计算相似度 const scored = allEvents.map(event => { const v = vectorMap.get(event.id); const baseSim = v ? cosineSimilarity(queryVector, v) : 0; - // 实体命中检查 const participants = (event.participants || []).map(p => normalize(p)); const hasEntityMatch = participants.some(p => focusSet.has(p)); - // 实体匹配加权 const bonus = hasEntityMatch ? 0.05 : 0; return { @@ -480,12 +394,10 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie }; }); - // 更新 metrics if (metrics) { metrics.l2.eventsInStore = allEvents.length; } - // 阈值过滤 let candidates = scored .filter(s => s.similarity >= CONFIG.L2_MIN_SIMILARITY) .sort((a, b) => b.similarity - a.similarity) @@ -495,14 +407,11 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie metrics.l2.eventsConsidered = candidates.length; } - // 实体过滤（可选） if (focusSet.size > 0) { const beforeFilter = candidates.length; candidates = candidates.filter(c => { - // 高相似度绕过 if (c.similarity >= 0.85) return true; - // 有实体匹配的保留 return c._hasEntityMatch; }); @@ -516,7 +425,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie } } - // MMR 去重 const selected = mmrSelect( candidates, CONFIG.L2_SELECT_MAX, @@ -525,7 +433,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie c => c.similarity ); - // 统计召回类型 let directCount = 0; let contextCount = 0; @@ -542,7 +449,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie }; }); - // 更新 metrics if (metrics) { metrics.l2.eventsSelected = results.length; metrics.l2.byRecallType = { direct: directCount, context: contextCount, causal: 0 }; @@ -553,14 +459,9 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie } // ═══════════════════════════════════════════════════════════════════════════ -// 因果链追溯 +// 因果链追溯（保持不变） // ═══════════════════════════════════════════════════════════════════════════ -/** - * 构建事件索引 - * @param {Array} allEvents - 所有事件 - * @returns {Map} 事件索引 - */ function buildEventIndex(allEvents) { const map = new Map(); for (const e of allEvents || []) { @@ -569,13 +470,6 @@ function buildEventIndex(allEvents) { return map; } -/** - * 追溯因果祖先 - * @param {Array} recalledEvents - 召回的事件 - * @param {Map} eventIndex - 事件索引 - * @param {number} maxDepth - 最大深度 - * @returns {object} {results, maxDepth} - */ function traceCausalAncestors(recalledEvents, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MAX_DEPTH) { const out = new Map(); const idRe = /^evt-\d+$/; @@ -626,19 +520,11 @@ function traceCausalAncestors(recalledEvents, eventIndex, maxDepth = CONFIG.CAUS // 辅助函数 // ═══════════════════════════════════════════════════════════════════════════ -/** - * 获取最近的消息 - * @param {Array} chat - 聊天数组 - * @param {number} count - 消息数量 - * @param {boolean} excludeLastAi - 是否排除最后一条 AI 消息 - * @returns {Array} 消息列表 - */ function getLastMessages(chat, count = 4, excludeLastAi = false) { if (!chat?.length) return []; let messages = [...chat]; - // 排除最后一条 AI 消息（swipe/regenerate 场景） if (excludeLastAi && messages.length > 0 && !messages[messages.length - 1]?.is_user) { messages = messages.slice(0, -1); } @@ -646,13 +532,6 @@ function getLastMessages(chat, count = 4, excludeLastAi = false) { return messages.slice(-count); } -/** - * 构建查询文本（降级用） - * @param {Array} chat - 聊天数组 - * @param {number} count - 消息数量 - * @param {boolean} excludeLastAi - 是否排除最后一条 AI 消息 - * @returns {string} 查询文本 - */ export function buildQueryText(chat, count = 2, excludeLastAi = false) { if (!chat?.length) return ''; @@ -672,14 +551,6 @@ export function buildQueryText(chat, count = 2, excludeLastAi = false) { // 主函数 // ═══════════════════════════════════════════════════════════════════════════ -/** - * 记忆召回主函数 - * @param {string} queryText - 查询文本（降级用） - * @param {Array} allEvents - 所有事件 - * @param {object} vectorConfig - 向量配置 - * @param {object} options - 选项 - * @returns {Promise} 召回结果 - */ export async function recallMemory(queryText, allEvents, vectorConfig, options = {}) { const T0 = performance.now(); const { chat, name1 } = getContext(); @@ -698,7 +569,6 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = const T_QE_Start = performance.now(); - // 获取最近对话 const lastMessages = getLastMessages(chat, 4, excludeLastAi); let expansion = { focus: [], queries: [] }; @@ -712,14 +582,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = xbLog.warn(MODULE_ID, 'Query Expansion 失败，降级使用原始文本', e); } - // 构建检索文本 const searchText = buildSearchText(expansion); const finalSearchText = searchText || queryText || lastMessages.map(m => cleanForRecall(m.mes || '').slice(0, 200)).join(' '); - // focusEntities（移除用户名） const focusEntities = removeUserNameFromFocus(expansion.focus, name1); - // 更新 L0 metrics metrics.l0.needRecall = true; metrics.l0.focusEntities = focusEntities; metrics.l0.queries = expansion.queries || []; @@ -746,7 +613,7 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = } // ═══════════════════════════════════════════════════════════════════════ - // Step 3: L0 检索 → L3 拉取（并行准备） + // Step 3: L0 检索 // ═══════════════════════════════════════════════════════════════════════ const T_L0_Start = performance.now(); @@ -756,15 +623,13 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = metrics.timing.l0Search = Math.round(performance.now() - T_L0_Start); // ═══════════════════════════════════════════════════════════════════════ - // Step 4: L3 从 L0 楼层拉取（带 Rerank） + // Step 4: L3 拉取 + L1 粗筛 + Rerank // ═══════════════════════════════════════════════════════════════════════ const T_L3_Start = performance.now(); - // 构建 rerank 用的查询文本 const rerankQuery = buildRerankQuery(expansion, lastMessages, pendingUserMessage); - - const chunks = await getChunksFromL0Floors(l0Floors, l0Atoms, rerankQuery, metrics); + const chunks = await getChunksFromL0Floors(l0Floors, l0Atoms, queryVector, rerankQuery, metrics); metrics.timing.l3Retrieval = Math.round(performance.now() - T_L3_Start); @@ -796,7 +661,6 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = chainFrom: x.chainFrom, })); - // 更新因果链 metrics if (metrics.l2.byRecallType) { metrics.l2.byRecallType.causal = causalEvents.length; } @@ -809,16 +673,14 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = metrics.timing.total = Math.round(performance.now() - T0); - // 实体信息 metrics.l2.entityNames = focusEntities; metrics.l2.entitiesLoaded = focusEntities.length; - // 日志 - console.group('%c[Recall v3]', 'color: #7c3aed; font-weight: bold'); + console.group('%c[Recall v4]', 'color: #7c3aed; font-weight: bold'); console.log(`Elapsed: ${metrics.timing.total}ms`); console.log(`Query Expansion: focus=[${expansion.focus.join(', ')}]`); console.log(`L0: ${l0Atoms.length} atoms → ${l0Floors.size} floors`); - console.log(`L3: ${chunks.length} chunks (L0=${metrics.l3.chunksSelectedByType?.l0Virtual || 0}, DB=${metrics.l3.chunksSelectedByType?.l1Real || 0})`); + console.log(`L3: ${metrics.l3.l1Total || 0} L1 → ${metrics.l3.l1AfterCoarse || 0} 粗筛 → ${chunks.length} final`); if (metrics.l3.rerankApplied) { console.log(`L3 Rerank: ${metrics.l3.beforeRerank} → ${metrics.l3.afterRerank} (${metrics.l3.rerankTime}ms)`); } diff --git a/modules/story-summary/vector/storage/chunk-store.js b/modules/story-summary/vector/storage/chunk-store.js index 8d33393..b40fc36 100644 --- a/modules/story-summary/vector/storage/chunk-store.js +++ b/modules/story-summary/vector/storage/chunk-store.js @@ -159,6 +159,20 @@ export async function getAllChunkVectors(chatId) { })); } +export async function getChunkVectorsByIds(chatId, chunkIds) { + if (!chatId || !chunkIds?.length) return []; + + const records = await chunkVectorsTable + .where('[chatId+chunkId]') + .anyOf(chunkIds.map(id => [chatId, id])) + .toArray(); + + return records.map(r => ({ + chunkId: r.chunkId, + vector: bufferToFloat32(r.vector), + })); +} + // ═══════════════════════════════════════════════════════════════════════════ // EventVectors 表操作 // ═══════════════════════════════════════════════════════════════════════════