X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update retrieval components

Browse Source
This commit is contained in:
bielie
2026-02-08 18:14:02 +08:00
parent 8fdce7b9a1
commit 8226c48624
3 changed files with 116 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
modules/story-summary/vector/retrieval/metrics.js
View File

@@ -48,17 +48,15 @@ export function createMetrics() {
// L3 Evidence Assembly
l3: {
floorsFromL0: 0,
// 候选规模（rerank 前）
l1Total: 0,
l1AfterCoarse: 0,
chunksInRange: 0,
chunksInRangeByType: { l0Virtual: 0, l1Real: 0 },
// 最终注入（rerank + sparse 后）
chunksSelected: 0,
chunksSelectedByType: { l0Virtual: 0, l1Real: 0 },
// 上下文配对
contextPairsAdded: 0,
tokens: 0,
assemblyTime: 0,
// Rerank 相关
rerankApplied: false,
beforeRerank: 0,
afterRerank: 0,
@@ -80,7 +78,6 @@ export function createMetrics() {
breakdown: {
constraints: 0,
events: 0,
entities: 0,
chunks: 0,
recentOrphans: 0,
arcs: 0,
@@ -204,8 +201,15 @@ export function formatMetricsLog(metrics) {
lines.push('[L3] Evidence Assembly');
lines.push(`├─ floors_from_l0: ${m.l3.floorsFromL0}`);
// 候选规模
lines.push(`├─ chunks_in_range: ${m.l3.chunksInRange}`);
// L1 粗筛信息
if (m.l3.l1Total > 0) {
lines.push(`├─ l1_coarse_filter:`);
lines.push(`│ ├─ total: ${m.l3.l1Total}`);
lines.push(`│ ├─ after: ${m.l3.l1AfterCoarse}`);
lines.push(`│ └─ filtered: ${m.l3.l1Total - m.l3.l1AfterCoarse}`);
}
lines.push(`├─ chunks_merged: ${m.l3.chunksInRange}`);
if (m.l3.chunksInRangeByType) {
const cir = m.l3.chunksInRangeByType;
lines.push(`│ ├─ l0_virtual: ${cir.l0Virtual || 0}`);
@@ -226,7 +230,6 @@ export function formatMetricsLog(metrics) {
lines.push(`├─ rerank_applied: false`);
}
// 最终注入规模
lines.push(`├─ chunks_selected: ${m.l3.chunksSelected}`);
if (m.l3.chunksSelectedByType) {
const cs = m.l3.chunksSelectedByType;
@@ -341,6 +344,14 @@ export function detectIssues(metrics) {
issues.push('L0 atoms not matched - may need to generate anchors');
}
// L1 粗筛问题
if (m.l3.l1Total > 0 && m.l3.l1AfterCoarse > 0) {
const coarseFilterRatio = 1 - (m.l3.l1AfterCoarse / m.l3.l1Total);
if (coarseFilterRatio > 0.9) {
issues.push(`Very high L1 coarse filter ratio (${(coarseFilterRatio * 100).toFixed(0)}%) - query may be too specific`);
}
}
// Rerank 相关问题
if (m.l3.rerankApplied) {
if (m.l3.beforeRerank > 0 && m.l3.afterRerank > 0) {
@@ -365,7 +376,7 @@ export function detectIssues(metrics) {
}
}
// 证据密度问题（基于 selected 的构成）
// 证据密度问题
if (m.l3.chunksSelected > 0 && m.l3.chunksSelectedByType) {
const l1Real = m.l3.chunksSelectedByType.l1Real || 0;
const density = l1Real / m.l3.chunksSelected;

302
modules/story-summary/vector/retrieval/recall.js
View File

@@ -1,13 +1,8 @@
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Recall Engine (v3 - L0 作为 L3 索引 + Rerank 精排)
//
// 架构：
// - Query Expansion → L0（主索引）→ L3（按楼层拉取）→ Rerank（精排）
// - Query Expansion → L2（独立检索）
// - L0 和 L2 不在同一抽象层，分开处理
// Story Summary - Recall Engine (v4 - L0 无上限 + L1 粗筛)
// ═══════════════════════════════════════════════════════════════════════════
import { getAllEventVectors, getChunksByFloors, getMeta } from '../storage/chunk-store.js';
import { getAllEventVectors, getChunksByFloors, getMeta, getChunkVectorsByIds } from '../storage/chunk-store.js';
import { getAllStateVectors, getStateAtoms } from '../storage/state-store.js';
import { getEngineFingerprint, embed } from '../utils/embedder.js';
import { xbLog } from '../../../../core/debug-core.js';
@@ -27,9 +22,11 @@ const CONFIG = {
// Query Expansion
QUERY_EXPANSION_TIMEOUT: 6000,
// L0 配置
L0_MAX_RESULTS: 30,
L0_MIN_SIMILARITY: 0.50,
// L0 配置 - 去掉硬上限，提高阈值
L0_MIN_SIMILARITY: 0.58,
// L1 粗筛配置
L1_MAX_CANDIDATES: 100,
// L2 配置
L2_CANDIDATE_MAX: 100,
@@ -37,11 +34,8 @@ const CONFIG = {
L2_MIN_SIMILARITY: 0.55,
L2_MMR_LAMBDA: 0.72,
// L3 配置（从 L0 楼层拉取）
L3_MAX_CHUNKS_PER_FLOOR: 3,
L3_MAX_TOTAL_CHUNKS: 60,
// Rerank 配置
RERANK_THRESHOLD: 80,
RERANK_TOP_N: 50,
RERANK_MIN_SCORE: 0.15,
@@ -49,6 +43,8 @@ const CONFIG = {
CAUSAL_CHAIN_MAX_DEPTH: 10,
CAUSAL_INJECT_MAX: 30,
};
// ═══════════════════════════════════════════════════════════════════════════
// 工具函数
// ═══════════════════════════════════════════════════════════════════════════
@@ -75,12 +71,6 @@ function cleanForRecall(text) {
return filterText(text).replace(/\[tts:[^\]]*\]/gi, '').trim();
}
/**
* 从 focusEntities 中移除用户名
* @param {Array} focusEntities - 焦点实体
* @param {string} userName - 用户名
* @returns {Array} 过滤后的实体
*/
function removeUserNameFromFocus(focusEntities, userName) {
const u = normalize(userName);
if (!u) return Array.isArray(focusEntities) ? focusEntities : [];
@@ -91,28 +81,17 @@ function removeUserNameFromFocus(focusEntities, userName) {
.filter(e => normalize(e) !== u);
}
/**
* 构建用于 Rerank 的查询文本
* 综合 Query Expansion 结果和最近对话
* @param {object} expansion - Query Expansion 结果
* @param {Array} lastMessages - 最近的消息
* @param {string} pendingUserMessage - 待发送的用户消息
* @returns {string} Rerank 用的查询文本
*/
function buildRerankQuery(expansion, lastMessages, pendingUserMessage) {
const parts = [];
// 1. focus entities
if (expansion?.focus?.length) {
parts.push(expansion.focus.join(' '));
}
// 2. DSL queries（取前3个）
if (expansion?.queries?.length) {
parts.push(...expansion.queries.slice(0, 3));
}
// 3. 最近对话的关键内容
const recentTexts = (lastMessages || [])
.slice(-2)
.map(m => cleanForRecall(m.mes || '').slice(0, 150))
@@ -122,7 +101,6 @@ function buildRerankQuery(expansion, lastMessages, pendingUserMessage) {
parts.push(...recentTexts);
}
// 4. 待发送消息
if (pendingUserMessage) {
parts.push(cleanForRecall(pendingUserMessage).slice(0, 200));
}
@@ -134,15 +112,6 @@ function buildRerankQuery(expansion, lastMessages, pendingUserMessage) {
// MMR 选择
// ═══════════════════════════════════════════════════════════════════════════
/**
* MMR 多样性选择
* @param {Array} candidates - 候选项
* @param {number} k - 选择数量
* @param {number} lambda - MMR 参数
* @param {Function} getVector - 获取向量函数
* @param {Function} getScore - 获取分数函数
* @returns {Array} 选中的项
*/
function mmrSelect(candidates, k, lambda, getVector, getScore) {
const selected = [];
const ids = new Set();
@@ -183,23 +152,15 @@ function mmrSelect(candidates, k, lambda, getVector, getScore) {
}
// ═══════════════════════════════════════════════════════════════════════════
// L0 检索：Query → L0 → 楼层集合
// L0 检索：无上限，阈值过滤
// ═══════════════════════════════════════════════════════════════════════════
/**
* L0 向量检索
* @param {Array} queryVector - 查询向量
* @param {object} vectorConfig - 向量配置
* @param {object} metrics - 指标对象
* @returns {Promise<object>} {atoms, floors}
*/
async function searchL0(queryVector, vectorConfig, metrics) {
const { chatId } = getContext();
if (!chatId || !queryVector?.length) {
return { atoms: [], floors: new Set() };
}
// 检查 fingerprint
const meta = await getMeta(chatId);
const fp = getEngineFingerprint(vectorConfig);
if (meta.fingerprint && meta.fingerprint !== fp) {
@@ -207,17 +168,15 @@ async function searchL0(queryVector, vectorConfig, metrics) {
return { atoms: [], floors: new Set() };
}
// 获取向量
const stateVectors = await getAllStateVectors(chatId);
if (!stateVectors.length) {
return { atoms: [], floors: new Set() };
}
// 获取 atoms 元数据
const atomsList = getStateAtoms();
const atomMap = new Map(atomsList.map(a => [a.atomId, a]));
// 计算相似度
// ★ 只按阈值过滤，不设硬上限
const scored = stateVectors
.map(sv => {
const atom = atomMap.get(sv.atomId);
@@ -232,13 +191,10 @@ async function searchL0(queryVector, vectorConfig, metrics) {
})
.filter(Boolean)
.filter(s => s.similarity >= CONFIG.L0_MIN_SIMILARITY)
.sort((a, b) => b.similarity - a.similarity)
.slice(0, CONFIG.L0_MAX_RESULTS);
.sort((a, b) => b.similarity - a.similarity);
// 收集楼层
const floors = new Set(scored.map(s => s.floor));
// 更新 metrics
if (metrics) {
metrics.l0.atomsMatched = scored.length;
metrics.l0.floorsHit = floors.size;
@@ -253,48 +209,9 @@ async function searchL0(queryVector, vectorConfig, metrics) {
}
// ═══════════════════════════════════════════════════════════════════════════
// L3 拉取：L0 楼层 → Chunks（带 Rerank 精排）
// 统计 chunks 类型构成
// ═══════════════════════════════════════════════════════════════════════════
/**
* 按楼层稀疏去重
* 每楼层最多保留 limit 个 chunk，优先保留分数高的
* @param {Array} chunks - chunk 列表（假设已按分数排序）
* @param {number} limit - 每楼层上限
* @returns {Array} 去重后的 chunks
*/
function sparseByFloor(chunks, limit = 3) {
const byFloor = new Map();
for (const c of chunks) {
const arr = byFloor.get(c.floor) || [];
if (arr.length < limit) {
arr.push(c);
byFloor.set(c.floor, arr);
}
}
const result = [];
const seen = new Set();
for (const c of chunks) {
if (!seen.has(c.chunkId)) {
const arr = byFloor.get(c.floor);
if (arr?.includes(c)) {
result.push(c);
seen.add(c.chunkId);
}
}
}
return result;
}
/**
* 统计 chunks 的类型构成
* @param {Array} chunks - chunk 列表
* @returns {object} {l0Virtual, l1Real}
*/
function countChunksByType(chunks) {
let l0Virtual = 0;
let l1Real = 0;
@@ -310,15 +227,11 @@ function countChunksByType(chunks) {
return { l0Virtual, l1Real };
}
/**
* 从 L0 命中楼层拉取 chunks，并用 Reranker 精排
* @param {Set} l0Floors - L0 命中的楼层
* @param {Array} l0Atoms - L0 atoms（用于构建虚拟 chunks）
* @param {string} queryText - 查询文本（用于 rerank）
* @param {object} metrics - 指标对象
* @returns {Promise<Array>} chunks 列表
*/
async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) {
// ═══════════════════════════════════════════════════════════════════════════
// L3 拉取 + L1 粗筛 + Rerank
// ═══════════════════════════════════════════════════════════════════════════
async function getChunksFromL0Floors(l0Floors, l0Atoms, queryVector, queryText, metrics) {
const { chatId } = getContext();
if (!chatId || !l0Floors.size) {
return [];
@@ -326,15 +239,7 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) {
const floorArray = Array.from(l0Floors);
// 从 DB 拉取 chunks
let dbChunks = [];
try {
dbChunks = await getChunksByFloors(chatId, floorArray);
} catch (e) {
xbLog.warn(MODULE_ID, '从 DB 拉取 chunks 失败', e);
}
// 构建 L0 虚拟 chunks
// 1. 构建 L0 虚拟 chunks
const l0VirtualChunks = (l0Atoms || []).map(a => ({
chunkId: `state-${a.atomId}`,
floor: a.floor,
@@ -347,40 +252,69 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) {
_atom: a.atom,
}));
// 合并所有 chunks
const allChunks = [...l0VirtualChunks, ...dbChunks.map(c => ({
...c,
isL0: false,
similarity: 0.5,
}))];
// 2. 拉取 L1 chunks
let dbChunks = [];
try {
dbChunks = await getChunksByFloors(chatId, floorArray);
} catch (e) {
xbLog.warn(MODULE_ID, '从 DB 拉取 chunks 失败', e);
}
// ★ 更新 metrics - 候选规模（rerank 前）
// 3. ★ L1 向量粗筛
let l1Filtered = [];
if (dbChunks.length > 0 && queryVector?.length) {
const chunkIds = dbChunks.map(c => c.chunkId);
let chunkVectors = [];
try {
chunkVectors = await getChunkVectorsByIds(chatId, chunkIds);
} catch (e) {
xbLog.warn(MODULE_ID, 'L1 向量获取失败', e);
}
const vectorMap = new Map(chunkVectors.map(v => [v.chunkId, v.vector]));
l1Filtered = dbChunks
.map(c => {
const vec = vectorMap.get(c.chunkId);
if (!vec?.length) return null;
return {
...c,
isL0: false,
similarity: cosineSimilarity(queryVector, vec),
};
})
.filter(Boolean)
.sort((a, b) => b.similarity - a.similarity)
.slice(0, CONFIG.L1_MAX_CANDIDATES);
}
// 4. 合并
const allChunks = [...l0VirtualChunks, ...l1Filtered];
// ★ 更新 metrics
if (metrics) {
metrics.l3.floorsFromL0 = floorArray.length;
metrics.l3.chunksInRange = allChunks.length;
metrics.l3.l1Total = dbChunks.length;
metrics.l3.l1AfterCoarse = l1Filtered.length;
metrics.l3.chunksInRange = l0VirtualChunks.length + l1Filtered.length;
metrics.l3.chunksInRangeByType = {
l0Virtual: l0VirtualChunks.length,
l1Real: dbChunks.length,
l1Real: l1Filtered.length,
};
}
// 如果数量不超限，直接按楼层去重返回
if (allChunks.length <= CONFIG.L3_MAX_TOTAL_CHUNKS) {
allChunks.sort((a, b) => (b.similarity || 0) - (a.similarity || 0));
const selected = sparseByFloor(allChunks, CONFIG.L3_MAX_CHUNKS_PER_FLOOR);
// ★ 更新 metrics - 最终注入规模
// 5. 是否需要 Rerank
if (allChunks.length <= CONFIG.RERANK_THRESHOLD) {
if (metrics) {
metrics.l3.rerankApplied = false;
metrics.l3.chunksSelected = selected.length;
metrics.l3.chunksSelectedByType = countChunksByType(selected);
metrics.l3.chunksSelected = allChunks.length;
metrics.l3.chunksSelectedByType = countChunksByType(allChunks);
}
return selected;
return allChunks;
}
// ★ Reranker 精排
// 6. Rerank 精排
const T_Rerank_Start = performance.now();
const reranked = await rerankChunks(queryText, allChunks, {
@@ -390,21 +324,16 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) {
const rerankTime = Math.round(performance.now() - T_Rerank_Start);
// 按楼层稀疏去重
const selected = sparseByFloor(reranked, CONFIG.L3_MAX_CHUNKS_PER_FLOOR);
// ★ 更新 metrics
if (metrics) {
metrics.l3.rerankApplied = true;
metrics.l3.beforeRerank = allChunks.length;
metrics.l3.afterRerank = reranked.length;
metrics.l3.chunksSelected = selected.length;
metrics.l3.chunksSelectedByType = countChunksByType(selected);
metrics.l3.chunksSelected = reranked.length;
metrics.l3.chunksSelectedByType = countChunksByType(reranked);
metrics.l3.rerankTime = rerankTime;
metrics.timing.l3Rerank = rerankTime;
// rerank 分数分布（基于 selected）
const scores = selected.map(c => c._rerankScore || 0).filter(s => s > 0);
const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0);
if (scores.length > 0) {
scores.sort((a, b) => a - b);
metrics.l3.rerankScoreDistribution = {
@@ -415,31 +344,21 @@ async function getChunksFromL0Floors(l0Floors, l0Atoms, queryText, metrics) {
}
}
xbLog.info(MODULE_ID, `L3 Rerank: ${allChunks.length} → ${reranked.length} → ${selected.length} (${rerankTime}ms)`);
xbLog.info(MODULE_ID, `L3: ${dbChunks.length} L1 → ${l1Filtered.length} 粗筛 → ${reranked.length} Rerank (${rerankTime}ms)`);
return selected;
return reranked;
}
// ═══════════════════════════════════════════════════════════════════════════
// L2 检索：Query → Events（独立）
// L2 检索（保持不变）
// ═══════════════════════════════════════════════════════════════════════════
/**
* L2 事件向量检索
* @param {Array} queryVector - 查询向量
* @param {Array} allEvents - 所有事件
* @param {object} vectorConfig - 向量配置
* @param {Array} focusEntities - 焦点实体（用于实体过滤）
* @param {object} metrics - 指标对象
* @returns {Promise<Array>} 事件列表
*/
async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntities, metrics) {
const { chatId } = getContext();
if (!chatId || !queryVector?.length || !allEvents?.length) {
return [];
}
// 检查 fingerprint
const meta = await getMeta(chatId);
const fp = getEngineFingerprint(vectorConfig);
if (meta.fingerprint && meta.fingerprint !== fp) {
@@ -447,7 +366,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
return [];
}
// 获取事件向量
const eventVectors = await getAllEventVectors(chatId);
const vectorMap = new Map(eventVectors.map(v => [v.eventId, v.vector]));
@@ -455,19 +373,15 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
return [];
}
// 实体匹配集合
const focusSet = new Set((focusEntities || []).map(normalize));
// 计算相似度
const scored = allEvents.map(event => {
const v = vectorMap.get(event.id);
const baseSim = v ? cosineSimilarity(queryVector, v) : 0;
// 实体命中检查
const participants = (event.participants || []).map(p => normalize(p));
const hasEntityMatch = participants.some(p => focusSet.has(p));
// 实体匹配加权
const bonus = hasEntityMatch ? 0.05 : 0;
return {
@@ -480,12 +394,10 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
};
});
// 更新 metrics
if (metrics) {
metrics.l2.eventsInStore = allEvents.length;
}
// 阈值过滤
let candidates = scored
.filter(s => s.similarity >= CONFIG.L2_MIN_SIMILARITY)
.sort((a, b) => b.similarity - a.similarity)
@@ -495,14 +407,11 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
metrics.l2.eventsConsidered = candidates.length;
}
// 实体过滤（可选）
if (focusSet.size > 0) {
const beforeFilter = candidates.length;
candidates = candidates.filter(c => {
// 高相似度绕过
if (c.similarity >= 0.85) return true;
// 有实体匹配的保留
return c._hasEntityMatch;
});
@@ -516,7 +425,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
}
}
// MMR 去重
const selected = mmrSelect(
candidates,
CONFIG.L2_SELECT_MAX,
@@ -525,7 +433,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
c => c.similarity
);
// 统计召回类型
let directCount = 0;
let contextCount = 0;
@@ -542,7 +449,6 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
};
});
// 更新 metrics
if (metrics) {
metrics.l2.eventsSelected = results.length;
metrics.l2.byRecallType = { direct: directCount, context: contextCount, causal: 0 };
@@ -553,14 +459,9 @@ async function searchL2Events(queryVector, allEvents, vectorConfig, focusEntitie
}
// ═══════════════════════════════════════════════════════════════════════════
// 因果链追溯
// 因果链追溯（保持不变）
// ═══════════════════════════════════════════════════════════════════════════
/**
* 构建事件索引
* @param {Array} allEvents - 所有事件
* @returns {Map} 事件索引
*/
function buildEventIndex(allEvents) {
const map = new Map();
for (const e of allEvents || []) {
@@ -569,13 +470,6 @@ function buildEventIndex(allEvents) {
return map;
}
/**
* 追溯因果祖先
* @param {Array} recalledEvents - 召回的事件
* @param {Map} eventIndex - 事件索引
* @param {number} maxDepth - 最大深度
* @returns {object} {results, maxDepth}
*/
function traceCausalAncestors(recalledEvents, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MAX_DEPTH) {
const out = new Map();
const idRe = /^evt-\d+$/;
@@ -626,19 +520,11 @@ function traceCausalAncestors(recalledEvents, eventIndex, maxDepth = CONFIG.CAUS
// 辅助函数
// ═══════════════════════════════════════════════════════════════════════════
/**
* 获取最近的消息
* @param {Array} chat - 聊天数组
* @param {number} count - 消息数量
* @param {boolean} excludeLastAi - 是否排除最后一条 AI 消息
* @returns {Array} 消息列表
*/
function getLastMessages(chat, count = 4, excludeLastAi = false) {
if (!chat?.length) return [];
let messages = [...chat];
// 排除最后一条 AI 消息（swipe/regenerate 场景）
if (excludeLastAi && messages.length > 0 && !messages[messages.length - 1]?.is_user) {
messages = messages.slice(0, -1);
}
@@ -646,13 +532,6 @@ function getLastMessages(chat, count = 4, excludeLastAi = false) {
return messages.slice(-count);
}
/**
* 构建查询文本（降级用）
* @param {Array} chat - 聊天数组
* @param {number} count - 消息数量
* @param {boolean} excludeLastAi - 是否排除最后一条 AI 消息
* @returns {string} 查询文本
*/
export function buildQueryText(chat, count = 2, excludeLastAi = false) {
if (!chat?.length) return '';
@@ -672,14 +551,6 @@ export function buildQueryText(chat, count = 2, excludeLastAi = false) {
// 主函数
// ═══════════════════════════════════════════════════════════════════════════
/**
* 记忆召回主函数
* @param {string} queryText - 查询文本（降级用）
* @param {Array} allEvents - 所有事件
* @param {object} vectorConfig - 向量配置
* @param {object} options - 选项
* @returns {Promise<object>} 召回结果
*/
export async function recallMemory(queryText, allEvents, vectorConfig, options = {}) {
const T0 = performance.now();
const { chat, name1 } = getContext();
@@ -698,7 +569,6 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
const T_QE_Start = performance.now();
// 获取最近对话
const lastMessages = getLastMessages(chat, 4, excludeLastAi);
let expansion = { focus: [], queries: [] };
@@ -712,14 +582,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
xbLog.warn(MODULE_ID, 'Query Expansion 失败，降级使用原始文本', e);
}
// 构建检索文本
const searchText = buildSearchText(expansion);
const finalSearchText = searchText || queryText || lastMessages.map(m => cleanForRecall(m.mes || '').slice(0, 200)).join(' ');
// focusEntities（移除用户名）
const focusEntities = removeUserNameFromFocus(expansion.focus, name1);
// 更新 L0 metrics
metrics.l0.needRecall = true;
metrics.l0.focusEntities = focusEntities;
metrics.l0.queries = expansion.queries || [];
@@ -746,7 +613,7 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
}
// ═══════════════════════════════════════════════════════════════════════
// Step 3: L0 检索 → L3 拉取（并行准备）
// Step 3: L0 检索
// ═══════════════════════════════════════════════════════════════════════
const T_L0_Start = performance.now();
@@ -756,15 +623,13 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
metrics.timing.l0Search = Math.round(performance.now() - T_L0_Start);
// ═══════════════════════════════════════════════════════════════════════
// Step 4: L3 从 L0 楼层拉取（带 Rerank）
// Step 4: L3 拉取 + L1 粗筛 + Rerank
// ═══════════════════════════════════════════════════════════════════════
const T_L3_Start = performance.now();
// 构建 rerank 用的查询文本
const rerankQuery = buildRerankQuery(expansion, lastMessages, pendingUserMessage);
const chunks = await getChunksFromL0Floors(l0Floors, l0Atoms, rerankQuery, metrics);
const chunks = await getChunksFromL0Floors(l0Floors, l0Atoms, queryVector, rerankQuery, metrics);
metrics.timing.l3Retrieval = Math.round(performance.now() - T_L3_Start);
@@ -796,7 +661,6 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
chainFrom: x.chainFrom,
}));
// 更新因果链 metrics
if (metrics.l2.byRecallType) {
metrics.l2.byRecallType.causal = causalEvents.length;
}
@@ -809,16 +673,14 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
metrics.timing.total = Math.round(performance.now() - T0);
// 实体信息
metrics.l2.entityNames = focusEntities;
metrics.l2.entitiesLoaded = focusEntities.length;
// 日志
console.group('%c[Recall v3]', 'color: #7c3aed; font-weight: bold');
console.group('%c[Recall v4]', 'color: #7c3aed; font-weight: bold');
console.log(`Elapsed: ${metrics.timing.total}ms`);
console.log(`Query Expansion: focus=[${expansion.focus.join(', ')}]`);
console.log(`L0: ${l0Atoms.length} atoms → ${l0Floors.size} floors`);
console.log(`L3: ${chunks.length} chunks (L0=${metrics.l3.chunksSelectedByType?.l0Virtual || 0}, DB=${metrics.l3.chunksSelectedByType?.l1Real || 0})`);
console.log(`L3: ${metrics.l3.l1Total || 0} L1 → ${metrics.l3.l1AfterCoarse || 0} 粗筛 → ${chunks.length} final`);
if (metrics.l3.rerankApplied) {
console.log(`L3 Rerank: ${metrics.l3.beforeRerank} → ${metrics.l3.afterRerank} (${metrics.l3.rerankTime}ms)`);
}

14
modules/story-summary/vector/storage/chunk-store.js
View File

@@ -159,6 +159,20 @@ export async function getAllChunkVectors(chatId) {
}));
}
export async function getChunkVectorsByIds(chatId, chunkIds) {
if (!chatId || !chunkIds?.length) return [];
const records = await chunkVectorsTable
.where('[chatId+chunkId]')
.anyOf(chunkIds.map(id => [chatId, id]))
.toArray();
return records.map(r => ({
chunkId: r.chunkId,
vector: bufferToFloat32(r.vector),
}));
}
// ═══════════════════════════════════════════════════════════════════════════
// EventVectors 表操作
// ═══════════════════════════════════════════════════════════════════════════