X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update retrieval, rerank, and indexing changes

Browse Source
This commit is contained in:
bielie
2026-02-11 13:55:19 +08:00
parent 8d062d39b5
commit 297cc03770
7 changed files with 501 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

338
modules/story-summary/vector/retrieval/recall.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence)
//
// 命名规范：
@@ -10,8 +10,8 @@
// 阶段 2: Round 1 Dense Retrieval（L0 + L2）
// 阶段 3: Query Refinement（用已命中记忆增强）
// 阶段 4: Round 2 Dense Retrieval（L0 + L2）
// 阶段 5: Lexical Retrieval + L0 Merge
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine 预筛选
// 阶段 5: Lexical Retrieval
// 阶段 6: Floor W-RRF Fusion + Rerank + L1 配对
// 阶段 7: L1 配对组装（L0 → top-1 AI L1 + top-1 USER L1）
// 阶段 8: Causation Trace
// ═══════════════════════════════════════════════════════════════════════════
@@ -49,10 +49,10 @@ const CONFIG = {
RRF_K: 60,
RRF_W_DENSE: 1.0,
RRF_W_LEX: 0.9,
FUSION_CAP: 100,
FUSION_CAP: 60,
// Rerank（L0-only）
RERANK_TOP_N: 50,
// Rerank（floor-level）
RERANK_TOP_N: 20,
RERANK_MIN_SCORE: 0.15,
// 因果链
@@ -421,14 +421,14 @@ function traceCausation(eventHits, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MA
*/
/**
* W-RRF 融合两路 L0 候选（dense + lexical）
* W-RRF 加权倒数排名融合（floor 粒度）
*
* @param {RankedItem[]} denseRank - Dense 路（cosine 降序）
* @param {RankedItem[]} lexRank - Lexical 路（MiniSearch score 降序）
* @param {{id: number, score: number}[]} denseRank - Dense 路（floor → max cosine，降序）
* @param {{id: number, score: number}[]} lexRank - Lexical 路（floor → max bm25，降序）
* @param {number} cap - 输出上限
* @returns {{top: {id: string, fusionScore: number}[], totalUnique: number}}
* @returns {{top: {id: number, fusionScore: number}[], totalUnique: number}}
*/
function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
function fuseByFloor(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
const k = CONFIG.RRF_K;
const wD = CONFIG.RRF_W_DENSE;
const wL = CONFIG.RRF_W_LEX;
@@ -445,141 +445,109 @@ function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
const denseMap = buildRankMap(denseRank || []);
const lexMap = buildRankMap(lexRank || []);
const allIds = new Set([
...denseMap.keys(),
...lexMap.keys(),
]);
const allIds = new Set([...denseMap.keys(), ...lexMap.keys()]);
const totalUnique = allIds.size;
const scored = [];
for (const id of allIds) {
let score = 0;
if (denseMap.has(id)) {
score += wD / (k + denseMap.get(id));
}
if (lexMap.has(id)) {
score += wL / (k + lexMap.get(id));
}
if (denseMap.has(id)) score += wD / (k + denseMap.get(id));
if (lexMap.has(id)) score += wL / (k + lexMap.get(id));
scored.push({ id, fusionScore: score });
}
scored.sort((a, b) => b.fusionScore - a.fusionScore);
return {
top: scored.slice(0, cap),
totalUnique,
};
return { top: scored.slice(0, cap), totalUnique };
}
// ═══════════════════════════════════════════════════════════════════════════
// [Stage 6] L0-only 融合 + Rerank ‖ 并发 L1 Cosine 预筛选
// [Stage 6] Floor 融合 + Rerank + L1 配对
// ═══════════════════════════════════════════════════════════════════════════
/**
* L0 融合 + rerank，并发拉取 L1 并 cosine 打分
* Floor 粒度融合 + Rerank + L1 配对
*
* @param {object[]} anchorHits - L0 dense 命中（Round 2）
* @param {Set<number>} anchorFloors - L0 命中楼层（含 lexical 扩展）
* @param {number[]} queryVector - 查询向量（v1）
* @param {string} rerankQuery - rerank 查询文本
* @param {string} rerankQuery - rerank 查询文本（纯自然语言）
* @param {object} lexicalResult - 词法检索结果
* @param {object} metrics
* @returns {Promise<{l0Selected: object[], l1ByFloor: Map<number, {aiTop1: object|null, userTop1: object|null}>}>}
*/
async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rerankQuery, lexicalResult, metrics) {
const { chatId, chat } = getContext();
async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexicalResult, metrics) {
const { chatId, chat, name1, name2 } = getContext();
if (!chatId) return { l0Selected: [], l1ByFloor: new Map() };
const T_Start = performance.now();
// ─────────────────────────────────────────────────────────────────
// 6a. 构建 L0 候选对象（用于 rerank）
//
// 重要：支持 lexical-only 的 L0（atom）进入候选池。
// 否则 hybrid 会退化为 dense-only：lexical 命中的 atom 若未被 dense 命中会被直接丢弃。
// 6a. Dense floor rank（每个 floor 取 max cosine）
// ─────────────────────────────────────────────────────────────────
const l0ObjectMap = new Map();
const denseFloorMap = new Map();
for (const a of (anchorHits || [])) {
const id = `anchor-${a.atomId}`;
l0ObjectMap.set(id, {
id,
atomId: a.atomId,
floor: a.floor,
similarity: a.similarity,
atom: a.atom,
text: a.atom?.semantic || '',
});
const cur = denseFloorMap.get(a.floor) || 0;
if (a.similarity > cur) denseFloorMap.set(a.floor, a.similarity);
}
// lexical-only atoms：从全量 StateAtoms 补齐（similarity 记为 0，靠 lex rank 贡献 W-RRF）
const lexAtomIds = lexicalResult?.atomIds || [];
if (lexAtomIds.length > 0) {
const atomsList = getStateAtoms();
const atomMap = new Map(atomsList.map(a => [a.atomId, a]));
const denseFloorRank = [...denseFloorMap.entries()]
.sort((a, b) => b[1] - a[1])
.map(([floor, score]) => ({ id: floor, score }));
for (const atomId of lexAtomIds) {
const id = `anchor-${atomId}`;
if (l0ObjectMap.has(id)) continue;
// ─────────────────────────────────────────────────────────────────
// 6b. Lexical floor rank（chunkScores → floor 聚合 + USER→AI 映射 + 预过滤）
// ─────────────────────────────────────────────────────────────────
const atom = atomMap.get(atomId);
if (!atom) continue;
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
const atomFloorSet = new Set(getStateAtoms().map(a => a.floor));
l0ObjectMap.set(id, {
id,
atomId,
floor: atom.floor,
similarity: 0,
atom,
text: atom.semantic || '',
});
const lexFloorScores = new Map();
for (const { chunkId, score } of (lexicalResult?.chunkScores || [])) {
const match = chunkId?.match(/^c-(\d+)-/);
if (!match) continue;
let floor = parseInt(match[1], 10);
// USER floor → AI floor 映射
if (chat?.[floor]?.is_user) {
const aiFloor = floor + 1;
if (aiFloor < chat.length && !chat[aiFloor]?.is_user) {
floor = aiFloor;
} else {
continue;
}
}
// 预过滤：必须有 L0 atoms
if (!atomFloorSet.has(floor)) continue;
const cur = lexFloorScores.get(floor) || 0;
if (score > cur) lexFloorScores.set(floor, score);
}
// ─────────────────────────────────────────────────────────────────
// 6b. 构建两路排名（L0-only）
// ─────────────────────────────────────────────────────────────────
// Dense 路：anchorHits 按 similarity 排序
const denseRank = (anchorHits || [])
.map(a => ({ id: `anchor-${a.atomId}`, score: a.similarity }))
.sort((a, b) => b.score - a.score);
// Lexical 路：从 lexicalResult.atomIds 构建排名（允许 lexical-only）
// atomIds 已按 MiniSearch score 排序（searchLexicalIndex 返回顺序）；W-RRF 依赖 rank，score 为占位
const lexRank = (lexAtomIds || [])
.map(atomId => ({ id: `anchor-${atomId}`, score: 1 }))
.filter(item => l0ObjectMap.has(item.id));
const lexFloorRank = [...lexFloorScores.entries()]
.sort((a, b) => b[1] - a[1])
.map(([floor, score]) => ({ id: floor, score }));
// ─────────────────────────────────────────────────────────────────
// 6c. W-RRF 融合（L0-only）
// 6c. Floor W-RRF 融合
// ─────────────────────────────────────────────────────────────────
const T_Fusion_Start = performance.now();
const { top: fusionResult, totalUnique } = fuseL0Candidates(denseRank, lexRank, CONFIG.FUSION_CAP);
const { top: fusedFloors, totalUnique } = fuseByFloor(denseFloorRank, lexFloorRank, CONFIG.FUSION_CAP);
const fusionTime = Math.round(performance.now() - T_Fusion_Start);
// 构建 rerank 候选列表
const rerankCandidates = fusionResult
.map(f => l0ObjectMap.get(f.id))
.filter(Boolean);
if (metrics) {
metrics.fusion.denseCount = denseRank.length;
metrics.fusion.lexCount = lexRank.length;
metrics.fusion.denseFloors = denseFloorRank.length;
metrics.fusion.lexFloors = lexFloorRank.length;
metrics.fusion.totalUnique = totalUnique;
metrics.fusion.afterCap = rerankCandidates.length;
metrics.fusion.afterCap = fusedFloors.length;
metrics.fusion.time = fusionTime;
metrics.evidence.l0Candidates = rerankCandidates.length;
metrics.evidence.floorCandidates = fusedFloors.length;
}
if (rerankCandidates.length === 0) {
if (fusedFloors.length === 0) {
if (metrics) {
metrics.evidence.l0Selected = 0;
metrics.evidence.floorsSelected = 0;
metrics.evidence.l0Collected = 0;
metrics.evidence.l1Pulled = 0;
metrics.evidence.l1Attached = 0;
metrics.evidence.l1CosineTime = 0;
@@ -589,54 +557,87 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
}
// ─────────────────────────────────────────────────────────────────
// 6d. 收集所有候选 L0 的楼层（用于并发拉取 L1）
// 包含 AI 楼层本身 + 上方 USER 楼层
// 6d. 拉取 L1 chunks + cosine 打分
// ─────────────────────────────────────────────────────────────────
const candidateFloors = new Set();
for (const c of rerankCandidates) {
candidateFloors.add(c.floor);
// 上方 USER 楼层
const userFloor = c.floor - 1;
const floorsToFetch = new Set();
for (const f of fusedFloors) {
floorsToFetch.add(f.id);
const userFloor = f.id - 1;
if (userFloor >= 0 && chat?.[userFloor]?.is_user) {
candidateFloors.add(userFloor);
floorsToFetch.add(userFloor);
}
}
const l1ScoredByFloor = await pullAndScoreL1(chatId, [...floorsToFetch], queryVector, chat);
if (metrics) {
let totalPulled = 0;
for (const [key, chunks] of l1ScoredByFloor) {
if (key === '_cosineTime') continue;
totalPulled += chunks.length;
}
metrics.evidence.l1Pulled = totalPulled;
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
}
// ─────────────────────────────────────────────────────────────────
// 6e. 并发：rerank L0 ‖ 拉取 L1 chunks + 向量 + cosine 打分
// 6e. 构建 rerank documents（每个 floor: USER chunks + AI chunks）
// ─────────────────────────────────────────────────────────────────
const rerankCandidates = [];
for (const f of fusedFloors) {
const aiFloor = f.id;
const userFloor = aiFloor - 1;
const aiChunks = l1ScoredByFloor.get(aiFloor) || [];
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
? (l1ScoredByFloor.get(userFloor) || [])
: [];
const parts = [];
const userName = chat?.[userFloor]?.name || name1 || '用户';
const aiName = chat?.[aiFloor]?.name || name2 || '角色';
if (userChunks.length > 0) {
parts.push(`${userName}：${userChunks.map(c => c.text).join(' ')}`);
}
if (aiChunks.length > 0) {
parts.push(`${aiName}：${aiChunks.map(c => c.text).join(' ')}`);
}
const text = parts.join('\n');
if (!text.trim()) continue;
rerankCandidates.push({
floor: aiFloor,
text,
fusionScore: f.fusionScore,
});
}
// ─────────────────────────────────────────────────────────────────
// 6f. 并发 Rerank
// ─────────────────────────────────────────────────────────────────
const T_Rerank_Start = performance.now();
// 并发任务 1：rerank L0
const rerankPromise = rerankChunks(rerankQuery, rerankCandidates, {
const reranked = await rerankChunks(rerankQuery, rerankCandidates, {
topN: CONFIG.RERANK_TOP_N,
minScore: CONFIG.RERANK_MIN_SCORE,
});
// 并发任务 2：拉取 L1 chunks + 向量 → cosine 打分
const l1Promise = pullAndScoreL1(chatId, Array.from(candidateFloors), queryVector, chat);
// 等待两个任务完成
const [rerankedL0, l1ScoredByFloor] = await Promise.all([rerankPromise, l1Promise]);
const rerankTime = Math.round(performance.now() - T_Rerank_Start);
// ─────────────────────────────────────────────────────────────────
// 6f. 记录 rerank metrics
// ─────────────────────────────────────────────────────────────────
if (metrics) {
metrics.evidence.rerankApplied = true;
metrics.evidence.beforeRerank = rerankCandidates.length;
metrics.evidence.afterRerank = rerankedL0.length;
metrics.evidence.rerankFailed = rerankedL0.some(c => c._rerankFailed);
metrics.evidence.l0Selected = rerankedL0.length;
metrics.evidence.afterRerank = reranked.length;
metrics.evidence.rerankFailed = reranked.some(c => c._rerankFailed);
metrics.evidence.rerankTime = rerankTime;
metrics.timing.evidenceRerank = rerankTime;
const scores = rerankedL0.map(c => c._rerankScore || 0).filter(s => s > 0);
const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0);
if (scores.length > 0) {
scores.sort((a, b) => a - b);
metrics.evidence.rerankScores = {
@@ -645,74 +646,82 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)),
};
}
// document 平均长度
if (rerankCandidates.length > 0) {
const totalLen = rerankCandidates.reduce((s, c) => s + (c.text?.length || 0), 0);
metrics.evidence.rerankDocAvgLength = Math.round(totalLen / rerankCandidates.length);
}
}
// ─────────────────────────────────────────────────────────────────
// 6g. 构建最终 l0Selected + l1ByFloor
// 6g. 收集 L0 atoms + L1 top-1 配对
// ─────────────────────────────────────────────────────────────────
const l0Selected = rerankedL0.map(item => ({
id: item.id,
atomId: item.atomId,
floor: item.floor,
similarity: item.similarity,
rerankScore: item._rerankScore || 0,
atom: item.atom,
text: item.text,
}));
const atomsList = getStateAtoms();
const atomsByFloor = new Map();
for (const atom of atomsList) {
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
if (!atomsByFloor.has(atom.floor)) atomsByFloor.set(atom.floor, []);
atomsByFloor.get(atom.floor).push(atom);
}
// 为每个选中的 L0 楼层组装 top-1 L1 配对
const selectedFloors = new Set(l0Selected.map(l => l.floor));
const l0Selected = [];
const l1ByFloor = new Map();
let contextPairsAdded = 0;
for (const floor of selectedFloors) {
for (const item of reranked) {
const floor = item.floor;
const rerankScore = item._rerankScore || 0;
const denseSim = denseFloorMap.get(floor) || 0;
// 收集该 floor 所有 L0 atoms，共享 floor 的 rerankScore
const floorAtoms = atomsByFloor.get(floor) || [];
for (const atom of floorAtoms) {
l0Selected.push({
id: `anchor-${atom.atomId}`,
atomId: atom.atomId,
floor: atom.floor,
similarity: denseSim,
rerankScore,
atom,
text: atom.semantic || '',
});
}
// L1 top-1 配对（cosine 最高）
const aiChunks = l1ScoredByFloor.get(floor) || [];
xbLog.info(
MODULE_ID,
`L1 attach check: floor=${floor}, l1ScoredByFloor.has=${l1ScoredByFloor.has(floor)}, aiChunks=${aiChunks.length}, l1ScoredByFloor.keys=[${[...l1ScoredByFloor.keys()].slice(0, 10)}...]`
);
const userFloor = floor - 1;
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
? (l1ScoredByFloor.get(userFloor) || [])
: [];
// top-1：取 cosine 最高的
const aiTop1 = aiChunks.length > 0
? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
: null;
const userTop1 = userChunks.length > 0
? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
: null;
// context pair = 上方 USER 楼层成功挂载（用于 metrics）
if (userTop1) contextPairsAdded++;
l1ByFloor.set(floor, { aiTop1, userTop1 });
}
// ─────────────────────────────────────────────────────────────────
// 6h. L1 metrics
// 6h. Metrics
// ─────────────────────────────────────────────────────────────────
if (metrics) {
let totalPulled = 0;
metrics.evidence.floorsSelected = reranked.length;
metrics.evidence.l0Collected = l0Selected.length;
let totalAttached = 0;
for (const [, scored] of l1ScoredByFloor) {
totalPulled += scored.length;
}
for (const [, pair] of l1ByFloor) {
if (pair.aiTop1) totalAttached++;
if (pair.userTop1) totalAttached++;
}
metrics.evidence.l1Pulled = totalPulled;
metrics.evidence.l1Attached = totalAttached;
metrics.evidence.contextPairsAdded = contextPairsAdded;
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
}
const totalTime = Math.round(performance.now() - T_Start);
@@ -721,13 +730,11 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
}
xbLog.info(MODULE_ID,
`Evidence: ${anchorHits?.length || 0} L0 dense → fusion=${rerankCandidates.length} → rerank=${rerankedL0.length} → L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
`Evidence: ${denseFloorRank.length} dense floors + ${lexFloorRank.length} lex floors → fusion=${fusedFloors.length} → rerank=${reranked.length} floors → L0=${l0Selected.length} L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
);
return { l0Selected, l1ByFloor };
}
// ═══════════════════════════════════════════════════════════════════════════
// [L1] 拉取 + Cosine 打分（并发子任务）
// ═══════════════════════════════════════════════════════════════════════════
@@ -973,7 +980,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
);
// ═══════════════════════════════════════════════════════════════════
// 阶段 5: Lexical Retrieval + L0 Merge
// 阶段 5: Lexical Retrieval
// ═══════════════════════════════════════════════════════════════════
const T_Lex_Start = performance.now();
@@ -1003,12 +1010,6 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10);
}
// 合并 L0 floors（dense + lexical）
const anchorFloors = new Set(anchorFloors_dense);
for (const f of lexicalResult.atomFloors) {
anchorFloors.add(f);
}
// 合并 L2 events（lexical 命中但 dense 未命中的 events）
const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean));
const eventIndex = buildEventIndex(allEvents);
@@ -1035,16 +1036,15 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
}
xbLog.info(MODULE_ID,
`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedFloors=${anchorFloors.size} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
);
// ═══════════════════════════════════════════════════════════════════
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine
// 阶段 6: Floor 粒度融合 + Rerank + L1 配对
// ═══════════════════════════════════════════════════════════════════
const { l0Selected, l1ByFloor } = await locateAndPullEvidence(
anchorHits,
anchorFloors,
queryVector_v1,
bundle.rerankQuery,
lexicalResult,
@@ -1086,11 +1086,11 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
console.log(`Total: ${metrics.timing.total}ms`);
console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`);
console.log(`Focus: [${bundle.focusEntities.join(', ')}]`);
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors.size} floors`);
console.log(`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
console.log(`Fusion (L0-only): dense=${metrics.fusion.denseCount} lex=${metrics.fusion.lexCount} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
console.log(`L0 Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.l0Selected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1 Pull: ${metrics.evidence.l1Pulled || 0} chunks → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors_dense.size} floors`);
console.log(`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
console.log(`Fusion (floor): dense=${metrics.fusion.denseFloors} lex=${metrics.fusion.lexFloors} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
console.log(`Floor Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.floorsSelected || 0} floors → L0=${metrics.evidence.l0Collected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1: ${metrics.evidence.l1Pulled || 0} pulled → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`);
console.groupEnd();