X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update retrieval, rerank, and indexing changes

Browse Source
This commit is contained in:
bielie
2026-02-11 13:55:19 +08:00
parent 8d062d39b5
commit 297cc03770
7 changed files with 501 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
modules/story-summary/vector/retrieval/lexical-index.js
View File

@@ -14,7 +14,6 @@
import MiniSearch from '../../../../libs/minisearch.mjs';
import { getContext } from '../../../../../../../extensions.js';
import { getSummaryStore } from '../../data/store.js';
import { getStateAtoms } from '../storage/state-store.js';
import { getAllChunks } from '../storage/chunk-store.js';
import { xbLog } from '../../../../core/debug-core.js';
import { tokenizeForIndex } from '../utils/tokenizer.js';
@@ -39,6 +38,8 @@ let building = false;
/** @type {Promise<MiniSearch|null>|null} 当前构建 Promise（防重入） */
let buildPromise = null;
/** @type {Map<number, string[]>} floor → 该楼层的 doc IDs（仅 L1 chunks） */
let floorDocIds = new Map();
// ─────────────────────────────────────────────────────────────────────────
// 工具函数
@@ -57,13 +58,12 @@ function cleanSummary(summary) {
/**
* 计算缓存指纹
* @param {number} atomCount
* @param {number} chunkCount
* @param {number} eventCount
* @returns {string}
*/
function computeFingerprint(atomCount, chunkCount, eventCount) {
return `${atomCount}:${chunkCount}:${eventCount}`;
function computeFingerprint(chunkCount, eventCount) {
return `${chunkCount}:${eventCount}`;
}
/**
@@ -81,34 +81,31 @@ function yieldToMain() {
/**
* 收集所有待索引文档
*
* @param {object[]} atoms - getStateAtoms() 返回值
* @param {object[]} chunks - getAllChunks(chatId) 返回值
* @param {object[]} events - store.json.events
* @returns {object[]} 文档数组
*/
function collectDocuments(atoms, chunks, events) {
function collectDocuments(chunks, events) {
const docs = [];
// L0 atoms
for (const atom of (atoms || [])) {
if (!atom?.atomId || !atom.semantic) continue;
docs.push({
id: atom.atomId,
type: 'atom',
floor: atom.floor ?? -1,
text: atom.semantic,
});
}
// L1 chunks
// L1 chunks + 填充 floorDocIds
for (const chunk of (chunks || [])) {
if (!chunk?.chunkId || !chunk.text) continue;
const floor = chunk.floor ?? -1;
docs.push({
id: chunk.chunkId,
type: 'chunk',
floor: chunk.floor ?? -1,
floor,
text: chunk.text,
});
if (floor >= 0) {
if (!floorDocIds.has(floor)) {
floorDocIds.set(floor, []);
}
floorDocIds.get(floor).push(chunk.chunkId);
}
}
// L2 events
@@ -244,7 +241,6 @@ export function searchLexicalIndex(index, terms) {
}
// 分类结果
const atomIdSet = new Set();
const chunkIdSet = new Set();
const eventIdSet = new Set();
@@ -254,16 +250,6 @@ export function searchLexicalIndex(index, terms) {
const floor = hit.floor;
switch (type) {
case 'atom':
if (!atomIdSet.has(id)) {
atomIdSet.add(id);
result.atomIds.push(id);
if (typeof floor === 'number' && floor >= 0) {
result.atomFloors.add(floor);
}
}
break;
case 'chunk':
if (!chunkIdSet.has(id)) {
chunkIdSet.add(id);
@@ -304,8 +290,10 @@ export function searchLexicalIndex(index, terms) {
* @returns {Promise<{index: MiniSearch, fingerprint: string}>}
*/
async function collectAndBuild(chatId) {
// 收集数据
const atoms = getStateAtoms() || [];
// 清空侧索引（全量重建）
floorDocIds = new Map();
// 收集数据（不含 L0 atoms）
const store = getSummaryStore();
const events = store?.json?.events || [];
@@ -316,15 +304,15 @@ async function collectAndBuild(chatId) {
xbLog.warn(MODULE_ID, '获取 chunks 失败', e);
}
const fp = computeFingerprint(atoms.length, chunks.length, events.length);
const fp = computeFingerprint(chunks.length, events.length);
// 检查是否在收集过程中缓存已被其他调用更新
if (cachedIndex && cachedChatId === chatId && cachedFingerprint === fp) {
return { index: cachedIndex, fingerprint: fp };
}
// 收集文档
const docs = collectDocuments(atoms, chunks, events);
// 收集文档（同时填充 floorDocIds）
const docs = collectDocuments(chunks, events);
// 异步分片构建
const index = await buildIndexAsync(docs);
@@ -438,4 +426,116 @@ export function invalidateLexicalIndex() {
cachedIndex = null;
cachedChatId = null;
cachedFingerprint = null;
floorDocIds = new Map();
}
// ─────────────────────────────────────────────────────────────────────────
// 增量更新接口
// ─────────────────────────────────────────────────────────────────────────
/**
* 为指定楼层添加 L1 chunks 到索引
*
* 先移除该楼层旧文档，再添加新文档。
* 如果索引不存在（缓存失效），静默跳过（下次 getLexicalIndex 全量重建）。
*
* @param {number} floor - 楼层号
* @param {object[]} chunks - chunk 对象列表（需有 chunkId、text、floor）
*/
export function addDocumentsForFloor(floor, chunks) {
if (!cachedIndex || !chunks?.length) return;
// 先移除旧文档
removeDocumentsByFloor(floor);
const docs = [];
const docIds = [];
for (const chunk of chunks) {
if (!chunk?.chunkId || !chunk.text) continue;
docs.push({
id: chunk.chunkId,
type: 'chunk',
floor: chunk.floor ?? floor,
text: chunk.text,
});
docIds.push(chunk.chunkId);
}
if (docs.length > 0) {
cachedIndex.addAll(docs);
floorDocIds.set(floor, docIds);
xbLog.info(MODULE_ID, `增量添加: floor ${floor}, ${docs.length} 个 chunk`);
}
}
/**
* 从索引中移除指定楼层的所有 L1 chunk 文档
*
* 使用 MiniSearch discard()（软删除）。
* 如果索引不存在，静默跳过。
*
* @param {number} floor - 楼层号
*/
export function removeDocumentsByFloor(floor) {
if (!cachedIndex) return;
const docIds = floorDocIds.get(floor);
if (!docIds?.length) return;
for (const id of docIds) {
try {
cachedIndex.discard(id);
} catch {
// 文档可能不存在（已被全量重建替换）
}
}
floorDocIds.delete(floor);
xbLog.info(MODULE_ID, `增量移除: floor ${floor}, ${docIds.length} 个文档`);
}
/**
* 将新 L2 事件添加到索引
*
* 如果事件 ID 已存在，先 discard 再 add（覆盖）。
* 如果索引不存在，静默跳过。
*
* @param {object[]} events - 事件对象列表（需有 id、title、summary 等）
*/
export function addEventDocuments(events) {
if (!cachedIndex || !events?.length) return;
const docs = [];
for (const ev of events) {
if (!ev?.id) continue;
const parts = [];
if (ev.title) parts.push(ev.title);
if (ev.participants?.length) parts.push(ev.participants.join(' '));
const summary = cleanSummary(ev.summary);
if (summary) parts.push(summary);
const text = parts.join(' ').trim();
if (!text) continue;
// 覆盖：先尝试移除旧的
try {
cachedIndex.discard(ev.id);
} catch {
// 不存在则忽略
}
docs.push({
id: ev.id,
type: 'event',
floor: null,
text,
});
}
if (docs.length > 0) {
cachedIndex.addAll(docs);
xbLog.info(MODULE_ID, `增量添加: ${docs.length} 个事件`);
}
}

109
modules/story-summary/vector/retrieval/metrics.js
View File

@@ -8,7 +8,7 @@
// 架构变更（v3 → v4）：
// - evidence 区块反映 L0-only 融合 + L1 按楼层拉取的两阶段架构
// - 删除 mergedByType / selectedByType（不再有混合池）
// - 新增 l0Candidates / l0Selected / l1Pulled / l1Attached / l1CosineTime
// - 新增 floorCandidates / floorsSelected / l0Collected / l1Pulled / l1Attached / l1CosineTime
// - fusion 区块明确标注 L0-only（删除 anchorCount）
// - quality.chunkRealRatio → quality.l1AttachRate
// ═══════════════════════════════════════════════════════════════════════════
@@ -48,10 +48,10 @@ export function createMetrics() {
searchTime: 0,
},
// Fusion (W-RRF, L0-only) - 多路融合
// Fusion (W-RRF, floor-level) - 多路融合
fusion: {
denseCount: 0,
lexCount: 0,
denseFloors: 0,
lexFloors: 0,
totalUnique: 0,
afterCap: 0,
time: 0,
@@ -80,25 +80,27 @@ export function createMetrics() {
entityNames: [],
},
// Evidence (Two-Stage: L0 rerank → L1 pull) - 原文证据
// Evidence (Two-Stage: Floor rerank → L1 pull) - 原文证据
evidence: {
// Stage 1: L0
l0Candidates: 0, // W-RRF 融合后的 L0 候选数
l0Selected: 0, // rerank 后选中的 L0 数
// Stage 1: Floor
floorCandidates: 0, // W-RRF 融合后的 floor 候选数
floorsSelected: 0, // rerank 后选中的 floor 数
l0Collected: 0, // 选中 floor 中收集的 L0 atom 总数
rerankApplied: false,
rerankFailed: false,
beforeRerank: 0,
afterRerank: 0,
rerankTime: 0,
rerankScores: null,
rerankDocAvgLength: 0, // rerank document 平均字符数
// Stage 2: L1
l1Pulled: 0, // 从 DB 拉取的 L1 chunk 总数
l1Attached: 0, // 实际挂载的 L1 数（top-1 × 楼层 × 2侧）
l1Attached: 0, // 实际挂载的 L1 数（top-1 × floor × 2侧）
l1CosineTime: 0, // L1 cosine 打分耗时
// 装配
contextPairsAdded: 0, // 保留兼容（= l1Attached 中 USER 侧数量）
contextPairsAdded: 0, // USER 侧挂载数量
tokens: 0,
assemblyTime: 0,
},
@@ -149,7 +151,7 @@ export function createMetrics() {
quality: {
constraintCoverage: 100,
eventPrecisionProxy: 0,
l1AttachRate: 0, // 有 L1 挂载的 L0 占比
l1AttachRate: 0, // 有 L1 挂载的 floor 占比
potentialIssues: [],
},
};
@@ -223,10 +225,10 @@ export function formatMetricsLog(metrics) {
lines.push(`└─ search_time: ${m.lexical.searchTime}ms`);
lines.push('');
// Fusion (W-RRF, L0-only)
lines.push('[Fusion] W-RRF (L0-only) - 多路融合');
lines.push(`├─ dense_count: ${m.fusion.denseCount}`);
lines.push(`├─ lex_count: ${m.fusion.lexCount}`);
// Fusion (W-RRF, floor-level)
lines.push('[Fusion] W-RRF (floor-level) - 多路融合');
lines.push(`├─ dense_floors: ${m.fusion.denseFloors}`);
lines.push(`├─ lex_floors: ${m.fusion.lexFloors}`);
lines.push(`├─ total_unique: ${m.fusion.totalUnique}`);
lines.push(`├─ after_cap: ${m.fusion.afterCap}`);
lines.push(`└─ time: ${m.fusion.time}ms`);
@@ -277,28 +279,32 @@ export function formatMetricsLog(metrics) {
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
lines.push('');
// Evidence (Two-Stage)
lines.push('[Evidence] Two-Stage: L0 Locate → L1 Pull');
lines.push(`├─ Stage 1 (L0):`);
lines.push(`│ ├─ candidates (post-fusion): ${m.evidence.l0Candidates}`);
// Evidence (Two-Stage: Floor Rerank → L1 Pull)
lines.push('[Evidence] Two-Stage: Floor Rerank → L1 Pull');
lines.push(`├─ Stage 1 (Floor Rerank):`);
lines.push(`│ ├─ floor_candidates (post-fusion): ${m.evidence.floorCandidates}`);
if (m.evidence.rerankApplied) {
lines.push(`│ ├─ rerank_applied: true`);
if (m.evidence.rerankFailed) {
lines.push(`│ ├─ rerank_failed: ⚠ YES (using fusion order)`);
lines.push(`│ │ ⚠ rerank_failed: using fusion order`);
}
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank}`);
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank}`);
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank} floors`);
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank} floors`);
lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`);
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
}
if (m.evidence.rerankDocAvgLength > 0) {
lines.push(`│ ├─ rerank_doc_avg_length: ${m.evidence.rerankDocAvgLength} chars`);
}
} else {
lines.push(`│ ├─ rerank_applied: false`);
}
lines.push(`│ └─ selected: ${m.evidence.l0Selected}`);
lines.push(`│ ├─ floors_selected: ${m.evidence.floorsSelected}`);
lines.push(`│ └─ l0_atoms_collected: ${m.evidence.l0Collected}`);
lines.push(`├─ Stage 2 (L1):`);
lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`);
lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`);
@@ -345,9 +351,7 @@ export function formatMetricsLog(metrics) {
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
if (m.timing.evidenceRerank > 0) {
lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`);
}
lines.push(`├─ floor_rerank: ${m.timing.evidenceRerank || 0}ms`);
lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`);
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
@@ -406,20 +410,20 @@ export function detectIssues(metrics) {
// 词法检索问题
// ─────────────────────────────────────────────────────────────────
if ((m.lexical.terms || []).length > 0 && m.lexical.atomHits === 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
if ((m.lexical.terms || []).length > 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
issues.push('Lexical search returned zero hits - terms may not match any indexed content');
}
// ─────────────────────────────────────────────────────────────────
// 融合问题（L0-only）
// 融合问题（floor-level）
// ─────────────────────────────────────────────────────────────────
if (m.fusion.lexCount === 0 && m.fusion.denseCount > 0) {
issues.push('No lexical L0 candidates in fusion - hybrid retrieval not contributing');
if (m.fusion.lexFloors === 0 && m.fusion.denseFloors > 0) {
issues.push('No lexical floors in fusion - hybrid retrieval not contributing');
}
if (m.fusion.afterCap === 0) {
issues.push('Fusion produced zero L0 candidates - all retrieval paths may have failed');
issues.push('Fusion produced zero floor candidates - all retrieval paths may have failed');
}
// ─────────────────────────────────────────────────────────────────
@@ -463,29 +467,30 @@ export function detectIssues(metrics) {
}
// ─────────────────────────────────────────────────────────────────
// L0 Rerank 问题
// Floor Rerank 问题
// ─────────────────────────────────────────────────────────────────
if (m.evidence.rerankApplied) {
if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) {
const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank);
if (filterRatio > 0.7) {
issues.push(`High L0 rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant L0 in fusion output`);
}
}
if (m.evidence.rerankFailed) {
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
}
if (m.evidence.rerankApplied && !m.evidence.rerankFailed) {
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
if (rs.max < 0.5) {
issues.push(`Low L0 rerank scores (max=${rs.max}) - query may be poorly matched`);
if (rs.max < 0.3) {
issues.push(`Low floor rerank scores (max=${rs.max}) - query-document domain mismatch`);
}
if (rs.mean < 0.3) {
issues.push(`Very low average L0 rerank score (mean=${rs.mean}) - context may be weak`);
if (rs.mean < 0.2) {
issues.push(`Very low average floor rerank score (mean=${rs.mean}) - context may be weak`);
}
}
if (m.evidence.rerankTime > 2000) {
issues.push(`Slow L0 rerank (${m.evidence.rerankTime}ms) - may affect response time`);
if (m.evidence.rerankTime > 3000) {
issues.push(`Slow floor rerank (${m.evidence.rerankTime}ms) - may affect response time`);
}
if (m.evidence.rerankDocAvgLength > 3000) {
issues.push(`Large rerank documents (avg ${m.evidence.rerankDocAvgLength} chars) - may reduce rerank precision`);
}
}
@@ -493,21 +498,17 @@ export function detectIssues(metrics) {
// L1 挂载问题
// ─────────────────────────────────────────────────────────────────
if (m.evidence.rerankFailed) {
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
}
if (m.evidence.l0Selected > 0 && m.evidence.l1Pulled === 0) {
if (m.evidence.floorsSelected > 0 && m.evidence.l1Pulled === 0) {
issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed');
}
if (m.evidence.l0Selected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
issues.push('L1 chunks pulled but none attached - cosine scores may be too low or floor mismatch');
if (m.evidence.floorsSelected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
issues.push('L1 chunks pulled but none attached - cosine scores may be too low');
}
const l1AttachRate = m.quality.l1AttachRate || 0;
if (m.evidence.l0Selected > 5 && l1AttachRate < 20) {
issues.push(`Low L1 attach rate (${l1AttachRate}%) - many L0 lack concrete dialogue evidence`);
if (m.evidence.floorsSelected > 3 && l1AttachRate < 50) {
issues.push(`Low L1 attach rate (${l1AttachRate}%) - selected floors lack L1 chunks`);
}
// ─────────────────────────────────────────────────────────────────

49
modules/story-summary/vector/retrieval/query-builder.js
View File

@@ -89,6 +89,46 @@ function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) {
.map(([term]) => term);
}
/**
* 构建 rerank 专用查询（纯自然语言，不带结构标签）
*
* reranker（bge-reranker-v2-m3）的 query 应为自然语言文本，
* 不含 [ENTITIES] [DIALOGUE] 等结构标签。
*
* @param {string[]} focusEntities - 焦点实体
* @param {object[]} lastMessages - 最近 K 条消息
* @param {string|null} pendingUserMessage - 待发送的用户消息
* @param {object} context - { name1, name2 }
* @returns {string}
*/
function buildRerankQuery(focusEntities, lastMessages, pendingUserMessage, context) {
const parts = [];
// 实体提示
if (focusEntities.length > 0) {
parts.push(`关于${focusEntities.join('、')}。`);
}
// 最近对话原文
for (const m of (lastMessages || [])) {
const speaker = m.is_user ? (context.name1 || '用户') : (m.name || context.name2 || '角色');
const clean = cleanMessageText(m.mes || '');
if (clean) {
parts.push(`${speaker}：${clean}`);
}
}
// 待发送消息
if (pendingUserMessage) {
const clean = cleanMessageText(pendingUserMessage);
if (clean) {
parts.push(`${context.name1 || '用户'}：${clean}`);
}
}
return parts.join('\n');
}
// ─────────────────────────────────────────────────────────────────────────
// QueryBundle 类型定义（JSDoc）
// ─────────────────────────────────────────────────────────────────────────
@@ -176,9 +216,8 @@ export function buildQueryBundle(lastMessages, pendingUserMessage, store = null,
const queryText_v0 = queryParts.join('\n\n');
// 6. rerankQuery 与 embedding query 同源（零暗箱）
// 后续 refine 会把它升级为与 queryText_v1 同源。
const rerankQuery = queryText_v0;
// 6. rerankQuery 独立构建（纯自然语言，供 reranker 使用）
const rerankQuery = buildRerankQuery(focusEntities, dialogueLines.length > 0 ? lastMessages : [], pendingUserMessage, context);
// 7. 构建 lexicalTerms
const entityTerms = focusEntities.map(e => e.toLowerCase());
@@ -281,8 +320,8 @@ export function refineQueryBundle(bundle, anchorHits, eventHits) {
}
}
// 5. rerankQuery 与最终 query 同源（零暗箱）
bundle.rerankQuery = bundle.queryText_v1 || bundle.queryText_v0;
// 5. rerankQuery 保持独立（不随 refinement 变更）
// reranker 需要纯自然语言 query，不受 memory hints 干扰
// 6. 增强 lexicalTerms
if (hints.length > 0) {

338
modules/story-summary/vector/retrieval/recall.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence)
//
// 命名规范：
@@ -10,8 +10,8 @@
// 阶段 2: Round 1 Dense Retrieval（L0 + L2）
// 阶段 3: Query Refinement（用已命中记忆增强）
// 阶段 4: Round 2 Dense Retrieval（L0 + L2）
// 阶段 5: Lexical Retrieval + L0 Merge
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine 预筛选
// 阶段 5: Lexical Retrieval
// 阶段 6: Floor W-RRF Fusion + Rerank + L1 配对
// 阶段 7: L1 配对组装（L0 → top-1 AI L1 + top-1 USER L1）
// 阶段 8: Causation Trace
// ═══════════════════════════════════════════════════════════════════════════
@@ -49,10 +49,10 @@ const CONFIG = {
RRF_K: 60,
RRF_W_DENSE: 1.0,
RRF_W_LEX: 0.9,
FUSION_CAP: 100,
FUSION_CAP: 60,
// Rerank（L0-only）
RERANK_TOP_N: 50,
// Rerank（floor-level）
RERANK_TOP_N: 20,
RERANK_MIN_SCORE: 0.15,
// 因果链
@@ -421,14 +421,14 @@ function traceCausation(eventHits, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MA
*/
/**
* W-RRF 融合两路 L0 候选（dense + lexical）
* W-RRF 加权倒数排名融合（floor 粒度）
*
* @param {RankedItem[]} denseRank - Dense 路（cosine 降序）
* @param {RankedItem[]} lexRank - Lexical 路（MiniSearch score 降序）
* @param {{id: number, score: number}[]} denseRank - Dense 路（floor → max cosine，降序）
* @param {{id: number, score: number}[]} lexRank - Lexical 路（floor → max bm25，降序）
* @param {number} cap - 输出上限
* @returns {{top: {id: string, fusionScore: number}[], totalUnique: number}}
* @returns {{top: {id: number, fusionScore: number}[], totalUnique: number}}
*/
function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
function fuseByFloor(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
const k = CONFIG.RRF_K;
const wD = CONFIG.RRF_W_DENSE;
const wL = CONFIG.RRF_W_LEX;
@@ -445,141 +445,109 @@ function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
const denseMap = buildRankMap(denseRank || []);
const lexMap = buildRankMap(lexRank || []);
const allIds = new Set([
...denseMap.keys(),
...lexMap.keys(),
]);
const allIds = new Set([...denseMap.keys(), ...lexMap.keys()]);
const totalUnique = allIds.size;
const scored = [];
for (const id of allIds) {
let score = 0;
if (denseMap.has(id)) {
score += wD / (k + denseMap.get(id));
}
if (lexMap.has(id)) {
score += wL / (k + lexMap.get(id));
}
if (denseMap.has(id)) score += wD / (k + denseMap.get(id));
if (lexMap.has(id)) score += wL / (k + lexMap.get(id));
scored.push({ id, fusionScore: score });
}
scored.sort((a, b) => b.fusionScore - a.fusionScore);
return {
top: scored.slice(0, cap),
totalUnique,
};
return { top: scored.slice(0, cap), totalUnique };
}
// ═══════════════════════════════════════════════════════════════════════════
// [Stage 6] L0-only 融合 + Rerank ‖ 并发 L1 Cosine 预筛选
// [Stage 6] Floor 融合 + Rerank + L1 配对
// ═══════════════════════════════════════════════════════════════════════════
/**
* L0 融合 + rerank，并发拉取 L1 并 cosine 打分
* Floor 粒度融合 + Rerank + L1 配对
*
* @param {object[]} anchorHits - L0 dense 命中（Round 2）
* @param {Set<number>} anchorFloors - L0 命中楼层（含 lexical 扩展）
* @param {number[]} queryVector - 查询向量（v1）
* @param {string} rerankQuery - rerank 查询文本
* @param {string} rerankQuery - rerank 查询文本（纯自然语言）
* @param {object} lexicalResult - 词法检索结果
* @param {object} metrics
* @returns {Promise<{l0Selected: object[], l1ByFloor: Map<number, {aiTop1: object|null, userTop1: object|null}>}>}
*/
async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rerankQuery, lexicalResult, metrics) {
const { chatId, chat } = getContext();
async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexicalResult, metrics) {
const { chatId, chat, name1, name2 } = getContext();
if (!chatId) return { l0Selected: [], l1ByFloor: new Map() };
const T_Start = performance.now();
// ─────────────────────────────────────────────────────────────────
// 6a. 构建 L0 候选对象（用于 rerank）
//
// 重要：支持 lexical-only 的 L0（atom）进入候选池。
// 否则 hybrid 会退化为 dense-only：lexical 命中的 atom 若未被 dense 命中会被直接丢弃。
// 6a. Dense floor rank（每个 floor 取 max cosine）
// ─────────────────────────────────────────────────────────────────
const l0ObjectMap = new Map();
const denseFloorMap = new Map();
for (const a of (anchorHits || [])) {
const id = `anchor-${a.atomId}`;
l0ObjectMap.set(id, {
id,
atomId: a.atomId,
floor: a.floor,
similarity: a.similarity,
atom: a.atom,
text: a.atom?.semantic || '',
});
const cur = denseFloorMap.get(a.floor) || 0;
if (a.similarity > cur) denseFloorMap.set(a.floor, a.similarity);
}
// lexical-only atoms：从全量 StateAtoms 补齐（similarity 记为 0，靠 lex rank 贡献 W-RRF）
const lexAtomIds = lexicalResult?.atomIds || [];
if (lexAtomIds.length > 0) {
const atomsList = getStateAtoms();
const atomMap = new Map(atomsList.map(a => [a.atomId, a]));
const denseFloorRank = [...denseFloorMap.entries()]
.sort((a, b) => b[1] - a[1])
.map(([floor, score]) => ({ id: floor, score }));
for (const atomId of lexAtomIds) {
const id = `anchor-${atomId}`;
if (l0ObjectMap.has(id)) continue;
// ─────────────────────────────────────────────────────────────────
// 6b. Lexical floor rank（chunkScores → floor 聚合 + USER→AI 映射 + 预过滤）
// ─────────────────────────────────────────────────────────────────
const atom = atomMap.get(atomId);
if (!atom) continue;
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
const atomFloorSet = new Set(getStateAtoms().map(a => a.floor));
l0ObjectMap.set(id, {
id,
atomId,
floor: atom.floor,
similarity: 0,
atom,
text: atom.semantic || '',
});
const lexFloorScores = new Map();
for (const { chunkId, score } of (lexicalResult?.chunkScores || [])) {
const match = chunkId?.match(/^c-(\d+)-/);
if (!match) continue;
let floor = parseInt(match[1], 10);
// USER floor → AI floor 映射
if (chat?.[floor]?.is_user) {
const aiFloor = floor + 1;
if (aiFloor < chat.length && !chat[aiFloor]?.is_user) {
floor = aiFloor;
} else {
continue;
}
}
// 预过滤：必须有 L0 atoms
if (!atomFloorSet.has(floor)) continue;
const cur = lexFloorScores.get(floor) || 0;
if (score > cur) lexFloorScores.set(floor, score);
}
// ─────────────────────────────────────────────────────────────────
// 6b. 构建两路排名（L0-only）
// ─────────────────────────────────────────────────────────────────
// Dense 路：anchorHits 按 similarity 排序
const denseRank = (anchorHits || [])
.map(a => ({ id: `anchor-${a.atomId}`, score: a.similarity }))
.sort((a, b) => b.score - a.score);
// Lexical 路：从 lexicalResult.atomIds 构建排名（允许 lexical-only）
// atomIds 已按 MiniSearch score 排序（searchLexicalIndex 返回顺序）；W-RRF 依赖 rank，score 为占位
const lexRank = (lexAtomIds || [])
.map(atomId => ({ id: `anchor-${atomId}`, score: 1 }))
.filter(item => l0ObjectMap.has(item.id));
const lexFloorRank = [...lexFloorScores.entries()]
.sort((a, b) => b[1] - a[1])
.map(([floor, score]) => ({ id: floor, score }));
// ─────────────────────────────────────────────────────────────────
// 6c. W-RRF 融合（L0-only）
// 6c. Floor W-RRF 融合
// ─────────────────────────────────────────────────────────────────
const T_Fusion_Start = performance.now();
const { top: fusionResult, totalUnique } = fuseL0Candidates(denseRank, lexRank, CONFIG.FUSION_CAP);
const { top: fusedFloors, totalUnique } = fuseByFloor(denseFloorRank, lexFloorRank, CONFIG.FUSION_CAP);
const fusionTime = Math.round(performance.now() - T_Fusion_Start);
// 构建 rerank 候选列表
const rerankCandidates = fusionResult
.map(f => l0ObjectMap.get(f.id))
.filter(Boolean);
if (metrics) {
metrics.fusion.denseCount = denseRank.length;
metrics.fusion.lexCount = lexRank.length;
metrics.fusion.denseFloors = denseFloorRank.length;
metrics.fusion.lexFloors = lexFloorRank.length;
metrics.fusion.totalUnique = totalUnique;
metrics.fusion.afterCap = rerankCandidates.length;
metrics.fusion.afterCap = fusedFloors.length;
metrics.fusion.time = fusionTime;
metrics.evidence.l0Candidates = rerankCandidates.length;
metrics.evidence.floorCandidates = fusedFloors.length;
}
if (rerankCandidates.length === 0) {
if (fusedFloors.length === 0) {
if (metrics) {
metrics.evidence.l0Selected = 0;
metrics.evidence.floorsSelected = 0;
metrics.evidence.l0Collected = 0;
metrics.evidence.l1Pulled = 0;
metrics.evidence.l1Attached = 0;
metrics.evidence.l1CosineTime = 0;
@@ -589,54 +557,87 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
}
// ─────────────────────────────────────────────────────────────────
// 6d. 收集所有候选 L0 的楼层（用于并发拉取 L1）
// 包含 AI 楼层本身 + 上方 USER 楼层
// 6d. 拉取 L1 chunks + cosine 打分
// ─────────────────────────────────────────────────────────────────
const candidateFloors = new Set();
for (const c of rerankCandidates) {
candidateFloors.add(c.floor);
// 上方 USER 楼层
const userFloor = c.floor - 1;
const floorsToFetch = new Set();
for (const f of fusedFloors) {
floorsToFetch.add(f.id);
const userFloor = f.id - 1;
if (userFloor >= 0 && chat?.[userFloor]?.is_user) {
candidateFloors.add(userFloor);
floorsToFetch.add(userFloor);
}
}
const l1ScoredByFloor = await pullAndScoreL1(chatId, [...floorsToFetch], queryVector, chat);
if (metrics) {
let totalPulled = 0;
for (const [key, chunks] of l1ScoredByFloor) {
if (key === '_cosineTime') continue;
totalPulled += chunks.length;
}
metrics.evidence.l1Pulled = totalPulled;
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
}
// ─────────────────────────────────────────────────────────────────
// 6e. 并发：rerank L0 ‖ 拉取 L1 chunks + 向量 + cosine 打分
// 6e. 构建 rerank documents（每个 floor: USER chunks + AI chunks）
// ─────────────────────────────────────────────────────────────────
const rerankCandidates = [];
for (const f of fusedFloors) {
const aiFloor = f.id;
const userFloor = aiFloor - 1;
const aiChunks = l1ScoredByFloor.get(aiFloor) || [];
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
? (l1ScoredByFloor.get(userFloor) || [])
: [];
const parts = [];
const userName = chat?.[userFloor]?.name || name1 || '用户';
const aiName = chat?.[aiFloor]?.name || name2 || '角色';
if (userChunks.length > 0) {
parts.push(`${userName}：${userChunks.map(c => c.text).join(' ')}`);
}
if (aiChunks.length > 0) {
parts.push(`${aiName}：${aiChunks.map(c => c.text).join(' ')}`);
}
const text = parts.join('\n');
if (!text.trim()) continue;
rerankCandidates.push({
floor: aiFloor,
text,
fusionScore: f.fusionScore,
});
}
// ─────────────────────────────────────────────────────────────────
// 6f. 并发 Rerank
// ─────────────────────────────────────────────────────────────────
const T_Rerank_Start = performance.now();
// 并发任务 1：rerank L0
const rerankPromise = rerankChunks(rerankQuery, rerankCandidates, {
const reranked = await rerankChunks(rerankQuery, rerankCandidates, {
topN: CONFIG.RERANK_TOP_N,
minScore: CONFIG.RERANK_MIN_SCORE,
});
// 并发任务 2：拉取 L1 chunks + 向量 → cosine 打分
const l1Promise = pullAndScoreL1(chatId, Array.from(candidateFloors), queryVector, chat);
// 等待两个任务完成
const [rerankedL0, l1ScoredByFloor] = await Promise.all([rerankPromise, l1Promise]);
const rerankTime = Math.round(performance.now() - T_Rerank_Start);
// ─────────────────────────────────────────────────────────────────
// 6f. 记录 rerank metrics
// ─────────────────────────────────────────────────────────────────
if (metrics) {
metrics.evidence.rerankApplied = true;
metrics.evidence.beforeRerank = rerankCandidates.length;
metrics.evidence.afterRerank = rerankedL0.length;
metrics.evidence.rerankFailed = rerankedL0.some(c => c._rerankFailed);
metrics.evidence.l0Selected = rerankedL0.length;
metrics.evidence.afterRerank = reranked.length;
metrics.evidence.rerankFailed = reranked.some(c => c._rerankFailed);
metrics.evidence.rerankTime = rerankTime;
metrics.timing.evidenceRerank = rerankTime;
const scores = rerankedL0.map(c => c._rerankScore || 0).filter(s => s > 0);
const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0);
if (scores.length > 0) {
scores.sort((a, b) => a - b);
metrics.evidence.rerankScores = {
@@ -645,74 +646,82 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)),
};
}
// document 平均长度
if (rerankCandidates.length > 0) {
const totalLen = rerankCandidates.reduce((s, c) => s + (c.text?.length || 0), 0);
metrics.evidence.rerankDocAvgLength = Math.round(totalLen / rerankCandidates.length);
}
}
// ─────────────────────────────────────────────────────────────────
// 6g. 构建最终 l0Selected + l1ByFloor
// 6g. 收集 L0 atoms + L1 top-1 配对
// ─────────────────────────────────────────────────────────────────
const l0Selected = rerankedL0.map(item => ({
id: item.id,
atomId: item.atomId,
floor: item.floor,
similarity: item.similarity,
rerankScore: item._rerankScore || 0,
atom: item.atom,
text: item.text,
}));
const atomsList = getStateAtoms();
const atomsByFloor = new Map();
for (const atom of atomsList) {
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
if (!atomsByFloor.has(atom.floor)) atomsByFloor.set(atom.floor, []);
atomsByFloor.get(atom.floor).push(atom);
}
// 为每个选中的 L0 楼层组装 top-1 L1 配对
const selectedFloors = new Set(l0Selected.map(l => l.floor));
const l0Selected = [];
const l1ByFloor = new Map();
let contextPairsAdded = 0;
for (const floor of selectedFloors) {
for (const item of reranked) {
const floor = item.floor;
const rerankScore = item._rerankScore || 0;
const denseSim = denseFloorMap.get(floor) || 0;
// 收集该 floor 所有 L0 atoms，共享 floor 的 rerankScore
const floorAtoms = atomsByFloor.get(floor) || [];
for (const atom of floorAtoms) {
l0Selected.push({
id: `anchor-${atom.atomId}`,
atomId: atom.atomId,
floor: atom.floor,
similarity: denseSim,
rerankScore,
atom,
text: atom.semantic || '',
});
}
// L1 top-1 配对（cosine 最高）
const aiChunks = l1ScoredByFloor.get(floor) || [];
xbLog.info(
MODULE_ID,
`L1 attach check: floor=${floor}, l1ScoredByFloor.has=${l1ScoredByFloor.has(floor)}, aiChunks=${aiChunks.length}, l1ScoredByFloor.keys=[${[...l1ScoredByFloor.keys()].slice(0, 10)}...]`
);
const userFloor = floor - 1;
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
? (l1ScoredByFloor.get(userFloor) || [])
: [];
// top-1：取 cosine 最高的
const aiTop1 = aiChunks.length > 0
? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
: null;
const userTop1 = userChunks.length > 0
? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
: null;
// context pair = 上方 USER 楼层成功挂载（用于 metrics）
if (userTop1) contextPairsAdded++;
l1ByFloor.set(floor, { aiTop1, userTop1 });
}
// ─────────────────────────────────────────────────────────────────
// 6h. L1 metrics
// 6h. Metrics
// ─────────────────────────────────────────────────────────────────
if (metrics) {
let totalPulled = 0;
metrics.evidence.floorsSelected = reranked.length;
metrics.evidence.l0Collected = l0Selected.length;
let totalAttached = 0;
for (const [, scored] of l1ScoredByFloor) {
totalPulled += scored.length;
}
for (const [, pair] of l1ByFloor) {
if (pair.aiTop1) totalAttached++;
if (pair.userTop1) totalAttached++;
}
metrics.evidence.l1Pulled = totalPulled;
metrics.evidence.l1Attached = totalAttached;
metrics.evidence.contextPairsAdded = contextPairsAdded;
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
}
const totalTime = Math.round(performance.now() - T_Start);
@@ -721,13 +730,11 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
}
xbLog.info(MODULE_ID,
`Evidence: ${anchorHits?.length || 0} L0 dense → fusion=${rerankCandidates.length} → rerank=${rerankedL0.length} → L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
`Evidence: ${denseFloorRank.length} dense floors + ${lexFloorRank.length} lex floors → fusion=${fusedFloors.length} → rerank=${reranked.length} floors → L0=${l0Selected.length} L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
);
return { l0Selected, l1ByFloor };
}
// ═══════════════════════════════════════════════════════════════════════════
// [L1] 拉取 + Cosine 打分（并发子任务）
// ═══════════════════════════════════════════════════════════════════════════
@@ -973,7 +980,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
);
// ═══════════════════════════════════════════════════════════════════
// 阶段 5: Lexical Retrieval + L0 Merge
// 阶段 5: Lexical Retrieval
// ═══════════════════════════════════════════════════════════════════
const T_Lex_Start = performance.now();
@@ -1003,12 +1010,6 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10);
}
// 合并 L0 floors（dense + lexical）
const anchorFloors = new Set(anchorFloors_dense);
for (const f of lexicalResult.atomFloors) {
anchorFloors.add(f);
}
// 合并 L2 events（lexical 命中但 dense 未命中的 events）
const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean));
const eventIndex = buildEventIndex(allEvents);
@@ -1035,16 +1036,15 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
}
xbLog.info(MODULE_ID,
`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedFloors=${anchorFloors.size} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
);
// ═══════════════════════════════════════════════════════════════════
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine
// 阶段 6: Floor 粒度融合 + Rerank + L1 配对
// ═══════════════════════════════════════════════════════════════════
const { l0Selected, l1ByFloor } = await locateAndPullEvidence(
anchorHits,
anchorFloors,
queryVector_v1,
bundle.rerankQuery,
lexicalResult,
@@ -1086,11 +1086,11 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
console.log(`Total: ${metrics.timing.total}ms`);
console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`);
console.log(`Focus: [${bundle.focusEntities.join(', ')}]`);
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors.size} floors`);
console.log(`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
console.log(`Fusion (L0-only): dense=${metrics.fusion.denseCount} lex=${metrics.fusion.lexCount} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
console.log(`L0 Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.l0Selected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1 Pull: ${metrics.evidence.l1Pulled || 0} chunks → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors_dense.size} floors`);
console.log(`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
console.log(`Fusion (floor): dense=${metrics.fusion.denseFloors} lex=${metrics.fusion.lexFloors} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
console.log(`Floor Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.floorsSelected || 0} floors → L0=${metrics.evidence.l0Collected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1: ${metrics.evidence.l1Pulled || 0} pulled → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`);
console.groupEnd();