From 297cc037701f8002cc9bf8b72e1d0993252a8f70 Mon Sep 17 00:00:00 2001 From: bielie Date: Wed, 11 Feb 2026 13:55:19 +0800 Subject: [PATCH] Update retrieval, rerank, and indexing changes --- modules/story-summary/generate/llm.js | 2 +- modules/story-summary/vector/llm/reranker.js | 110 ++++-- .../vector/pipeline/chunk-builder.js | 10 +- .../vector/retrieval/lexical-index.js | 170 +++++++-- .../story-summary/vector/retrieval/metrics.js | 109 +++--- .../vector/retrieval/query-builder.js | 49 ++- .../story-summary/vector/retrieval/recall.js | 338 +++++++++--------- 7 files changed, 501 insertions(+), 287 deletions(-) diff --git a/modules/story-summary/generate/llm.js b/modules/story-summary/generate/llm.js index 4371a61..28c7e49 100644 --- a/modules/story-summary/generate/llm.js +++ b/modules/story-summary/generate/llm.js @@ -11,7 +11,7 @@ const PROVIDER_MAP = { custom: "custom", }; -const JSON_PREFILL = '{"mindful_prelude": {'; +const JSON_PREFILL = '下面重新生成完整JSON。'; const LLM_PROMPT_CONFIG = { topSystem: `Story Analyst: This task involves narrative comprehension and structured incremental summarization, representing creative story analysis at the intersection of plot tracking and character development. As a story analyst, you will conduct systematic evaluation of provided dialogue content to generate structured incremental summary data. diff --git a/modules/story-summary/vector/llm/reranker.js b/modules/story-summary/vector/llm/reranker.js index 191ad54..2702076 100644 --- a/modules/story-summary/vector/llm/reranker.js +++ b/modules/story-summary/vector/llm/reranker.js @@ -1,4 +1,4 @@ -// ═══════════════════════════════════════════════════════════════════════════ +// ═══════════════════════════════════════════════════════════════════════════ // Reranker - 硅基 bge-reranker-v2-m3 // 对候选文档进行精排，过滤与 query 不相关的内容 // ═══════════════════════════════════════════════════════════════════════════ @@ -11,6 +11,8 @@ const RERANK_URL = 'https://api.siliconflow.cn/v1/rerank'; const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3'; const DEFAULT_TIMEOUT = 15000; const MAX_DOCUMENTS = 100; // API 限制 +const RERANK_BATCH_SIZE = 20; +const RERANK_MAX_CONCURRENCY = 5; /** * 对文档列表进行 Rerank 精排 @@ -140,25 +142,83 @@ export async function rerankChunks(query, chunks, options = {}) { const { topN = 40, minScore = 0.1 } = options; if (!chunks?.length) return []; - if (chunks.length <= topN) { - const texts = chunks.map(c => c.text || c.semantic || ''); - const { results, failed } = await rerank(query, texts, { topN: chunks.length, ...options }); + + const texts = chunks.map(c => c.text || c.semantic || ''); + + // ─── 单批：直接调用 ─── + if (texts.length <= RERANK_BATCH_SIZE) { + const { results, failed } = await rerank(query, texts, { + topN: Math.min(topN, texts.length), + timeout: options.timeout, + signal: options.signal, + }); if (failed) { return chunks.map(c => ({ ...c, _rerankScore: 0, _rerankFailed: true })); } - const scoreMap = new Map(results.map(r => [r.index, r.relevance_score])); - return chunks.map((c, i) => ({ - ...c, - _rerankScore: scoreMap.get(i) ?? 0, - })).sort((a, b) => b._rerankScore - a._rerankScore); + return results + .filter(r => r.relevance_score >= minScore) + .sort((a, b) => b.relevance_score - a.relevance_score) + .slice(0, topN) + .map(r => ({ + ...chunks[r.index], + _rerankScore: r.relevance_score, + })); } - const texts = chunks.map(c => c.text || c.semantic || ''); - const { results, failed } = await rerank(query, texts, { topN, ...options }); + // ─── 多批：拆分 → 并发 → 合并 ─── + const batches = []; + for (let i = 0; i < texts.length; i += RERANK_BATCH_SIZE) { + batches.push({ + texts: texts.slice(i, i + RERANK_BATCH_SIZE), + offset: i, + }); + } - if (failed) { + const concurrency = Math.min(batches.length, RERANK_MAX_CONCURRENCY); + xbLog.info(MODULE_ID, `并发 Rerank: ${batches.length} 批 × ≤${RERANK_BATCH_SIZE} docs, concurrency=${concurrency}`); + + const batchResults = new Array(batches.length); + let failedBatches = 0; + + const runBatch = async (batchIdx) => { + const batch = batches[batchIdx]; + const { results, failed } = await rerank(query, batch.texts, { + topN: batch.texts.length, + timeout: options.timeout, + signal: options.signal, + }); + + if (failed) { + failedBatches++; + // 单批降级：保留原始顺序，score=0 + batchResults[batchIdx] = batch.texts.map((_, i) => ({ + globalIndex: batch.offset + i, + relevance_score: 0, + _batchFailed: true, + })); + } else { + batchResults[batchIdx] = results.map(r => ({ + globalIndex: batch.offset + r.index, + relevance_score: r.relevance_score, + })); + } + }; + + // 并发池 + let nextIdx = 0; + const worker = async () => { + while (nextIdx < batches.length) { + const idx = nextIdx++; + await runBatch(idx); + } + }; + await Promise.all(Array.from({ length: concurrency }, () => worker())); + + // 全部失败 → 整体降级 + if (failedBatches === batches.length) { + xbLog.warn(MODULE_ID, `全部 ${batches.length} 批 rerank 失败，整体降级`); return chunks.slice(0, topN).map(c => ({ ...c, _rerankScore: 0, @@ -166,15 +226,25 @@ export async function rerankChunks(query, chunks, options = {}) { })); } - return results - .filter(r => r.relevance_score >= minScore) - .sort((a, b) => b.relevance_score - a.relevance_score) - .map(r => ({ - ...chunks[r.index], - _rerankScore: r.relevance_score, - })); -} + // 合并所有批次结果 + const merged = batchResults.flat(); + const selected = merged + .filter(r => r._batchFailed || r.relevance_score >= minScore) + .sort((a, b) => b.relevance_score - a.relevance_score) + .slice(0, topN) + .map(r => ({ + ...chunks[r.globalIndex], + _rerankScore: r.relevance_score, + ...(r._batchFailed ? { _rerankFailed: true } : {}), + })); + + xbLog.info(MODULE_ID, + `Rerank 合并: ${merged.length} candidates, ${failedBatches}/${batches.length} 批失败, 选中 ${selected.length}` + ); + + return selected; +} /** * 测试 Rerank 服务连接 */ diff --git a/modules/story-summary/vector/pipeline/chunk-builder.js b/modules/story-summary/vector/pipeline/chunk-builder.js index 9119458..c7dcb35 100644 --- a/modules/story-summary/vector/pipeline/chunk-builder.js +++ b/modules/story-summary/vector/pipeline/chunk-builder.js @@ -340,15 +340,15 @@ export async function syncOnMessageSwiped(chatId, lastFloor) { * 新消息后同步：删除 + 重建最后楼层 */ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, onL0Complete) { - if (!chatId || lastFloor < 0 || !message) return; - if (!vectorConfig?.enabled) return; + if (!chatId || lastFloor < 0 || !message) return { built: 0, chunks: [] }; + if (!vectorConfig?.enabled) return { built: 0, chunks: [] }; // 删除该楼层旧的 await deleteChunksAtFloor(chatId, lastFloor); // 重建 const chunks = chunkMessage(lastFloor, message); - if (chunks.length === 0) return; + if (chunks.length === 0) return { built: 0, chunks: [] }; await saveChunks(chatId, chunks); @@ -356,12 +356,14 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo const fingerprint = getEngineFingerprint(vectorConfig); const texts = chunks.map(c => c.text); + let vectorized = false; try { const vectors = await embed(texts, vectorConfig); const items = chunks.map((c, i) => ({ chunkId: c.chunkId, vector: vectors[i] })); await saveChunkVectors(chatId, items, fingerprint); await updateMeta(chatId, { lastChunkFloor: lastFloor }); + vectorized = true; xbLog.info(MODULE_ID, `消息同步：重建 floor ${lastFloor}，${chunks.length} 个 chunk`); } catch (e) { xbLog.error(MODULE_ID, `消息同步失败：floor ${lastFloor}`, e); @@ -384,4 +386,6 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo xbLog.warn(MODULE_ID, `Atom 提取失败: floor ${lastFloor}`, e); } } + + return { built: vectorized ? chunks.length : 0, chunks }; } diff --git a/modules/story-summary/vector/retrieval/lexical-index.js b/modules/story-summary/vector/retrieval/lexical-index.js index c0861e1..83124d6 100644 --- a/modules/story-summary/vector/retrieval/lexical-index.js +++ b/modules/story-summary/vector/retrieval/lexical-index.js @@ -14,7 +14,6 @@ import MiniSearch from '../../../../libs/minisearch.mjs'; import { getContext } from '../../../../../../../extensions.js'; import { getSummaryStore } from '../../data/store.js'; -import { getStateAtoms } from '../storage/state-store.js'; import { getAllChunks } from '../storage/chunk-store.js'; import { xbLog } from '../../../../core/debug-core.js'; import { tokenizeForIndex } from '../utils/tokenizer.js'; @@ -39,6 +38,8 @@ let building = false; /** @type {Promise|null} 当前构建 Promise（防重入） */ let buildPromise = null; +/** @type {Map} floor → 该楼层的 doc IDs（仅 L1 chunks） */ +let floorDocIds = new Map(); // ───────────────────────────────────────────────────────────────────────── // 工具函数 @@ -57,13 +58,12 @@ function cleanSummary(summary) { /** * 计算缓存指纹 - * @param {number} atomCount * @param {number} chunkCount * @param {number} eventCount * @returns {string} */ -function computeFingerprint(atomCount, chunkCount, eventCount) { - return `${atomCount}:${chunkCount}:${eventCount}`; +function computeFingerprint(chunkCount, eventCount) { + return `${chunkCount}:${eventCount}`; } /** @@ -81,34 +81,31 @@ function yieldToMain() { /** * 收集所有待索引文档 * - * @param {object[]} atoms - getStateAtoms() 返回值 * @param {object[]} chunks - getAllChunks(chatId) 返回值 * @param {object[]} events - store.json.events * @returns {object[]} 文档数组 */ -function collectDocuments(atoms, chunks, events) { +function collectDocuments(chunks, events) { const docs = []; - // L0 atoms - for (const atom of (atoms || [])) { - if (!atom?.atomId || !atom.semantic) continue; - docs.push({ - id: atom.atomId, - type: 'atom', - floor: atom.floor ?? -1, - text: atom.semantic, - }); - } - - // L1 chunks + // L1 chunks + 填充 floorDocIds for (const chunk of (chunks || [])) { if (!chunk?.chunkId || !chunk.text) continue; + + const floor = chunk.floor ?? -1; docs.push({ id: chunk.chunkId, type: 'chunk', - floor: chunk.floor ?? -1, + floor, text: chunk.text, }); + + if (floor >= 0) { + if (!floorDocIds.has(floor)) { + floorDocIds.set(floor, []); + } + floorDocIds.get(floor).push(chunk.chunkId); + } } // L2 events @@ -244,7 +241,6 @@ export function searchLexicalIndex(index, terms) { } // 分类结果 - const atomIdSet = new Set(); const chunkIdSet = new Set(); const eventIdSet = new Set(); @@ -254,16 +250,6 @@ export function searchLexicalIndex(index, terms) { const floor = hit.floor; switch (type) { - case 'atom': - if (!atomIdSet.has(id)) { - atomIdSet.add(id); - result.atomIds.push(id); - if (typeof floor === 'number' && floor >= 0) { - result.atomFloors.add(floor); - } - } - break; - case 'chunk': if (!chunkIdSet.has(id)) { chunkIdSet.add(id); @@ -304,8 +290,10 @@ export function searchLexicalIndex(index, terms) { * @returns {Promise<{index: MiniSearch, fingerprint: string}>} */ async function collectAndBuild(chatId) { - // 收集数据 - const atoms = getStateAtoms() || []; + // 清空侧索引（全量重建） + floorDocIds = new Map(); + + // 收集数据（不含 L0 atoms） const store = getSummaryStore(); const events = store?.json?.events || []; @@ -316,15 +304,15 @@ async function collectAndBuild(chatId) { xbLog.warn(MODULE_ID, '获取 chunks 失败', e); } - const fp = computeFingerprint(atoms.length, chunks.length, events.length); + const fp = computeFingerprint(chunks.length, events.length); // 检查是否在收集过程中缓存已被其他调用更新 if (cachedIndex && cachedChatId === chatId && cachedFingerprint === fp) { return { index: cachedIndex, fingerprint: fp }; } - // 收集文档 - const docs = collectDocuments(atoms, chunks, events); + // 收集文档（同时填充 floorDocIds） + const docs = collectDocuments(chunks, events); // 异步分片构建 const index = await buildIndexAsync(docs); @@ -438,4 +426,116 @@ export function invalidateLexicalIndex() { cachedIndex = null; cachedChatId = null; cachedFingerprint = null; + floorDocIds = new Map(); +} + +// ───────────────────────────────────────────────────────────────────────── +// 增量更新接口 +// ───────────────────────────────────────────────────────────────────────── + +/** + * 为指定楼层添加 L1 chunks 到索引 + * + * 先移除该楼层旧文档，再添加新文档。 + * 如果索引不存在（缓存失效），静默跳过（下次 getLexicalIndex 全量重建）。 + * + * @param {number} floor - 楼层号 + * @param {object[]} chunks - chunk 对象列表（需有 chunkId、text、floor） + */ +export function addDocumentsForFloor(floor, chunks) { + if (!cachedIndex || !chunks?.length) return; + + // 先移除旧文档 + removeDocumentsByFloor(floor); + + const docs = []; + const docIds = []; + + for (const chunk of chunks) { + if (!chunk?.chunkId || !chunk.text) continue; + docs.push({ + id: chunk.chunkId, + type: 'chunk', + floor: chunk.floor ?? floor, + text: chunk.text, + }); + docIds.push(chunk.chunkId); + } + + if (docs.length > 0) { + cachedIndex.addAll(docs); + floorDocIds.set(floor, docIds); + xbLog.info(MODULE_ID, `增量添加: floor ${floor}, ${docs.length} 个 chunk`); + } +} + +/** + * 从索引中移除指定楼层的所有 L1 chunk 文档 + * + * 使用 MiniSearch discard()（软删除）。 + * 如果索引不存在，静默跳过。 + * + * @param {number} floor - 楼层号 + */ +export function removeDocumentsByFloor(floor) { + if (!cachedIndex) return; + + const docIds = floorDocIds.get(floor); + if (!docIds?.length) return; + + for (const id of docIds) { + try { + cachedIndex.discard(id); + } catch { + // 文档可能不存在（已被全量重建替换） + } + } + + floorDocIds.delete(floor); + xbLog.info(MODULE_ID, `增量移除: floor ${floor}, ${docIds.length} 个文档`); +} + +/** + * 将新 L2 事件添加到索引 + * + * 如果事件 ID 已存在，先 discard 再 add（覆盖）。 + * 如果索引不存在，静默跳过。 + * + * @param {object[]} events - 事件对象列表（需有 id、title、summary 等） + */ +export function addEventDocuments(events) { + if (!cachedIndex || !events?.length) return; + + const docs = []; + + for (const ev of events) { + if (!ev?.id) continue; + + const parts = []; + if (ev.title) parts.push(ev.title); + if (ev.participants?.length) parts.push(ev.participants.join(' ')); + const summary = cleanSummary(ev.summary); + if (summary) parts.push(summary); + const text = parts.join(' ').trim(); + if (!text) continue; + + // 覆盖：先尝试移除旧的 + try { + cachedIndex.discard(ev.id); + } catch { + // 不存在则忽略 + } + + docs.push({ + id: ev.id, + type: 'event', + floor: null, + text, + }); + } + + if (docs.length > 0) { + cachedIndex.addAll(docs); + xbLog.info(MODULE_ID, `增量添加: ${docs.length} 个事件`); + } } diff --git a/modules/story-summary/vector/retrieval/metrics.js b/modules/story-summary/vector/retrieval/metrics.js index fde9c3f..527d6d1 100644 --- a/modules/story-summary/vector/retrieval/metrics.js +++ b/modules/story-summary/vector/retrieval/metrics.js @@ -8,7 +8,7 @@ // 架构变更（v3 → v4）： // - evidence 区块反映 L0-only 融合 + L1 按楼层拉取的两阶段架构 // - 删除 mergedByType / selectedByType（不再有混合池） -// - 新增 l0Candidates / l0Selected / l1Pulled / l1Attached / l1CosineTime +// - 新增 floorCandidates / floorsSelected / l0Collected / l1Pulled / l1Attached / l1CosineTime // - fusion 区块明确标注 L0-only（删除 anchorCount） // - quality.chunkRealRatio → quality.l1AttachRate // ═══════════════════════════════════════════════════════════════════════════ @@ -48,10 +48,10 @@ export function createMetrics() { searchTime: 0, }, - // Fusion (W-RRF, L0-only) - 多路融合 + // Fusion (W-RRF, floor-level) - 多路融合 fusion: { - denseCount: 0, - lexCount: 0, + denseFloors: 0, + lexFloors: 0, totalUnique: 0, afterCap: 0, time: 0, @@ -80,25 +80,27 @@ export function createMetrics() { entityNames: [], }, - // Evidence (Two-Stage: L0 rerank → L1 pull) - 原文证据 + // Evidence (Two-Stage: Floor rerank → L1 pull) - 原文证据 evidence: { - // Stage 1: L0 - l0Candidates: 0, // W-RRF 融合后的 L0 候选数 - l0Selected: 0, // rerank 后选中的 L0 数 + // Stage 1: Floor + floorCandidates: 0, // W-RRF 融合后的 floor 候选数 + floorsSelected: 0, // rerank 后选中的 floor 数 + l0Collected: 0, // 选中 floor 中收集的 L0 atom 总数 rerankApplied: false, rerankFailed: false, beforeRerank: 0, afterRerank: 0, rerankTime: 0, rerankScores: null, + rerankDocAvgLength: 0, // rerank document 平均字符数 // Stage 2: L1 l1Pulled: 0, // 从 DB 拉取的 L1 chunk 总数 - l1Attached: 0, // 实际挂载的 L1 数（top-1 × 楼层 × 2侧） + l1Attached: 0, // 实际挂载的 L1 数（top-1 × floor × 2侧） l1CosineTime: 0, // L1 cosine 打分耗时 // 装配 - contextPairsAdded: 0, // 保留兼容（= l1Attached 中 USER 侧数量） + contextPairsAdded: 0, // USER 侧挂载数量 tokens: 0, assemblyTime: 0, }, @@ -149,7 +151,7 @@ export function createMetrics() { quality: { constraintCoverage: 100, eventPrecisionProxy: 0, - l1AttachRate: 0, // 有 L1 挂载的 L0 占比 + l1AttachRate: 0, // 有 L1 挂载的 floor 占比 potentialIssues: [], }, }; @@ -223,10 +225,10 @@ export function formatMetricsLog(metrics) { lines.push(`└─ search_time: ${m.lexical.searchTime}ms`); lines.push(''); - // Fusion (W-RRF, L0-only) - lines.push('[Fusion] W-RRF (L0-only) - 多路融合'); - lines.push(`├─ dense_count: ${m.fusion.denseCount}`); - lines.push(`├─ lex_count: ${m.fusion.lexCount}`); + // Fusion (W-RRF, floor-level) + lines.push('[Fusion] W-RRF (floor-level) - 多路融合'); + lines.push(`├─ dense_floors: ${m.fusion.denseFloors}`); + lines.push(`├─ lex_floors: ${m.fusion.lexFloors}`); lines.push(`├─ total_unique: ${m.fusion.totalUnique}`); lines.push(`├─ after_cap: ${m.fusion.afterCap}`); lines.push(`└─ time: ${m.fusion.time}ms`); @@ -277,28 +279,32 @@ export function formatMetricsLog(metrics) { lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`); lines.push(''); - // Evidence (Two-Stage) - lines.push('[Evidence] Two-Stage: L0 Locate → L1 Pull'); - lines.push(`├─ Stage 1 (L0):`); - lines.push(`│ ├─ candidates (post-fusion): ${m.evidence.l0Candidates}`); + // Evidence (Two-Stage: Floor Rerank → L1 Pull) + lines.push('[Evidence] Two-Stage: Floor Rerank → L1 Pull'); + lines.push(`├─ Stage 1 (Floor Rerank):`); + lines.push(`│ ├─ floor_candidates (post-fusion): ${m.evidence.floorCandidates}`); if (m.evidence.rerankApplied) { lines.push(`│ ├─ rerank_applied: true`); if (m.evidence.rerankFailed) { - lines.push(`│ ├─ rerank_failed: ⚠ YES (using fusion order)`); + lines.push(`│ │ ⚠ rerank_failed: using fusion order`); } - lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank}`); - lines.push(`│ │ ├─ after: ${m.evidence.afterRerank}`); + lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank} floors`); + lines.push(`│ │ ├─ after: ${m.evidence.afterRerank} floors`); lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`); if (m.evidence.rerankScores) { const rs = m.evidence.rerankScores; lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`); } + if (m.evidence.rerankDocAvgLength > 0) { + lines.push(`│ ├─ rerank_doc_avg_length: ${m.evidence.rerankDocAvgLength} chars`); + } } else { lines.push(`│ ├─ rerank_applied: false`); } - lines.push(`│ └─ selected: ${m.evidence.l0Selected}`); + lines.push(`│ ├─ floors_selected: ${m.evidence.floorsSelected}`); + lines.push(`│ └─ l0_atoms_collected: ${m.evidence.l0Collected}`); lines.push(`├─ Stage 2 (L1):`); lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`); lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`); @@ -345,9 +351,7 @@ export function formatMetricsLog(metrics) { lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`); lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`); lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`); - if (m.timing.evidenceRerank > 0) { - lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`); - } + lines.push(`├─ floor_rerank: ${m.timing.evidenceRerank || 0}ms`); lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`); lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`); lines.push(`├─ formatting: ${m.timing.formatting}ms`); @@ -406,20 +410,20 @@ export function detectIssues(metrics) { // 词法检索问题 // ───────────────────────────────────────────────────────────────── - if ((m.lexical.terms || []).length > 0 && m.lexical.atomHits === 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) { + if ((m.lexical.terms || []).length > 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) { issues.push('Lexical search returned zero hits - terms may not match any indexed content'); } // ───────────────────────────────────────────────────────────────── - // 融合问题（L0-only） + // 融合问题（floor-level） // ───────────────────────────────────────────────────────────────── - if (m.fusion.lexCount === 0 && m.fusion.denseCount > 0) { - issues.push('No lexical L0 candidates in fusion - hybrid retrieval not contributing'); + if (m.fusion.lexFloors === 0 && m.fusion.denseFloors > 0) { + issues.push('No lexical floors in fusion - hybrid retrieval not contributing'); } if (m.fusion.afterCap === 0) { - issues.push('Fusion produced zero L0 candidates - all retrieval paths may have failed'); + issues.push('Fusion produced zero floor candidates - all retrieval paths may have failed'); } // ───────────────────────────────────────────────────────────────── @@ -463,29 +467,30 @@ export function detectIssues(metrics) { } // ───────────────────────────────────────────────────────────────── - // L0 Rerank 问题 + // Floor Rerank 问题 // ───────────────────────────────────────────────────────────────── - if (m.evidence.rerankApplied) { - if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) { - const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank); - if (filterRatio > 0.7) { - issues.push(`High L0 rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant L0 in fusion output`); - } - } + if (m.evidence.rerankFailed) { + issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero'); + } + if (m.evidence.rerankApplied && !m.evidence.rerankFailed) { if (m.evidence.rerankScores) { const rs = m.evidence.rerankScores; - if (rs.max < 0.5) { - issues.push(`Low L0 rerank scores (max=${rs.max}) - query may be poorly matched`); + if (rs.max < 0.3) { + issues.push(`Low floor rerank scores (max=${rs.max}) - query-document domain mismatch`); } - if (rs.mean < 0.3) { - issues.push(`Very low average L0 rerank score (mean=${rs.mean}) - context may be weak`); + if (rs.mean < 0.2) { + issues.push(`Very low average floor rerank score (mean=${rs.mean}) - context may be weak`); } } - if (m.evidence.rerankTime > 2000) { - issues.push(`Slow L0 rerank (${m.evidence.rerankTime}ms) - may affect response time`); + if (m.evidence.rerankTime > 3000) { + issues.push(`Slow floor rerank (${m.evidence.rerankTime}ms) - may affect response time`); + } + + if (m.evidence.rerankDocAvgLength > 3000) { + issues.push(`Large rerank documents (avg ${m.evidence.rerankDocAvgLength} chars) - may reduce rerank precision`); } } @@ -493,21 +498,17 @@ export function detectIssues(metrics) { // L1 挂载问题 // ───────────────────────────────────────────────────────────────── - if (m.evidence.rerankFailed) { - issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero'); - } - - if (m.evidence.l0Selected > 0 && m.evidence.l1Pulled === 0) { + if (m.evidence.floorsSelected > 0 && m.evidence.l1Pulled === 0) { issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed'); } - if (m.evidence.l0Selected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) { - issues.push('L1 chunks pulled but none attached - cosine scores may be too low or floor mismatch'); + if (m.evidence.floorsSelected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) { + issues.push('L1 chunks pulled but none attached - cosine scores may be too low'); } const l1AttachRate = m.quality.l1AttachRate || 0; - if (m.evidence.l0Selected > 5 && l1AttachRate < 20) { - issues.push(`Low L1 attach rate (${l1AttachRate}%) - many L0 lack concrete dialogue evidence`); + if (m.evidence.floorsSelected > 3 && l1AttachRate < 50) { + issues.push(`Low L1 attach rate (${l1AttachRate}%) - selected floors lack L1 chunks`); } // ───────────────────────────────────────────────────────────────── diff --git a/modules/story-summary/vector/retrieval/query-builder.js b/modules/story-summary/vector/retrieval/query-builder.js index d6f80b2..4f21b3e 100644 --- a/modules/story-summary/vector/retrieval/query-builder.js +++ b/modules/story-summary/vector/retrieval/query-builder.js @@ -89,6 +89,46 @@ function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) { .map(([term]) => term); } +/** + * 构建 rerank 专用查询（纯自然语言，不带结构标签） + * + * reranker（bge-reranker-v2-m3）的 query 应为自然语言文本， + * 不含 [ENTITIES] [DIALOGUE] 等结构标签。 + * + * @param {string[]} focusEntities - 焦点实体 + * @param {object[]} lastMessages - 最近 K 条消息 + * @param {string|null} pendingUserMessage - 待发送的用户消息 + * @param {object} context - { name1, name2 } + * @returns {string} + */ +function buildRerankQuery(focusEntities, lastMessages, pendingUserMessage, context) { + const parts = []; + + // 实体提示 + if (focusEntities.length > 0) { + parts.push(`关于${focusEntities.join('、')}。`); + } + + // 最近对话原文 + for (const m of (lastMessages || [])) { + const speaker = m.is_user ? (context.name1 || '用户') : (m.name || context.name2 || '角色'); + const clean = cleanMessageText(m.mes || ''); + if (clean) { + parts.push(`${speaker}：${clean}`); + } + } + + // 待发送消息 + if (pendingUserMessage) { + const clean = cleanMessageText(pendingUserMessage); + if (clean) { + parts.push(`${context.name1 || '用户'}：${clean}`); + } + } + + return parts.join('\n'); +} + // ───────────────────────────────────────────────────────────────────────── // QueryBundle 类型定义（JSDoc） // ───────────────────────────────────────────────────────────────────────── @@ -176,9 +216,8 @@ export function buildQueryBundle(lastMessages, pendingUserMessage, store = null, const queryText_v0 = queryParts.join('\n\n'); - // 6. rerankQuery 与 embedding query 同源（零暗箱） - // 后续 refine 会把它升级为与 queryText_v1 同源。 - const rerankQuery = queryText_v0; + // 6. rerankQuery 独立构建（纯自然语言，供 reranker 使用） + const rerankQuery = buildRerankQuery(focusEntities, dialogueLines.length > 0 ? lastMessages : [], pendingUserMessage, context); // 7. 构建 lexicalTerms const entityTerms = focusEntities.map(e => e.toLowerCase()); @@ -281,8 +320,8 @@ export function refineQueryBundle(bundle, anchorHits, eventHits) { } } - // 5. rerankQuery 与最终 query 同源（零暗箱） - bundle.rerankQuery = bundle.queryText_v1 || bundle.queryText_v0; + // 5. rerankQuery 保持独立（不随 refinement 变更） + // reranker 需要纯自然语言 query，不受 memory hints 干扰 // 6. 增强 lexicalTerms if (hints.length > 0) { diff --git a/modules/story-summary/vector/retrieval/recall.js b/modules/story-summary/vector/retrieval/recall.js index 318b14d..8f6d156 100644 --- a/modules/story-summary/vector/retrieval/recall.js +++ b/modules/story-summary/vector/retrieval/recall.js @@ -1,4 +1,4 @@ -// ═══════════════════════════════════════════════════════════════════════════ +// ═══════════════════════════════════════════════════════════════════════════ // Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence) // // 命名规范： @@ -10,8 +10,8 @@ // 阶段 2: Round 1 Dense Retrieval（L0 + L2） // 阶段 3: Query Refinement（用已命中记忆增强） // 阶段 4: Round 2 Dense Retrieval（L0 + L2） -// 阶段 5: Lexical Retrieval + L0 Merge -// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine 预筛选 +// 阶段 5: Lexical Retrieval +// 阶段 6: Floor W-RRF Fusion + Rerank + L1 配对 // 阶段 7: L1 配对组装（L0 → top-1 AI L1 + top-1 USER L1） // 阶段 8: Causation Trace // ═══════════════════════════════════════════════════════════════════════════ @@ -49,10 +49,10 @@ const CONFIG = { RRF_K: 60, RRF_W_DENSE: 1.0, RRF_W_LEX: 0.9, - FUSION_CAP: 100, + FUSION_CAP: 60, - // Rerank（L0-only） - RERANK_TOP_N: 50, + // Rerank（floor-level） + RERANK_TOP_N: 20, RERANK_MIN_SCORE: 0.15, // 因果链 @@ -421,14 +421,14 @@ function traceCausation(eventHits, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MA */ /** - * W-RRF 融合两路 L0 候选（dense + lexical） + * W-RRF 加权倒数排名融合（floor 粒度） * - * @param {RankedItem[]} denseRank - Dense 路（cosine 降序） - * @param {RankedItem[]} lexRank - Lexical 路（MiniSearch score 降序） + * @param {{id: number, score: number}[]} denseRank - Dense 路（floor → max cosine，降序） + * @param {{id: number, score: number}[]} lexRank - Lexical 路（floor → max bm25，降序） * @param {number} cap - 输出上限 - * @returns {{top: {id: string, fusionScore: number}[], totalUnique: number}} + * @returns {{top: {id: number, fusionScore: number}[], totalUnique: number}} */ -function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) { +function fuseByFloor(denseRank, lexRank, cap = CONFIG.FUSION_CAP) { const k = CONFIG.RRF_K; const wD = CONFIG.RRF_W_DENSE; const wL = CONFIG.RRF_W_LEX; @@ -445,141 +445,109 @@ function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) { const denseMap = buildRankMap(denseRank || []); const lexMap = buildRankMap(lexRank || []); - const allIds = new Set([ - ...denseMap.keys(), - ...lexMap.keys(), - ]); - + const allIds = new Set([...denseMap.keys(), ...lexMap.keys()]); const totalUnique = allIds.size; const scored = []; for (const id of allIds) { let score = 0; - if (denseMap.has(id)) { - score += wD / (k + denseMap.get(id)); - } - if (lexMap.has(id)) { - score += wL / (k + lexMap.get(id)); - } + if (denseMap.has(id)) score += wD / (k + denseMap.get(id)); + if (lexMap.has(id)) score += wL / (k + lexMap.get(id)); scored.push({ id, fusionScore: score }); } scored.sort((a, b) => b.fusionScore - a.fusionScore); - - return { - top: scored.slice(0, cap), - totalUnique, - }; + return { top: scored.slice(0, cap), totalUnique }; } // ═══════════════════════════════════════════════════════════════════════════ -// [Stage 6] L0-only 融合 + Rerank ‖ 并发 L1 Cosine 预筛选 +// [Stage 6] Floor 融合 + Rerank + L1 配对 // ═══════════════════════════════════════════════════════════════════════════ /** - * L0 融合 + rerank，并发拉取 L1 并 cosine 打分 + * Floor 粒度融合 + Rerank + L1 配对 * * @param {object[]} anchorHits - L0 dense 命中（Round 2） - * @param {Set} anchorFloors - L0 命中楼层（含 lexical 扩展） * @param {number[]} queryVector - 查询向量（v1） - * @param {string} rerankQuery - rerank 查询文本 + * @param {string} rerankQuery - rerank 查询文本（纯自然语言） * @param {object} lexicalResult - 词法检索结果 * @param {object} metrics * @returns {Promise<{l0Selected: object[], l1ByFloor: Map}>} */ -async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rerankQuery, lexicalResult, metrics) { - const { chatId, chat } = getContext(); +async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexicalResult, metrics) { + const { chatId, chat, name1, name2 } = getContext(); if (!chatId) return { l0Selected: [], l1ByFloor: new Map() }; const T_Start = performance.now(); // ───────────────────────────────────────────────────────────────── - // 6a. 构建 L0 候选对象（用于 rerank） - // - // 重要：支持 lexical-only 的 L0（atom）进入候选池。 - // 否则 hybrid 会退化为 dense-only：lexical 命中的 atom 若未被 dense 命中会被直接丢弃。 + // 6a. Dense floor rank（每个 floor 取 max cosine） // ───────────────────────────────────────────────────────────────── - const l0ObjectMap = new Map(); + const denseFloorMap = new Map(); for (const a of (anchorHits || [])) { - const id = `anchor-${a.atomId}`; - l0ObjectMap.set(id, { - id, - atomId: a.atomId, - floor: a.floor, - similarity: a.similarity, - atom: a.atom, - text: a.atom?.semantic || '', - }); + const cur = denseFloorMap.get(a.floor) || 0; + if (a.similarity > cur) denseFloorMap.set(a.floor, a.similarity); } - // lexical-only atoms：从全量 StateAtoms 补齐（similarity 记为 0，靠 lex rank 贡献 W-RRF） - const lexAtomIds = lexicalResult?.atomIds || []; - if (lexAtomIds.length > 0) { - const atomsList = getStateAtoms(); - const atomMap = new Map(atomsList.map(a => [a.atomId, a])); + const denseFloorRank = [...denseFloorMap.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([floor, score]) => ({ id: floor, score })); - for (const atomId of lexAtomIds) { - const id = `anchor-${atomId}`; - if (l0ObjectMap.has(id)) continue; + // ───────────────────────────────────────────────────────────────── + // 6b. Lexical floor rank（chunkScores → floor 聚合 + USER→AI 映射 + 预过滤） + // ───────────────────────────────────────────────────────────────── - const atom = atomMap.get(atomId); - if (!atom) continue; - if (typeof atom.floor !== 'number' || atom.floor < 0) continue; + const atomFloorSet = new Set(getStateAtoms().map(a => a.floor)); - l0ObjectMap.set(id, { - id, - atomId, - floor: atom.floor, - similarity: 0, - atom, - text: atom.semantic || '', - }); + const lexFloorScores = new Map(); + for (const { chunkId, score } of (lexicalResult?.chunkScores || [])) { + const match = chunkId?.match(/^c-(\d+)-/); + if (!match) continue; + let floor = parseInt(match[1], 10); + + // USER floor → AI floor 映射 + if (chat?.[floor]?.is_user) { + const aiFloor = floor + 1; + if (aiFloor < chat.length && !chat[aiFloor]?.is_user) { + floor = aiFloor; + } else { + continue; + } } + + // 预过滤：必须有 L0 atoms + if (!atomFloorSet.has(floor)) continue; + + const cur = lexFloorScores.get(floor) || 0; + if (score > cur) lexFloorScores.set(floor, score); } - // ───────────────────────────────────────────────────────────────── - // 6b. 构建两路排名（L0-only） - // ───────────────────────────────────────────────────────────────── - - // Dense 路：anchorHits 按 similarity 排序 - const denseRank = (anchorHits || []) - .map(a => ({ id: `anchor-${a.atomId}`, score: a.similarity })) - .sort((a, b) => b.score - a.score); - - // Lexical 路：从 lexicalResult.atomIds 构建排名（允许 lexical-only） - // atomIds 已按 MiniSearch score 排序（searchLexicalIndex 返回顺序）；W-RRF 依赖 rank，score 为占位 - const lexRank = (lexAtomIds || []) - .map(atomId => ({ id: `anchor-${atomId}`, score: 1 })) - .filter(item => l0ObjectMap.has(item.id)); + const lexFloorRank = [...lexFloorScores.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([floor, score]) => ({ id: floor, score })); // ───────────────────────────────────────────────────────────────── - // 6c. W-RRF 融合（L0-only） + // 6c. Floor W-RRF 融合 // ───────────────────────────────────────────────────────────────── const T_Fusion_Start = performance.now(); - - const { top: fusionResult, totalUnique } = fuseL0Candidates(denseRank, lexRank, CONFIG.FUSION_CAP); - + const { top: fusedFloors, totalUnique } = fuseByFloor(denseFloorRank, lexFloorRank, CONFIG.FUSION_CAP); const fusionTime = Math.round(performance.now() - T_Fusion_Start); - // 构建 rerank 候选列表 - const rerankCandidates = fusionResult - .map(f => l0ObjectMap.get(f.id)) - .filter(Boolean); - if (metrics) { - metrics.fusion.denseCount = denseRank.length; - metrics.fusion.lexCount = lexRank.length; + metrics.fusion.denseFloors = denseFloorRank.length; + metrics.fusion.lexFloors = lexFloorRank.length; metrics.fusion.totalUnique = totalUnique; - metrics.fusion.afterCap = rerankCandidates.length; + metrics.fusion.afterCap = fusedFloors.length; metrics.fusion.time = fusionTime; - metrics.evidence.l0Candidates = rerankCandidates.length; + metrics.evidence.floorCandidates = fusedFloors.length; } - if (rerankCandidates.length === 0) { + if (fusedFloors.length === 0) { if (metrics) { - metrics.evidence.l0Selected = 0; + metrics.evidence.floorsSelected = 0; + metrics.evidence.l0Collected = 0; metrics.evidence.l1Pulled = 0; metrics.evidence.l1Attached = 0; metrics.evidence.l1CosineTime = 0; @@ -589,54 +557,87 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera } // ───────────────────────────────────────────────────────────────── - // 6d. 收集所有候选 L0 的楼层（用于并发拉取 L1） - // 包含 AI 楼层本身 + 上方 USER 楼层 + // 6d. 拉取 L1 chunks + cosine 打分 // ───────────────────────────────────────────────────────────────── - const candidateFloors = new Set(); - for (const c of rerankCandidates) { - candidateFloors.add(c.floor); - // 上方 USER 楼层 - const userFloor = c.floor - 1; + const floorsToFetch = new Set(); + for (const f of fusedFloors) { + floorsToFetch.add(f.id); + const userFloor = f.id - 1; if (userFloor >= 0 && chat?.[userFloor]?.is_user) { - candidateFloors.add(userFloor); + floorsToFetch.add(userFloor); } } + const l1ScoredByFloor = await pullAndScoreL1(chatId, [...floorsToFetch], queryVector, chat); + + if (metrics) { + let totalPulled = 0; + for (const [key, chunks] of l1ScoredByFloor) { + if (key === '_cosineTime') continue; + totalPulled += chunks.length; + } + metrics.evidence.l1Pulled = totalPulled; + metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0; + } + // ───────────────────────────────────────────────────────────────── - // 6e. 并发：rerank L0 ‖ 拉取 L1 chunks + 向量 + cosine 打分 + // 6e. 构建 rerank documents（每个 floor: USER chunks + AI chunks） + // ───────────────────────────────────────────────────────────────── + + const rerankCandidates = []; + for (const f of fusedFloors) { + const aiFloor = f.id; + const userFloor = aiFloor - 1; + + const aiChunks = l1ScoredByFloor.get(aiFloor) || []; + const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user) + ? (l1ScoredByFloor.get(userFloor) || []) + : []; + + const parts = []; + const userName = chat?.[userFloor]?.name || name1 || '用户'; + const aiName = chat?.[aiFloor]?.name || name2 || '角色'; + + if (userChunks.length > 0) { + parts.push(`${userName}：${userChunks.map(c => c.text).join(' ')}`); + } + if (aiChunks.length > 0) { + parts.push(`${aiName}：${aiChunks.map(c => c.text).join(' ')}`); + } + + const text = parts.join('\n'); + if (!text.trim()) continue; + + rerankCandidates.push({ + floor: aiFloor, + text, + fusionScore: f.fusionScore, + }); + } + + // ───────────────────────────────────────────────────────────────── + // 6f. 并发 Rerank // ───────────────────────────────────────────────────────────────── const T_Rerank_Start = performance.now(); - // 并发任务 1：rerank L0 - const rerankPromise = rerankChunks(rerankQuery, rerankCandidates, { + const reranked = await rerankChunks(rerankQuery, rerankCandidates, { topN: CONFIG.RERANK_TOP_N, minScore: CONFIG.RERANK_MIN_SCORE, }); - // 并发任务 2：拉取 L1 chunks + 向量 → cosine 打分 - const l1Promise = pullAndScoreL1(chatId, Array.from(candidateFloors), queryVector, chat); - - // 等待两个任务完成 - const [rerankedL0, l1ScoredByFloor] = await Promise.all([rerankPromise, l1Promise]); - const rerankTime = Math.round(performance.now() - T_Rerank_Start); - // ───────────────────────────────────────────────────────────────── - // 6f. 记录 rerank metrics - // ───────────────────────────────────────────────────────────────── - if (metrics) { metrics.evidence.rerankApplied = true; metrics.evidence.beforeRerank = rerankCandidates.length; - metrics.evidence.afterRerank = rerankedL0.length; - metrics.evidence.rerankFailed = rerankedL0.some(c => c._rerankFailed); - metrics.evidence.l0Selected = rerankedL0.length; + metrics.evidence.afterRerank = reranked.length; + metrics.evidence.rerankFailed = reranked.some(c => c._rerankFailed); metrics.evidence.rerankTime = rerankTime; metrics.timing.evidenceRerank = rerankTime; - const scores = rerankedL0.map(c => c._rerankScore || 0).filter(s => s > 0); + const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0); if (scores.length > 0) { scores.sort((a, b) => a - b); metrics.evidence.rerankScores = { @@ -645,74 +646,82 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)), }; } + + // document 平均长度 + if (rerankCandidates.length > 0) { + const totalLen = rerankCandidates.reduce((s, c) => s + (c.text?.length || 0), 0); + metrics.evidence.rerankDocAvgLength = Math.round(totalLen / rerankCandidates.length); + } } // ───────────────────────────────────────────────────────────────── - // 6g. 构建最终 l0Selected + l1ByFloor + // 6g. 收集 L0 atoms + L1 top-1 配对 // ───────────────────────────────────────────────────────────────── - const l0Selected = rerankedL0.map(item => ({ - id: item.id, - atomId: item.atomId, - floor: item.floor, - similarity: item.similarity, - rerankScore: item._rerankScore || 0, - atom: item.atom, - text: item.text, - })); + const atomsList = getStateAtoms(); + const atomsByFloor = new Map(); + for (const atom of atomsList) { + if (typeof atom.floor !== 'number' || atom.floor < 0) continue; + if (!atomsByFloor.has(atom.floor)) atomsByFloor.set(atom.floor, []); + atomsByFloor.get(atom.floor).push(atom); + } - // 为每个选中的 L0 楼层组装 top-1 L1 配对 - const selectedFloors = new Set(l0Selected.map(l => l.floor)); + const l0Selected = []; const l1ByFloor = new Map(); let contextPairsAdded = 0; - for (const floor of selectedFloors) { + for (const item of reranked) { + const floor = item.floor; + const rerankScore = item._rerankScore || 0; + const denseSim = denseFloorMap.get(floor) || 0; + + // 收集该 floor 所有 L0 atoms，共享 floor 的 rerankScore + const floorAtoms = atomsByFloor.get(floor) || []; + for (const atom of floorAtoms) { + l0Selected.push({ + id: `anchor-${atom.atomId}`, + atomId: atom.atomId, + floor: atom.floor, + similarity: denseSim, + rerankScore, + atom, + text: atom.semantic || '', + }); + } + + // L1 top-1 配对（cosine 最高） const aiChunks = l1ScoredByFloor.get(floor) || []; - xbLog.info( - MODULE_ID, - `L1 attach check: floor=${floor}, l1ScoredByFloor.has=${l1ScoredByFloor.has(floor)}, aiChunks=${aiChunks.length}, l1ScoredByFloor.keys=[${[...l1ScoredByFloor.keys()].slice(0, 10)}...]` - ); const userFloor = floor - 1; const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user) ? (l1ScoredByFloor.get(userFloor) || []) : []; - // top-1：取 cosine 最高的 const aiTop1 = aiChunks.length > 0 ? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best)) : null; - const userTop1 = userChunks.length > 0 ? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best)) : null; - // context pair = 上方 USER 楼层成功挂载（用于 metrics） if (userTop1) contextPairsAdded++; - l1ByFloor.set(floor, { aiTop1, userTop1 }); } // ───────────────────────────────────────────────────────────────── - // 6h. L1 metrics + // 6h. Metrics // ───────────────────────────────────────────────────────────────── if (metrics) { - let totalPulled = 0; + metrics.evidence.floorsSelected = reranked.length; + metrics.evidence.l0Collected = l0Selected.length; + let totalAttached = 0; - - for (const [, scored] of l1ScoredByFloor) { - totalPulled += scored.length; - } - for (const [, pair] of l1ByFloor) { if (pair.aiTop1) totalAttached++; if (pair.userTop1) totalAttached++; } - - metrics.evidence.l1Pulled = totalPulled; metrics.evidence.l1Attached = totalAttached; metrics.evidence.contextPairsAdded = contextPairsAdded; - metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0; } const totalTime = Math.round(performance.now() - T_Start); @@ -721,13 +730,11 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera } xbLog.info(MODULE_ID, - `Evidence: ${anchorHits?.length || 0} L0 dense → fusion=${rerankCandidates.length} → rerank=${rerankedL0.length} → L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)` + `Evidence: ${denseFloorRank.length} dense floors + ${lexFloorRank.length} lex floors → fusion=${fusedFloors.length} → rerank=${reranked.length} floors → L0=${l0Selected.length} L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)` ); return { l0Selected, l1ByFloor }; } - -// ═══════════════════════════════════════════════════════════════════════════ // [L1] 拉取 + Cosine 打分（并发子任务） // ═══════════════════════════════════════════════════════════════════════════ @@ -973,7 +980,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) { ); // ═══════════════════════════════════════════════════════════════════ - // 阶段 5: Lexical Retrieval + L0 Merge + // 阶段 5: Lexical Retrieval // ═══════════════════════════════════════════════════════════════════ const T_Lex_Start = performance.now(); @@ -1003,12 +1010,6 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) { metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10); } - // 合并 L0 floors（dense + lexical） - const anchorFloors = new Set(anchorFloors_dense); - for (const f of lexicalResult.atomFloors) { - anchorFloors.add(f); - } - // 合并 L2 events（lexical 命中但 dense 未命中的 events） const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean)); const eventIndex = buildEventIndex(allEvents); @@ -1035,16 +1036,15 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) { } xbLog.info(MODULE_ID, - `Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedFloors=${anchorFloors.size} mergedEvents=+${lexicalEventCount} (${lexTime}ms)` + `Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} (${lexTime}ms)` ); // ═══════════════════════════════════════════════════════════════════ - // 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine + // 阶段 6: Floor 粒度融合 + Rerank + L1 配对 // ═══════════════════════════════════════════════════════════════════ const { l0Selected, l1ByFloor } = await locateAndPullEvidence( anchorHits, - anchorFloors, queryVector_v1, bundle.rerankQuery, lexicalResult, @@ -1086,11 +1086,11 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) { console.log(`Total: ${metrics.timing.total}ms`); console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`); console.log(`Focus: [${bundle.focusEntities.join(', ')}]`); - console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors.size} floors`); - console.log(`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`); - console.log(`Fusion (L0-only): dense=${metrics.fusion.denseCount} lex=${metrics.fusion.lexCount} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`); - console.log(`L0 Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.l0Selected || 0} (${metrics.evidence.rerankTime || 0}ms)`); - console.log(`L1 Pull: ${metrics.evidence.l1Pulled || 0} chunks → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`); + console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors_dense.size} floors`); + console.log(`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`); + console.log(`Fusion (floor): dense=${metrics.fusion.denseFloors} lex=${metrics.fusion.lexFloors} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`); + console.log(`Floor Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.floorsSelected || 0} floors → L0=${metrics.evidence.l0Collected || 0} (${metrics.evidence.rerankTime || 0}ms)`); + console.log(`L1: ${metrics.evidence.l1Pulled || 0} pulled → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`); console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`); console.groupEnd();