X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: X:6c6091a942f61087cae7607154c11f50ab81a739
Branches Tags
X:main
...
compare: X:297cc037701f8002cc9bf8b72e1d0993252a8f70
Branches Tags
X:main

2 Commits
6c6091a942 ... 297cc03770

Author SHA1 Message Date
bielie
297cc03770 Update retrieval, rerank, and indexing changes 2026-02-11 13:55:19 +08:00
bielie
8d062d39b5 Fix vector build and lexical index updates 2026-02-11 13:54:29 +08:00
8 changed files with 555 additions and 317 deletions
Expand all files Collapse all files

2
modules/story-summary/generate/llm.js
View File

@@ -11,7 +11,7 @@ const PROVIDER_MAP = {
custom: "custom",
};
const JSON_PREFILL = '{"mindful_prelude": {';
const JSON_PREFILL = '下面重新生成完整JSON。';
const LLM_PROMPT_CONFIG = {
topSystem: `Story Analyst: This task involves narrative comprehension and structured incremental summarization, representing creative story analysis at the intersection of plot tracking and character development. As a story analyst, you will conduct systematic evaluation of provided dialogue content to generate structured incremental summary data.

54
modules/story-summary/story-summary.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - 主入口
//
// 稳定目标：
@@ -91,7 +91,7 @@ import {
// vector io
import { exportVectors, importVectors } from "./vector/storage/vector-io.js";
import { invalidateLexicalIndex, warmupIndex } from "./vector/retrieval/lexical-index.js";
import { invalidateLexicalIndex, warmupIndex, addDocumentsForFloor, removeDocumentsByFloor, addEventDocuments } from "./vector/retrieval/lexical-index.js";
// ═══════════════════════════════════════════════════════════════════════════
// 常量
@@ -351,9 +351,12 @@ async function handleAnchorGenerate() {
});
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "向量化 L1..." });
await buildIncrementalChunks({ vectorConfig: vectorCfg });
const chunkResult = await buildIncrementalChunks({ vectorConfig: vectorCfg });
// L1 rebuild only if new chunks were added (usually 0 in normal chat)
if (chunkResult.built > 0) {
invalidateLexicalIndex();
}
await sendAnchorStatsToFrame();
await sendVectorStatsToFrame();
@@ -589,8 +592,6 @@ function refreshEntityLexiconAndWarmup() {
injectEntities(lexicon, displayMap);
// 异步预建词法索引（不阻塞）
invalidateLexicalIndex();
warmupIndex();
}
// ═══════════════════════════════════════════════════════════════════════════
@@ -615,9 +616,12 @@ async function maybeAutoExtractL0() {
await incrementalExtractAtoms(chatId, chat, null, { maxFloors: 20 });
// 为新提取的 L0 楼层构建 L1 chunks
await buildIncrementalChunks({ vectorConfig: vectorCfg });
const chunkResult = await buildIncrementalChunks({ vectorConfig: vectorCfg });
// L1 rebuild only if new chunks were added
if (chunkResult.built > 0) {
invalidateLexicalIndex();
}
await sendAnchorStatsToFrame();
await sendVectorStatsToFrame();
@@ -637,7 +641,7 @@ async function maybeAutoExtractL0() {
function warmupEmbeddingConnection() {
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return;
embed(['.'], vectorCfg, { timeout: 5000 }).catch(() => {});
embed(['.'], vectorCfg, { timeout: 5000 }).catch(() => { });
}
async function autoVectorizeNewEvents(newEventIds) {
@@ -1043,7 +1047,12 @@ async function autoRunSummaryWithRetry(targetMesId, configForRun) {
const store = getSummaryStore();
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
invalidateLexicalIndex();
// Incrementally add new events to the lexical index
if (newEventIds?.length) {
const allEvents = store?.json?.events || [];
const idSet = new Set(newEventIds);
addEventDocuments(allEvents.filter(e => idSet.has(e.id)));
}
applyHideStateDebounced();
updateFrameStatsAfterSummary(endMesId, store.json || {});
@@ -1150,7 +1159,7 @@ function handleFrameMessage(event) {
case "VECTOR_CANCEL_GENERATE":
vectorCancelled = true;
cancelL0Extraction();
try { vectorAbortController?.abort?.(); } catch {}
try { vectorAbortController?.abort?.(); } catch { }
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
break;
@@ -1338,7 +1347,12 @@ async function handleManualGenerate(mesId, config) {
const store = getSummaryStore();
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
invalidateLexicalIndex();
// Incrementally add new events to the lexical index
if (newEventIds?.length) {
const allEvents = store?.json?.events || [];
const idSet = new Set(newEventIds);
addEventDocuments(allEvents.filter(e => idSet.has(e.id)));
}
applyHideStateDebounced();
updateFrameStatsAfterSummary(endMesId, store.json || {});
@@ -1381,6 +1395,10 @@ async function handleChatChanged() {
// 实体词典注入 + 索引预热
refreshEntityLexiconAndWarmup();
// Full lexical index rebuild on chat change
invalidateLexicalIndex();
warmupIndex();
// Embedding 连接预热（保持 TCP keep-alive，减少首次召回超时）
warmupEmbeddingConnection();
@@ -1421,7 +1439,7 @@ async function handleMessageSwiped() {
await deleteStateVectorsFromFloor(chatId, lastFloor);
}
invalidateLexicalIndex();
removeDocumentsByFloor(lastFloor);
initButtonsForAll();
applyHideStateDebounced();
@@ -1437,22 +1455,28 @@ async function handleMessageReceived() {
initButtonsForAll();
// 向量全量生成中时跳过 L1 sync（避免竞争写入）
// Skip L1 sync while full vector generation is running
if (guard.isRunning('vector')) return;
await syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, () => {
const syncResult = await syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, () => {
sendAnchorStatsToFrame();
sendVectorStatsToFrame();
});
// Incrementally update lexical index with built chunks (avoid re-read)
if (syncResult?.chunks?.length) {
addDocumentsForFloor(lastFloor, syncResult.chunks);
}
await maybeAutoBuildChunks();
applyHideStateDebounced();
setTimeout(() => maybeAutoRunSummary("after_ai"), 1000);
// 新消息后刷新实体词典（可能有新角色）
// Refresh entity lexicon after new message (new roles may appear)
refreshEntityLexiconAndWarmup();
// 自动补提取缺失的 L0（延迟执行，避免与当前楼提取竞争）
// Auto backfill missing L0 (delay to avoid contention with current floor)
setTimeout(() => maybeAutoExtractL0(), 2000);
}

106
modules/story-summary/vector/llm/reranker.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Reranker - 硅基 bge-reranker-v2-m3
// 对候选文档进行精排，过滤与 query 不相关的内容
// ═══════════════════════════════════════════════════════════════════════════
@@ -11,6 +11,8 @@ const RERANK_URL = 'https://api.siliconflow.cn/v1/rerank';
const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3';
const DEFAULT_TIMEOUT = 15000;
const MAX_DOCUMENTS = 100; // API 限制
const RERANK_BATCH_SIZE = 20;
const RERANK_MAX_CONCURRENCY = 5;
/**
* 对文档列表进行 Rerank 精排
@@ -140,25 +142,83 @@ export async function rerankChunks(query, chunks, options = {}) {
const { topN = 40, minScore = 0.1 } = options;
if (!chunks?.length) return [];
if (chunks.length <= topN) {
const texts = chunks.map(c => c.text || c.semantic || '');
const { results, failed } = await rerank(query, texts, { topN: chunks.length, ...options });
// ─── 单批：直接调用 ───
if (texts.length <= RERANK_BATCH_SIZE) {
const { results, failed } = await rerank(query, texts, {
topN: Math.min(topN, texts.length),
timeout: options.timeout,
signal: options.signal,
});
if (failed) {
return chunks.map(c => ({ ...c, _rerankScore: 0, _rerankFailed: true }));
}
const scoreMap = new Map(results.map(r => [r.index, r.relevance_score]));
return chunks.map((c, i) => ({
...c,
_rerankScore: scoreMap.get(i) ?? 0,
})).sort((a, b) => b._rerankScore - a._rerankScore);
return results
.filter(r => r.relevance_score >= minScore)
.sort((a, b) => b.relevance_score - a.relevance_score)
.slice(0, topN)
.map(r => ({
...chunks[r.index],
_rerankScore: r.relevance_score,
}));
}
const texts = chunks.map(c => c.text || c.semantic || '');
const { results, failed } = await rerank(query, texts, { topN, ...options });
// ─── 多批：拆分 → 并发 → 合并 ───
const batches = [];
for (let i = 0; i < texts.length; i += RERANK_BATCH_SIZE) {
batches.push({
texts: texts.slice(i, i + RERANK_BATCH_SIZE),
offset: i,
});
}
const concurrency = Math.min(batches.length, RERANK_MAX_CONCURRENCY);
xbLog.info(MODULE_ID, `并发 Rerank: ${batches.length} 批 × ≤${RERANK_BATCH_SIZE} docs, concurrency=${concurrency}`);
const batchResults = new Array(batches.length);
let failedBatches = 0;
const runBatch = async (batchIdx) => {
const batch = batches[batchIdx];
const { results, failed } = await rerank(query, batch.texts, {
topN: batch.texts.length,
timeout: options.timeout,
signal: options.signal,
});
if (failed) {
failedBatches++;
// 单批降级：保留原始顺序，score=0
batchResults[batchIdx] = batch.texts.map((_, i) => ({
globalIndex: batch.offset + i,
relevance_score: 0,
_batchFailed: true,
}));
} else {
batchResults[batchIdx] = results.map(r => ({
globalIndex: batch.offset + r.index,
relevance_score: r.relevance_score,
}));
}
};
// 并发池
let nextIdx = 0;
const worker = async () => {
while (nextIdx < batches.length) {
const idx = nextIdx++;
await runBatch(idx);
}
};
await Promise.all(Array.from({ length: concurrency }, () => worker()));
// 全部失败 → 整体降级
if (failedBatches === batches.length) {
xbLog.warn(MODULE_ID, `全部 ${batches.length} 批 rerank 失败，整体降级`);
return chunks.slice(0, topN).map(c => ({
...c,
_rerankScore: 0,
@@ -166,15 +226,25 @@ export async function rerankChunks(query, chunks, options = {}) {
}));
}
return results
.filter(r => r.relevance_score >= minScore)
.sort((a, b) => b.relevance_score - a.relevance_score)
.map(r => ({
...chunks[r.index],
_rerankScore: r.relevance_score,
}));
}
// 合并所有批次结果
const merged = batchResults.flat();
const selected = merged
.filter(r => r._batchFailed || r.relevance_score >= minScore)
.sort((a, b) => b.relevance_score - a.relevance_score)
.slice(0, topN)
.map(r => ({
...chunks[r.globalIndex],
_rerankScore: r.relevance_score,
...(r._batchFailed ? { _rerankFailed: true } : {}),
}));
xbLog.info(MODULE_ID,
`Rerank 合并: ${merged.length} candidates, ${failedBatches}/${batches.length} 批失败, 选中 ${selected.length}`
);
return selected;
}
/**
* 测试 Rerank 服务连接
*/

10
modules/story-summary/vector/pipeline/chunk-builder.js
View File

@@ -340,15 +340,15 @@ export async function syncOnMessageSwiped(chatId, lastFloor) {
* 新消息后同步：删除 + 重建最后楼层
*/
export async function syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, onL0Complete) {
if (!chatId || lastFloor < 0 || !message) return;
if (!vectorConfig?.enabled) return;
if (!chatId || lastFloor < 0 || !message) return { built: 0, chunks: [] };
if (!vectorConfig?.enabled) return { built: 0, chunks: [] };
// 删除该楼层旧的
await deleteChunksAtFloor(chatId, lastFloor);
// 重建
const chunks = chunkMessage(lastFloor, message);
if (chunks.length === 0) return;
if (chunks.length === 0) return { built: 0, chunks: [] };
await saveChunks(chatId, chunks);
@@ -356,12 +356,14 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
const fingerprint = getEngineFingerprint(vectorConfig);
const texts = chunks.map(c => c.text);
let vectorized = false;
try {
const vectors = await embed(texts, vectorConfig);
const items = chunks.map((c, i) => ({ chunkId: c.chunkId, vector: vectors[i] }));
await saveChunkVectors(chatId, items, fingerprint);
await updateMeta(chatId, { lastChunkFloor: lastFloor });
vectorized = true;
xbLog.info(MODULE_ID, `消息同步：重建 floor ${lastFloor}，${chunks.length} 个 chunk`);
} catch (e) {
xbLog.error(MODULE_ID, `消息同步失败：floor ${lastFloor}`, e);
@@ -384,4 +386,6 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
xbLog.warn(MODULE_ID, `Atom 提取失败: floor ${lastFloor}`, e);
}
}
return { built: vectorized ? chunks.length : 0, chunks };
}

170
modules/story-summary/vector/retrieval/lexical-index.js
View File

@@ -14,7 +14,6 @@
import MiniSearch from '../../../../libs/minisearch.mjs';
import { getContext } from '../../../../../../../extensions.js';
import { getSummaryStore } from '../../data/store.js';
import { getStateAtoms } from '../storage/state-store.js';
import { getAllChunks } from '../storage/chunk-store.js';
import { xbLog } from '../../../../core/debug-core.js';
import { tokenizeForIndex } from '../utils/tokenizer.js';
@@ -39,6 +38,8 @@ let building = false;
/** @type {Promise<MiniSearch|null>|null} 当前构建 Promise（防重入） */
let buildPromise = null;
/** @type {Map<number, string[]>} floor → 该楼层的 doc IDs（仅 L1 chunks） */
let floorDocIds = new Map();
// ─────────────────────────────────────────────────────────────────────────
// 工具函数
@@ -57,13 +58,12 @@ function cleanSummary(summary) {
/**
* 计算缓存指纹
* @param {number} atomCount
* @param {number} chunkCount
* @param {number} eventCount
* @returns {string}
*/
function computeFingerprint(atomCount, chunkCount, eventCount) {
return `${atomCount}:${chunkCount}:${eventCount}`;
function computeFingerprint(chunkCount, eventCount) {
return `${chunkCount}:${eventCount}`;
}
/**
@@ -81,34 +81,31 @@ function yieldToMain() {
/**
* 收集所有待索引文档
*
* @param {object[]} atoms - getStateAtoms() 返回值
* @param {object[]} chunks - getAllChunks(chatId) 返回值
* @param {object[]} events - store.json.events
* @returns {object[]} 文档数组
*/
function collectDocuments(atoms, chunks, events) {
function collectDocuments(chunks, events) {
const docs = [];
// L0 atoms
for (const atom of (atoms || [])) {
if (!atom?.atomId || !atom.semantic) continue;
docs.push({
id: atom.atomId,
type: 'atom',
floor: atom.floor ?? -1,
text: atom.semantic,
});
}
// L1 chunks
// L1 chunks + 填充 floorDocIds
for (const chunk of (chunks || [])) {
if (!chunk?.chunkId || !chunk.text) continue;
const floor = chunk.floor ?? -1;
docs.push({
id: chunk.chunkId,
type: 'chunk',
floor: chunk.floor ?? -1,
floor,
text: chunk.text,
});
if (floor >= 0) {
if (!floorDocIds.has(floor)) {
floorDocIds.set(floor, []);
}
floorDocIds.get(floor).push(chunk.chunkId);
}
}
// L2 events
@@ -244,7 +241,6 @@ export function searchLexicalIndex(index, terms) {
}
// 分类结果
const atomIdSet = new Set();
const chunkIdSet = new Set();
const eventIdSet = new Set();
@@ -254,16 +250,6 @@ export function searchLexicalIndex(index, terms) {
const floor = hit.floor;
switch (type) {
case 'atom':
if (!atomIdSet.has(id)) {
atomIdSet.add(id);
result.atomIds.push(id);
if (typeof floor === 'number' && floor >= 0) {
result.atomFloors.add(floor);
}
}
break;
case 'chunk':
if (!chunkIdSet.has(id)) {
chunkIdSet.add(id);
@@ -304,8 +290,10 @@ export function searchLexicalIndex(index, terms) {
* @returns {Promise<{index: MiniSearch, fingerprint: string}>}
*/
async function collectAndBuild(chatId) {
// 收集数据
const atoms = getStateAtoms() || [];
// 清空侧索引（全量重建）
floorDocIds = new Map();
// 收集数据（不含 L0 atoms）
const store = getSummaryStore();
const events = store?.json?.events || [];
@@ -316,15 +304,15 @@ async function collectAndBuild(chatId) {
xbLog.warn(MODULE_ID, '获取 chunks 失败', e);
}
const fp = computeFingerprint(atoms.length, chunks.length, events.length);
const fp = computeFingerprint(chunks.length, events.length);
// 检查是否在收集过程中缓存已被其他调用更新
if (cachedIndex && cachedChatId === chatId && cachedFingerprint === fp) {
return { index: cachedIndex, fingerprint: fp };
}
// 收集文档
const docs = collectDocuments(atoms, chunks, events);
// 收集文档（同时填充 floorDocIds）
const docs = collectDocuments(chunks, events);
// 异步分片构建
const index = await buildIndexAsync(docs);
@@ -438,4 +426,116 @@ export function invalidateLexicalIndex() {
cachedIndex = null;
cachedChatId = null;
cachedFingerprint = null;
floorDocIds = new Map();
}
// ─────────────────────────────────────────────────────────────────────────
// 增量更新接口
// ─────────────────────────────────────────────────────────────────────────
/**
* 为指定楼层添加 L1 chunks 到索引
*
* 先移除该楼层旧文档，再添加新文档。
* 如果索引不存在（缓存失效），静默跳过（下次 getLexicalIndex 全量重建）。
*
* @param {number} floor - 楼层号
* @param {object[]} chunks - chunk 对象列表（需有 chunkId、text、floor）
*/
export function addDocumentsForFloor(floor, chunks) {
if (!cachedIndex || !chunks?.length) return;
// 先移除旧文档
removeDocumentsByFloor(floor);
const docs = [];
const docIds = [];
for (const chunk of chunks) {
if (!chunk?.chunkId || !chunk.text) continue;
docs.push({
id: chunk.chunkId,
type: 'chunk',
floor: chunk.floor ?? floor,
text: chunk.text,
});
docIds.push(chunk.chunkId);
}
if (docs.length > 0) {
cachedIndex.addAll(docs);
floorDocIds.set(floor, docIds);
xbLog.info(MODULE_ID, `增量添加: floor ${floor}, ${docs.length} 个 chunk`);
}
}
/**
* 从索引中移除指定楼层的所有 L1 chunk 文档
*
* 使用 MiniSearch discard()（软删除）。
* 如果索引不存在，静默跳过。
*
* @param {number} floor - 楼层号
*/
export function removeDocumentsByFloor(floor) {
if (!cachedIndex) return;
const docIds = floorDocIds.get(floor);
if (!docIds?.length) return;
for (const id of docIds) {
try {
cachedIndex.discard(id);
} catch {
// 文档可能不存在（已被全量重建替换）
}
}
floorDocIds.delete(floor);
xbLog.info(MODULE_ID, `增量移除: floor ${floor}, ${docIds.length} 个文档`);
}
/**
* 将新 L2 事件添加到索引
*
* 如果事件 ID 已存在，先 discard 再 add（覆盖）。
* 如果索引不存在，静默跳过。
*
* @param {object[]} events - 事件对象列表（需有 id、title、summary 等）
*/
export function addEventDocuments(events) {
if (!cachedIndex || !events?.length) return;
const docs = [];
for (const ev of events) {
if (!ev?.id) continue;
const parts = [];
if (ev.title) parts.push(ev.title);
if (ev.participants?.length) parts.push(ev.participants.join(' '));
const summary = cleanSummary(ev.summary);
if (summary) parts.push(summary);
const text = parts.join(' ').trim();
if (!text) continue;
// 覆盖：先尝试移除旧的
try {
cachedIndex.discard(ev.id);
} catch {
// 不存在则忽略
}
docs.push({
id: ev.id,
type: 'event',
floor: null,
text,
});
}
if (docs.length > 0) {
cachedIndex.addAll(docs);
xbLog.info(MODULE_ID, `增量添加: ${docs.length} 个事件`);
}
}

107
modules/story-summary/vector/retrieval/metrics.js
View File

@@ -8,7 +8,7 @@
// 架构变更（v3 → v4）：
// - evidence 区块反映 L0-only 融合 + L1 按楼层拉取的两阶段架构
// - 删除 mergedByType / selectedByType（不再有混合池）
// - 新增 l0Candidates / l0Selected / l1Pulled / l1Attached / l1CosineTime
// - 新增 floorCandidates / floorsSelected / l0Collected / l1Pulled / l1Attached / l1CosineTime
// - fusion 区块明确标注 L0-only（删除 anchorCount）
// - quality.chunkRealRatio → quality.l1AttachRate
// ═══════════════════════════════════════════════════════════════════════════
@@ -48,10 +48,10 @@ export function createMetrics() {
searchTime: 0,
},
// Fusion (W-RRF, L0-only) - 多路融合
// Fusion (W-RRF, floor-level) - 多路融合
fusion: {
denseCount: 0,
lexCount: 0,
denseFloors: 0,
lexFloors: 0,
totalUnique: 0,
afterCap: 0,
time: 0,
@@ -80,25 +80,27 @@ export function createMetrics() {
entityNames: [],
},
// Evidence (Two-Stage: L0 rerank → L1 pull) - 原文证据
// Evidence (Two-Stage: Floor rerank → L1 pull) - 原文证据
evidence: {
// Stage 1: L0
l0Candidates: 0, // W-RRF 融合后的 L0 候选数
l0Selected: 0, // rerank 后选中的 L0 数
// Stage 1: Floor
floorCandidates: 0, // W-RRF 融合后的 floor 候选数
floorsSelected: 0, // rerank 后选中的 floor 数
l0Collected: 0, // 选中 floor 中收集的 L0 atom 总数
rerankApplied: false,
rerankFailed: false,
beforeRerank: 0,
afterRerank: 0,
rerankTime: 0,
rerankScores: null,
rerankDocAvgLength: 0, // rerank document 平均字符数
// Stage 2: L1
l1Pulled: 0, // 从 DB 拉取的 L1 chunk 总数
l1Attached: 0, // 实际挂载的 L1 数（top-1 × 楼层 × 2侧）
l1Attached: 0, // 实际挂载的 L1 数（top-1 × floor × 2侧）
l1CosineTime: 0, // L1 cosine 打分耗时
// 装配
contextPairsAdded: 0, // 保留兼容（= l1Attached 中 USER 侧数量）
contextPairsAdded: 0, // USER 侧挂载数量
tokens: 0,
assemblyTime: 0,
},
@@ -149,7 +151,7 @@ export function createMetrics() {
quality: {
constraintCoverage: 100,
eventPrecisionProxy: 0,
l1AttachRate: 0, // 有 L1 挂载的 L0 占比
l1AttachRate: 0, // 有 L1 挂载的 floor 占比
potentialIssues: [],
},
};
@@ -223,10 +225,10 @@ export function formatMetricsLog(metrics) {
lines.push(`└─ search_time: ${m.lexical.searchTime}ms`);
lines.push('');
// Fusion (W-RRF, L0-only)
lines.push('[Fusion] W-RRF (L0-only) - 多路融合');
lines.push(`├─ dense_count: ${m.fusion.denseCount}`);
lines.push(`├─ lex_count: ${m.fusion.lexCount}`);
// Fusion (W-RRF, floor-level)
lines.push('[Fusion] W-RRF (floor-level) - 多路融合');
lines.push(`├─ dense_floors: ${m.fusion.denseFloors}`);
lines.push(`├─ lex_floors: ${m.fusion.lexFloors}`);
lines.push(`├─ total_unique: ${m.fusion.totalUnique}`);
lines.push(`├─ after_cap: ${m.fusion.afterCap}`);
lines.push(`└─ time: ${m.fusion.time}ms`);
@@ -277,28 +279,32 @@ export function formatMetricsLog(metrics) {
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
lines.push('');
// Evidence (Two-Stage)
lines.push('[Evidence] Two-Stage: L0 Locate → L1 Pull');
lines.push(`├─ Stage 1 (L0):`);
lines.push(`│ ├─ candidates (post-fusion): ${m.evidence.l0Candidates}`);
// Evidence (Two-Stage: Floor Rerank → L1 Pull)
lines.push('[Evidence] Two-Stage: Floor Rerank → L1 Pull');
lines.push(`├─ Stage 1 (Floor Rerank):`);
lines.push(`│ ├─ floor_candidates (post-fusion): ${m.evidence.floorCandidates}`);
if (m.evidence.rerankApplied) {
lines.push(`│ ├─ rerank_applied: true`);
if (m.evidence.rerankFailed) {
lines.push(`│ ├─ rerank_failed: ⚠ YES (using fusion order)`);
lines.push(`│ │ ⚠ rerank_failed: using fusion order`);
}
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank}`);
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank}`);
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank} floors`);
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank} floors`);
lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`);
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
}
if (m.evidence.rerankDocAvgLength > 0) {
lines.push(`│ ├─ rerank_doc_avg_length: ${m.evidence.rerankDocAvgLength} chars`);
}
} else {
lines.push(`│ ├─ rerank_applied: false`);
}
lines.push(`│ └─ selected: ${m.evidence.l0Selected}`);
lines.push(`│ ├─ floors_selected: ${m.evidence.floorsSelected}`);
lines.push(`│ └─ l0_atoms_collected: ${m.evidence.l0Collected}`);
lines.push(`├─ Stage 2 (L1):`);
lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`);
lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`);
@@ -345,9 +351,7 @@ export function formatMetricsLog(metrics) {
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
if (m.timing.evidenceRerank > 0) {
lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`);
}
lines.push(`├─ floor_rerank: ${m.timing.evidenceRerank || 0}ms`);
lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`);
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
@@ -406,20 +410,20 @@ export function detectIssues(metrics) {
// 词法检索问题
// ─────────────────────────────────────────────────────────────────
if ((m.lexical.terms || []).length > 0 && m.lexical.atomHits === 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
if ((m.lexical.terms || []).length > 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
issues.push('Lexical search returned zero hits - terms may not match any indexed content');
}
// ─────────────────────────────────────────────────────────────────
// 融合问题（L0-only）
// 融合问题（floor-level）
// ─────────────────────────────────────────────────────────────────
if (m.fusion.lexCount === 0 && m.fusion.denseCount > 0) {
issues.push('No lexical L0 candidates in fusion - hybrid retrieval not contributing');
if (m.fusion.lexFloors === 0 && m.fusion.denseFloors > 0) {
issues.push('No lexical floors in fusion - hybrid retrieval not contributing');
}
if (m.fusion.afterCap === 0) {
issues.push('Fusion produced zero L0 candidates - all retrieval paths may have failed');
issues.push('Fusion produced zero floor candidates - all retrieval paths may have failed');
}
// ─────────────────────────────────────────────────────────────────
@@ -463,29 +467,30 @@ export function detectIssues(metrics) {
}
// ─────────────────────────────────────────────────────────────────
// L0 Rerank 问题
// Floor Rerank 问题
// ─────────────────────────────────────────────────────────────────
if (m.evidence.rerankApplied) {
if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) {
const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank);
if (filterRatio > 0.7) {
issues.push(`High L0 rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant L0 in fusion output`);
}
if (m.evidence.rerankFailed) {
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
}
if (m.evidence.rerankApplied && !m.evidence.rerankFailed) {
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
if (rs.max < 0.5) {
issues.push(`Low L0 rerank scores (max=${rs.max}) - query may be poorly matched`);
if (rs.max < 0.3) {
issues.push(`Low floor rerank scores (max=${rs.max}) - query-document domain mismatch`);
}
if (rs.mean < 0.3) {
issues.push(`Very low average L0 rerank score (mean=${rs.mean}) - context may be weak`);
if (rs.mean < 0.2) {
issues.push(`Very low average floor rerank score (mean=${rs.mean}) - context may be weak`);
}
}
if (m.evidence.rerankTime > 2000) {
issues.push(`Slow L0 rerank (${m.evidence.rerankTime}ms) - may affect response time`);
if (m.evidence.rerankTime > 3000) {
issues.push(`Slow floor rerank (${m.evidence.rerankTime}ms) - may affect response time`);
}
if (m.evidence.rerankDocAvgLength > 3000) {
issues.push(`Large rerank documents (avg ${m.evidence.rerankDocAvgLength} chars) - may reduce rerank precision`);
}
}
@@ -493,21 +498,17 @@ export function detectIssues(metrics) {
// L1 挂载问题
// ─────────────────────────────────────────────────────────────────
if (m.evidence.rerankFailed) {
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
}
if (m.evidence.l0Selected > 0 && m.evidence.l1Pulled === 0) {
if (m.evidence.floorsSelected > 0 && m.evidence.l1Pulled === 0) {
issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed');
}
if (m.evidence.l0Selected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
issues.push('L1 chunks pulled but none attached - cosine scores may be too low or floor mismatch');
if (m.evidence.floorsSelected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
issues.push('L1 chunks pulled but none attached - cosine scores may be too low');
}
const l1AttachRate = m.quality.l1AttachRate || 0;
if (m.evidence.l0Selected > 5 && l1AttachRate < 20) {
issues.push(`Low L1 attach rate (${l1AttachRate}%) - many L0 lack concrete dialogue evidence`);
if (m.evidence.floorsSelected > 3 && l1AttachRate < 50) {
issues.push(`Low L1 attach rate (${l1AttachRate}%) - selected floors lack L1 chunks`);
}
// ─────────────────────────────────────────────────────────────────

49
modules/story-summary/vector/retrieval/query-builder.js
View File

@@ -89,6 +89,46 @@ function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) {
.map(([term]) => term);
}
/**
* 构建 rerank 专用查询（纯自然语言，不带结构标签）
*
* reranker（bge-reranker-v2-m3）的 query 应为自然语言文本，
* 不含 [ENTITIES] [DIALOGUE] 等结构标签。
*
* @param {string[]} focusEntities - 焦点实体
* @param {object[]} lastMessages - 最近 K 条消息
* @param {string|null} pendingUserMessage - 待发送的用户消息
* @param {object} context - { name1, name2 }
* @returns {string}
*/
function buildRerankQuery(focusEntities, lastMessages, pendingUserMessage, context) {
const parts = [];
// 实体提示
if (focusEntities.length > 0) {
parts.push(`关于${focusEntities.join('、')}。`);
}
// 最近对话原文
for (const m of (lastMessages || [])) {
const speaker = m.is_user ? (context.name1 || '用户') : (m.name || context.name2 || '角色');
const clean = cleanMessageText(m.mes || '');
if (clean) {
parts.push(`${speaker}：${clean}`);
}
}
// 待发送消息
if (pendingUserMessage) {
const clean = cleanMessageText(pendingUserMessage);
if (clean) {
parts.push(`${context.name1 || '用户'}：${clean}`);
}
}
return parts.join('\n');
}
// ─────────────────────────────────────────────────────────────────────────
// QueryBundle 类型定义（JSDoc）
// ─────────────────────────────────────────────────────────────────────────
@@ -176,9 +216,8 @@ export function buildQueryBundle(lastMessages, pendingUserMessage, store = null,
const queryText_v0 = queryParts.join('\n\n');
// 6. rerankQuery 与 embedding query 同源（零暗箱）
// 后续 refine 会把它升级为与 queryText_v1 同源。
const rerankQuery = queryText_v0;
// 6. rerankQuery 独立构建（纯自然语言，供 reranker 使用）
const rerankQuery = buildRerankQuery(focusEntities, dialogueLines.length > 0 ? lastMessages : [], pendingUserMessage, context);
// 7. 构建 lexicalTerms
const entityTerms = focusEntities.map(e => e.toLowerCase());
@@ -281,8 +320,8 @@ export function refineQueryBundle(bundle, anchorHits, eventHits) {
}
}
// 5. rerankQuery 与最终 query 同源（零暗箱）
bundle.rerankQuery = bundle.queryText_v1 || bundle.queryText_v0;
// 5. rerankQuery 保持独立（不随 refinement 变更）
// reranker 需要纯自然语言 query，不受 memory hints 干扰
// 6. 增强 lexicalTerms
if (hints.length > 0) {

334
modules/story-summary/vector/retrieval/recall.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence)
//
// 命名规范：
@@ -10,8 +10,8 @@
// 阶段 2: Round 1 Dense Retrieval（L0 + L2）
// 阶段 3: Query Refinement（用已命中记忆增强）
// 阶段 4: Round 2 Dense Retrieval（L0 + L2）
// 阶段 5: Lexical Retrieval + L0 Merge
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine 预筛选
// 阶段 5: Lexical Retrieval
// 阶段 6: Floor W-RRF Fusion + Rerank + L1 配对
// 阶段 7: L1 配对组装（L0 → top-1 AI L1 + top-1 USER L1）
// 阶段 8: Causation Trace
// ═══════════════════════════════════════════════════════════════════════════
@@ -49,10 +49,10 @@ const CONFIG = {
RRF_K: 60,
RRF_W_DENSE: 1.0,
RRF_W_LEX: 0.9,
FUSION_CAP: 100,
FUSION_CAP: 60,
// Rerank（L0-only）
RERANK_TOP_N: 50,
// Rerank（floor-level）
RERANK_TOP_N: 20,
RERANK_MIN_SCORE: 0.15,
// 因果链
@@ -421,14 +421,14 @@ function traceCausation(eventHits, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MA
*/
/**
* W-RRF 融合两路 L0 候选（dense + lexical）
* W-RRF 加权倒数排名融合（floor 粒度）
*
* @param {RankedItem[]} denseRank - Dense 路（cosine 降序）
* @param {RankedItem[]} lexRank - Lexical 路（MiniSearch score 降序）
* @param {{id: number, score: number}[]} denseRank - Dense 路（floor → max cosine，降序）
* @param {{id: number, score: number}[]} lexRank - Lexical 路（floor → max bm25，降序）
* @param {number} cap - 输出上限
* @returns {{top: {id: string, fusionScore: number}[], totalUnique: number}}
* @returns {{top: {id: number, fusionScore: number}[], totalUnique: number}}
*/
function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
function fuseByFloor(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
const k = CONFIG.RRF_K;
const wD = CONFIG.RRF_W_DENSE;
const wL = CONFIG.RRF_W_LEX;
@@ -445,141 +445,109 @@ function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
const denseMap = buildRankMap(denseRank || []);
const lexMap = buildRankMap(lexRank || []);
const allIds = new Set([
...denseMap.keys(),
...lexMap.keys(),
]);
const allIds = new Set([...denseMap.keys(), ...lexMap.keys()]);
const totalUnique = allIds.size;
const scored = [];
for (const id of allIds) {
let score = 0;
if (denseMap.has(id)) {
score += wD / (k + denseMap.get(id));
}
if (lexMap.has(id)) {
score += wL / (k + lexMap.get(id));
}
if (denseMap.has(id)) score += wD / (k + denseMap.get(id));
if (lexMap.has(id)) score += wL / (k + lexMap.get(id));
scored.push({ id, fusionScore: score });
}
scored.sort((a, b) => b.fusionScore - a.fusionScore);
return {
top: scored.slice(0, cap),
totalUnique,
};
return { top: scored.slice(0, cap), totalUnique };
}
// ═══════════════════════════════════════════════════════════════════════════
// [Stage 6] L0-only 融合 + Rerank ‖ 并发 L1 Cosine 预筛选
// [Stage 6] Floor 融合 + Rerank + L1 配对
// ═══════════════════════════════════════════════════════════════════════════
/**
* L0 融合 + rerank，并发拉取 L1 并 cosine 打分
* Floor 粒度融合 + Rerank + L1 配对
*
* @param {object[]} anchorHits - L0 dense 命中（Round 2）
* @param {Set<number>} anchorFloors - L0 命中楼层（含 lexical 扩展）
* @param {number[]} queryVector - 查询向量（v1）
* @param {string} rerankQuery - rerank 查询文本
* @param {string} rerankQuery - rerank 查询文本（纯自然语言）
* @param {object} lexicalResult - 词法检索结果
* @param {object} metrics
* @returns {Promise<{l0Selected: object[], l1ByFloor: Map<number, {aiTop1: object|null, userTop1: object|null}>}>}
*/
async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rerankQuery, lexicalResult, metrics) {
const { chatId, chat } = getContext();
async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexicalResult, metrics) {
const { chatId, chat, name1, name2 } = getContext();
if (!chatId) return { l0Selected: [], l1ByFloor: new Map() };
const T_Start = performance.now();
// ─────────────────────────────────────────────────────────────────
// 6a. 构建 L0 候选对象（用于 rerank）
//
// 重要：支持 lexical-only 的 L0（atom）进入候选池。
// 否则 hybrid 会退化为 dense-only：lexical 命中的 atom 若未被 dense 命中会被直接丢弃。
// 6a. Dense floor rank（每个 floor 取 max cosine）
// ─────────────────────────────────────────────────────────────────
const l0ObjectMap = new Map();
const denseFloorMap = new Map();
for (const a of (anchorHits || [])) {
const id = `anchor-${a.atomId}`;
l0ObjectMap.set(id, {
id,
atomId: a.atomId,
floor: a.floor,
similarity: a.similarity,
atom: a.atom,
text: a.atom?.semantic || '',
});
const cur = denseFloorMap.get(a.floor) || 0;
if (a.similarity > cur) denseFloorMap.set(a.floor, a.similarity);
}
// lexical-only atoms：从全量 StateAtoms 补齐（similarity 记为 0，靠 lex rank 贡献 W-RRF）
const lexAtomIds = lexicalResult?.atomIds || [];
if (lexAtomIds.length > 0) {
const atomsList = getStateAtoms();
const atomMap = new Map(atomsList.map(a => [a.atomId, a]));
const denseFloorRank = [...denseFloorMap.entries()]
.sort((a, b) => b[1] - a[1])
.map(([floor, score]) => ({ id: floor, score }));
for (const atomId of lexAtomIds) {
const id = `anchor-${atomId}`;
if (l0ObjectMap.has(id)) continue;
// ─────────────────────────────────────────────────────────────────
// 6b. Lexical floor rank（chunkScores → floor 聚合 + USER→AI 映射 + 预过滤）
// ─────────────────────────────────────────────────────────────────
const atom = atomMap.get(atomId);
if (!atom) continue;
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
const atomFloorSet = new Set(getStateAtoms().map(a => a.floor));
l0ObjectMap.set(id, {
id,
atomId,
floor: atom.floor,
similarity: 0,
atom,
text: atom.semantic || '',
});
const lexFloorScores = new Map();
for (const { chunkId, score } of (lexicalResult?.chunkScores || [])) {
const match = chunkId?.match(/^c-(\d+)-/);
if (!match) continue;
let floor = parseInt(match[1], 10);
// USER floor → AI floor 映射
if (chat?.[floor]?.is_user) {
const aiFloor = floor + 1;
if (aiFloor < chat.length && !chat[aiFloor]?.is_user) {
floor = aiFloor;
} else {
continue;
}
}
// ─────────────────────────────────────────────────────────────────
// 6b. 构建两路排名（L0-only）
// ─────────────────────────────────────────────────────────────────
// 预过滤：必须有 L0 atoms
if (!atomFloorSet.has(floor)) continue;
// Dense 路：anchorHits 按 similarity 排序
const denseRank = (anchorHits || [])
.map(a => ({ id: `anchor-${a.atomId}`, score: a.similarity }))
.sort((a, b) => b.score - a.score);
const cur = lexFloorScores.get(floor) || 0;
if (score > cur) lexFloorScores.set(floor, score);
}
// Lexical 路：从 lexicalResult.atomIds 构建排名（允许 lexical-only）
// atomIds 已按 MiniSearch score 排序（searchLexicalIndex 返回顺序）；W-RRF 依赖 rank，score 为占位
const lexRank = (lexAtomIds || [])
.map(atomId => ({ id: `anchor-${atomId}`, score: 1 }))
.filter(item => l0ObjectMap.has(item.id));
const lexFloorRank = [...lexFloorScores.entries()]
.sort((a, b) => b[1] - a[1])
.map(([floor, score]) => ({ id: floor, score }));
// ─────────────────────────────────────────────────────────────────
// 6c. W-RRF 融合（L0-only）
// 6c. Floor W-RRF 融合
// ─────────────────────────────────────────────────────────────────
const T_Fusion_Start = performance.now();
const { top: fusionResult, totalUnique } = fuseL0Candidates(denseRank, lexRank, CONFIG.FUSION_CAP);
const { top: fusedFloors, totalUnique } = fuseByFloor(denseFloorRank, lexFloorRank, CONFIG.FUSION_CAP);
const fusionTime = Math.round(performance.now() - T_Fusion_Start);
// 构建 rerank 候选列表
const rerankCandidates = fusionResult
.map(f => l0ObjectMap.get(f.id))
.filter(Boolean);
if (metrics) {
metrics.fusion.denseCount = denseRank.length;
metrics.fusion.lexCount = lexRank.length;
metrics.fusion.denseFloors = denseFloorRank.length;
metrics.fusion.lexFloors = lexFloorRank.length;
metrics.fusion.totalUnique = totalUnique;
metrics.fusion.afterCap = rerankCandidates.length;
metrics.fusion.afterCap = fusedFloors.length;
metrics.fusion.time = fusionTime;
metrics.evidence.l0Candidates = rerankCandidates.length;
metrics.evidence.floorCandidates = fusedFloors.length;
}
if (rerankCandidates.length === 0) {
if (fusedFloors.length === 0) {
if (metrics) {
metrics.evidence.l0Selected = 0;
metrics.evidence.floorsSelected = 0;
metrics.evidence.l0Collected = 0;
metrics.evidence.l1Pulled = 0;
metrics.evidence.l1Attached = 0;
metrics.evidence.l1CosineTime = 0;
@@ -589,54 +557,87 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
}
// ─────────────────────────────────────────────────────────────────
// 6d. 收集所有候选 L0 的楼层（用于并发拉取 L1）
// 包含 AI 楼层本身 + 上方 USER 楼层
// 6d. 拉取 L1 chunks + cosine 打分
// ─────────────────────────────────────────────────────────────────
const candidateFloors = new Set();
for (const c of rerankCandidates) {
candidateFloors.add(c.floor);
// 上方 USER 楼层
const userFloor = c.floor - 1;
const floorsToFetch = new Set();
for (const f of fusedFloors) {
floorsToFetch.add(f.id);
const userFloor = f.id - 1;
if (userFloor >= 0 && chat?.[userFloor]?.is_user) {
candidateFloors.add(userFloor);
floorsToFetch.add(userFloor);
}
}
const l1ScoredByFloor = await pullAndScoreL1(chatId, [...floorsToFetch], queryVector, chat);
if (metrics) {
let totalPulled = 0;
for (const [key, chunks] of l1ScoredByFloor) {
if (key === '_cosineTime') continue;
totalPulled += chunks.length;
}
metrics.evidence.l1Pulled = totalPulled;
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
}
// ─────────────────────────────────────────────────────────────────
// 6e. 并发：rerank L0 ‖ 拉取 L1 chunks + 向量 + cosine 打分
// 6e. 构建 rerank documents（每个 floor: USER chunks + AI chunks）
// ─────────────────────────────────────────────────────────────────
const rerankCandidates = [];
for (const f of fusedFloors) {
const aiFloor = f.id;
const userFloor = aiFloor - 1;
const aiChunks = l1ScoredByFloor.get(aiFloor) || [];
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
? (l1ScoredByFloor.get(userFloor) || [])
: [];
const parts = [];
const userName = chat?.[userFloor]?.name || name1 || '用户';
const aiName = chat?.[aiFloor]?.name || name2 || '角色';
if (userChunks.length > 0) {
parts.push(`${userName}：${userChunks.map(c => c.text).join(' ')}`);
}
if (aiChunks.length > 0) {
parts.push(`${aiName}：${aiChunks.map(c => c.text).join(' ')}`);
}
const text = parts.join('\n');
if (!text.trim()) continue;
rerankCandidates.push({
floor: aiFloor,
text,
fusionScore: f.fusionScore,
});
}
// ─────────────────────────────────────────────────────────────────
// 6f. 并发 Rerank
// ─────────────────────────────────────────────────────────────────
const T_Rerank_Start = performance.now();
// 并发任务 1：rerank L0
const rerankPromise = rerankChunks(rerankQuery, rerankCandidates, {
const reranked = await rerankChunks(rerankQuery, rerankCandidates, {
topN: CONFIG.RERANK_TOP_N,
minScore: CONFIG.RERANK_MIN_SCORE,
});
// 并发任务 2：拉取 L1 chunks + 向量 → cosine 打分
const l1Promise = pullAndScoreL1(chatId, Array.from(candidateFloors), queryVector, chat);
// 等待两个任务完成
const [rerankedL0, l1ScoredByFloor] = await Promise.all([rerankPromise, l1Promise]);
const rerankTime = Math.round(performance.now() - T_Rerank_Start);
// ─────────────────────────────────────────────────────────────────
// 6f. 记录 rerank metrics
// ─────────────────────────────────────────────────────────────────
if (metrics) {
metrics.evidence.rerankApplied = true;
metrics.evidence.beforeRerank = rerankCandidates.length;
metrics.evidence.afterRerank = rerankedL0.length;
metrics.evidence.rerankFailed = rerankedL0.some(c => c._rerankFailed);
metrics.evidence.l0Selected = rerankedL0.length;
metrics.evidence.afterRerank = reranked.length;
metrics.evidence.rerankFailed = reranked.some(c => c._rerankFailed);
metrics.evidence.rerankTime = rerankTime;
metrics.timing.evidenceRerank = rerankTime;
const scores = rerankedL0.map(c => c._rerankScore || 0).filter(s => s > 0);
const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0);
if (scores.length > 0) {
scores.sort((a, b) => a - b);
metrics.evidence.rerankScores = {
@@ -645,74 +646,82 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)),
};
}
// document 平均长度
if (rerankCandidates.length > 0) {
const totalLen = rerankCandidates.reduce((s, c) => s + (c.text?.length || 0), 0);
metrics.evidence.rerankDocAvgLength = Math.round(totalLen / rerankCandidates.length);
}
}
// ─────────────────────────────────────────────────────────────────
// 6g. 构建最终 l0Selected + l1ByFloor
// 6g. 收集 L0 atoms + L1 top-1 配对
// ─────────────────────────────────────────────────────────────────
const l0Selected = rerankedL0.map(item => ({
id: item.id,
atomId: item.atomId,
floor: item.floor,
similarity: item.similarity,
rerankScore: item._rerankScore || 0,
atom: item.atom,
text: item.text,
}));
const atomsList = getStateAtoms();
const atomsByFloor = new Map();
for (const atom of atomsList) {
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
if (!atomsByFloor.has(atom.floor)) atomsByFloor.set(atom.floor, []);
atomsByFloor.get(atom.floor).push(atom);
}
// 为每个选中的 L0 楼层组装 top-1 L1 配对
const selectedFloors = new Set(l0Selected.map(l => l.floor));
const l0Selected = [];
const l1ByFloor = new Map();
let contextPairsAdded = 0;
for (const floor of selectedFloors) {
for (const item of reranked) {
const floor = item.floor;
const rerankScore = item._rerankScore || 0;
const denseSim = denseFloorMap.get(floor) || 0;
// 收集该 floor 所有 L0 atoms，共享 floor 的 rerankScore
const floorAtoms = atomsByFloor.get(floor) || [];
for (const atom of floorAtoms) {
l0Selected.push({
id: `anchor-${atom.atomId}`,
atomId: atom.atomId,
floor: atom.floor,
similarity: denseSim,
rerankScore,
atom,
text: atom.semantic || '',
});
}
// L1 top-1 配对（cosine 最高）
const aiChunks = l1ScoredByFloor.get(floor) || [];
xbLog.info(
MODULE_ID,
`L1 attach check: floor=${floor}, l1ScoredByFloor.has=${l1ScoredByFloor.has(floor)}, aiChunks=${aiChunks.length}, l1ScoredByFloor.keys=[${[...l1ScoredByFloor.keys()].slice(0, 10)}...]`
);
const userFloor = floor - 1;
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
? (l1ScoredByFloor.get(userFloor) || [])
: [];
// top-1：取 cosine 最高的
const aiTop1 = aiChunks.length > 0
? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
: null;
const userTop1 = userChunks.length > 0
? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
: null;
// context pair = 上方 USER 楼层成功挂载（用于 metrics）
if (userTop1) contextPairsAdded++;
l1ByFloor.set(floor, { aiTop1, userTop1 });
}
// ─────────────────────────────────────────────────────────────────
// 6h. L1 metrics
// 6h. Metrics
// ─────────────────────────────────────────────────────────────────
if (metrics) {
let totalPulled = 0;
metrics.evidence.floorsSelected = reranked.length;
metrics.evidence.l0Collected = l0Selected.length;
let totalAttached = 0;
for (const [, scored] of l1ScoredByFloor) {
totalPulled += scored.length;
}
for (const [, pair] of l1ByFloor) {
if (pair.aiTop1) totalAttached++;
if (pair.userTop1) totalAttached++;
}
metrics.evidence.l1Pulled = totalPulled;
metrics.evidence.l1Attached = totalAttached;
metrics.evidence.contextPairsAdded = contextPairsAdded;
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
}
const totalTime = Math.round(performance.now() - T_Start);
@@ -721,13 +730,11 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
}
xbLog.info(MODULE_ID,
`Evidence: ${anchorHits?.length || 0} L0 dense → fusion=${rerankCandidates.length} → rerank=${rerankedL0.length} → L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
`Evidence: ${denseFloorRank.length} dense floors + ${lexFloorRank.length} lex floors → fusion=${fusedFloors.length} → rerank=${reranked.length} floors → L0=${l0Selected.length} L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
);
return { l0Selected, l1ByFloor };
}
// ═══════════════════════════════════════════════════════════════════════════
// [L1] 拉取 + Cosine 打分（并发子任务）
// ═══════════════════════════════════════════════════════════════════════════
@@ -973,7 +980,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
);
// ═══════════════════════════════════════════════════════════════════
// 阶段 5: Lexical Retrieval + L0 Merge
// 阶段 5: Lexical Retrieval
// ═══════════════════════════════════════════════════════════════════
const T_Lex_Start = performance.now();
@@ -1003,12 +1010,6 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10);
}
// 合并 L0 floors（dense + lexical）
const anchorFloors = new Set(anchorFloors_dense);
for (const f of lexicalResult.atomFloors) {
anchorFloors.add(f);
}
// 合并 L2 events（lexical 命中但 dense 未命中的 events）
const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean));
const eventIndex = buildEventIndex(allEvents);
@@ -1035,16 +1036,15 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
}
xbLog.info(MODULE_ID,
`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedFloors=${anchorFloors.size} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
);
// ═══════════════════════════════════════════════════════════════════
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine
// 阶段 6: Floor 粒度融合 + Rerank + L1 配对
// ═══════════════════════════════════════════════════════════════════
const { l0Selected, l1ByFloor } = await locateAndPullEvidence(
anchorHits,
anchorFloors,
queryVector_v1,
bundle.rerankQuery,
lexicalResult,
@@ -1086,11 +1086,11 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
console.log(`Total: ${metrics.timing.total}ms`);
console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`);
console.log(`Focus: [${bundle.focusEntities.join(', ')}]`);
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors.size} floors`);
console.log(`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
console.log(`Fusion (L0-only): dense=${metrics.fusion.denseCount} lex=${metrics.fusion.lexCount} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
console.log(`L0 Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.l0Selected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1 Pull: ${metrics.evidence.l1Pulled || 0} chunks → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors_dense.size} floors`);
console.log(`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
console.log(`Fusion (floor): dense=${metrics.fusion.denseFloors} lex=${metrics.fusion.lexFloors} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
console.log(`Floor Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.floorsSelected || 0} floors → L0=${metrics.evidence.l0Collected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1: ${metrics.evidence.l1Pulled || 0} pulled → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`);
console.groupEnd();