Update retrieval, rerank, and indexing changes
This commit is contained in:
@@ -11,7 +11,7 @@ const PROVIDER_MAP = {
|
|||||||
custom: "custom",
|
custom: "custom",
|
||||||
};
|
};
|
||||||
|
|
||||||
const JSON_PREFILL = '{"mindful_prelude": {';
|
const JSON_PREFILL = '下面重新生成完整JSON。';
|
||||||
|
|
||||||
const LLM_PROMPT_CONFIG = {
|
const LLM_PROMPT_CONFIG = {
|
||||||
topSystem: `Story Analyst: This task involves narrative comprehension and structured incremental summarization, representing creative story analysis at the intersection of plot tracking and character development. As a story analyst, you will conduct systematic evaluation of provided dialogue content to generate structured incremental summary data.
|
topSystem: `Story Analyst: This task involves narrative comprehension and structured incremental summarization, representing creative story analysis at the intersection of plot tracking and character development. As a story analyst, you will conduct systematic evaluation of provided dialogue content to generate structured incremental summary data.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
// Reranker - 硅基 bge-reranker-v2-m3
|
// Reranker - 硅基 bge-reranker-v2-m3
|
||||||
// 对候选文档进行精排,过滤与 query 不相关的内容
|
// 对候选文档进行精排,过滤与 query 不相关的内容
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
@@ -11,6 +11,8 @@ const RERANK_URL = 'https://api.siliconflow.cn/v1/rerank';
|
|||||||
const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3';
|
const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3';
|
||||||
const DEFAULT_TIMEOUT = 15000;
|
const DEFAULT_TIMEOUT = 15000;
|
||||||
const MAX_DOCUMENTS = 100; // API 限制
|
const MAX_DOCUMENTS = 100; // API 限制
|
||||||
|
const RERANK_BATCH_SIZE = 20;
|
||||||
|
const RERANK_MAX_CONCURRENCY = 5;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 对文档列表进行 Rerank 精排
|
* 对文档列表进行 Rerank 精排
|
||||||
@@ -140,25 +142,83 @@ export async function rerankChunks(query, chunks, options = {}) {
|
|||||||
const { topN = 40, minScore = 0.1 } = options;
|
const { topN = 40, minScore = 0.1 } = options;
|
||||||
|
|
||||||
if (!chunks?.length) return [];
|
if (!chunks?.length) return [];
|
||||||
if (chunks.length <= topN) {
|
|
||||||
const texts = chunks.map(c => c.text || c.semantic || '');
|
const texts = chunks.map(c => c.text || c.semantic || '');
|
||||||
const { results, failed } = await rerank(query, texts, { topN: chunks.length, ...options });
|
|
||||||
|
// ─── 单批:直接调用 ───
|
||||||
|
if (texts.length <= RERANK_BATCH_SIZE) {
|
||||||
|
const { results, failed } = await rerank(query, texts, {
|
||||||
|
topN: Math.min(topN, texts.length),
|
||||||
|
timeout: options.timeout,
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
|
||||||
if (failed) {
|
if (failed) {
|
||||||
return chunks.map(c => ({ ...c, _rerankScore: 0, _rerankFailed: true }));
|
return chunks.map(c => ({ ...c, _rerankScore: 0, _rerankFailed: true }));
|
||||||
}
|
}
|
||||||
|
|
||||||
const scoreMap = new Map(results.map(r => [r.index, r.relevance_score]));
|
return results
|
||||||
return chunks.map((c, i) => ({
|
.filter(r => r.relevance_score >= minScore)
|
||||||
...c,
|
.sort((a, b) => b.relevance_score - a.relevance_score)
|
||||||
_rerankScore: scoreMap.get(i) ?? 0,
|
.slice(0, topN)
|
||||||
})).sort((a, b) => b._rerankScore - a._rerankScore);
|
.map(r => ({
|
||||||
|
...chunks[r.index],
|
||||||
|
_rerankScore: r.relevance_score,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
const texts = chunks.map(c => c.text || c.semantic || '');
|
// ─── 多批:拆分 → 并发 → 合并 ───
|
||||||
const { results, failed } = await rerank(query, texts, { topN, ...options });
|
const batches = [];
|
||||||
|
for (let i = 0; i < texts.length; i += RERANK_BATCH_SIZE) {
|
||||||
|
batches.push({
|
||||||
|
texts: texts.slice(i, i + RERANK_BATCH_SIZE),
|
||||||
|
offset: i,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (failed) {
|
const concurrency = Math.min(batches.length, RERANK_MAX_CONCURRENCY);
|
||||||
|
xbLog.info(MODULE_ID, `并发 Rerank: ${batches.length} 批 × ≤${RERANK_BATCH_SIZE} docs, concurrency=${concurrency}`);
|
||||||
|
|
||||||
|
const batchResults = new Array(batches.length);
|
||||||
|
let failedBatches = 0;
|
||||||
|
|
||||||
|
const runBatch = async (batchIdx) => {
|
||||||
|
const batch = batches[batchIdx];
|
||||||
|
const { results, failed } = await rerank(query, batch.texts, {
|
||||||
|
topN: batch.texts.length,
|
||||||
|
timeout: options.timeout,
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (failed) {
|
||||||
|
failedBatches++;
|
||||||
|
// 单批降级:保留原始顺序,score=0
|
||||||
|
batchResults[batchIdx] = batch.texts.map((_, i) => ({
|
||||||
|
globalIndex: batch.offset + i,
|
||||||
|
relevance_score: 0,
|
||||||
|
_batchFailed: true,
|
||||||
|
}));
|
||||||
|
} else {
|
||||||
|
batchResults[batchIdx] = results.map(r => ({
|
||||||
|
globalIndex: batch.offset + r.index,
|
||||||
|
relevance_score: r.relevance_score,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// 并发池
|
||||||
|
let nextIdx = 0;
|
||||||
|
const worker = async () => {
|
||||||
|
while (nextIdx < batches.length) {
|
||||||
|
const idx = nextIdx++;
|
||||||
|
await runBatch(idx);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
||||||
|
|
||||||
|
// 全部失败 → 整体降级
|
||||||
|
if (failedBatches === batches.length) {
|
||||||
|
xbLog.warn(MODULE_ID, `全部 ${batches.length} 批 rerank 失败,整体降级`);
|
||||||
return chunks.slice(0, topN).map(c => ({
|
return chunks.slice(0, topN).map(c => ({
|
||||||
...c,
|
...c,
|
||||||
_rerankScore: 0,
|
_rerankScore: 0,
|
||||||
@@ -166,15 +226,25 @@ export async function rerankChunks(query, chunks, options = {}) {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
// 合并所有批次结果
|
||||||
.filter(r => r.relevance_score >= minScore)
|
const merged = batchResults.flat();
|
||||||
.sort((a, b) => b.relevance_score - a.relevance_score)
|
|
||||||
.map(r => ({
|
|
||||||
...chunks[r.index],
|
|
||||||
_rerankScore: r.relevance_score,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
const selected = merged
|
||||||
|
.filter(r => r._batchFailed || r.relevance_score >= minScore)
|
||||||
|
.sort((a, b) => b.relevance_score - a.relevance_score)
|
||||||
|
.slice(0, topN)
|
||||||
|
.map(r => ({
|
||||||
|
...chunks[r.globalIndex],
|
||||||
|
_rerankScore: r.relevance_score,
|
||||||
|
...(r._batchFailed ? { _rerankFailed: true } : {}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
xbLog.info(MODULE_ID,
|
||||||
|
`Rerank 合并: ${merged.length} candidates, ${failedBatches}/${batches.length} 批失败, 选中 ${selected.length}`
|
||||||
|
);
|
||||||
|
|
||||||
|
return selected;
|
||||||
|
}
|
||||||
/**
|
/**
|
||||||
* 测试 Rerank 服务连接
|
* 测试 Rerank 服务连接
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -340,15 +340,15 @@ export async function syncOnMessageSwiped(chatId, lastFloor) {
|
|||||||
* 新消息后同步:删除 + 重建最后楼层
|
* 新消息后同步:删除 + 重建最后楼层
|
||||||
*/
|
*/
|
||||||
export async function syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, onL0Complete) {
|
export async function syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, onL0Complete) {
|
||||||
if (!chatId || lastFloor < 0 || !message) return;
|
if (!chatId || lastFloor < 0 || !message) return { built: 0, chunks: [] };
|
||||||
if (!vectorConfig?.enabled) return;
|
if (!vectorConfig?.enabled) return { built: 0, chunks: [] };
|
||||||
|
|
||||||
// 删除该楼层旧的
|
// 删除该楼层旧的
|
||||||
await deleteChunksAtFloor(chatId, lastFloor);
|
await deleteChunksAtFloor(chatId, lastFloor);
|
||||||
|
|
||||||
// 重建
|
// 重建
|
||||||
const chunks = chunkMessage(lastFloor, message);
|
const chunks = chunkMessage(lastFloor, message);
|
||||||
if (chunks.length === 0) return;
|
if (chunks.length === 0) return { built: 0, chunks: [] };
|
||||||
|
|
||||||
await saveChunks(chatId, chunks);
|
await saveChunks(chatId, chunks);
|
||||||
|
|
||||||
@@ -356,12 +356,14 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
|
|||||||
const fingerprint = getEngineFingerprint(vectorConfig);
|
const fingerprint = getEngineFingerprint(vectorConfig);
|
||||||
const texts = chunks.map(c => c.text);
|
const texts = chunks.map(c => c.text);
|
||||||
|
|
||||||
|
let vectorized = false;
|
||||||
try {
|
try {
|
||||||
const vectors = await embed(texts, vectorConfig);
|
const vectors = await embed(texts, vectorConfig);
|
||||||
const items = chunks.map((c, i) => ({ chunkId: c.chunkId, vector: vectors[i] }));
|
const items = chunks.map((c, i) => ({ chunkId: c.chunkId, vector: vectors[i] }));
|
||||||
await saveChunkVectors(chatId, items, fingerprint);
|
await saveChunkVectors(chatId, items, fingerprint);
|
||||||
await updateMeta(chatId, { lastChunkFloor: lastFloor });
|
await updateMeta(chatId, { lastChunkFloor: lastFloor });
|
||||||
|
|
||||||
|
vectorized = true;
|
||||||
xbLog.info(MODULE_ID, `消息同步:重建 floor ${lastFloor},${chunks.length} 个 chunk`);
|
xbLog.info(MODULE_ID, `消息同步:重建 floor ${lastFloor},${chunks.length} 个 chunk`);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
xbLog.error(MODULE_ID, `消息同步失败:floor ${lastFloor}`, e);
|
xbLog.error(MODULE_ID, `消息同步失败:floor ${lastFloor}`, e);
|
||||||
@@ -384,4 +386,6 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
|
|||||||
xbLog.warn(MODULE_ID, `Atom 提取失败: floor ${lastFloor}`, e);
|
xbLog.warn(MODULE_ID, `Atom 提取失败: floor ${lastFloor}`, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return { built: vectorized ? chunks.length : 0, chunks };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
import MiniSearch from '../../../../libs/minisearch.mjs';
|
import MiniSearch from '../../../../libs/minisearch.mjs';
|
||||||
import { getContext } from '../../../../../../../extensions.js';
|
import { getContext } from '../../../../../../../extensions.js';
|
||||||
import { getSummaryStore } from '../../data/store.js';
|
import { getSummaryStore } from '../../data/store.js';
|
||||||
import { getStateAtoms } from '../storage/state-store.js';
|
|
||||||
import { getAllChunks } from '../storage/chunk-store.js';
|
import { getAllChunks } from '../storage/chunk-store.js';
|
||||||
import { xbLog } from '../../../../core/debug-core.js';
|
import { xbLog } from '../../../../core/debug-core.js';
|
||||||
import { tokenizeForIndex } from '../utils/tokenizer.js';
|
import { tokenizeForIndex } from '../utils/tokenizer.js';
|
||||||
@@ -39,6 +38,8 @@ let building = false;
|
|||||||
|
|
||||||
/** @type {Promise<MiniSearch|null>|null} 当前构建 Promise(防重入) */
|
/** @type {Promise<MiniSearch|null>|null} 当前构建 Promise(防重入) */
|
||||||
let buildPromise = null;
|
let buildPromise = null;
|
||||||
|
/** @type {Map<number, string[]>} floor → 该楼层的 doc IDs(仅 L1 chunks) */
|
||||||
|
let floorDocIds = new Map();
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
// 工具函数
|
// 工具函数
|
||||||
@@ -57,13 +58,12 @@ function cleanSummary(summary) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 计算缓存指纹
|
* 计算缓存指纹
|
||||||
* @param {number} atomCount
|
|
||||||
* @param {number} chunkCount
|
* @param {number} chunkCount
|
||||||
* @param {number} eventCount
|
* @param {number} eventCount
|
||||||
* @returns {string}
|
* @returns {string}
|
||||||
*/
|
*/
|
||||||
function computeFingerprint(atomCount, chunkCount, eventCount) {
|
function computeFingerprint(chunkCount, eventCount) {
|
||||||
return `${atomCount}:${chunkCount}:${eventCount}`;
|
return `${chunkCount}:${eventCount}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -81,34 +81,31 @@ function yieldToMain() {
|
|||||||
/**
|
/**
|
||||||
* 收集所有待索引文档
|
* 收集所有待索引文档
|
||||||
*
|
*
|
||||||
* @param {object[]} atoms - getStateAtoms() 返回值
|
|
||||||
* @param {object[]} chunks - getAllChunks(chatId) 返回值
|
* @param {object[]} chunks - getAllChunks(chatId) 返回值
|
||||||
* @param {object[]} events - store.json.events
|
* @param {object[]} events - store.json.events
|
||||||
* @returns {object[]} 文档数组
|
* @returns {object[]} 文档数组
|
||||||
*/
|
*/
|
||||||
function collectDocuments(atoms, chunks, events) {
|
function collectDocuments(chunks, events) {
|
||||||
const docs = [];
|
const docs = [];
|
||||||
|
|
||||||
// L0 atoms
|
// L1 chunks + 填充 floorDocIds
|
||||||
for (const atom of (atoms || [])) {
|
|
||||||
if (!atom?.atomId || !atom.semantic) continue;
|
|
||||||
docs.push({
|
|
||||||
id: atom.atomId,
|
|
||||||
type: 'atom',
|
|
||||||
floor: atom.floor ?? -1,
|
|
||||||
text: atom.semantic,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// L1 chunks
|
|
||||||
for (const chunk of (chunks || [])) {
|
for (const chunk of (chunks || [])) {
|
||||||
if (!chunk?.chunkId || !chunk.text) continue;
|
if (!chunk?.chunkId || !chunk.text) continue;
|
||||||
|
|
||||||
|
const floor = chunk.floor ?? -1;
|
||||||
docs.push({
|
docs.push({
|
||||||
id: chunk.chunkId,
|
id: chunk.chunkId,
|
||||||
type: 'chunk',
|
type: 'chunk',
|
||||||
floor: chunk.floor ?? -1,
|
floor,
|
||||||
text: chunk.text,
|
text: chunk.text,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (floor >= 0) {
|
||||||
|
if (!floorDocIds.has(floor)) {
|
||||||
|
floorDocIds.set(floor, []);
|
||||||
|
}
|
||||||
|
floorDocIds.get(floor).push(chunk.chunkId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// L2 events
|
// L2 events
|
||||||
@@ -244,7 +241,6 @@ export function searchLexicalIndex(index, terms) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 分类结果
|
// 分类结果
|
||||||
const atomIdSet = new Set();
|
|
||||||
const chunkIdSet = new Set();
|
const chunkIdSet = new Set();
|
||||||
const eventIdSet = new Set();
|
const eventIdSet = new Set();
|
||||||
|
|
||||||
@@ -254,16 +250,6 @@ export function searchLexicalIndex(index, terms) {
|
|||||||
const floor = hit.floor;
|
const floor = hit.floor;
|
||||||
|
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case 'atom':
|
|
||||||
if (!atomIdSet.has(id)) {
|
|
||||||
atomIdSet.add(id);
|
|
||||||
result.atomIds.push(id);
|
|
||||||
if (typeof floor === 'number' && floor >= 0) {
|
|
||||||
result.atomFloors.add(floor);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
|
|
||||||
case 'chunk':
|
case 'chunk':
|
||||||
if (!chunkIdSet.has(id)) {
|
if (!chunkIdSet.has(id)) {
|
||||||
chunkIdSet.add(id);
|
chunkIdSet.add(id);
|
||||||
@@ -304,8 +290,10 @@ export function searchLexicalIndex(index, terms) {
|
|||||||
* @returns {Promise<{index: MiniSearch, fingerprint: string}>}
|
* @returns {Promise<{index: MiniSearch, fingerprint: string}>}
|
||||||
*/
|
*/
|
||||||
async function collectAndBuild(chatId) {
|
async function collectAndBuild(chatId) {
|
||||||
// 收集数据
|
// 清空侧索引(全量重建)
|
||||||
const atoms = getStateAtoms() || [];
|
floorDocIds = new Map();
|
||||||
|
|
||||||
|
// 收集数据(不含 L0 atoms)
|
||||||
const store = getSummaryStore();
|
const store = getSummaryStore();
|
||||||
const events = store?.json?.events || [];
|
const events = store?.json?.events || [];
|
||||||
|
|
||||||
@@ -316,15 +304,15 @@ async function collectAndBuild(chatId) {
|
|||||||
xbLog.warn(MODULE_ID, '获取 chunks 失败', e);
|
xbLog.warn(MODULE_ID, '获取 chunks 失败', e);
|
||||||
}
|
}
|
||||||
|
|
||||||
const fp = computeFingerprint(atoms.length, chunks.length, events.length);
|
const fp = computeFingerprint(chunks.length, events.length);
|
||||||
|
|
||||||
// 检查是否在收集过程中缓存已被其他调用更新
|
// 检查是否在收集过程中缓存已被其他调用更新
|
||||||
if (cachedIndex && cachedChatId === chatId && cachedFingerprint === fp) {
|
if (cachedIndex && cachedChatId === chatId && cachedFingerprint === fp) {
|
||||||
return { index: cachedIndex, fingerprint: fp };
|
return { index: cachedIndex, fingerprint: fp };
|
||||||
}
|
}
|
||||||
|
|
||||||
// 收集文档
|
// 收集文档(同时填充 floorDocIds)
|
||||||
const docs = collectDocuments(atoms, chunks, events);
|
const docs = collectDocuments(chunks, events);
|
||||||
|
|
||||||
// 异步分片构建
|
// 异步分片构建
|
||||||
const index = await buildIndexAsync(docs);
|
const index = await buildIndexAsync(docs);
|
||||||
@@ -438,4 +426,116 @@ export function invalidateLexicalIndex() {
|
|||||||
cachedIndex = null;
|
cachedIndex = null;
|
||||||
cachedChatId = null;
|
cachedChatId = null;
|
||||||
cachedFingerprint = null;
|
cachedFingerprint = null;
|
||||||
|
floorDocIds = new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
|
// 增量更新接口
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 为指定楼层添加 L1 chunks 到索引
|
||||||
|
*
|
||||||
|
* 先移除该楼层旧文档,再添加新文档。
|
||||||
|
* 如果索引不存在(缓存失效),静默跳过(下次 getLexicalIndex 全量重建)。
|
||||||
|
*
|
||||||
|
* @param {number} floor - 楼层号
|
||||||
|
* @param {object[]} chunks - chunk 对象列表(需有 chunkId、text、floor)
|
||||||
|
*/
|
||||||
|
export function addDocumentsForFloor(floor, chunks) {
|
||||||
|
if (!cachedIndex || !chunks?.length) return;
|
||||||
|
|
||||||
|
// 先移除旧文档
|
||||||
|
removeDocumentsByFloor(floor);
|
||||||
|
|
||||||
|
const docs = [];
|
||||||
|
const docIds = [];
|
||||||
|
|
||||||
|
for (const chunk of chunks) {
|
||||||
|
if (!chunk?.chunkId || !chunk.text) continue;
|
||||||
|
docs.push({
|
||||||
|
id: chunk.chunkId,
|
||||||
|
type: 'chunk',
|
||||||
|
floor: chunk.floor ?? floor,
|
||||||
|
text: chunk.text,
|
||||||
|
});
|
||||||
|
docIds.push(chunk.chunkId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (docs.length > 0) {
|
||||||
|
cachedIndex.addAll(docs);
|
||||||
|
floorDocIds.set(floor, docIds);
|
||||||
|
xbLog.info(MODULE_ID, `增量添加: floor ${floor}, ${docs.length} 个 chunk`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从索引中移除指定楼层的所有 L1 chunk 文档
|
||||||
|
*
|
||||||
|
* 使用 MiniSearch discard()(软删除)。
|
||||||
|
* 如果索引不存在,静默跳过。
|
||||||
|
*
|
||||||
|
* @param {number} floor - 楼层号
|
||||||
|
*/
|
||||||
|
export function removeDocumentsByFloor(floor) {
|
||||||
|
if (!cachedIndex) return;
|
||||||
|
|
||||||
|
const docIds = floorDocIds.get(floor);
|
||||||
|
if (!docIds?.length) return;
|
||||||
|
|
||||||
|
for (const id of docIds) {
|
||||||
|
try {
|
||||||
|
cachedIndex.discard(id);
|
||||||
|
} catch {
|
||||||
|
// 文档可能不存在(已被全量重建替换)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
floorDocIds.delete(floor);
|
||||||
|
xbLog.info(MODULE_ID, `增量移除: floor ${floor}, ${docIds.length} 个文档`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将新 L2 事件添加到索引
|
||||||
|
*
|
||||||
|
* 如果事件 ID 已存在,先 discard 再 add(覆盖)。
|
||||||
|
* 如果索引不存在,静默跳过。
|
||||||
|
*
|
||||||
|
* @param {object[]} events - 事件对象列表(需有 id、title、summary 等)
|
||||||
|
*/
|
||||||
|
export function addEventDocuments(events) {
|
||||||
|
if (!cachedIndex || !events?.length) return;
|
||||||
|
|
||||||
|
const docs = [];
|
||||||
|
|
||||||
|
for (const ev of events) {
|
||||||
|
if (!ev?.id) continue;
|
||||||
|
|
||||||
|
const parts = [];
|
||||||
|
if (ev.title) parts.push(ev.title);
|
||||||
|
if (ev.participants?.length) parts.push(ev.participants.join(' '));
|
||||||
|
const summary = cleanSummary(ev.summary);
|
||||||
|
if (summary) parts.push(summary);
|
||||||
|
const text = parts.join(' ').trim();
|
||||||
|
if (!text) continue;
|
||||||
|
|
||||||
|
// 覆盖:先尝试移除旧的
|
||||||
|
try {
|
||||||
|
cachedIndex.discard(ev.id);
|
||||||
|
} catch {
|
||||||
|
// 不存在则忽略
|
||||||
|
}
|
||||||
|
|
||||||
|
docs.push({
|
||||||
|
id: ev.id,
|
||||||
|
type: 'event',
|
||||||
|
floor: null,
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (docs.length > 0) {
|
||||||
|
cachedIndex.addAll(docs);
|
||||||
|
xbLog.info(MODULE_ID, `增量添加: ${docs.length} 个事件`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
// 架构变更(v3 → v4):
|
// 架构变更(v3 → v4):
|
||||||
// - evidence 区块反映 L0-only 融合 + L1 按楼层拉取的两阶段架构
|
// - evidence 区块反映 L0-only 融合 + L1 按楼层拉取的两阶段架构
|
||||||
// - 删除 mergedByType / selectedByType(不再有混合池)
|
// - 删除 mergedByType / selectedByType(不再有混合池)
|
||||||
// - 新增 l0Candidates / l0Selected / l1Pulled / l1Attached / l1CosineTime
|
// - 新增 floorCandidates / floorsSelected / l0Collected / l1Pulled / l1Attached / l1CosineTime
|
||||||
// - fusion 区块明确标注 L0-only(删除 anchorCount)
|
// - fusion 区块明确标注 L0-only(删除 anchorCount)
|
||||||
// - quality.chunkRealRatio → quality.l1AttachRate
|
// - quality.chunkRealRatio → quality.l1AttachRate
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
@@ -48,10 +48,10 @@ export function createMetrics() {
|
|||||||
searchTime: 0,
|
searchTime: 0,
|
||||||
},
|
},
|
||||||
|
|
||||||
// Fusion (W-RRF, L0-only) - 多路融合
|
// Fusion (W-RRF, floor-level) - 多路融合
|
||||||
fusion: {
|
fusion: {
|
||||||
denseCount: 0,
|
denseFloors: 0,
|
||||||
lexCount: 0,
|
lexFloors: 0,
|
||||||
totalUnique: 0,
|
totalUnique: 0,
|
||||||
afterCap: 0,
|
afterCap: 0,
|
||||||
time: 0,
|
time: 0,
|
||||||
@@ -80,25 +80,27 @@ export function createMetrics() {
|
|||||||
entityNames: [],
|
entityNames: [],
|
||||||
},
|
},
|
||||||
|
|
||||||
// Evidence (Two-Stage: L0 rerank → L1 pull) - 原文证据
|
// Evidence (Two-Stage: Floor rerank → L1 pull) - 原文证据
|
||||||
evidence: {
|
evidence: {
|
||||||
// Stage 1: L0
|
// Stage 1: Floor
|
||||||
l0Candidates: 0, // W-RRF 融合后的 L0 候选数
|
floorCandidates: 0, // W-RRF 融合后的 floor 候选数
|
||||||
l0Selected: 0, // rerank 后选中的 L0 数
|
floorsSelected: 0, // rerank 后选中的 floor 数
|
||||||
|
l0Collected: 0, // 选中 floor 中收集的 L0 atom 总数
|
||||||
rerankApplied: false,
|
rerankApplied: false,
|
||||||
rerankFailed: false,
|
rerankFailed: false,
|
||||||
beforeRerank: 0,
|
beforeRerank: 0,
|
||||||
afterRerank: 0,
|
afterRerank: 0,
|
||||||
rerankTime: 0,
|
rerankTime: 0,
|
||||||
rerankScores: null,
|
rerankScores: null,
|
||||||
|
rerankDocAvgLength: 0, // rerank document 平均字符数
|
||||||
|
|
||||||
// Stage 2: L1
|
// Stage 2: L1
|
||||||
l1Pulled: 0, // 从 DB 拉取的 L1 chunk 总数
|
l1Pulled: 0, // 从 DB 拉取的 L1 chunk 总数
|
||||||
l1Attached: 0, // 实际挂载的 L1 数(top-1 × 楼层 × 2侧)
|
l1Attached: 0, // 实际挂载的 L1 数(top-1 × floor × 2侧)
|
||||||
l1CosineTime: 0, // L1 cosine 打分耗时
|
l1CosineTime: 0, // L1 cosine 打分耗时
|
||||||
|
|
||||||
// 装配
|
// 装配
|
||||||
contextPairsAdded: 0, // 保留兼容(= l1Attached 中 USER 侧数量)
|
contextPairsAdded: 0, // USER 侧挂载数量
|
||||||
tokens: 0,
|
tokens: 0,
|
||||||
assemblyTime: 0,
|
assemblyTime: 0,
|
||||||
},
|
},
|
||||||
@@ -149,7 +151,7 @@ export function createMetrics() {
|
|||||||
quality: {
|
quality: {
|
||||||
constraintCoverage: 100,
|
constraintCoverage: 100,
|
||||||
eventPrecisionProxy: 0,
|
eventPrecisionProxy: 0,
|
||||||
l1AttachRate: 0, // 有 L1 挂载的 L0 占比
|
l1AttachRate: 0, // 有 L1 挂载的 floor 占比
|
||||||
potentialIssues: [],
|
potentialIssues: [],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -223,10 +225,10 @@ export function formatMetricsLog(metrics) {
|
|||||||
lines.push(`└─ search_time: ${m.lexical.searchTime}ms`);
|
lines.push(`└─ search_time: ${m.lexical.searchTime}ms`);
|
||||||
lines.push('');
|
lines.push('');
|
||||||
|
|
||||||
// Fusion (W-RRF, L0-only)
|
// Fusion (W-RRF, floor-level)
|
||||||
lines.push('[Fusion] W-RRF (L0-only) - 多路融合');
|
lines.push('[Fusion] W-RRF (floor-level) - 多路融合');
|
||||||
lines.push(`├─ dense_count: ${m.fusion.denseCount}`);
|
lines.push(`├─ dense_floors: ${m.fusion.denseFloors}`);
|
||||||
lines.push(`├─ lex_count: ${m.fusion.lexCount}`);
|
lines.push(`├─ lex_floors: ${m.fusion.lexFloors}`);
|
||||||
lines.push(`├─ total_unique: ${m.fusion.totalUnique}`);
|
lines.push(`├─ total_unique: ${m.fusion.totalUnique}`);
|
||||||
lines.push(`├─ after_cap: ${m.fusion.afterCap}`);
|
lines.push(`├─ after_cap: ${m.fusion.afterCap}`);
|
||||||
lines.push(`└─ time: ${m.fusion.time}ms`);
|
lines.push(`└─ time: ${m.fusion.time}ms`);
|
||||||
@@ -277,28 +279,32 @@ export function formatMetricsLog(metrics) {
|
|||||||
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
|
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
|
||||||
lines.push('');
|
lines.push('');
|
||||||
|
|
||||||
// Evidence (Two-Stage)
|
// Evidence (Two-Stage: Floor Rerank → L1 Pull)
|
||||||
lines.push('[Evidence] Two-Stage: L0 Locate → L1 Pull');
|
lines.push('[Evidence] Two-Stage: Floor Rerank → L1 Pull');
|
||||||
lines.push(`├─ Stage 1 (L0):`);
|
lines.push(`├─ Stage 1 (Floor Rerank):`);
|
||||||
lines.push(`│ ├─ candidates (post-fusion): ${m.evidence.l0Candidates}`);
|
lines.push(`│ ├─ floor_candidates (post-fusion): ${m.evidence.floorCandidates}`);
|
||||||
|
|
||||||
if (m.evidence.rerankApplied) {
|
if (m.evidence.rerankApplied) {
|
||||||
lines.push(`│ ├─ rerank_applied: true`);
|
lines.push(`│ ├─ rerank_applied: true`);
|
||||||
if (m.evidence.rerankFailed) {
|
if (m.evidence.rerankFailed) {
|
||||||
lines.push(`│ ├─ rerank_failed: ⚠ YES (using fusion order)`);
|
lines.push(`│ │ ⚠ rerank_failed: using fusion order`);
|
||||||
}
|
}
|
||||||
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank}`);
|
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank} floors`);
|
||||||
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank}`);
|
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank} floors`);
|
||||||
lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`);
|
lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`);
|
||||||
if (m.evidence.rerankScores) {
|
if (m.evidence.rerankScores) {
|
||||||
const rs = m.evidence.rerankScores;
|
const rs = m.evidence.rerankScores;
|
||||||
lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
|
lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
|
||||||
}
|
}
|
||||||
|
if (m.evidence.rerankDocAvgLength > 0) {
|
||||||
|
lines.push(`│ ├─ rerank_doc_avg_length: ${m.evidence.rerankDocAvgLength} chars`);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
lines.push(`│ ├─ rerank_applied: false`);
|
lines.push(`│ ├─ rerank_applied: false`);
|
||||||
}
|
}
|
||||||
|
|
||||||
lines.push(`│ └─ selected: ${m.evidence.l0Selected}`);
|
lines.push(`│ ├─ floors_selected: ${m.evidence.floorsSelected}`);
|
||||||
|
lines.push(`│ └─ l0_atoms_collected: ${m.evidence.l0Collected}`);
|
||||||
lines.push(`├─ Stage 2 (L1):`);
|
lines.push(`├─ Stage 2 (L1):`);
|
||||||
lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`);
|
lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`);
|
||||||
lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`);
|
lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`);
|
||||||
@@ -345,9 +351,7 @@ export function formatMetricsLog(metrics) {
|
|||||||
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
|
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
|
||||||
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
|
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
|
||||||
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
|
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
|
||||||
if (m.timing.evidenceRerank > 0) {
|
lines.push(`├─ floor_rerank: ${m.timing.evidenceRerank || 0}ms`);
|
||||||
lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`);
|
|
||||||
}
|
|
||||||
lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`);
|
lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`);
|
||||||
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
|
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
|
||||||
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
|
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
|
||||||
@@ -406,20 +410,20 @@ export function detectIssues(metrics) {
|
|||||||
// 词法检索问题
|
// 词法检索问题
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
if ((m.lexical.terms || []).length > 0 && m.lexical.atomHits === 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
|
if ((m.lexical.terms || []).length > 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
|
||||||
issues.push('Lexical search returned zero hits - terms may not match any indexed content');
|
issues.push('Lexical search returned zero hits - terms may not match any indexed content');
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 融合问题(L0-only)
|
// 融合问题(floor-level)
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
if (m.fusion.lexCount === 0 && m.fusion.denseCount > 0) {
|
if (m.fusion.lexFloors === 0 && m.fusion.denseFloors > 0) {
|
||||||
issues.push('No lexical L0 candidates in fusion - hybrid retrieval not contributing');
|
issues.push('No lexical floors in fusion - hybrid retrieval not contributing');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m.fusion.afterCap === 0) {
|
if (m.fusion.afterCap === 0) {
|
||||||
issues.push('Fusion produced zero L0 candidates - all retrieval paths may have failed');
|
issues.push('Fusion produced zero floor candidates - all retrieval paths may have failed');
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
@@ -463,29 +467,30 @@ export function detectIssues(metrics) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// L0 Rerank 问题
|
// Floor Rerank 问题
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
if (m.evidence.rerankApplied) {
|
if (m.evidence.rerankFailed) {
|
||||||
if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) {
|
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
|
||||||
const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank);
|
}
|
||||||
if (filterRatio > 0.7) {
|
|
||||||
issues.push(`High L0 rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant L0 in fusion output`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
if (m.evidence.rerankApplied && !m.evidence.rerankFailed) {
|
||||||
if (m.evidence.rerankScores) {
|
if (m.evidence.rerankScores) {
|
||||||
const rs = m.evidence.rerankScores;
|
const rs = m.evidence.rerankScores;
|
||||||
if (rs.max < 0.5) {
|
if (rs.max < 0.3) {
|
||||||
issues.push(`Low L0 rerank scores (max=${rs.max}) - query may be poorly matched`);
|
issues.push(`Low floor rerank scores (max=${rs.max}) - query-document domain mismatch`);
|
||||||
}
|
}
|
||||||
if (rs.mean < 0.3) {
|
if (rs.mean < 0.2) {
|
||||||
issues.push(`Very low average L0 rerank score (mean=${rs.mean}) - context may be weak`);
|
issues.push(`Very low average floor rerank score (mean=${rs.mean}) - context may be weak`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m.evidence.rerankTime > 2000) {
|
if (m.evidence.rerankTime > 3000) {
|
||||||
issues.push(`Slow L0 rerank (${m.evidence.rerankTime}ms) - may affect response time`);
|
issues.push(`Slow floor rerank (${m.evidence.rerankTime}ms) - may affect response time`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m.evidence.rerankDocAvgLength > 3000) {
|
||||||
|
issues.push(`Large rerank documents (avg ${m.evidence.rerankDocAvgLength} chars) - may reduce rerank precision`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -493,21 +498,17 @@ export function detectIssues(metrics) {
|
|||||||
// L1 挂载问题
|
// L1 挂载问题
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
if (m.evidence.rerankFailed) {
|
if (m.evidence.floorsSelected > 0 && m.evidence.l1Pulled === 0) {
|
||||||
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (m.evidence.l0Selected > 0 && m.evidence.l1Pulled === 0) {
|
|
||||||
issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed');
|
issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m.evidence.l0Selected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
|
if (m.evidence.floorsSelected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
|
||||||
issues.push('L1 chunks pulled but none attached - cosine scores may be too low or floor mismatch');
|
issues.push('L1 chunks pulled but none attached - cosine scores may be too low');
|
||||||
}
|
}
|
||||||
|
|
||||||
const l1AttachRate = m.quality.l1AttachRate || 0;
|
const l1AttachRate = m.quality.l1AttachRate || 0;
|
||||||
if (m.evidence.l0Selected > 5 && l1AttachRate < 20) {
|
if (m.evidence.floorsSelected > 3 && l1AttachRate < 50) {
|
||||||
issues.push(`Low L1 attach rate (${l1AttachRate}%) - many L0 lack concrete dialogue evidence`);
|
issues.push(`Low L1 attach rate (${l1AttachRate}%) - selected floors lack L1 chunks`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -89,6 +89,46 @@ function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) {
|
|||||||
.map(([term]) => term);
|
.map(([term]) => term);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 构建 rerank 专用查询(纯自然语言,不带结构标签)
|
||||||
|
*
|
||||||
|
* reranker(bge-reranker-v2-m3)的 query 应为自然语言文本,
|
||||||
|
* 不含 [ENTITIES] [DIALOGUE] 等结构标签。
|
||||||
|
*
|
||||||
|
* @param {string[]} focusEntities - 焦点实体
|
||||||
|
* @param {object[]} lastMessages - 最近 K 条消息
|
||||||
|
* @param {string|null} pendingUserMessage - 待发送的用户消息
|
||||||
|
* @param {object} context - { name1, name2 }
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function buildRerankQuery(focusEntities, lastMessages, pendingUserMessage, context) {
|
||||||
|
const parts = [];
|
||||||
|
|
||||||
|
// 实体提示
|
||||||
|
if (focusEntities.length > 0) {
|
||||||
|
parts.push(`关于${focusEntities.join('、')}。`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 最近对话原文
|
||||||
|
for (const m of (lastMessages || [])) {
|
||||||
|
const speaker = m.is_user ? (context.name1 || '用户') : (m.name || context.name2 || '角色');
|
||||||
|
const clean = cleanMessageText(m.mes || '');
|
||||||
|
if (clean) {
|
||||||
|
parts.push(`${speaker}:${clean}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 待发送消息
|
||||||
|
if (pendingUserMessage) {
|
||||||
|
const clean = cleanMessageText(pendingUserMessage);
|
||||||
|
if (clean) {
|
||||||
|
parts.push(`${context.name1 || '用户'}:${clean}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
// QueryBundle 类型定义(JSDoc)
|
// QueryBundle 类型定义(JSDoc)
|
||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
@@ -176,9 +216,8 @@ export function buildQueryBundle(lastMessages, pendingUserMessage, store = null,
|
|||||||
|
|
||||||
const queryText_v0 = queryParts.join('\n\n');
|
const queryText_v0 = queryParts.join('\n\n');
|
||||||
|
|
||||||
// 6. rerankQuery 与 embedding query 同源(零暗箱)
|
// 6. rerankQuery 独立构建(纯自然语言,供 reranker 使用)
|
||||||
// 后续 refine 会把它升级为与 queryText_v1 同源。
|
const rerankQuery = buildRerankQuery(focusEntities, dialogueLines.length > 0 ? lastMessages : [], pendingUserMessage, context);
|
||||||
const rerankQuery = queryText_v0;
|
|
||||||
|
|
||||||
// 7. 构建 lexicalTerms
|
// 7. 构建 lexicalTerms
|
||||||
const entityTerms = focusEntities.map(e => e.toLowerCase());
|
const entityTerms = focusEntities.map(e => e.toLowerCase());
|
||||||
@@ -281,8 +320,8 @@ export function refineQueryBundle(bundle, anchorHits, eventHits) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5. rerankQuery 与最终 query 同源(零暗箱)
|
// 5. rerankQuery 保持独立(不随 refinement 变更)
|
||||||
bundle.rerankQuery = bundle.queryText_v1 || bundle.queryText_v0;
|
// reranker 需要纯自然语言 query,不受 memory hints 干扰
|
||||||
|
|
||||||
// 6. 增强 lexicalTerms
|
// 6. 增强 lexicalTerms
|
||||||
if (hints.length > 0) {
|
if (hints.length > 0) {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
// Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence)
|
// Story Summary - Recall Engine (v7 - Two-Stage: L0 Locate → L1 Evidence)
|
||||||
//
|
//
|
||||||
// 命名规范:
|
// 命名规范:
|
||||||
@@ -10,8 +10,8 @@
|
|||||||
// 阶段 2: Round 1 Dense Retrieval(L0 + L2)
|
// 阶段 2: Round 1 Dense Retrieval(L0 + L2)
|
||||||
// 阶段 3: Query Refinement(用已命中记忆增强)
|
// 阶段 3: Query Refinement(用已命中记忆增强)
|
||||||
// 阶段 4: Round 2 Dense Retrieval(L0 + L2)
|
// 阶段 4: Round 2 Dense Retrieval(L0 + L2)
|
||||||
// 阶段 5: Lexical Retrieval + L0 Merge
|
// 阶段 5: Lexical Retrieval
|
||||||
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine 预筛选
|
// 阶段 6: Floor W-RRF Fusion + Rerank + L1 配对
|
||||||
// 阶段 7: L1 配对组装(L0 → top-1 AI L1 + top-1 USER L1)
|
// 阶段 7: L1 配对组装(L0 → top-1 AI L1 + top-1 USER L1)
|
||||||
// 阶段 8: Causation Trace
|
// 阶段 8: Causation Trace
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
@@ -49,10 +49,10 @@ const CONFIG = {
|
|||||||
RRF_K: 60,
|
RRF_K: 60,
|
||||||
RRF_W_DENSE: 1.0,
|
RRF_W_DENSE: 1.0,
|
||||||
RRF_W_LEX: 0.9,
|
RRF_W_LEX: 0.9,
|
||||||
FUSION_CAP: 100,
|
FUSION_CAP: 60,
|
||||||
|
|
||||||
// Rerank(L0-only)
|
// Rerank(floor-level)
|
||||||
RERANK_TOP_N: 50,
|
RERANK_TOP_N: 20,
|
||||||
RERANK_MIN_SCORE: 0.15,
|
RERANK_MIN_SCORE: 0.15,
|
||||||
|
|
||||||
// 因果链
|
// 因果链
|
||||||
@@ -421,14 +421,14 @@ function traceCausation(eventHits, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MA
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* W-RRF 融合两路 L0 候选(dense + lexical)
|
* W-RRF 加权倒数排名融合(floor 粒度)
|
||||||
*
|
*
|
||||||
* @param {RankedItem[]} denseRank - Dense 路(cosine 降序)
|
* @param {{id: number, score: number}[]} denseRank - Dense 路(floor → max cosine,降序)
|
||||||
* @param {RankedItem[]} lexRank - Lexical 路(MiniSearch score 降序)
|
* @param {{id: number, score: number}[]} lexRank - Lexical 路(floor → max bm25,降序)
|
||||||
* @param {number} cap - 输出上限
|
* @param {number} cap - 输出上限
|
||||||
* @returns {{top: {id: string, fusionScore: number}[], totalUnique: number}}
|
* @returns {{top: {id: number, fusionScore: number}[], totalUnique: number}}
|
||||||
*/
|
*/
|
||||||
function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
|
function fuseByFloor(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
|
||||||
const k = CONFIG.RRF_K;
|
const k = CONFIG.RRF_K;
|
||||||
const wD = CONFIG.RRF_W_DENSE;
|
const wD = CONFIG.RRF_W_DENSE;
|
||||||
const wL = CONFIG.RRF_W_LEX;
|
const wL = CONFIG.RRF_W_LEX;
|
||||||
@@ -445,141 +445,109 @@ function fuseL0Candidates(denseRank, lexRank, cap = CONFIG.FUSION_CAP) {
|
|||||||
const denseMap = buildRankMap(denseRank || []);
|
const denseMap = buildRankMap(denseRank || []);
|
||||||
const lexMap = buildRankMap(lexRank || []);
|
const lexMap = buildRankMap(lexRank || []);
|
||||||
|
|
||||||
const allIds = new Set([
|
const allIds = new Set([...denseMap.keys(), ...lexMap.keys()]);
|
||||||
...denseMap.keys(),
|
|
||||||
...lexMap.keys(),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const totalUnique = allIds.size;
|
const totalUnique = allIds.size;
|
||||||
|
|
||||||
const scored = [];
|
const scored = [];
|
||||||
for (const id of allIds) {
|
for (const id of allIds) {
|
||||||
let score = 0;
|
let score = 0;
|
||||||
if (denseMap.has(id)) {
|
if (denseMap.has(id)) score += wD / (k + denseMap.get(id));
|
||||||
score += wD / (k + denseMap.get(id));
|
if (lexMap.has(id)) score += wL / (k + lexMap.get(id));
|
||||||
}
|
|
||||||
if (lexMap.has(id)) {
|
|
||||||
score += wL / (k + lexMap.get(id));
|
|
||||||
}
|
|
||||||
scored.push({ id, fusionScore: score });
|
scored.push({ id, fusionScore: score });
|
||||||
}
|
}
|
||||||
|
|
||||||
scored.sort((a, b) => b.fusionScore - a.fusionScore);
|
scored.sort((a, b) => b.fusionScore - a.fusionScore);
|
||||||
|
return { top: scored.slice(0, cap), totalUnique };
|
||||||
return {
|
|
||||||
top: scored.slice(0, cap),
|
|
||||||
totalUnique,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
// [Stage 6] L0-only 融合 + Rerank ‖ 并发 L1 Cosine 预筛选
|
// [Stage 6] Floor 融合 + Rerank + L1 配对
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* L0 融合 + rerank,并发拉取 L1 并 cosine 打分
|
* Floor 粒度融合 + Rerank + L1 配对
|
||||||
*
|
*
|
||||||
* @param {object[]} anchorHits - L0 dense 命中(Round 2)
|
* @param {object[]} anchorHits - L0 dense 命中(Round 2)
|
||||||
* @param {Set<number>} anchorFloors - L0 命中楼层(含 lexical 扩展)
|
|
||||||
* @param {number[]} queryVector - 查询向量(v1)
|
* @param {number[]} queryVector - 查询向量(v1)
|
||||||
* @param {string} rerankQuery - rerank 查询文本
|
* @param {string} rerankQuery - rerank 查询文本(纯自然语言)
|
||||||
* @param {object} lexicalResult - 词法检索结果
|
* @param {object} lexicalResult - 词法检索结果
|
||||||
* @param {object} metrics
|
* @param {object} metrics
|
||||||
* @returns {Promise<{l0Selected: object[], l1ByFloor: Map<number, {aiTop1: object|null, userTop1: object|null}>}>}
|
* @returns {Promise<{l0Selected: object[], l1ByFloor: Map<number, {aiTop1: object|null, userTop1: object|null}>}>}
|
||||||
*/
|
*/
|
||||||
async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rerankQuery, lexicalResult, metrics) {
|
async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexicalResult, metrics) {
|
||||||
const { chatId, chat } = getContext();
|
const { chatId, chat, name1, name2 } = getContext();
|
||||||
if (!chatId) return { l0Selected: [], l1ByFloor: new Map() };
|
if (!chatId) return { l0Selected: [], l1ByFloor: new Map() };
|
||||||
|
|
||||||
const T_Start = performance.now();
|
const T_Start = performance.now();
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 6a. 构建 L0 候选对象(用于 rerank)
|
// 6a. Dense floor rank(每个 floor 取 max cosine)
|
||||||
//
|
|
||||||
// 重要:支持 lexical-only 的 L0(atom)进入候选池。
|
|
||||||
// 否则 hybrid 会退化为 dense-only:lexical 命中的 atom 若未被 dense 命中会被直接丢弃。
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const l0ObjectMap = new Map();
|
const denseFloorMap = new Map();
|
||||||
for (const a of (anchorHits || [])) {
|
for (const a of (anchorHits || [])) {
|
||||||
const id = `anchor-${a.atomId}`;
|
const cur = denseFloorMap.get(a.floor) || 0;
|
||||||
l0ObjectMap.set(id, {
|
if (a.similarity > cur) denseFloorMap.set(a.floor, a.similarity);
|
||||||
id,
|
|
||||||
atomId: a.atomId,
|
|
||||||
floor: a.floor,
|
|
||||||
similarity: a.similarity,
|
|
||||||
atom: a.atom,
|
|
||||||
text: a.atom?.semantic || '',
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lexical-only atoms:从全量 StateAtoms 补齐(similarity 记为 0,靠 lex rank 贡献 W-RRF)
|
const denseFloorRank = [...denseFloorMap.entries()]
|
||||||
const lexAtomIds = lexicalResult?.atomIds || [];
|
.sort((a, b) => b[1] - a[1])
|
||||||
if (lexAtomIds.length > 0) {
|
.map(([floor, score]) => ({ id: floor, score }));
|
||||||
const atomsList = getStateAtoms();
|
|
||||||
const atomMap = new Map(atomsList.map(a => [a.atomId, a]));
|
|
||||||
|
|
||||||
for (const atomId of lexAtomIds) {
|
// ─────────────────────────────────────────────────────────────────
|
||||||
const id = `anchor-${atomId}`;
|
// 6b. Lexical floor rank(chunkScores → floor 聚合 + USER→AI 映射 + 预过滤)
|
||||||
if (l0ObjectMap.has(id)) continue;
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const atom = atomMap.get(atomId);
|
const atomFloorSet = new Set(getStateAtoms().map(a => a.floor));
|
||||||
if (!atom) continue;
|
|
||||||
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
|
|
||||||
|
|
||||||
l0ObjectMap.set(id, {
|
const lexFloorScores = new Map();
|
||||||
id,
|
for (const { chunkId, score } of (lexicalResult?.chunkScores || [])) {
|
||||||
atomId,
|
const match = chunkId?.match(/^c-(\d+)-/);
|
||||||
floor: atom.floor,
|
if (!match) continue;
|
||||||
similarity: 0,
|
let floor = parseInt(match[1], 10);
|
||||||
atom,
|
|
||||||
text: atom.semantic || '',
|
// USER floor → AI floor 映射
|
||||||
});
|
if (chat?.[floor]?.is_user) {
|
||||||
|
const aiFloor = floor + 1;
|
||||||
|
if (aiFloor < chat.length && !chat[aiFloor]?.is_user) {
|
||||||
|
floor = aiFloor;
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 预过滤:必须有 L0 atoms
|
||||||
|
if (!atomFloorSet.has(floor)) continue;
|
||||||
|
|
||||||
|
const cur = lexFloorScores.get(floor) || 0;
|
||||||
|
if (score > cur) lexFloorScores.set(floor, score);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
const lexFloorRank = [...lexFloorScores.entries()]
|
||||||
// 6b. 构建两路排名(L0-only)
|
.sort((a, b) => b[1] - a[1])
|
||||||
// ─────────────────────────────────────────────────────────────────
|
.map(([floor, score]) => ({ id: floor, score }));
|
||||||
|
|
||||||
// Dense 路:anchorHits 按 similarity 排序
|
|
||||||
const denseRank = (anchorHits || [])
|
|
||||||
.map(a => ({ id: `anchor-${a.atomId}`, score: a.similarity }))
|
|
||||||
.sort((a, b) => b.score - a.score);
|
|
||||||
|
|
||||||
// Lexical 路:从 lexicalResult.atomIds 构建排名(允许 lexical-only)
|
|
||||||
// atomIds 已按 MiniSearch score 排序(searchLexicalIndex 返回顺序);W-RRF 依赖 rank,score 为占位
|
|
||||||
const lexRank = (lexAtomIds || [])
|
|
||||||
.map(atomId => ({ id: `anchor-${atomId}`, score: 1 }))
|
|
||||||
.filter(item => l0ObjectMap.has(item.id));
|
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 6c. W-RRF 融合(L0-only)
|
// 6c. Floor W-RRF 融合
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const T_Fusion_Start = performance.now();
|
const T_Fusion_Start = performance.now();
|
||||||
|
const { top: fusedFloors, totalUnique } = fuseByFloor(denseFloorRank, lexFloorRank, CONFIG.FUSION_CAP);
|
||||||
const { top: fusionResult, totalUnique } = fuseL0Candidates(denseRank, lexRank, CONFIG.FUSION_CAP);
|
|
||||||
|
|
||||||
const fusionTime = Math.round(performance.now() - T_Fusion_Start);
|
const fusionTime = Math.round(performance.now() - T_Fusion_Start);
|
||||||
|
|
||||||
// 构建 rerank 候选列表
|
|
||||||
const rerankCandidates = fusionResult
|
|
||||||
.map(f => l0ObjectMap.get(f.id))
|
|
||||||
.filter(Boolean);
|
|
||||||
|
|
||||||
if (metrics) {
|
if (metrics) {
|
||||||
metrics.fusion.denseCount = denseRank.length;
|
metrics.fusion.denseFloors = denseFloorRank.length;
|
||||||
metrics.fusion.lexCount = lexRank.length;
|
metrics.fusion.lexFloors = lexFloorRank.length;
|
||||||
metrics.fusion.totalUnique = totalUnique;
|
metrics.fusion.totalUnique = totalUnique;
|
||||||
metrics.fusion.afterCap = rerankCandidates.length;
|
metrics.fusion.afterCap = fusedFloors.length;
|
||||||
metrics.fusion.time = fusionTime;
|
metrics.fusion.time = fusionTime;
|
||||||
metrics.evidence.l0Candidates = rerankCandidates.length;
|
metrics.evidence.floorCandidates = fusedFloors.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rerankCandidates.length === 0) {
|
if (fusedFloors.length === 0) {
|
||||||
if (metrics) {
|
if (metrics) {
|
||||||
metrics.evidence.l0Selected = 0;
|
metrics.evidence.floorsSelected = 0;
|
||||||
|
metrics.evidence.l0Collected = 0;
|
||||||
metrics.evidence.l1Pulled = 0;
|
metrics.evidence.l1Pulled = 0;
|
||||||
metrics.evidence.l1Attached = 0;
|
metrics.evidence.l1Attached = 0;
|
||||||
metrics.evidence.l1CosineTime = 0;
|
metrics.evidence.l1CosineTime = 0;
|
||||||
@@ -589,54 +557,87 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 6d. 收集所有候选 L0 的楼层(用于并发拉取 L1)
|
// 6d. 拉取 L1 chunks + cosine 打分
|
||||||
// 包含 AI 楼层本身 + 上方 USER 楼层
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const candidateFloors = new Set();
|
const floorsToFetch = new Set();
|
||||||
for (const c of rerankCandidates) {
|
for (const f of fusedFloors) {
|
||||||
candidateFloors.add(c.floor);
|
floorsToFetch.add(f.id);
|
||||||
// 上方 USER 楼层
|
const userFloor = f.id - 1;
|
||||||
const userFloor = c.floor - 1;
|
|
||||||
if (userFloor >= 0 && chat?.[userFloor]?.is_user) {
|
if (userFloor >= 0 && chat?.[userFloor]?.is_user) {
|
||||||
candidateFloors.add(userFloor);
|
floorsToFetch.add(userFloor);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const l1ScoredByFloor = await pullAndScoreL1(chatId, [...floorsToFetch], queryVector, chat);
|
||||||
|
|
||||||
|
if (metrics) {
|
||||||
|
let totalPulled = 0;
|
||||||
|
for (const [key, chunks] of l1ScoredByFloor) {
|
||||||
|
if (key === '_cosineTime') continue;
|
||||||
|
totalPulled += chunks.length;
|
||||||
|
}
|
||||||
|
metrics.evidence.l1Pulled = totalPulled;
|
||||||
|
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 6e. 并发:rerank L0 ‖ 拉取 L1 chunks + 向量 + cosine 打分
|
// 6e. 构建 rerank documents(每个 floor: USER chunks + AI chunks)
|
||||||
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const rerankCandidates = [];
|
||||||
|
for (const f of fusedFloors) {
|
||||||
|
const aiFloor = f.id;
|
||||||
|
const userFloor = aiFloor - 1;
|
||||||
|
|
||||||
|
const aiChunks = l1ScoredByFloor.get(aiFloor) || [];
|
||||||
|
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
|
||||||
|
? (l1ScoredByFloor.get(userFloor) || [])
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const parts = [];
|
||||||
|
const userName = chat?.[userFloor]?.name || name1 || '用户';
|
||||||
|
const aiName = chat?.[aiFloor]?.name || name2 || '角色';
|
||||||
|
|
||||||
|
if (userChunks.length > 0) {
|
||||||
|
parts.push(`${userName}:${userChunks.map(c => c.text).join(' ')}`);
|
||||||
|
}
|
||||||
|
if (aiChunks.length > 0) {
|
||||||
|
parts.push(`${aiName}:${aiChunks.map(c => c.text).join(' ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = parts.join('\n');
|
||||||
|
if (!text.trim()) continue;
|
||||||
|
|
||||||
|
rerankCandidates.push({
|
||||||
|
floor: aiFloor,
|
||||||
|
text,
|
||||||
|
fusionScore: f.fusionScore,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
// 6f. 并发 Rerank
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const T_Rerank_Start = performance.now();
|
const T_Rerank_Start = performance.now();
|
||||||
|
|
||||||
// 并发任务 1:rerank L0
|
const reranked = await rerankChunks(rerankQuery, rerankCandidates, {
|
||||||
const rerankPromise = rerankChunks(rerankQuery, rerankCandidates, {
|
|
||||||
topN: CONFIG.RERANK_TOP_N,
|
topN: CONFIG.RERANK_TOP_N,
|
||||||
minScore: CONFIG.RERANK_MIN_SCORE,
|
minScore: CONFIG.RERANK_MIN_SCORE,
|
||||||
});
|
});
|
||||||
|
|
||||||
// 并发任务 2:拉取 L1 chunks + 向量 → cosine 打分
|
|
||||||
const l1Promise = pullAndScoreL1(chatId, Array.from(candidateFloors), queryVector, chat);
|
|
||||||
|
|
||||||
// 等待两个任务完成
|
|
||||||
const [rerankedL0, l1ScoredByFloor] = await Promise.all([rerankPromise, l1Promise]);
|
|
||||||
|
|
||||||
const rerankTime = Math.round(performance.now() - T_Rerank_Start);
|
const rerankTime = Math.round(performance.now() - T_Rerank_Start);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
|
||||||
// 6f. 记录 rerank metrics
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
if (metrics) {
|
if (metrics) {
|
||||||
metrics.evidence.rerankApplied = true;
|
metrics.evidence.rerankApplied = true;
|
||||||
metrics.evidence.beforeRerank = rerankCandidates.length;
|
metrics.evidence.beforeRerank = rerankCandidates.length;
|
||||||
metrics.evidence.afterRerank = rerankedL0.length;
|
metrics.evidence.afterRerank = reranked.length;
|
||||||
metrics.evidence.rerankFailed = rerankedL0.some(c => c._rerankFailed);
|
metrics.evidence.rerankFailed = reranked.some(c => c._rerankFailed);
|
||||||
metrics.evidence.l0Selected = rerankedL0.length;
|
|
||||||
metrics.evidence.rerankTime = rerankTime;
|
metrics.evidence.rerankTime = rerankTime;
|
||||||
metrics.timing.evidenceRerank = rerankTime;
|
metrics.timing.evidenceRerank = rerankTime;
|
||||||
|
|
||||||
const scores = rerankedL0.map(c => c._rerankScore || 0).filter(s => s > 0);
|
const scores = reranked.map(c => c._rerankScore || 0).filter(s => s > 0);
|
||||||
if (scores.length > 0) {
|
if (scores.length > 0) {
|
||||||
scores.sort((a, b) => a - b);
|
scores.sort((a, b) => a - b);
|
||||||
metrics.evidence.rerankScores = {
|
metrics.evidence.rerankScores = {
|
||||||
@@ -645,74 +646,82 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
|
|||||||
mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)),
|
mean: Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// document 平均长度
|
||||||
|
if (rerankCandidates.length > 0) {
|
||||||
|
const totalLen = rerankCandidates.reduce((s, c) => s + (c.text?.length || 0), 0);
|
||||||
|
metrics.evidence.rerankDocAvgLength = Math.round(totalLen / rerankCandidates.length);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 6g. 构建最终 l0Selected + l1ByFloor
|
// 6g. 收集 L0 atoms + L1 top-1 配对
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const l0Selected = rerankedL0.map(item => ({
|
const atomsList = getStateAtoms();
|
||||||
id: item.id,
|
const atomsByFloor = new Map();
|
||||||
atomId: item.atomId,
|
for (const atom of atomsList) {
|
||||||
floor: item.floor,
|
if (typeof atom.floor !== 'number' || atom.floor < 0) continue;
|
||||||
similarity: item.similarity,
|
if (!atomsByFloor.has(atom.floor)) atomsByFloor.set(atom.floor, []);
|
||||||
rerankScore: item._rerankScore || 0,
|
atomsByFloor.get(atom.floor).push(atom);
|
||||||
atom: item.atom,
|
}
|
||||||
text: item.text,
|
|
||||||
}));
|
|
||||||
|
|
||||||
// 为每个选中的 L0 楼层组装 top-1 L1 配对
|
const l0Selected = [];
|
||||||
const selectedFloors = new Set(l0Selected.map(l => l.floor));
|
|
||||||
const l1ByFloor = new Map();
|
const l1ByFloor = new Map();
|
||||||
let contextPairsAdded = 0;
|
let contextPairsAdded = 0;
|
||||||
|
|
||||||
for (const floor of selectedFloors) {
|
for (const item of reranked) {
|
||||||
|
const floor = item.floor;
|
||||||
|
const rerankScore = item._rerankScore || 0;
|
||||||
|
const denseSim = denseFloorMap.get(floor) || 0;
|
||||||
|
|
||||||
|
// 收集该 floor 所有 L0 atoms,共享 floor 的 rerankScore
|
||||||
|
const floorAtoms = atomsByFloor.get(floor) || [];
|
||||||
|
for (const atom of floorAtoms) {
|
||||||
|
l0Selected.push({
|
||||||
|
id: `anchor-${atom.atomId}`,
|
||||||
|
atomId: atom.atomId,
|
||||||
|
floor: atom.floor,
|
||||||
|
similarity: denseSim,
|
||||||
|
rerankScore,
|
||||||
|
atom,
|
||||||
|
text: atom.semantic || '',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// L1 top-1 配对(cosine 最高)
|
||||||
const aiChunks = l1ScoredByFloor.get(floor) || [];
|
const aiChunks = l1ScoredByFloor.get(floor) || [];
|
||||||
xbLog.info(
|
|
||||||
MODULE_ID,
|
|
||||||
`L1 attach check: floor=${floor}, l1ScoredByFloor.has=${l1ScoredByFloor.has(floor)}, aiChunks=${aiChunks.length}, l1ScoredByFloor.keys=[${[...l1ScoredByFloor.keys()].slice(0, 10)}...]`
|
|
||||||
);
|
|
||||||
const userFloor = floor - 1;
|
const userFloor = floor - 1;
|
||||||
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
|
const userChunks = (userFloor >= 0 && chat?.[userFloor]?.is_user)
|
||||||
? (l1ScoredByFloor.get(userFloor) || [])
|
? (l1ScoredByFloor.get(userFloor) || [])
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
// top-1:取 cosine 最高的
|
|
||||||
const aiTop1 = aiChunks.length > 0
|
const aiTop1 = aiChunks.length > 0
|
||||||
? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
|
? aiChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
const userTop1 = userChunks.length > 0
|
const userTop1 = userChunks.length > 0
|
||||||
? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
|
? userChunks.reduce((best, c) => (c._cosineScore > best._cosineScore ? c : best))
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
// context pair = 上方 USER 楼层成功挂载(用于 metrics)
|
|
||||||
if (userTop1) contextPairsAdded++;
|
if (userTop1) contextPairsAdded++;
|
||||||
|
|
||||||
l1ByFloor.set(floor, { aiTop1, userTop1 });
|
l1ByFloor.set(floor, { aiTop1, userTop1 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
// 6h. L1 metrics
|
// 6h. Metrics
|
||||||
// ─────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
if (metrics) {
|
if (metrics) {
|
||||||
let totalPulled = 0;
|
metrics.evidence.floorsSelected = reranked.length;
|
||||||
|
metrics.evidence.l0Collected = l0Selected.length;
|
||||||
|
|
||||||
let totalAttached = 0;
|
let totalAttached = 0;
|
||||||
|
|
||||||
for (const [, scored] of l1ScoredByFloor) {
|
|
||||||
totalPulled += scored.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [, pair] of l1ByFloor) {
|
for (const [, pair] of l1ByFloor) {
|
||||||
if (pair.aiTop1) totalAttached++;
|
if (pair.aiTop1) totalAttached++;
|
||||||
if (pair.userTop1) totalAttached++;
|
if (pair.userTop1) totalAttached++;
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics.evidence.l1Pulled = totalPulled;
|
|
||||||
metrics.evidence.l1Attached = totalAttached;
|
metrics.evidence.l1Attached = totalAttached;
|
||||||
metrics.evidence.contextPairsAdded = contextPairsAdded;
|
metrics.evidence.contextPairsAdded = contextPairsAdded;
|
||||||
metrics.evidence.l1CosineTime = l1ScoredByFloor._cosineTime || 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const totalTime = Math.round(performance.now() - T_Start);
|
const totalTime = Math.round(performance.now() - T_Start);
|
||||||
@@ -721,13 +730,11 @@ async function locateAndPullEvidence(anchorHits, anchorFloors, queryVector, rera
|
|||||||
}
|
}
|
||||||
|
|
||||||
xbLog.info(MODULE_ID,
|
xbLog.info(MODULE_ID,
|
||||||
`Evidence: ${anchorHits?.length || 0} L0 dense → fusion=${rerankCandidates.length} → rerank=${rerankedL0.length} → L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
|
`Evidence: ${denseFloorRank.length} dense floors + ${lexFloorRank.length} lex floors → fusion=${fusedFloors.length} → rerank=${reranked.length} floors → L0=${l0Selected.length} L1 attached=${metrics?.evidence?.l1Attached || 0} (${totalTime}ms)`
|
||||||
);
|
);
|
||||||
|
|
||||||
return { l0Selected, l1ByFloor };
|
return { l0Selected, l1ByFloor };
|
||||||
}
|
}
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
|
||||||
// [L1] 拉取 + Cosine 打分(并发子任务)
|
// [L1] 拉取 + Cosine 打分(并发子任务)
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
@@ -973,7 +980,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
// 阶段 5: Lexical Retrieval + L0 Merge
|
// 阶段 5: Lexical Retrieval
|
||||||
// ═══════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
const T_Lex_Start = performance.now();
|
const T_Lex_Start = performance.now();
|
||||||
@@ -1003,12 +1010,6 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
|
|||||||
metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10);
|
metrics.lexical.terms = bundle.lexicalTerms.slice(0, 10);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 合并 L0 floors(dense + lexical)
|
|
||||||
const anchorFloors = new Set(anchorFloors_dense);
|
|
||||||
for (const f of lexicalResult.atomFloors) {
|
|
||||||
anchorFloors.add(f);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 合并 L2 events(lexical 命中但 dense 未命中的 events)
|
// 合并 L2 events(lexical 命中但 dense 未命中的 events)
|
||||||
const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean));
|
const existingEventIds = new Set(eventHits.map(e => e.event?.id).filter(Boolean));
|
||||||
const eventIndex = buildEventIndex(allEvents);
|
const eventIndex = buildEventIndex(allEvents);
|
||||||
@@ -1035,16 +1036,15 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
xbLog.info(MODULE_ID,
|
xbLog.info(MODULE_ID,
|
||||||
`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedFloors=${anchorFloors.size} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
|
`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} (${lexTime}ms)`
|
||||||
);
|
);
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
// 阶段 6: L0-only W-RRF Fusion + Rerank ‖ 并发 L1 Cosine
|
// 阶段 6: Floor 粒度融合 + Rerank + L1 配对
|
||||||
// ═══════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
const { l0Selected, l1ByFloor } = await locateAndPullEvidence(
|
const { l0Selected, l1ByFloor } = await locateAndPullEvidence(
|
||||||
anchorHits,
|
anchorHits,
|
||||||
anchorFloors,
|
|
||||||
queryVector_v1,
|
queryVector_v1,
|
||||||
bundle.rerankQuery,
|
bundle.rerankQuery,
|
||||||
lexicalResult,
|
lexicalResult,
|
||||||
@@ -1086,11 +1086,11 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
|
|||||||
console.log(`Total: ${metrics.timing.total}ms`);
|
console.log(`Total: ${metrics.timing.total}ms`);
|
||||||
console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`);
|
console.log(`Query Build: ${metrics.query.buildTime}ms | Refine: ${metrics.query.refineTime}ms`);
|
||||||
console.log(`Focus: [${bundle.focusEntities.join(', ')}]`);
|
console.log(`Focus: [${bundle.focusEntities.join(', ')}]`);
|
||||||
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors.size} floors`);
|
console.log(`Round 2 Anchors: ${anchorHits.length} hits → ${anchorFloors_dense.size} floors`);
|
||||||
console.log(`Lexical: atoms=${lexicalResult.atomIds.length} chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
|
console.log(`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length}`);
|
||||||
console.log(`Fusion (L0-only): dense=${metrics.fusion.denseCount} lex=${metrics.fusion.lexCount} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
|
console.log(`Fusion (floor): dense=${metrics.fusion.denseFloors} lex=${metrics.fusion.lexFloors} → cap=${metrics.fusion.afterCap} (${metrics.fusion.time}ms)`);
|
||||||
console.log(`L0 Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.l0Selected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
|
console.log(`Floor Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.floorsSelected || 0} floors → L0=${metrics.evidence.l0Collected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
|
||||||
console.log(`L1 Pull: ${metrics.evidence.l1Pulled || 0} chunks → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
|
console.log(`L1: ${metrics.evidence.l1Pulled || 0} pulled → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
|
||||||
console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`);
|
console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`);
|
||||||
console.groupEnd();
|
console.groupEnd();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user