X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update story summary recall and prompt injection

Browse Source
This commit is contained in:
bielie
2026-01-26 23:50:48 +08:00
parent 13bcf5a0bb
commit a010681ea6
5 changed files with 1139 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
modules/story-summary/vector/recall.js
View File

@@ -20,13 +20,18 @@ const CONFIG = {
QUERY_MAX_CHARS: 600,
QUERY_CONTEXT_CHARS: 240,
CANDIDATE_CHUNKS: 120,
CANDIDATE_EVENTS: 100,
// 因果链
CAUSAL_CHAIN_MAX_DEPTH: 10, // 放宽跳数，让图自然终止
CAUSAL_INJECT_MAX: 30, // 放宽上限，由 prompt token 预算最终控制
TOP_K_CHUNKS: 40,
TOP_K_EVENTS: 35,
CANDIDATE_CHUNKS: 200,
CANDIDATE_EVENTS: 150,
MIN_SIMILARITY: 0.35,
MAX_CHUNKS: 40,
MAX_EVENTS: 120,
MIN_SIMILARITY_CHUNK: 0.55,
MIN_SIMILARITY_EVENT: 0.6,
MMR_LAMBDA: 0.72,
BONUS_PARTICIPANT_HIT: 0.08,
@@ -58,6 +63,78 @@ function normalizeVec(v) {
return v.map(x => x / s);
}
// ═══════════════════════════════════════════════════════════════════════════
// 因果链追溯（Graph-augmented retrieval）
// - 从已召回事件出发，沿 causedBy 向上追溯祖先事件
// - 记录边：chainFrom = 哪个召回事件需要它
// - 不在这里决定“是否额外注入”，只负责遍历与结构化结果
// ═══════════════════════════════════════════════════════════════════════════
function buildEventIndex(allEvents) {
const map = new Map();
for (const e of allEvents || []) {
if (e?.id) map.set(e.id, e);
}
return map;
}
/**
* @returns {Map<string, {event, depth, chainFrom}>}
*/
function traceCausalAncestors(recalledEvents, eventIndex, maxDepth = CONFIG.CAUSAL_CHAIN_MAX_DEPTH) {
const out = new Map();
const idRe = /^evt-\d+$/;
function visit(parentId, depth, chainFrom) {
if (depth > maxDepth) return;
if (!idRe.test(parentId)) return;
const ev = eventIndex.get(parentId);
if (!ev) return;
// 如果同一个祖先被多个召回事件引用：保留更“近”的深度或追加来源
const existed = out.get(parentId);
if (!existed) {
out.set(parentId, { event: ev, depth, chainFrom: [chainFrom] });
} else {
if (depth < existed.depth) existed.depth = depth;
if (!existed.chainFrom.includes(chainFrom)) existed.chainFrom.push(chainFrom);
}
for (const next of (ev.causedBy || [])) {
visit(String(next || '').trim(), depth + 1, chainFrom);
}
}
for (const r of recalledEvents || []) {
const rid = r?.event?.id;
if (!rid) continue;
for (const cid of (r.event?.causedBy || [])) {
visit(String(cid || '').trim(), 1, rid);
}
}
return out;
}
/**
* 因果事件排序：引用数 > 深度 > 编号
*/
function sortCausalEvents(causalArray) {
return causalArray.sort((a, b) => {
// 1. 被多条召回链引用的优先
const refDiff = b.chainFrom.length - a.chainFrom.length;
if (refDiff !== 0) return refDiff;
// 2. 深度浅的优先
const depthDiff = a.depth - b.depth;
if (depthDiff !== 0) return depthDiff;
// 3. 事件编号排序
return String(a.event.id).localeCompare(String(b.event.id));
});
}
function normalize(s) {
return String(s || '').normalize('NFKC').replace(/[\u200B-\u200D\uFEFF]/g, '').trim();
}
@@ -243,14 +320,31 @@ async function searchChunks(queryVector, vectorConfig) {
};
});
// Pre-filter stats for logging
const preFilterStats = {
total: scored.length,
passThreshold: scored.filter(s => s.similarity >= CONFIG.MIN_SIMILARITY_CHUNK).length,
threshold: CONFIG.MIN_SIMILARITY_CHUNK,
distribution: {
'0.8+': scored.filter(s => s.similarity >= 0.8).length,
'0.7-0.8': scored.filter(s => s.similarity >= 0.7 && s.similarity < 0.8).length,
'0.6-0.7': scored.filter(s => s.similarity >= 0.6 && s.similarity < 0.7).length,
'0.55-0.6': scored.filter(s => s.similarity >= 0.55 && s.similarity < 0.6).length,
'<0.55': scored.filter(s => s.similarity < 0.55).length,
},
};
const candidates = scored
.filter(s => s.similarity >= CONFIG.MIN_SIMILARITY)
.filter(s => s.similarity >= CONFIG.MIN_SIMILARITY_CHUNK)
.sort((a, b) => b.similarity - a.similarity)
.slice(0, CONFIG.CANDIDATE_CHUNKS);
// 动态 K：质量不够就少拿
const dynamicK = Math.min(CONFIG.MAX_CHUNKS, candidates.length);
const selected = mmrSelect(
candidates,
CONFIG.TOP_K_CHUNKS,
dynamicK,
CONFIG.MMR_LAMBDA,
c => c.vector,
c => c.similarity
@@ -270,7 +364,7 @@ async function searchChunks(queryVector, vectorConfig) {
const chunks = await getChunksByFloors(chatId, floors);
const chunkMap = new Map(chunks.map(c => [c.chunkId, c]));
return sparse.map(item => {
const results = sparse.map(item => {
const chunk = chunkMap.get(item.chunkId);
if (!chunk) return null;
return {
@@ -283,6 +377,13 @@ async function searchChunks(queryVector, vectorConfig) {
similarity: item.similarity,
};
}).filter(Boolean);
// Attach stats for logging
if (results.length > 0) {
results._preFilterStats = preFilterStats;
}
return results;
}
// ═══════════════════════════════════════════════════════════════════════════
@@ -291,14 +392,27 @@ async function searchChunks(queryVector, vectorConfig) {
async function searchEvents(queryVector, allEvents, vectorConfig, store, queryEntities) {
const { chatId, name1 } = getContext();
if (!chatId || !queryVector?.length) return [];
if (!chatId || !queryVector?.length) {
console.warn('[searchEvents] 早期返回: chatId或queryVector为空');
return [];
}
const meta = await getMeta(chatId);
const fp = getEngineFingerprint(vectorConfig);
console.log('[searchEvents] fingerprint检查:', {
metaFp: meta.fingerprint,
currentFp: fp,
match: meta.fingerprint === fp || !meta.fingerprint,
});
if (meta.fingerprint && meta.fingerprint !== fp) return [];
const eventVectors = await getAllEventVectors(chatId);
const vectorMap = new Map(eventVectors.map(v => [v.eventId, v.vector]));
console.log('[searchEvents] 向量数据:', {
eventVectorsCount: eventVectors.length,
vectorMapSize: vectorMap.size,
allEventsCount: allEvents?.length,
});
if (!vectorMap.size) return [];
const userName = normalize(name1);
@@ -350,14 +464,40 @@ async function searchEvents(queryVector, allEvents, vectorConfig, store, queryEn
};
});
// 相似度分布日志
const simValues = scored.map(s => s.similarity).sort((a, b) => b - a);
console.log('[searchEvents] 相似度分布（前20）:', simValues.slice(0, 20));
console.log('[searchEvents] 相似度分布（后20）:', simValues.slice(-20));
console.log('[searchEvents] 有向量的事件数:', scored.filter(s => s.similarity > 0).length);
console.log('[searchEvents] sim >= 0.6:', scored.filter(s => s.similarity >= 0.6).length);
console.log('[searchEvents] sim >= 0.5:', scored.filter(s => s.similarity >= 0.5).length);
console.log('[searchEvents] sim >= 0.3:', scored.filter(s => s.similarity >= 0.3).length);
// ★ 记录过滤前的分布（用 finalScore，与显示一致）
const preFilterDistribution = {
total: scored.length,
'0.85+': scored.filter(s => s.finalScore >= 0.85).length,
'0.7-0.85': scored.filter(s => s.finalScore >= 0.7 && s.finalScore < 0.85).length,
'0.6-0.7': scored.filter(s => s.finalScore >= 0.6 && s.finalScore < 0.7).length,
'0.5-0.6': scored.filter(s => s.finalScore >= 0.5 && s.finalScore < 0.6).length,
'<0.5': scored.filter(s => s.finalScore < 0.5).length,
passThreshold: scored.filter(s => s.finalScore >= CONFIG.MIN_SIMILARITY_EVENT).length,
threshold: CONFIG.MIN_SIMILARITY_EVENT,
};
// ★ 过滤改成用 finalScore（包含 bonus）
const candidates = scored
.filter(s => s.similarity >= CONFIG.MIN_SIMILARITY)
.filter(s => s.finalScore >= CONFIG.MIN_SIMILARITY_EVENT)
.sort((a, b) => b.finalScore - a.finalScore)
.slice(0, CONFIG.CANDIDATE_EVENTS);
console.log('[searchEvents] 过滤后candidates:', candidates.length);
// 动态 K：质量不够就少拿
const dynamicK = Math.min(CONFIG.MAX_EVENTS, candidates.length);
const selected = mmrSelect(
candidates,
CONFIG.TOP_K_EVENTS,
dynamicK,
CONFIG.MMR_LAMBDA,
c => c.vector,
c => c.finalScore
@@ -370,14 +510,59 @@ async function searchEvents(queryVector, allEvents, vectorConfig, store, queryEn
similarity: s.finalScore,
_recallType: s.isDirect ? 'DIRECT' : 'SIMILAR',
_recallReason: s.reasons.length ? s.reasons.join('+') : '相似',
_preFilterDistribution: preFilterDistribution,
}));
}
// ═══════════════════════════════════════════════════════════════════════════
// 日志
// 日志：因果树格式化
// ═══════════════════════════════════════════════════════════════════════════
function formatRecallLog({ elapsed, segments, weights, chunkResults, eventResults, allEvents, queryEntities }) {
function formatCausalTree(causalEvents, recalledEvents) {
if (!causalEvents?.length) return '';
const lines = [
'',
'┌─────────────────────────────────────────────────────────────┐',
'│ 【因果链追溯】 │',
'└─────────────────────────────────────────────────────────────┘',
];
// 按 chainFrom 分组展示
const bySource = new Map();
for (const c of causalEvents) {
for (const src of c.chainFrom || []) {
if (!bySource.has(src)) bySource.set(src, []);
bySource.get(src).push(c);
}
}
for (const [sourceId, ancestors] of bySource) {
const sourceEvent = recalledEvents.find(e => e.event?.id === sourceId);
const sourceTitle = sourceEvent?.event?.title || sourceId;
lines.push(` ${sourceId} "${sourceTitle}" 的前因链：`);
// 按深度排序
ancestors.sort((a, b) => a.depth - b.depth);
for (const c of ancestors) {
const indent = ' ' + ' '.repeat(c.depth - 1);
const ev = c.event;
const title = ev.title || '(无标题)';
const refs = c.chainFrom.length > 1 ? ` [被${c.chainFrom.length}条链引用]` : '';
lines.push(`${indent}└─ [depth=${c.depth}] ${ev.id} "${title}"${refs}`);
}
}
lines.push('');
return lines.join('\n');
}
// ═══════════════════════════════════════════════════════════════════════════
// 日志：主报告
// ═══════════════════════════════════════════════════════════════════════════
function formatRecallLog({ elapsed, segments, weights, chunkResults, eventResults, allEvents, queryEntities, causalEvents = [], chunkPreFilterStats = null }) {
const lines = [
'╔══════════════════════════════════════════════════════════════╗',
'║ 记忆召回报告 ║',
@@ -413,9 +598,21 @@ function formatRecallLog({ elapsed, segments, weights, chunkResults, eventResult
lines.push('');
lines.push('┌─────────────────────────────────────────────────────────────┐');
lines.push(`│ 【L1 原文片段】召回 ${chunkResults.length} 条`);
lines.push('│ 【L1 原文片段】 │');
lines.push('└─────────────────────────────────────────────────────────────┘');
if (chunkPreFilterStats) {
const dist = chunkPreFilterStats.distribution || {};
lines.push(` 过滤前: ${chunkPreFilterStats.total} 条`);
lines.push(' 相似度分布:');
lines.push(` 0.8+: ${dist['0.8+'] || 0} | 0.7-0.8: ${dist['0.7-0.8'] || 0} | 0.6-0.7: ${dist['0.6-0.7'] || 0}`);
lines.push(` 0.55-0.6: ${dist['0.55-0.6'] || 0} | <0.55: ${dist['<0.55'] || 0}`);
lines.push(` 通过阈值(>=${chunkPreFilterStats.threshold}): ${chunkPreFilterStats.passThreshold} 条`);
lines.push(` MMR+Floor去重后: ${chunkResults.length} 条`);
} else {
lines.push(` 召回: ${chunkResults.length} 条`);
}
chunkResults.slice(0, 15).forEach((c, i) => {
const preview = c.text.length > 50 ? c.text.slice(0, 50) + '...' : c.text;
lines.push(` ${String(i + 1).padStart(2)}. #${String(c.floor).padStart(3)} [${c.speaker}] ${preview}`);
@@ -428,7 +625,7 @@ function formatRecallLog({ elapsed, segments, weights, chunkResults, eventResult
lines.push('');
lines.push('┌─────────────────────────────────────────────────────────────┐');
lines.push(`│ 【L2 事件记忆】召回 ${eventResults.length} / ${allEvents.length} 条`);
lines.push('│ 【L2 事件记忆】 │');
lines.push('│ DIRECT=亲身经历 SIMILAR=相关背景 │');
lines.push('└─────────────────────────────────────────────────────────────┘');
@@ -442,16 +639,27 @@ function formatRecallLog({ elapsed, segments, weights, chunkResults, eventResult
// 统计
const directCount = eventResults.filter(e => e._recallType === 'DIRECT').length;
const similarCount = eventResults.filter(e => e._recallType === 'SIMILAR').length;
const preFilterDist = eventResults[0]?._preFilterDistribution || {};
lines.push('');
lines.push('┌─────────────────────────────────────────────────────────────┐');
lines.push('│ 【统计】 │');
lines.push('└─────────────────────────────────────────────────────────────┘');
lines.push(` L1 片段: ${chunkResults.length} 条`);
lines.push(` L2 事件: ${eventResults.length} 条 (DIRECT: ${directCount}, SIMILAR: ${similarCount})`);
lines.push(` L2 事件: ${eventResults.length} / ${allEvents.length} 条 (DIRECT: ${directCount}, SIMILAR: ${similarCount})`);
if (preFilterDist.total) {
lines.push(` L2 过滤前分布（${preFilterDist.total} 条，含bonus）:`);
lines.push(` 0.85+: ${preFilterDist['0.85+'] || 0} | 0.7-0.85: ${preFilterDist['0.7-0.85'] || 0} | 0.6-0.7: ${preFilterDist['0.6-0.7'] || 0}`);
lines.push(` 0.5-0.6: ${preFilterDist['0.5-0.6'] || 0} | <0.5: ${preFilterDist['<0.5'] || 0}`);
lines.push(` 通过阈值(>=${preFilterDist.threshold || 0.6}): ${preFilterDist.passThreshold || 0} 条`);
}
lines.push(` 实体命中: ${queryEntities?.length || 0} 个`);
if (causalEvents.length) lines.push(` 因果链追溯: ${causalEvents.length} 条`);
lines.push('');
// 追加因果树详情
lines.push(formatCausalTree(causalEvents, eventResults));
return lines.join('\n');
}
@@ -492,15 +700,53 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
searchEvents(queryVector, allEvents, vectorConfig, store, queryEntities),
]);
const chunkPreFilterStats = chunkResults._preFilterStats || null;
// ─────────────────────────────────────────────────────────────────────
// 因果链追溯：从 eventResults 出发找祖先事件
// 注意：是否“额外注入”要去重（如果祖先事件本来已召回，就不额外注入）
// ─────────────────────────────────────────────────────────────────────
const eventIndex = buildEventIndex(allEvents);
const causalMap = traceCausalAncestors(eventResults, eventIndex);
const recalledIdSet = new Set(eventResults.map(x => x?.event?.id).filter(Boolean));
const causalEvents = Array.from(causalMap.values())
.filter(x => x?.event?.id && !recalledIdSet.has(x.event.id))
.map(x => ({
event: x.event,
similarity: 0,
_recallType: 'CAUSAL',
_recallReason: `因果链(${x.chainFrom.join(',')})`,
_causalDepth: x.depth,
_chainFrom: x.chainFrom,
chainFrom: x.chainFrom,
depth: x.depth,
}));
// 排序：引用数 > 深度 > 编号，然后截断
sortCausalEvents(causalEvents);
const causalEventsTruncated = causalEvents.slice(0, CONFIG.CAUSAL_INJECT_MAX);
const elapsed = Math.round(performance.now() - T0);
const logText = formatRecallLog({ elapsed, queryText, segments, weights, chunkResults, eventResults, allEvents, queryEntities });
const logText = formatRecallLog({
elapsed,
queryText,
segments,
weights,
chunkResults,
eventResults,
allEvents,
queryEntities,
causalEvents: causalEventsTruncated,
chunkPreFilterStats,
});
console.group('%c[Recall]', 'color: #7c3aed; font-weight: bold');
console.log(`Elapsed: ${elapsed}ms | Entities: ${queryEntities.join(', ') || '(none)'}`);
console.log(`L1: ${chunkResults.length} | L2: ${eventResults.length}/${allEvents.length}`);
console.log(`L1: ${chunkResults.length} | L2: ${eventResults.length}/${allEvents.length} | Causal: ${causalEventsTruncated.length}`);
console.groupEnd();
return { events: eventResults, chunks: chunkResults, elapsed, logText };
return { events: eventResults, causalEvents: causalEventsTruncated, chunks: chunkResults, elapsed, logText, queryEntities };
}
export function buildQueryText(chat, count = 2, excludeLastAi = false) {