X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(recall): add diffusion stage and improve retrieval metrics

Browse Source
This commit is contained in:
bielie
2026-02-12 15:36:07 +08:00
parent 111cd081f6
commit a646a70224
6 changed files with 1084 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
modules/story-summary/vector/retrieval/recall.js
View File

@@ -39,6 +39,7 @@ import {
import { getLexicalIndex, searchLexicalIndex } from './lexical-index.js';
import { rerankChunks } from '../llm/reranker.js';
import { createMetrics, calcSimilarityStats } from './metrics.js';
import { diffuseFromSeeds } from './diffusion.js';
const MODULE_ID = 'recall';
@@ -59,10 +60,10 @@ const CONFIG = {
EVENT_SELECT_MAX: 50,
EVENT_MIN_SIMILARITY: 0.55,
EVENT_MMR_LAMBDA: 0.72,
EVENT_ENTITY_BYPASS_SIM: 0.80,
EVENT_ENTITY_BYPASS_SIM: 0.70,
// Lexical Dense 门槛
LEXICAL_EVENT_DENSE_MIN: 0.50,
LEXICAL_EVENT_DENSE_MIN: 0.60,
LEXICAL_FLOOR_DENSE_MIN: 0.50,
// W-RRF 融合（L0-only）
@@ -71,10 +72,6 @@ const CONFIG = {
RRF_W_LEX: 0.9,
FUSION_CAP: 60,
// Dense floor 聚合权重
DENSE_AGG_W_MAX: 0.6,
DENSE_AGG_W_MEAN: 0.4,
// Lexical floor 聚合密度加成
LEX_DENSITY_BONUS: 0.3,
@@ -102,6 +99,20 @@ function cosineSimilarity(a, b) {
return nA && nB ? dot / (Math.sqrt(nA) * Math.sqrt(nB)) : 0;
}
/**
* 从事件 summary 末尾解析楼层范围 (#X) 或 (#X-Y)
* @param {string} summary
* @returns {{start: number, end: number}|null}
*/
function parseFloorRange(summary) {
if (!summary) return null;
const match = String(summary).match(/\(#(\d+)(?:-(\d+))?\)/);
if (!match) return null;
const start = Math.max(0, parseInt(match[1], 10) - 1);
const end = Math.max(0, (match[2] ? parseInt(match[2], 10) : parseInt(match[1], 10)) - 1);
return { start, end };
}
function normalize(s) {
return String(s || '')
.normalize('NFKC')
@@ -253,19 +264,19 @@ function mmrSelect(candidates, k, lambda, getVector, getScore) {
async function recallAnchors(queryVector, vectorConfig, metrics) {
const { chatId } = getContext();
if (!chatId || !queryVector?.length) {
return { hits: [], floors: new Set() };
return { hits: [], floors: new Set(), stateVectors: [] };
}
const meta = await getMeta(chatId);
const fp = getEngineFingerprint(vectorConfig);
if (meta.fingerprint && meta.fingerprint !== fp) {
xbLog.warn(MODULE_ID, 'Anchor fingerprint 不匹配');
return { hits: [], floors: new Set() };
return { hits: [], floors: new Set(), stateVectors: [] };
}
const stateVectors = await getAllStateVectors(chatId);
if (!stateVectors.length) {
return { hits: [], floors: new Set() };
return { hits: [], floors: new Set(), stateVectors: [] };
}
const atomsList = getStateAtoms();
@@ -298,7 +309,7 @@ async function recallAnchors(queryVector, vectorConfig, metrics) {
}));
}
return { hits: scored, floors };
return { hits: scored, floors, stateVectors };
}
// ═══════════════════════════════════════════════════════════════════════════
@@ -402,7 +413,7 @@ async function recallEvents(queryVector, allEvents, vectorConfig, focusEntities,
if (metrics) {
metrics.event.selected = results.length;
metrics.event.byRecallType = { direct: directCount, related: relatedCount, causal: 0, lexical: 0 };
metrics.event.byRecallType = { direct: directCount, related: relatedCount, causal: 0, lexical: 0, l0Linked: 0 };
metrics.event.similarityDistribution = calcSimilarityStats(results.map(r => r.similarity));
}
@@ -517,23 +528,18 @@ async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexic
// 6a. Dense floor rank（加权聚合：maxSim×0.6 + meanSim×0.4）
// ─────────────────────────────────────────────────────────────────
const denseFloorAgg = new Map();
const denseFloorMax = new Map();
for (const a of (anchorHits || [])) {
const cur = denseFloorAgg.get(a.floor);
if (!cur) {
denseFloorAgg.set(a.floor, { maxSim: a.similarity, hitCount: 1, sumSim: a.similarity });
} else {
cur.maxSim = Math.max(cur.maxSim, a.similarity);
cur.hitCount++;
cur.sumSim += a.similarity;
const cur = denseFloorMax.get(a.floor);
if (!cur || a.similarity > cur) {
denseFloorMax.set(a.floor, a.similarity);
}
}
const denseFloorRank = [...denseFloorAgg.entries()]
.map(([floor, info]) => ({
const denseFloorRank = [...denseFloorMax.entries()]
.map(([floor, maxSim]) => ({
id: floor,
score: info.maxSim * CONFIG.DENSE_AGG_W_MAX
+ (info.sumSim / info.hitCount) * CONFIG.DENSE_AGG_W_MEAN,
score: maxSim,
}))
.sort((a, b) => b.score - a.score);
@@ -565,8 +571,8 @@ async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexic
if (!atomFloorSet.has(floor)) continue;
// Dense 门槛：lexical floor 必须有最低 dense 相关性
const denseInfo = denseFloorAgg.get(floor);
if (!denseInfo || denseInfo.maxSim < CONFIG.LEXICAL_FLOOR_DENSE_MIN) {
const denseMax = denseFloorMax.get(floor);
if (!denseMax || denseMax < CONFIG.LEXICAL_FLOOR_DENSE_MIN) {
lexFloorFilteredByDense++;
continue;
}
@@ -605,7 +611,7 @@ async function locateAndPullEvidence(anchorHits, queryVector, rerankQuery, lexic
metrics.fusion.totalUnique = totalUnique;
metrics.fusion.afterCap = fusedFloors.length;
metrics.fusion.time = fusionTime;
metrics.fusion.denseAggMethod = `max×${CONFIG.DENSE_AGG_W_MAX}+mean×${CONFIG.DENSE_AGG_W_MEAN}`;
metrics.fusion.denseAggMethod = 'maxSim';
metrics.fusion.lexDensityBonus = CONFIG.LEX_DENSITY_BONUS;
metrics.evidence.floorCandidates = fusedFloors.length;
}
@@ -1060,7 +1066,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
}
const T_R2_Anchor_Start = performance.now();
const { hits: anchorHits, floors: anchorFloors_dense } = await recallAnchors(queryVector_v1, vectorConfig, metrics);
const { hits: anchorHits, floors: anchorFloors_dense, stateVectors: allStateVectors } = await recallAnchors(queryVector_v1, vectorConfig, metrics);
metrics.timing.anchorSearch = Math.round(performance.now() - T_R2_Anchor_Start);
const T_R2_Event_Start = performance.now();
@@ -1108,6 +1114,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
const eventIndex = buildEventIndex(allEvents);
let lexicalEventCount = 0;
let lexicalEventFilteredByDense = 0;
const focusSetForLexical = new Set((bundle.focusEntities || []).map(normalize));
for (const eid of lexicalResult.eventIds) {
if (existingEventIds.has(eid)) continue;
@@ -1129,16 +1136,59 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
continue;
}
// 通过门槛，使用实际 dense similarity（而非硬编码 0）
// 实体分类：与 Dense 路径统一标准
const participants = (ev.participants || []).map(p => normalize(p));
const hasEntityMatch = focusSetForLexical.size > 0 && participants.some(p => focusSetForLexical.has(p));
eventHits.push({
event: ev,
similarity: sim,
_recallType: 'LEXICAL',
_recallType: hasEntityMatch ? 'DIRECT' : 'RELATED',
});
existingEventIds.add(eid);
lexicalEventCount++;
}
// ═══════════════════════════════════════════════════════════════════
// 阶段 5.5: L0 → L2 反向查找
// 已召回的 L0 楼层落在某 L2 事件范围内，但该 L2 自身未被召回
// ═══════════════════════════════════════════════════════════════════
const recalledL0Floors = new Set(anchorHits.map(h => h.floor));
let l0LinkedCount = 0;
for (const event of allEvents) {
if (existingEventIds.has(event.id)) continue;
const range = parseFloorRange(event.summary);
if (!range) continue;
let hasOverlap = false;
for (const floor of recalledL0Floors) {
if (floor >= range.start && floor <= range.end) {
hasOverlap = true;
break;
}
}
if (!hasOverlap) continue;
// 实体分类：与所有路径统一标准
const participants = (event.participants || []).map(p => normalize(p));
const hasEntityMatch = focusSetForLexical.size > 0
&& participants.some(p => focusSetForLexical.has(p));
const evVec = eventVectorMap.get(event.id);
const sim = evVec?.length ? cosineSimilarity(queryVector_v1, evVec) : 0;
eventHits.push({
event,
similarity: sim,
_recallType: hasEntityMatch ? 'DIRECT' : 'RELATED',
});
existingEventIds.add(event.id);
l0LinkedCount++;
}
if (metrics) {
metrics.lexical.eventFilteredByDense = lexicalEventFilteredByDense;
@@ -1146,10 +1196,14 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
metrics.event.byRecallType.lexical = lexicalEventCount;
metrics.event.selected += lexicalEventCount;
}
if (l0LinkedCount > 0) {
metrics.event.byRecallType.l0Linked = l0LinkedCount;
metrics.event.selected += l0LinkedCount;
}
}
xbLog.info(MODULE_ID,
`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} filteredByDense=${lexicalEventFilteredByDense} (${lexTime}ms)`
`Lexical: chunks=${lexicalResult.chunkIds.length} events=${lexicalResult.eventIds.length} mergedEvents=+${lexicalEventCount} filteredByDense=${lexicalEventFilteredByDense} l0Linked=+${l0LinkedCount} (${lexTime}ms)`
);
// ═══════════════════════════════════════════════════════════════════
@@ -1164,6 +1218,35 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
metrics
);
// ═══════════════════════════════════════════════════════════════════
// Stage 7.5: PPR Diffusion Activation
//
// Spread from reranked seeds through entity co-occurrence graph.
// Diffused atoms merge into l0Selected at lower scores than seeds,
// consumed by prompt.js through the same budget pipeline.
// ═══════════════════════════════════════════════════════════════════
const diffused = diffuseFromSeeds(
l0Selected, // seeds (rerank-verified)
getStateAtoms(), // all L0 atoms
allStateVectors, // all L0 vectors (already read by recallAnchors)
queryVector_v1, // R2 query vector (for cosine gate)
metrics, // metrics collector
);
for (const da of diffused) {
l0Selected.push({
id: `diffused-${da.atomId}`,
atomId: da.atomId,
floor: da.floor,
similarity: da.finalScore,
rerankScore: da.finalScore,
atom: da.atom,
text: da.atom.semantic || '',
});
}
metrics.timing.diffusion = metrics.diffusion?.time || 0;
// ═══════════════════════════════════════════════════════════════════
// 阶段 7: Causation Trace
// ═══════════════════════════════════════════════════════════════════
@@ -1206,6 +1289,7 @@ export async function recallMemory(allEvents, vectorConfig, options = {}) {
console.log(`Floor Rerank: ${metrics.evidence.beforeRerank || 0} → ${metrics.evidence.floorsSelected || 0} floors → L0=${metrics.evidence.l0Collected || 0} (${metrics.evidence.rerankTime || 0}ms)`);
console.log(`L1: ${metrics.evidence.l1Pulled || 0} pulled → ${metrics.evidence.l1Attached || 0} attached (${metrics.evidence.l1CosineTime || 0}ms)`);
console.log(`Events: ${eventHits.length} hits, ${causalChain.length} causal`);
console.log(`Diffusion: ${metrics.diffusion?.seedCount || 0} seeds → ${metrics.diffusion?.pprActivated || 0} activated → ${metrics.diffusion?.finalCount || 0} final (${metrics.diffusion?.time || 0}ms)`);
console.groupEnd();
return {