X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update recall metrics and context pairing

Browse Source
This commit is contained in:
bielie
2026-02-10 00:18:51 +08:00
parent da1e3088eb
commit 3af76a9651
3 changed files with 657 additions and 750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
modules/story-summary/vector/retrieval/metrics.js
View File

@@ -1,9 +1,16 @@
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Metrics Collector (v3 - Deterministic Query + Hybrid + W-RRF)
// Story Summary - Metrics Collector (v4 - Two-Stage: L0 Locate → L1 Evidence)
//
// 命名规范：
// - 存储层用 L0/L1/L2/L3（StateAtom/Chunk/Event/Fact）
// - 指标层用语义名称：anchor/evidence/event/constraint/arc
//
// 架构变更（v3 → v4）：
// - evidence 区块反映 L0-only 融合 + L1 按楼层拉取的两阶段架构
// - 删除 mergedByType / selectedByType（不再有混合池）
// - 新增 l0Candidates / l0Selected / l1Pulled / l1Attached / l1CosineTime
// - fusion 区块明确标注 L0-only（删除 anchorCount）
// - quality.chunkRealRatio → quality.l1AttachRate
// ═══════════════════════════════════════════════════════════════════════════
/**
@@ -41,11 +48,10 @@ export function createMetrics() {
searchTime: 0,
},
// Fusion (W-RRF) - 多路融合
// Fusion (W-RRF, L0-only) - 多路融合
fusion: {
denseCount: 0,
lexCount: 0,
anchorCount: 0,
totalUnique: 0,
afterCap: 0,
time: 0,
@@ -74,23 +80,26 @@ export function createMetrics() {
entityNames: [],
},
// Evidence (L1 Chunks) - 原文证据
// Evidence (Two-Stage: L0 rerank → L1 pull) - 原文证据
evidence: {
floorsFromAnchors: 0,
chunkTotal: 0,
denseCoarse: 0,
merged: 0,
mergedByType: { anchorVirtual: 0, chunkReal: 0 },
selected: 0,
selectedByType: { anchorVirtual: 0, chunkReal: 0 },
contextPairsAdded: 0,
tokens: 0,
assemblyTime: 0,
// Stage 1: L0
l0Candidates: 0, // W-RRF 融合后的 L0 候选数
l0Selected: 0, // rerank 后选中的 L0 数
rerankApplied: false,
beforeRerank: 0,
afterRerank: 0,
rerankTime: 0,
rerankScores: null,
// Stage 2: L1
l1Pulled: 0, // 从 DB 拉取的 L1 chunk 总数
l1Attached: 0, // 实际挂载的 L1 数（top-1 × 楼层 × 2侧）
l1CosineTime: 0, // L1 cosine 打分耗时
// 装配
contextPairsAdded: 0, // 保留兼容（= l1Attached 中 USER 侧数量）
tokens: 0,
assemblyTime: 0,
},
// Arc - 人物弧光
@@ -139,8 +148,7 @@ export function createMetrics() {
quality: {
constraintCoverage: 100,
eventPrecisionProxy: 0,
evidenceDensity: 0,
chunkRealRatio: 0,
l1AttachRate: 0, // 有 L1 挂载的 L0 占比
potentialIssues: [],
},
};
@@ -178,7 +186,7 @@ export function formatMetricsLog(metrics) {
lines.push('');
lines.push('════════════════════════════════════════');
lines.push(' Recall Metrics Report ');
lines.push(' Recall Metrics Report (v4) ');
lines.push('════════════════════════════════════════');
lines.push('');
@@ -214,11 +222,10 @@ export function formatMetricsLog(metrics) {
lines.push(`└─ search_time: ${m.lexical.searchTime}ms`);
lines.push('');
// Fusion (W-RRF)
lines.push('[Fusion] W-RRF - 多路融合');
// Fusion (W-RRF, L0-only)
lines.push('[Fusion] W-RRF (L0-only) - 多路融合');
lines.push(`├─ dense_count: ${m.fusion.denseCount}`);
lines.push(`├─ lex_count: ${m.fusion.lexCount}`);
lines.push(`├─ anchor_count: ${m.fusion.anchorCount}`);
lines.push(`├─ total_unique: ${m.fusion.totalUnique}`);
lines.push(`├─ after_cap: ${m.fusion.afterCap}`);
lines.push(`└─ time: ${m.fusion.time}ms`);
@@ -269,43 +276,29 @@ export function formatMetricsLog(metrics) {
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
lines.push('');
// Evidence (L1 Chunks)
lines.push('[Evidence] L1 Chunks - 原文证据');
lines.push(`├─ floors_from_anchors: ${m.evidence.floorsFromAnchors}`);
if (m.evidence.chunkTotal > 0) {
lines.push(`├─ chunk_total: ${m.evidence.chunkTotal}`);
lines.push(`├─ dense_coarse: ${m.evidence.denseCoarse}`);
}
lines.push(`├─ merged: ${m.evidence.merged}`);
if (m.evidence.mergedByType) {
const mt = m.evidence.mergedByType;
lines.push(`│ ├─ anchor_virtual: ${mt.anchorVirtual || 0}`);
lines.push(`│ └─ chunk_real: ${mt.chunkReal || 0}`);
}
// Evidence (Two-Stage)
lines.push('[Evidence] Two-Stage: L0 Locate → L1 Pull');
lines.push(`├─ Stage 1 (L0):`);
lines.push(`│ ├─ candidates (post-fusion): ${m.evidence.l0Candidates}`);
if (m.evidence.rerankApplied) {
lines.push(`├─ rerank_applied: true`);
lines.push(`│ ├─ before: ${m.evidence.beforeRerank}`);
lines.push(`│ ├─ after: ${m.evidence.afterRerank}`);
lines.push(`│ └─ time: ${m.evidence.rerankTime}ms`);
lines.push(`│ ├─ rerank_applied: true`);
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank}`);
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank}`);
lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`);
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
lines.push(`├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
}
} else {
lines.push(`├─ rerank_applied: false`);
lines.push(`│ ├─ rerank_applied: false`);
}
lines.push(`├─ selected: ${m.evidence.selected}`);
if (m.evidence.selectedByType) {
const st = m.evidence.selectedByType;
lines.push(`│ ├─ anchor_virtual: ${st.anchorVirtual || 0}`);
lines.push(`│ └─ chunk_real: ${st.chunkReal || 0}`);
}
lines.push(`├─ context_pairs_added: ${m.evidence.contextPairsAdded}`);
lines.push(`│ └─ selected: ${m.evidence.l0Selected}`);
lines.push(`├─ Stage 2 (L1):`);
lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`);
lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`);
lines.push(`│ └─ cosine_time: ${m.evidence.l1CosineTime}ms`);
lines.push(`├─ tokens: ${m.evidence.tokens}`);
lines.push(`└─ assembly_time: ${m.evidence.assemblyTime}ms`);
lines.push('');
@@ -351,6 +344,7 @@ export function formatMetricsLog(metrics) {
if (m.timing.evidenceRerank > 0) {
lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`);
}
lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`);
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
lines.push(`└─ total: ${m.timing.total}ms`);
@@ -360,8 +354,7 @@ export function formatMetricsLog(metrics) {
lines.push('[Quality] 质量指标');
lines.push(`├─ constraint_coverage: ${m.quality.constraintCoverage}%`);
lines.push(`├─ event_precision_proxy: ${m.quality.eventPrecisionProxy}`);
lines.push(`├─ evidence_density: ${m.quality.evidenceDensity}%`);
lines.push(`├─ chunk_real_ratio: ${m.quality.chunkRealRatio}%`);
lines.push(`├─ l1_attach_rate: ${m.quality.l1AttachRate}%`);
if (m.quality.potentialIssues && m.quality.potentialIssues.length > 0) {
lines.push(`└─ potential_issues:`);
@@ -414,15 +407,15 @@ export function detectIssues(metrics) {
}
// ─────────────────────────────────────────────────────────────────
// 融合问题
// 融合问题（L0-only）
// ─────────────────────────────────────────────────────────────────
if (m.fusion.lexCount === 0 && m.fusion.denseCount > 0) {
issues.push('No lexical candidates in fusion - hybrid retrieval not contributing');
issues.push('No lexical L0 candidates in fusion - hybrid retrieval not contributing');
}
if (m.fusion.afterCap === 0) {
issues.push('Fusion produced zero candidates - all retrieval paths may have failed');
issues.push('Fusion produced zero L0 candidates - all retrieval paths may have failed');
}
// ─────────────────────────────────────────────────────────────────
@@ -430,7 +423,6 @@ export function detectIssues(metrics) {
// ─────────────────────────────────────────────────────────────────
if (m.event.considered > 0) {
// 只统计 Dense 路选中（direct + related），Lexical 是额外补充不计入
const denseSelected =
(m.event.byRecallType?.direct || 0) +
(m.event.byRecallType?.related || 0);
@@ -467,50 +459,47 @@ export function detectIssues(metrics) {
}
// ─────────────────────────────────────────────────────────────────
// 证据问题
// L0 Rerank 问题
// ─────────────────────────────────────────────────────────────────
// Dense 粗筛比例
if (m.evidence.chunkTotal > 0 && m.evidence.denseCoarse > 0) {
const coarseFilterRatio = 1 - (m.evidence.denseCoarse / m.evidence.chunkTotal);
if (coarseFilterRatio > 0.95) {
issues.push(`Very high dense coarse filter ratio (${(coarseFilterRatio * 100).toFixed(0)}%) - query vector may be poorly aligned`);
}
}
// Rerank 相关问题
if (m.evidence.rerankApplied) {
if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) {
const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank);
if (filterRatio > 0.7) {
issues.push(`High rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant chunks in fusion output`);
issues.push(`High L0 rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant L0 in fusion output`);
}
}
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
if (rs.max < 0.5) {
issues.push(`Low rerank scores (max=${rs.max}) - query may be poorly matched`);
issues.push(`Low L0 rerank scores (max=${rs.max}) - query may be poorly matched`);
}
if (rs.mean < 0.3) {
issues.push(`Very low average rerank score (mean=${rs.mean}) - context may be weak`);
issues.push(`Very low average L0 rerank score (mean=${rs.mean}) - context may be weak`);
}
}
if (m.evidence.rerankTime > 2000) {
issues.push(`Slow rerank (${m.evidence.rerankTime}ms) - may affect response time`);
issues.push(`Slow L0 rerank (${m.evidence.rerankTime}ms) - may affect response time`);
}
}
// chunk_real 比例（核心质量指标）
if (m.evidence.selected > 0 && m.evidence.selectedByType) {
const chunkReal = m.evidence.selectedByType.chunkReal || 0;
const ratio = chunkReal / m.evidence.selected;
if (ratio === 0 && m.evidence.selected > 5) {
issues.push('Zero real chunks in selected evidence - only anchor virtual chunks present');
} else if (ratio < 0.2 && m.evidence.selected > 10) {
issues.push(`Low real chunk ratio (${(ratio * 100).toFixed(0)}%) - may lack concrete dialogue evidence`);
}
// ─────────────────────────────────────────────────────────────────
// L1 挂载问题
// ─────────────────────────────────────────────────────────────────
if (m.evidence.l0Selected > 0 && m.evidence.l1Pulled === 0) {
issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed');
}
if (m.evidence.l0Selected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
issues.push('L1 chunks pulled but none attached - cosine scores may be too low or floor mismatch');
}
const l1AttachRate = m.quality.l1AttachRate || 0;
if (m.evidence.l0Selected > 5 && l1AttachRate < 20) {
issues.push(`Low L1 attach rate (${l1AttachRate}%) - many L0 lack concrete dialogue evidence`);
}
// ─────────────────────────────────────────────────────────────────
@@ -533,5 +522,9 @@ export function detectIssues(metrics) {
issues.push(`Slow query build (${m.query.buildTime}ms) - entity lexicon may be too large`);
}
if (m.evidence.l1CosineTime > 1000) {
issues.push(`Slow L1 cosine scoring (${m.evidence.l1CosineTime}ms) - too many chunks pulled`);
}
return issues;
}