X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update recall logic and remove unused state-recall

Browse Source
This commit is contained in:
bielie
2026-02-09 10:09:16 +08:00
parent 24f97f605e
commit 5b9da7aaf4
5 changed files with 827 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

347
modules/story-summary/vector/retrieval/metrics.js
View File

@@ -1,59 +1,60 @@
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Metrics Collector
// 召回质量指标收集与格式化
// Story Summary - Metrics Collector (v2 - 统一命名)
//
// 命名规范：
// - 存储层用 L0/L1/L2/L3（StateAtom/Chunk/Event/Fact）
// - 指标层用语义名称：anchor/evidence/event/constraint/arc
// ═══════════════════════════════════════════════════════════════════════════
/**
* 创建空的指标对象
* @returns {object} 指标对象
*/
export function createMetrics() {
return {
// L0 Query Understanding
l0: {
// Anchor (L0 StateAtoms) - 语义锚点
anchor: {
needRecall: false,
intent: '',
focusEntities: [],
queries: [],
implicitTopics: [],
queryExpansionTime: 0,
atomsMatched: 0,
matched: 0,
floorsHit: 0,
topAtoms: [],
topHits: [],
},
// L1 Constraints (Facts)
l1: {
factsTotal: 0,
factsInjected: 0,
factsFiltered: 0,
// Constraint (L3 Facts) - 世界约束
constraint: {
total: 0,
filtered: 0,
injected: 0,
tokens: 0,
samples: [],
},
// L2 Narrative Retrieval
l2: {
eventsInStore: 0,
eventsConsidered: 0,
eventsSelected: 0,
byRecallType: { direct: 0, causal: 0, context: 0 },
// Event (L2 Events) - 事件摘要
event: {
inStore: 0,
considered: 0,
selected: 0,
byRecallType: { direct: 0, related: 0, causal: 0 },
similarityDistribution: { min: 0, max: 0, mean: 0, median: 0 },
entityFilterStats: null,
entityFilter: null,
causalChainDepth: 0,
causalEventsCount: 0,
entitiesLoaded: 0,
causalCount: 0,
entitiesUsed: 0,
entityNames: [],
retrievalTime: 0,
},
// L3 Evidence Assembly
l3: {
floorsFromL0: 0,
l1Total: 0,
l1AfterCoarse: 0,
chunksInRange: 0,
chunksInRangeByType: { l0Virtual: 0, l1Real: 0 },
chunksSelected: 0,
chunksSelectedByType: { l0Virtual: 0, l1Real: 0 },
// Evidence (L1 Chunks) - 原文证据
evidence: {
floorsFromAnchors: 0,
chunkTotal: 0,
chunkAfterCoarse: 0,
merged: 0,
mergedByType: { anchorVirtual: 0, chunkReal: 0 },
selected: 0,
selectedByType: { anchorVirtual: 0, chunkReal: 0 },
contextPairsAdded: 0,
tokens: 0,
assemblyTime: 0,
@@ -61,16 +62,22 @@ export function createMetrics() {
beforeRerank: 0,
afterRerank: 0,
rerankTime: 0,
rerankScoreDistribution: null,
rerankScores: null,
},
// L4 Formatting
l4: {
// Arc - 人物弧光
arc: {
injected: 0,
tokens: 0,
},
// Formatting - 格式化
formatting: {
sectionsIncluded: [],
formattingTime: 0,
time: 0,
},
// Budget Summary
// Budget Summary - 预算
budget: {
total: 0,
limit: 0,
@@ -78,26 +85,26 @@ export function createMetrics() {
breakdown: {
constraints: 0,
events: 0,
chunks: 0,
recentOrphans: 0,
distantEvidence: 0,
recentEvidence: 0,
arcs: 0,
},
},
// Total Timing
// Timing - 计时
timing: {
queryExpansion: 0,
l0Search: 0,
l1Constraints: 0,
l2Retrieval: 0,
l3Retrieval: 0,
l3Rerank: 0,
l3Assembly: 0,
l4Formatting: 0,
anchorSearch: 0,
constraintFilter: 0,
eventRetrieval: 0,
evidenceRetrieval: 0,
evidenceRerank: 0,
evidenceAssembly: 0,
formatting: 0,
total: 0,
},
// Quality Indicators
// Quality Indicators - 质量指标
quality: {
constraintCoverage: 100,
eventPrecisionProxy: 0,
@@ -109,6 +116,8 @@ export function createMetrics() {
/**
* 计算相似度分布统计
* @param {number[]} similarities - 相似度数组
* @returns {{min: number, max: number, mean: number, median: number}}
*/
export function calcSimilarityStats(similarities) {
if (!similarities?.length) {
@@ -128,6 +137,8 @@ export function calcSimilarityStats(similarities) {
/**
* 格式化指标为可读日志
* @param {object} metrics - 指标对象
* @returns {string} 格式化后的日志
*/
export function formatMetricsLog(metrics) {
const m = metrics;
@@ -139,51 +150,50 @@ export function formatMetricsLog(metrics) {
lines.push('════════════════════════════════════════');
lines.push('');
// L0 Query Understanding
lines.push('[L0] Query Understanding');
lines.push(`├─ need_recall: ${m.l0.needRecall}`);
if (m.l0.needRecall) {
lines.push(`├─ intent: ${m.l0.intent || 'mixed'}`);
lines.push(`├─ focus_entities: [${(m.l0.focusEntities || []).join(', ')}]`);
lines.push(`├─ queries: [${(m.l0.queries || []).slice(0, 3).join(', ')}]`);
lines.push(`├─ query_expansion_time: ${m.l0.queryExpansionTime}ms`);
lines.push(`├─ atoms_matched: ${m.l0.atomsMatched || 0}`);
lines.push(`└─ floors_hit: ${m.l0.floorsHit || 0}`);
// Anchor (L0 StateAtoms)
lines.push('[Anchor] L0 StateAtoms - 语义锚点');
lines.push(`├─ need_recall: ${m.anchor.needRecall}`);
if (m.anchor.needRecall) {
lines.push(`├─ focus_entities: [${(m.anchor.focusEntities || []).join(', ')}]`);
lines.push(`├─ queries: [${(m.anchor.queries || []).slice(0, 3).join(', ')}]`);
lines.push(`├─ query_expansion_time: ${m.anchor.queryExpansionTime}ms`);
lines.push(`├─ matched: ${m.anchor.matched || 0}`);
lines.push(`└─ floors_hit: ${m.anchor.floorsHit || 0}`);
}
lines.push('');
// L1 Constraints
lines.push('[L1] Constraints (Facts)');
lines.push(`├─ facts_total: ${m.l1.factsTotal}`);
lines.push(`├─ facts_filtered: ${m.l1.factsFiltered || 0}`);
lines.push(`├─ facts_injected: ${m.l1.factsInjected}`);
lines.push(`├─ tokens: ${m.l1.tokens}`);
if (m.l1.samples && m.l1.samples.length > 0) {
lines.push(`└─ samples: "${m.l1.samples.slice(0, 2).join('", "')}"`);
// Constraint (L3 Facts)
lines.push('[Constraint] L3 Facts - 世界约束');
lines.push(`├─ total: ${m.constraint.total}`);
lines.push(`├─ filtered: ${m.constraint.filtered || 0}`);
lines.push(`├─ injected: ${m.constraint.injected}`);
lines.push(`├─ tokens: ${m.constraint.tokens}`);
if (m.constraint.samples && m.constraint.samples.length > 0) {
lines.push(`└─ samples: "${m.constraint.samples.slice(0, 2).join('", "')}"`);
}
lines.push('');
// L2 Narrative Retrieval
lines.push('[L2] Narrative Retrieval');
lines.push(`├─ events_in_store: ${m.l2.eventsInStore}`);
lines.push(`├─ events_considered: ${m.l2.eventsConsidered}`);
// Event (L2 Events)
lines.push('[Event] L2 Events - 事件摘要');
lines.push(`├─ in_store: ${m.event.inStore}`);
lines.push(`├─ considered: ${m.event.considered}`);
if (m.l2.entityFilterStats) {
const ef = m.l2.entityFilterStats;
if (m.event.entityFilter) {
const ef = m.event.entityFilter;
lines.push(`├─ entity_filter:`);
lines.push(`│ ├─ focus_entities: [${(ef.focusEntities || []).join(', ')}]`);
lines.push(`│ ├─ before_filter: ${ef.before}`);
lines.push(`│ ├─ after_filter: ${ef.after}`);
lines.push(`│ └─ filtered_out: ${ef.filtered}`);
lines.push(`│ ├─ before: ${ef.before}`);
lines.push(`│ ├─ after: ${ef.after}`);
lines.push(`│ └─ filtered: ${ef.filtered}`);
}
lines.push(`├─ events_selected: ${m.l2.eventsSelected}`);
lines.push(`├─ selected: ${m.event.selected}`);
lines.push(`├─ by_recall_type:`);
lines.push(`│ ├─ direct: ${m.l2.byRecallType.direct}`);
lines.push(`│ ├─ causal: ${m.l2.byRecallType.causal}`);
lines.push(`│ └─ context: ${m.l2.byRecallType.context}`);
lines.push(`│ ├─ direct: ${m.event.byRecallType.direct}`);
lines.push(`│ ├─ related: ${m.event.byRecallType.related}`);
lines.push(`│ └─ causal: ${m.event.byRecallType.causal}`);
const sim = m.l2.similarityDistribution;
const sim = m.event.similarityDistribution;
if (sim && sim.max > 0) {
lines.push(`├─ similarity_distribution:`);
lines.push(`│ ├─ min: ${sim.min}`);
@@ -192,93 +202,100 @@ export function formatMetricsLog(metrics) {
lines.push(`│ └─ median: ${sim.median}`);
}
lines.push(`├─ causal_chain: depth=${m.l2.causalChainDepth}, events=${m.l2.causalEventsCount}`);
lines.push(`├─ entities_loaded: ${m.l2.entitiesLoaded} [${(m.l2.entityNames || []).join(', ')}]`);
lines.push(`└─ retrieval_time: ${m.l2.retrievalTime}ms`);
lines.push(`├─ causal_chain: depth=${m.event.causalChainDepth}, count=${m.event.causalCount}`);
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
lines.push('');
// L3 Evidence Assembly
lines.push('[L3] Evidence Assembly');
lines.push(`├─ floors_from_l0: ${m.l3.floorsFromL0}`);
// Evidence (L1 Chunks)
lines.push('[Evidence] L1 Chunks - 原文证据');
lines.push(`├─ floors_from_anchors: ${m.evidence.floorsFromAnchors}`);
// L1 粗筛信息
if (m.l3.l1Total > 0) {
lines.push(`├─ l1_coarse_filter:`);
lines.push(`│ ├─ total: ${m.l3.l1Total}`);
lines.push(`│ ├─ after: ${m.l3.l1AfterCoarse}`);
lines.push(`│ └─ filtered: ${m.l3.l1Total - m.l3.l1AfterCoarse}`);
// 粗筛信息
if (m.evidence.chunkTotal > 0) {
lines.push(`├─ coarse_filter:`);
lines.push(`│ ├─ total: ${m.evidence.chunkTotal}`);
lines.push(`│ ├─ after: ${m.evidence.chunkAfterCoarse}`);
lines.push(`│ └─ filtered: ${m.evidence.chunkTotal - m.evidence.chunkAfterCoarse}`);
}
lines.push(`├─ chunks_merged: ${m.l3.chunksInRange}`);
if (m.l3.chunksInRangeByType) {
const cir = m.l3.chunksInRangeByType;
lines.push(`│ ├─ l0_virtual: ${cir.l0Virtual || 0}`);
lines.push(`│ └─ l1_real: ${cir.l1Real || 0}`);
lines.push(`├─ merged: ${m.evidence.merged}`);
if (m.evidence.mergedByType) {
const mt = m.evidence.mergedByType;
lines.push(`│ ├─ anchor_virtual: ${mt.anchorVirtual || 0}`);
lines.push(`│ └─ chunk_real: ${mt.chunkReal || 0}`);
}
// Rerank 信息
if (m.l3.rerankApplied) {
if (m.evidence.rerankApplied) {
lines.push(`├─ rerank_applied: true`);
lines.push(`│ ├─ before: ${m.l3.beforeRerank}`);
lines.push(`│ ├─ after: ${m.l3.afterRerank}`);
lines.push(`│ └─ time: ${m.l3.rerankTime}ms`);
if (m.l3.rerankScoreDistribution) {
const rd = m.l3.rerankScoreDistribution;
lines.push(`├─ rerank_scores: min=${rd.min}, max=${rd.max}, mean=${rd.mean}`);
lines.push(`│ ├─ before: ${m.evidence.beforeRerank}`);
lines.push(`│ ├─ after: ${m.evidence.afterRerank}`);
lines.push(`│ └─ time: ${m.evidence.rerankTime}ms`);
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
lines.push(`├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
}
} else {
lines.push(`├─ rerank_applied: false`);
}
lines.push(`├─ chunks_selected: ${m.l3.chunksSelected}`);
if (m.l3.chunksSelectedByType) {
const cs = m.l3.chunksSelectedByType;
lines.push(`│ ├─ l0_virtual: ${cs.l0Virtual || 0}`);
lines.push(`│ └─ l1_real: ${cs.l1Real || 0}`);
lines.push(`├─ selected: ${m.evidence.selected}`);
if (m.evidence.selectedByType) {
const st = m.evidence.selectedByType;
lines.push(`│ ├─ anchor_virtual: ${st.anchorVirtual || 0}`);
lines.push(`│ └─ chunk_real: ${st.chunkReal || 0}`);
}
lines.push(`├─ context_pairs_added: ${m.l3.contextPairsAdded}`);
lines.push(`├─ tokens: ${m.l3.tokens}`);
lines.push(`└─ assembly_time: ${m.l3.assemblyTime}ms`);
lines.push(`├─ context_pairs_added: ${m.evidence.contextPairsAdded}`);
lines.push(`├─ tokens: ${m.evidence.tokens}`);
lines.push(`└─ assembly_time: ${m.evidence.assemblyTime}ms`);
lines.push('');
// L4 Formatting
lines.push('[L4] Prompt Formatting');
lines.push(`├─ sections: [${(m.l4.sectionsIncluded || []).join(', ')}]`);
lines.push(`└─ formatting_time: ${m.l4.formattingTime}ms`);
// Arc
if (m.arc.injected > 0) {
lines.push('[Arc] 人物弧光');
lines.push(`├─ injected: ${m.arc.injected}`);
lines.push(`└─ tokens: ${m.arc.tokens}`);
lines.push('');
}
// Formatting
lines.push('[Formatting] 格式化');
lines.push(`├─ sections: [${(m.formatting.sectionsIncluded || []).join(', ')}]`);
lines.push(`└─ time: ${m.formatting.time}ms`);
lines.push('');
// Budget Summary
lines.push('[Budget Summary]');
lines.push('[Budget] 预算');
lines.push(`├─ total_tokens: ${m.budget.total}`);
lines.push(`├─ budget_limit: ${m.budget.limit}`);
lines.push(`├─ limit: ${m.budget.limit}`);
lines.push(`├─ utilization: ${m.budget.utilization}%`);
lines.push(`└─ breakdown:`);
const bd = m.budget.breakdown || {};
lines.push(` ├─ constraints (L1): ${bd.constraints || 0}`);
lines.push(` ├─ events (L2): ${bd.events || 0}`);
lines.push(` ├─ chunks (L3): ${bd.chunks || 0}`);
lines.push(` ├─ recent_orphans: ${bd.recentOrphans || 0}`);
lines.push(` ├─ constraints: ${bd.constraints || 0}`);
lines.push(` ├─ events: ${bd.events || 0}`);
lines.push(` ├─ distant_evidence: ${bd.distantEvidence || 0}`);
lines.push(` ├─ recent_evidence: ${bd.recentEvidence || 0}`);
lines.push(` └─ arcs: ${bd.arcs || 0}`);
lines.push('');
// Timing
lines.push('[Timing]');
lines.push('[Timing] 计时');
lines.push(`├─ query_expansion: ${m.timing.queryExpansion}ms`);
lines.push(`├─ l0_search: ${m.timing.l0Search}ms`);
lines.push(`├─ l1_constraints: ${m.timing.l1Constraints}ms`);
lines.push(`├─ l2_retrieval: ${m.timing.l2Retrieval}ms`);
lines.push(`├─ l3_retrieval: ${m.timing.l3Retrieval}ms`);
if (m.timing.l3Rerank > 0) {
lines.push(`├─ l3_rerank: ${m.timing.l3Rerank}ms`);
lines.push(`├─ anchor_search: ${m.timing.anchorSearch}ms`);
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
if (m.timing.evidenceRerank > 0) {
lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`);
}
lines.push(`├─ l3_assembly: ${m.timing.l3Assembly}ms`);
lines.push(`├─ l4_formatting: ${m.timing.l4Formatting}ms`);
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
lines.push(`└─ total: ${m.timing.total}ms`);
lines.push('');
// Quality Indicators
lines.push('[Quality Indicators]');
lines.push('[Quality] 质量指标');
lines.push(`├─ constraint_coverage: ${m.quality.constraintCoverage}%`);
lines.push(`├─ event_precision_proxy: ${m.quality.eventPrecisionProxy}`);
lines.push(`├─ evidence_density: ${m.quality.evidenceDensity}%`);
@@ -302,25 +319,27 @@ export function formatMetricsLog(metrics) {
/**
* 检测潜在问题
* @param {object} metrics - 指标对象
* @returns {string[]} 问题列表
*/
export function detectIssues(metrics) {
const issues = [];
const m = metrics;
// 召回比例问题
if (m.l2.eventsConsidered > 0) {
const selectRatio = m.l2.eventsSelected / m.l2.eventsConsidered;
// 事件召回比例问题
if (m.event.considered > 0) {
const selectRatio = m.event.selected / m.event.considered;
if (selectRatio < 0.1) {
issues.push(`Event selection ratio too low (${(selectRatio * 100).toFixed(1)}%) - threshold may be too high`);
}
if (selectRatio > 0.6 && m.l2.eventsConsidered > 10) {
if (selectRatio > 0.6 && m.event.considered > 10) {
issues.push(`Event selection ratio high (${(selectRatio * 100).toFixed(1)}%) - may include noise`);
}
}
// 实体过滤问题
if (m.l2.entityFilterStats) {
const ef = m.l2.entityFilterStats;
if (m.event.entityFilter) {
const ef = m.event.entityFilter;
if (ef.filtered === 0 && ef.before > 10) {
issues.push(`No events filtered by entity - focus entities may be too broad or missing`);
}
@@ -330,58 +349,58 @@ export function detectIssues(metrics) {
}
// 相似度问题
if (m.l2.similarityDistribution && m.l2.similarityDistribution.min > 0 && m.l2.similarityDistribution.min < 0.5) {
issues.push(`Low similarity events included (min=${m.l2.similarityDistribution.min})`);
if (m.event.similarityDistribution && m.event.similarityDistribution.min > 0 && m.event.similarityDistribution.min < 0.5) {
issues.push(`Low similarity events included (min=${m.event.similarityDistribution.min})`);
}
// 因果链问题
if (m.l2.eventsSelected > 0 && m.l2.causalEventsCount === 0 && m.l2.byRecallType.direct === 0) {
if (m.event.selected > 0 && m.event.causalCount === 0 && m.event.byRecallType.direct === 0) {
issues.push('No direct or causal events - query expansion may be inaccurate');
}
// L0 atoms 问题
if ((m.l0.atomsMatched || 0) === 0) {
issues.push('L0 atoms not matched - may need to generate anchors');
// 锚点匹配问题
if ((m.anchor.matched || 0) === 0) {
issues.push('No anchors matched - may need to generate anchors');
}
// L1 粗筛问题
if (m.l3.l1Total > 0 && m.l3.l1AfterCoarse > 0) {
const coarseFilterRatio = 1 - (m.l3.l1AfterCoarse / m.l3.l1Total);
// 证据粗筛问题
if (m.evidence.chunkTotal > 0 && m.evidence.chunkAfterCoarse > 0) {
const coarseFilterRatio = 1 - (m.evidence.chunkAfterCoarse / m.evidence.chunkTotal);
if (coarseFilterRatio > 0.9) {
issues.push(`Very high L1 coarse filter ratio (${(coarseFilterRatio * 100).toFixed(0)}%) - query may be too specific`);
issues.push(`Very high evidence coarse filter ratio (${(coarseFilterRatio * 100).toFixed(0)}%) - query may be too specific`);
}
}
// Rerank 相关问题
if (m.l3.rerankApplied) {
if (m.l3.beforeRerank > 0 && m.l3.afterRerank > 0) {
const filterRatio = 1 - (m.l3.afterRerank / m.l3.beforeRerank);
if (m.evidence.rerankApplied) {
if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) {
const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank);
if (filterRatio > 0.7) {
issues.push(`High rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant chunks removed`);
}
}
if (m.l3.rerankScoreDistribution) {
const rd = m.l3.rerankScoreDistribution;
if (rd.max < 0.5) {
issues.push(`Low rerank scores (max=${rd.max}) - query may be poorly matched`);
if (m.evidence.rerankScores) {
const rs = m.evidence.rerankScores;
if (rs.max < 0.5) {
issues.push(`Low rerank scores (max=${rs.max}) - query may be poorly matched`);
}
if (rd.mean < 0.3) {
issues.push(`Very low average rerank score (mean=${rd.mean}) - context may be weak`);
if (rs.mean < 0.3) {
issues.push(`Very low average rerank score (mean=${rs.mean}) - context may be weak`);
}
}
if (m.l3.rerankTime > 2000) {
issues.push(`Slow rerank (${m.l3.rerankTime}ms) - may affect response time`);
if (m.evidence.rerankTime > 2000) {
issues.push(`Slow rerank (${m.evidence.rerankTime}ms) - may affect response time`);
}
}
// 证据密度问题
if (m.l3.chunksSelected > 0 && m.l3.chunksSelectedByType) {
const l1Real = m.l3.chunksSelectedByType.l1Real || 0;
const density = l1Real / m.l3.chunksSelected;
if (density < 0.3 && m.l3.chunksSelected > 10) {
issues.push(`Low L1 chunk ratio in selected (${(density * 100).toFixed(0)}%) - may lack concrete evidence`);
if (m.evidence.selected > 0 && m.evidence.selectedByType) {
const chunkReal = m.evidence.selectedByType.chunkReal || 0;
const density = chunkReal / m.evidence.selected;
if (density < 0.3 && m.evidence.selected > 10) {
issues.push(`Low real chunk ratio in selected (${(density * 100).toFixed(0)}%) - may lack concrete evidence`);
}
}