// ═══════════════════════════════════════════════════════════════════════════ // Story Summary - Metrics Collector // 召回质量指标收集与格式化 // ═══════════════════════════════════════════════════════════════════════════ /** * 创建空的指标对象 */ export function createMetrics() { return { // L0 Query Understanding l0: { needRecall: false, intent: '', focusEntities: [], queries: [], implicitTopics: [], queryExpansionTime: 0, atomsMatched: 0, floorsHit: 0, topAtoms: [], }, // L1 Constraints (Facts) l1: { factsTotal: 0, factsInjected: 0, factsFiltered: 0, tokens: 0, samples: [], }, // L2 Narrative Retrieval l2: { eventsInStore: 0, eventsConsidered: 0, eventsSelected: 0, byRecallType: { direct: 0, causal: 0, context: 0 }, similarityDistribution: { min: 0, max: 0, mean: 0, median: 0 }, entityFilterStats: null, causalChainDepth: 0, causalEventsCount: 0, entitiesLoaded: 0, entityNames: [], retrievalTime: 0, }, // L3 Evidence Assembly l3: { floorsFromL0: 0, // 候选规模（rerank 前） chunksInRange: 0, chunksInRangeByType: { l0Virtual: 0, l1Real: 0 }, // 最终注入（rerank + sparse 后） chunksSelected: 0, chunksSelectedByType: { l0Virtual: 0, l1Real: 0 }, // 上下文配对 contextPairsAdded: 0, tokens: 0, assemblyTime: 0, // Rerank 相关 rerankApplied: false, beforeRerank: 0, afterRerank: 0, rerankTime: 0, rerankScoreDistribution: null, }, // L4 Formatting l4: { sectionsIncluded: [], formattingTime: 0, }, // Budget Summary budget: { total: 0, limit: 0, utilization: 0, breakdown: { constraints: 0, events: 0, entities: 0, chunks: 0, recentOrphans: 0, arcs: 0, }, }, // Total Timing timing: { queryExpansion: 0, l0Search: 0, l1Constraints: 0, l2Retrieval: 0, l3Retrieval: 0, l3Rerank: 0, l3Assembly: 0, l4Formatting: 0, total: 0, }, // Quality Indicators quality: { constraintCoverage: 100, eventPrecisionProxy: 0, evidenceDensity: 0, potentialIssues: [], }, }; } /** * 计算相似度分布统计 */ export function calcSimilarityStats(similarities) { if (!similarities?.length) { return { min: 0, max: 0, mean: 0, median: 0 }; } const sorted = [...similarities].sort((a, b) => a - b); const sum = sorted.reduce((a, b) => a + b, 0); return { min: Number(sorted[0].toFixed(3)), max: Number(sorted[sorted.length - 1].toFixed(3)), mean: Number((sum / sorted.length).toFixed(3)), median: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)), }; } /** * 格式化指标为可读日志 */ export function formatMetricsLog(metrics) { const m = metrics; const lines = []; lines.push(''); lines.push('════════════════════════════════════════'); lines.push(' Recall Metrics Report '); lines.push('════════════════════════════════════════'); lines.push(''); // L0 Query Understanding lines.push('[L0] Query Understanding'); lines.push(`├─ need_recall: ${m.l0.needRecall}`); if (m.l0.needRecall) { lines.push(`├─ intent: ${m.l0.intent || 'mixed'}`); lines.push(`├─ focus_entities: [${(m.l0.focusEntities || []).join(', ')}]`); lines.push(`├─ queries: [${(m.l0.queries || []).slice(0, 3).join(', ')}]`); lines.push(`├─ query_expansion_time: ${m.l0.queryExpansionTime}ms`); lines.push(`├─ atoms_matched: ${m.l0.atomsMatched || 0}`); lines.push(`└─ floors_hit: ${m.l0.floorsHit || 0}`); } lines.push(''); // L1 Constraints lines.push('[L1] Constraints (Facts)'); lines.push(`├─ facts_total: ${m.l1.factsTotal}`); lines.push(`├─ facts_filtered: ${m.l1.factsFiltered || 0}`); lines.push(`├─ facts_injected: ${m.l1.factsInjected}`); lines.push(`├─ tokens: ${m.l1.tokens}`); if (m.l1.samples && m.l1.samples.length > 0) { lines.push(`└─ samples: "${m.l1.samples.slice(0, 2).join('", "')}"`); } lines.push(''); // L2 Narrative Retrieval lines.push('[L2] Narrative Retrieval'); lines.push(`├─ events_in_store: ${m.l2.eventsInStore}`); lines.push(`├─ events_considered: ${m.l2.eventsConsidered}`); if (m.l2.entityFilterStats) { const ef = m.l2.entityFilterStats; lines.push(`├─ entity_filter:`); lines.push(`│ ├─ focus_entities: [${(ef.focusEntities || []).join(', ')}]`); lines.push(`│ ├─ before_filter: ${ef.before}`); lines.push(`│ ├─ after_filter: ${ef.after}`); lines.push(`│ └─ filtered_out: ${ef.filtered}`); } lines.push(`├─ events_selected: ${m.l2.eventsSelected}`); lines.push(`├─ by_recall_type:`); lines.push(`│ ├─ direct: ${m.l2.byRecallType.direct}`); lines.push(`│ ├─ causal: ${m.l2.byRecallType.causal}`); lines.push(`│ └─ context: ${m.l2.byRecallType.context}`); const sim = m.l2.similarityDistribution; if (sim && sim.max > 0) { lines.push(`├─ similarity_distribution:`); lines.push(`│ ├─ min: ${sim.min}`); lines.push(`│ ├─ max: ${sim.max}`); lines.push(`│ ├─ mean: ${sim.mean}`); lines.push(`│ └─ median: ${sim.median}`); } lines.push(`├─ causal_chain: depth=${m.l2.causalChainDepth}, events=${m.l2.causalEventsCount}`); lines.push(`├─ entities_loaded: ${m.l2.entitiesLoaded} [${(m.l2.entityNames || []).join(', ')}]`); lines.push(`└─ retrieval_time: ${m.l2.retrievalTime}ms`); lines.push(''); // L3 Evidence Assembly lines.push('[L3] Evidence Assembly'); lines.push(`├─ floors_from_l0: ${m.l3.floorsFromL0}`); // 候选规模 lines.push(`├─ chunks_in_range: ${m.l3.chunksInRange}`); if (m.l3.chunksInRangeByType) { const cir = m.l3.chunksInRangeByType; lines.push(`│ ├─ l0_virtual: ${cir.l0Virtual || 0}`); lines.push(`│ └─ l1_real: ${cir.l1Real || 0}`); } // Rerank 信息 if (m.l3.rerankApplied) { lines.push(`├─ rerank_applied: true`); lines.push(`│ ├─ before: ${m.l3.beforeRerank}`); lines.push(`│ ├─ after: ${m.l3.afterRerank}`); lines.push(`│ └─ time: ${m.l3.rerankTime}ms`); if (m.l3.rerankScoreDistribution) { const rd = m.l3.rerankScoreDistribution; lines.push(`├─ rerank_scores: min=${rd.min}, max=${rd.max}, mean=${rd.mean}`); } } else { lines.push(`├─ rerank_applied: false`); } // 最终注入规模 lines.push(`├─ chunks_selected: ${m.l3.chunksSelected}`); if (m.l3.chunksSelectedByType) { const cs = m.l3.chunksSelectedByType; lines.push(`│ ├─ l0_virtual: ${cs.l0Virtual || 0}`); lines.push(`│ └─ l1_real: ${cs.l1Real || 0}`); } lines.push(`├─ context_pairs_added: ${m.l3.contextPairsAdded}`); lines.push(`├─ tokens: ${m.l3.tokens}`); lines.push(`└─ assembly_time: ${m.l3.assemblyTime}ms`); lines.push(''); // L4 Formatting lines.push('[L4] Prompt Formatting'); lines.push(`├─ sections: [${(m.l4.sectionsIncluded || []).join(', ')}]`); lines.push(`└─ formatting_time: ${m.l4.formattingTime}ms`); lines.push(''); // Budget Summary lines.push('[Budget Summary]'); lines.push(`├─ total_tokens: ${m.budget.total}`); lines.push(`├─ budget_limit: ${m.budget.limit}`); lines.push(`├─ utilization: ${m.budget.utilization}%`); lines.push(`└─ breakdown:`); const bd = m.budget.breakdown || {}; lines.push(` ├─ constraints (L1): ${bd.constraints || 0}`); lines.push(` ├─ events (L2): ${bd.events || 0}`); lines.push(` ├─ chunks (L3): ${bd.chunks || 0}`); lines.push(` ├─ recent_orphans: ${bd.recentOrphans || 0}`); lines.push(` └─ arcs: ${bd.arcs || 0}`); lines.push(''); // Timing lines.push('[Timing]'); lines.push(`├─ query_expansion: ${m.timing.queryExpansion}ms`); lines.push(`├─ l0_search: ${m.timing.l0Search}ms`); lines.push(`├─ l1_constraints: ${m.timing.l1Constraints}ms`); lines.push(`├─ l2_retrieval: ${m.timing.l2Retrieval}ms`); lines.push(`├─ l3_retrieval: ${m.timing.l3Retrieval}ms`); if (m.timing.l3Rerank > 0) { lines.push(`├─ l3_rerank: ${m.timing.l3Rerank}ms`); } lines.push(`├─ l3_assembly: ${m.timing.l3Assembly}ms`); lines.push(`├─ l4_formatting: ${m.timing.l4Formatting}ms`); lines.push(`└─ total: ${m.timing.total}ms`); lines.push(''); // Quality Indicators lines.push('[Quality Indicators]'); lines.push(`├─ constraint_coverage: ${m.quality.constraintCoverage}%`); lines.push(`├─ event_precision_proxy: ${m.quality.eventPrecisionProxy}`); lines.push(`├─ evidence_density: ${m.quality.evidenceDensity}%`); if (m.quality.potentialIssues && m.quality.potentialIssues.length > 0) { lines.push(`└─ potential_issues:`); m.quality.potentialIssues.forEach((issue, i) => { const prefix = i === m.quality.potentialIssues.length - 1 ? ' └─' : ' ├─'; lines.push(`${prefix} ⚠ ${issue}`); }); } else { lines.push(`└─ potential_issues: none`); } lines.push(''); lines.push('════════════════════════════════════════'); lines.push(''); return lines.join('\n'); } /** * 检测潜在问题 */ export function detectIssues(metrics) { const issues = []; const m = metrics; // 召回比例问题 if (m.l2.eventsConsidered > 0) { const selectRatio = m.l2.eventsSelected / m.l2.eventsConsidered; if (selectRatio < 0.1) { issues.push(`Event selection ratio too low (${(selectRatio * 100).toFixed(1)}%) - threshold may be too high`); } if (selectRatio > 0.6 && m.l2.eventsConsidered > 10) { issues.push(`Event selection ratio high (${(selectRatio * 100).toFixed(1)}%) - may include noise`); } } // 实体过滤问题 if (m.l2.entityFilterStats) { const ef = m.l2.entityFilterStats; if (ef.filtered === 0 && ef.before > 10) { issues.push(`No events filtered by entity - focus entities may be too broad or missing`); } if (ef.before > 0 && ef.filtered > ef.before * 0.8) { issues.push(`Too many events filtered (${ef.filtered}/${ef.before}) - focus may be too narrow`); } } // 相似度问题 if (m.l2.similarityDistribution && m.l2.similarityDistribution.min > 0 && m.l2.similarityDistribution.min < 0.5) { issues.push(`Low similarity events included (min=${m.l2.similarityDistribution.min})`); } // 因果链问题 if (m.l2.eventsSelected > 0 && m.l2.causalEventsCount === 0 && m.l2.byRecallType.direct === 0) { issues.push('No direct or causal events - query expansion may be inaccurate'); } // L0 atoms 问题 if ((m.l0.atomsMatched || 0) === 0) { issues.push('L0 atoms not matched - may need to generate anchors'); } // Rerank 相关问题 if (m.l3.rerankApplied) { if (m.l3.beforeRerank > 0 && m.l3.afterRerank > 0) { const filterRatio = 1 - (m.l3.afterRerank / m.l3.beforeRerank); if (filterRatio > 0.7) { issues.push(`High rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant chunks removed`); } } if (m.l3.rerankScoreDistribution) { const rd = m.l3.rerankScoreDistribution; if (rd.max < 0.5) { issues.push(`Low rerank scores (max=${rd.max}) - query may be poorly matched`); } if (rd.mean < 0.3) { issues.push(`Very low average rerank score (mean=${rd.mean}) - context may be weak`); } } if (m.l3.rerankTime > 2000) { issues.push(`Slow rerank (${m.l3.rerankTime}ms) - may affect response time`); } } // 证据密度问题（基于 selected 的构成） if (m.l3.chunksSelected > 0 && m.l3.chunksSelectedByType) { const l1Real = m.l3.chunksSelectedByType.l1Real || 0; const density = l1Real / m.l3.chunksSelected; if (density < 0.3 && m.l3.chunksSelected > 10) { issues.push(`Low L1 chunk ratio in selected (${(density * 100).toFixed(0)}%) - may lack concrete evidence`); } } // 预算问题 if (m.budget.utilization > 90) { issues.push(`High budget utilization (${m.budget.utilization}%) - may be truncating content`); } // 性能问题 if (m.timing.total > 5000) { issues.push(`Slow recall (${m.timing.total}ms) - consider optimization`); } return issues; }