419 lines
15 KiB
JavaScript
419 lines
15 KiB
JavaScript
// ═══════════════════════════════════════════════════════════════════════════
|
||
// Story Summary - Metrics Collector (v2 - 统一命名)
|
||
//
|
||
// 命名规范：
|
||
// - 存储层用 L0/L1/L2/L3（StateAtom/Chunk/Event/Fact）
|
||
// - 指标层用语义名称：anchor/evidence/event/constraint/arc
|
||
// ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
/**
|
||
* 创建空的指标对象
|
||
* @returns {object} 指标对象
|
||
*/
|
||
export function createMetrics() {
|
||
return {
|
||
// Anchor (L0 StateAtoms) - 语义锚点
|
||
anchor: {
|
||
needRecall: false,
|
||
focusEntities: [],
|
||
queries: [],
|
||
queryExpansionTime: 0,
|
||
matched: 0,
|
||
floorsHit: 0,
|
||
topHits: [],
|
||
},
|
||
|
||
// Constraint (L3 Facts) - 世界约束
|
||
constraint: {
|
||
total: 0,
|
||
filtered: 0,
|
||
injected: 0,
|
||
tokens: 0,
|
||
samples: [],
|
||
},
|
||
|
||
// Event (L2 Events) - 事件摘要
|
||
event: {
|
||
inStore: 0,
|
||
considered: 0,
|
||
selected: 0,
|
||
byRecallType: { direct: 0, related: 0, causal: 0 },
|
||
similarityDistribution: { min: 0, max: 0, mean: 0, median: 0 },
|
||
entityFilter: null,
|
||
causalChainDepth: 0,
|
||
causalCount: 0,
|
||
entitiesUsed: 0,
|
||
entityNames: [],
|
||
},
|
||
|
||
// Evidence (L1 Chunks) - 原文证据
|
||
evidence: {
|
||
floorsFromAnchors: 0,
|
||
chunkTotal: 0,
|
||
chunkAfterCoarse: 0,
|
||
merged: 0,
|
||
mergedByType: { anchorVirtual: 0, chunkReal: 0 },
|
||
selected: 0,
|
||
selectedByType: { anchorVirtual: 0, chunkReal: 0 },
|
||
contextPairsAdded: 0,
|
||
tokens: 0,
|
||
assemblyTime: 0,
|
||
rerankApplied: false,
|
||
beforeRerank: 0,
|
||
afterRerank: 0,
|
||
rerankTime: 0,
|
||
rerankScores: null,
|
||
},
|
||
|
||
// Arc - 人物弧光
|
||
arc: {
|
||
injected: 0,
|
||
tokens: 0,
|
||
},
|
||
|
||
// Formatting - 格式化
|
||
formatting: {
|
||
sectionsIncluded: [],
|
||
time: 0,
|
||
},
|
||
|
||
// Budget Summary - 预算
|
||
budget: {
|
||
total: 0,
|
||
limit: 0,
|
||
utilization: 0,
|
||
breakdown: {
|
||
constraints: 0,
|
||
events: 0,
|
||
distantEvidence: 0,
|
||
recentEvidence: 0,
|
||
arcs: 0,
|
||
},
|
||
},
|
||
|
||
// Timing - 计时
|
||
timing: {
|
||
queryExpansion: 0,
|
||
anchorSearch: 0,
|
||
constraintFilter: 0,
|
||
eventRetrieval: 0,
|
||
evidenceRetrieval: 0,
|
||
evidenceRerank: 0,
|
||
evidenceAssembly: 0,
|
||
formatting: 0,
|
||
total: 0,
|
||
},
|
||
|
||
// Quality Indicators - 质量指标
|
||
quality: {
|
||
constraintCoverage: 100,
|
||
eventPrecisionProxy: 0,
|
||
evidenceDensity: 0,
|
||
potentialIssues: [],
|
||
},
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 计算相似度分布统计
|
||
* @param {number[]} similarities - 相似度数组
|
||
* @returns {{min: number, max: number, mean: number, median: number}}
|
||
*/
|
||
export function calcSimilarityStats(similarities) {
|
||
if (!similarities?.length) {
|
||
return { min: 0, max: 0, mean: 0, median: 0 };
|
||
}
|
||
|
||
const sorted = [...similarities].sort((a, b) => a - b);
|
||
const sum = sorted.reduce((a, b) => a + b, 0);
|
||
|
||
return {
|
||
min: Number(sorted[0].toFixed(3)),
|
||
max: Number(sorted[sorted.length - 1].toFixed(3)),
|
||
mean: Number((sum / sorted.length).toFixed(3)),
|
||
median: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 格式化指标为可读日志
|
||
* @param {object} metrics - 指标对象
|
||
* @returns {string} 格式化后的日志
|
||
*/
|
||
export function formatMetricsLog(metrics) {
|
||
const m = metrics;
|
||
const lines = [];
|
||
|
||
lines.push('');
|
||
lines.push('════════════════════════════════════════');
|
||
lines.push(' Recall Metrics Report ');
|
||
lines.push('════════════════════════════════════════');
|
||
lines.push('');
|
||
|
||
// Anchor (L0 StateAtoms)
|
||
lines.push('[Anchor] L0 StateAtoms - 语义锚点');
|
||
lines.push(`├─ need_recall: ${m.anchor.needRecall}`);
|
||
if (m.anchor.needRecall) {
|
||
lines.push(`├─ focus_entities: [${(m.anchor.focusEntities || []).join(', ')}]`);
|
||
lines.push(`├─ queries: [${(m.anchor.queries || []).slice(0, 3).join(', ')}]`);
|
||
lines.push(`├─ query_expansion_time: ${m.anchor.queryExpansionTime}ms`);
|
||
lines.push(`├─ matched: ${m.anchor.matched || 0}`);
|
||
lines.push(`└─ floors_hit: ${m.anchor.floorsHit || 0}`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Constraint (L3 Facts)
|
||
lines.push('[Constraint] L3 Facts - 世界约束');
|
||
lines.push(`├─ total: ${m.constraint.total}`);
|
||
lines.push(`├─ filtered: ${m.constraint.filtered || 0}`);
|
||
lines.push(`├─ injected: ${m.constraint.injected}`);
|
||
lines.push(`├─ tokens: ${m.constraint.tokens}`);
|
||
if (m.constraint.samples && m.constraint.samples.length > 0) {
|
||
lines.push(`└─ samples: "${m.constraint.samples.slice(0, 2).join('", "')}"`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Event (L2 Events)
|
||
lines.push('[Event] L2 Events - 事件摘要');
|
||
lines.push(`├─ in_store: ${m.event.inStore}`);
|
||
lines.push(`├─ considered: ${m.event.considered}`);
|
||
|
||
if (m.event.entityFilter) {
|
||
const ef = m.event.entityFilter;
|
||
lines.push(`├─ entity_filter:`);
|
||
lines.push(`│ ├─ focus_entities: [${(ef.focusEntities || []).join(', ')}]`);
|
||
lines.push(`│ ├─ before: ${ef.before}`);
|
||
lines.push(`│ ├─ after: ${ef.after}`);
|
||
lines.push(`│ └─ filtered: ${ef.filtered}`);
|
||
}
|
||
|
||
lines.push(`├─ selected: ${m.event.selected}`);
|
||
lines.push(`├─ by_recall_type:`);
|
||
lines.push(`│ ├─ direct: ${m.event.byRecallType.direct}`);
|
||
lines.push(`│ ├─ related: ${m.event.byRecallType.related}`);
|
||
lines.push(`│ └─ causal: ${m.event.byRecallType.causal}`);
|
||
|
||
const sim = m.event.similarityDistribution;
|
||
if (sim && sim.max > 0) {
|
||
lines.push(`├─ similarity_distribution:`);
|
||
lines.push(`│ ├─ min: ${sim.min}`);
|
||
lines.push(`│ ├─ max: ${sim.max}`);
|
||
lines.push(`│ ├─ mean: ${sim.mean}`);
|
||
lines.push(`│ └─ median: ${sim.median}`);
|
||
}
|
||
|
||
lines.push(`├─ causal_chain: depth=${m.event.causalChainDepth}, count=${m.event.causalCount}`);
|
||
lines.push(`└─ entities_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}]`);
|
||
lines.push('');
|
||
|
||
// Evidence (L1 Chunks)
|
||
lines.push('[Evidence] L1 Chunks - 原文证据');
|
||
lines.push(`├─ floors_from_anchors: ${m.evidence.floorsFromAnchors}`);
|
||
|
||
// 粗筛信息
|
||
if (m.evidence.chunkTotal > 0) {
|
||
lines.push(`├─ coarse_filter:`);
|
||
lines.push(`│ ├─ total: ${m.evidence.chunkTotal}`);
|
||
lines.push(`│ ├─ after: ${m.evidence.chunkAfterCoarse}`);
|
||
lines.push(`│ └─ filtered: ${m.evidence.chunkTotal - m.evidence.chunkAfterCoarse}`);
|
||
}
|
||
|
||
lines.push(`├─ merged: ${m.evidence.merged}`);
|
||
if (m.evidence.mergedByType) {
|
||
const mt = m.evidence.mergedByType;
|
||
lines.push(`│ ├─ anchor_virtual: ${mt.anchorVirtual || 0}`);
|
||
lines.push(`│ └─ chunk_real: ${mt.chunkReal || 0}`);
|
||
}
|
||
|
||
// Rerank 信息
|
||
if (m.evidence.rerankApplied) {
|
||
lines.push(`├─ rerank_applied: true`);
|
||
lines.push(`│ ├─ before: ${m.evidence.beforeRerank}`);
|
||
lines.push(`│ ├─ after: ${m.evidence.afterRerank}`);
|
||
lines.push(`│ └─ time: ${m.evidence.rerankTime}ms`);
|
||
if (m.evidence.rerankScores) {
|
||
const rs = m.evidence.rerankScores;
|
||
lines.push(`├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
|
||
}
|
||
} else {
|
||
lines.push(`├─ rerank_applied: false`);
|
||
}
|
||
|
||
lines.push(`├─ selected: ${m.evidence.selected}`);
|
||
if (m.evidence.selectedByType) {
|
||
const st = m.evidence.selectedByType;
|
||
lines.push(`│ ├─ anchor_virtual: ${st.anchorVirtual || 0}`);
|
||
lines.push(`│ └─ chunk_real: ${st.chunkReal || 0}`);
|
||
}
|
||
|
||
lines.push(`├─ context_pairs_added: ${m.evidence.contextPairsAdded}`);
|
||
lines.push(`├─ tokens: ${m.evidence.tokens}`);
|
||
lines.push(`└─ assembly_time: ${m.evidence.assemblyTime}ms`);
|
||
lines.push('');
|
||
|
||
// Arc
|
||
if (m.arc.injected > 0) {
|
||
lines.push('[Arc] 人物弧光');
|
||
lines.push(`├─ injected: ${m.arc.injected}`);
|
||
lines.push(`└─ tokens: ${m.arc.tokens}`);
|
||
lines.push('');
|
||
}
|
||
|
||
// Formatting
|
||
lines.push('[Formatting] 格式化');
|
||
lines.push(`├─ sections: [${(m.formatting.sectionsIncluded || []).join(', ')}]`);
|
||
lines.push(`└─ time: ${m.formatting.time}ms`);
|
||
lines.push('');
|
||
|
||
// Budget Summary
|
||
lines.push('[Budget] 预算');
|
||
lines.push(`├─ total_tokens: ${m.budget.total}`);
|
||
lines.push(`├─ limit: ${m.budget.limit}`);
|
||
lines.push(`├─ utilization: ${m.budget.utilization}%`);
|
||
lines.push(`└─ breakdown:`);
|
||
const bd = m.budget.breakdown || {};
|
||
lines.push(` ├─ constraints: ${bd.constraints || 0}`);
|
||
lines.push(` ├─ events: ${bd.events || 0}`);
|
||
lines.push(` ├─ distant_evidence: ${bd.distantEvidence || 0}`);
|
||
lines.push(` ├─ recent_evidence: ${bd.recentEvidence || 0}`);
|
||
lines.push(` └─ arcs: ${bd.arcs || 0}`);
|
||
lines.push('');
|
||
|
||
// Timing
|
||
lines.push('[Timing] 计时');
|
||
lines.push(`├─ query_expansion: ${m.timing.queryExpansion}ms`);
|
||
lines.push(`├─ anchor_search: ${m.timing.anchorSearch}ms`);
|
||
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
|
||
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
|
||
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
|
||
if (m.timing.evidenceRerank > 0) {
|
||
lines.push(`├─ evidence_rerank: ${m.timing.evidenceRerank}ms`);
|
||
}
|
||
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
|
||
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
|
||
lines.push(`└─ total: ${m.timing.total}ms`);
|
||
lines.push('');
|
||
|
||
// Quality Indicators
|
||
lines.push('[Quality] 质量指标');
|
||
lines.push(`├─ constraint_coverage: ${m.quality.constraintCoverage}%`);
|
||
lines.push(`├─ event_precision_proxy: ${m.quality.eventPrecisionProxy}`);
|
||
lines.push(`├─ evidence_density: ${m.quality.evidenceDensity}%`);
|
||
|
||
if (m.quality.potentialIssues && m.quality.potentialIssues.length > 0) {
|
||
lines.push(`└─ potential_issues:`);
|
||
m.quality.potentialIssues.forEach((issue, i) => {
|
||
const prefix = i === m.quality.potentialIssues.length - 1 ? ' └─' : ' ├─';
|
||
lines.push(`${prefix} ⚠ ${issue}`);
|
||
});
|
||
} else {
|
||
lines.push(`└─ potential_issues: none`);
|
||
}
|
||
|
||
lines.push('');
|
||
lines.push('════════════════════════════════════════');
|
||
lines.push('');
|
||
|
||
return lines.join('\n');
|
||
}
|
||
|
||
/**
|
||
* 检测潜在问题
|
||
* @param {object} metrics - 指标对象
|
||
* @returns {string[]} 问题列表
|
||
*/
|
||
export function detectIssues(metrics) {
|
||
const issues = [];
|
||
const m = metrics;
|
||
|
||
// 事件召回比例问题
|
||
if (m.event.considered > 0) {
|
||
const selectRatio = m.event.selected / m.event.considered;
|
||
if (selectRatio < 0.1) {
|
||
issues.push(`Event selection ratio too low (${(selectRatio * 100).toFixed(1)}%) - threshold may be too high`);
|
||
}
|
||
if (selectRatio > 0.6 && m.event.considered > 10) {
|
||
issues.push(`Event selection ratio high (${(selectRatio * 100).toFixed(1)}%) - may include noise`);
|
||
}
|
||
}
|
||
|
||
// 实体过滤问题
|
||
if (m.event.entityFilter) {
|
||
const ef = m.event.entityFilter;
|
||
if (ef.filtered === 0 && ef.before > 10) {
|
||
issues.push(`No events filtered by entity - focus entities may be too broad or missing`);
|
||
}
|
||
if (ef.before > 0 && ef.filtered > ef.before * 0.8) {
|
||
issues.push(`Too many events filtered (${ef.filtered}/${ef.before}) - focus may be too narrow`);
|
||
}
|
||
}
|
||
|
||
// 相似度问题
|
||
if (m.event.similarityDistribution && m.event.similarityDistribution.min > 0 && m.event.similarityDistribution.min < 0.5) {
|
||
issues.push(`Low similarity events included (min=${m.event.similarityDistribution.min})`);
|
||
}
|
||
|
||
// 因果链问题
|
||
if (m.event.selected > 0 && m.event.causalCount === 0 && m.event.byRecallType.direct === 0) {
|
||
issues.push('No direct or causal events - query expansion may be inaccurate');
|
||
}
|
||
|
||
// 锚点匹配问题
|
||
if ((m.anchor.matched || 0) === 0) {
|
||
issues.push('No anchors matched - may need to generate anchors');
|
||
}
|
||
|
||
// 证据粗筛问题
|
||
if (m.evidence.chunkTotal > 0 && m.evidence.chunkAfterCoarse > 0) {
|
||
const coarseFilterRatio = 1 - (m.evidence.chunkAfterCoarse / m.evidence.chunkTotal);
|
||
if (coarseFilterRatio > 0.9) {
|
||
issues.push(`Very high evidence coarse filter ratio (${(coarseFilterRatio * 100).toFixed(0)}%) - query may be too specific`);
|
||
}
|
||
}
|
||
|
||
// Rerank 相关问题
|
||
if (m.evidence.rerankApplied) {
|
||
if (m.evidence.beforeRerank > 0 && m.evidence.afterRerank > 0) {
|
||
const filterRatio = 1 - (m.evidence.afterRerank / m.evidence.beforeRerank);
|
||
if (filterRatio > 0.7) {
|
||
issues.push(`High rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant chunks removed`);
|
||
}
|
||
}
|
||
|
||
if (m.evidence.rerankScores) {
|
||
const rs = m.evidence.rerankScores;
|
||
if (rs.max < 0.5) {
|
||
issues.push(`Low rerank scores (max=${rs.max}) - query may be poorly matched`);
|
||
}
|
||
if (rs.mean < 0.3) {
|
||
issues.push(`Very low average rerank score (mean=${rs.mean}) - context may be weak`);
|
||
}
|
||
}
|
||
|
||
if (m.evidence.rerankTime > 2000) {
|
||
issues.push(`Slow rerank (${m.evidence.rerankTime}ms) - may affect response time`);
|
||
}
|
||
}
|
||
|
||
// 证据密度问题
|
||
if (m.evidence.selected > 0 && m.evidence.selectedByType) {
|
||
const chunkReal = m.evidence.selectedByType.chunkReal || 0;
|
||
const density = chunkReal / m.evidence.selected;
|
||
if (density < 0.3 && m.evidence.selected > 10) {
|
||
issues.push(`Low real chunk ratio in selected (${(density * 100).toFixed(0)}%) - may lack concrete evidence`);
|
||
}
|
||
}
|
||
|
||
// 预算问题
|
||
if (m.budget.utilization > 90) {
|
||
issues.push(`High budget utilization (${m.budget.utilization}%) - may be truncating content`);
|
||
}
|
||
|
||
// 性能问题
|
||
if (m.timing.total > 5000) {
|
||
issues.push(`Slow recall (${m.timing.total}ms) - consider optimization`);
|
||
}
|
||
|
||
return issues;
|
||
}
|