X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix tokenizer jieba tag flow and debug logging

Browse Source
This commit is contained in:
bielie
2026-02-03 22:13:51 +08:00
parent b0ed876cb0
commit 12db08abe0
10 changed files with 1357 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
modules/story-summary/vector/recall.js
View File

@@ -21,6 +21,10 @@ import {
mergeAndSparsify,
} from './state-recall.js';
import { ensureEventTextIndex, searchEventsByText } from './text-search.js';
import {
extractRareTerms,
extractNounsFromFactsO,
} from './tokenizer.js';
const MODULE_ID = 'recall';
@@ -50,6 +54,10 @@ const CONFIG = {
RRF_K: 60,
TEXT_SEARCH_LIMIT: 80,
// TEXT-only 质量控制
TEXT_SOFT_MIN_SIM: 0.50,
TEXT_TOTAL_MAX: 6,
};
// ═══════════════════════════════════════════════════════════════════════════
@@ -83,7 +91,7 @@ function fuseEventsByRRF(vectorRanked, textRanked, eventById, k = CONFIG.RRF_K)
const upsert = (id) => {
if (!map.has(id)) {
map.set(id, { id, rrf: 0, vRank: Infinity, tRank: Infinity, type: 'TEXT' });
map.set(id, { id, rrf: 0, vRank: Infinity, tRank: Infinity, type: 'TEXT', rawSim: 0, vector: null });
}
return map.get(id);
};
@@ -96,6 +104,7 @@ function fuseEventsByRRF(vectorRanked, textRanked, eventById, k = CONFIG.RRF_K)
o.rrf += 1 / (k + i + 1);
o.type = o.tRank !== Infinity ? 'HYBRID' : 'VECTOR';
o.vector = r.vector;
o.rawSim = r.rawSim || 0;
});
textRanked.forEach((r) => {
@@ -381,6 +390,70 @@ function normalizeEntityWeights(queryEntityWeights) {
return normalized;
}
// ═══════════════════════════════════════════════════════════════════════════
// 文本路 Query 构建（分层高信号词）
// ═══════════════════════════════════════════════════════════════════════════
async function buildTextSearchQuery(segments, queryEntities, facts, expandedTerms) {
const breakdown = {
entities: [],
rareTerms: [],
factsO: [],
expanded: [],
};
breakdown.entities = [...(queryEntities || [])];
const q2Segments = segments.slice(-2);
const q2Text = q2Segments.join(' ');
try {
breakdown.rareTerms = await extractRareTerms(q2Text, 15);
} catch (e) {
xbLog.warn(MODULE_ID, '稀有词提取失败', e);
breakdown.rareTerms = [];
}
const entitySet = new Set(breakdown.entities.map(e => e.toLowerCase()));
breakdown.rareTerms = breakdown.rareTerms.filter(t => !entitySet.has(t.toLowerCase()));
const relevantSubjects = new Set(queryEntities || []);
try {
breakdown.factsO = await extractNounsFromFactsO(facts, relevantSubjects, 5);
} catch (e) {
xbLog.warn(MODULE_ID, 'facts O 提取失败', e);
breakdown.factsO = [];
}
const existingSet = new Set([
...breakdown.entities,
...breakdown.rareTerms,
].map(e => e.toLowerCase()));
breakdown.factsO = breakdown.factsO.filter(t => !existingSet.has(t.toLowerCase()));
const allExistingSet = new Set([
...breakdown.entities,
...breakdown.rareTerms,
...breakdown.factsO,
].map(e => e.toLowerCase()));
breakdown.expanded = (expandedTerms || [])
.filter(t => !allExistingSet.has(t.toLowerCase()))
.slice(0, 3);
const queryParts = [
...breakdown.entities,
...breakdown.entities,
...breakdown.rareTerms,
...breakdown.factsO,
...breakdown.expanded,
];
const query = queryParts.join(' ');
return { query, breakdown };
}
function stripFloorTag(s) {
return String(s || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').trim();
}
@@ -605,7 +678,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
const scored = (allEvents || []).map((event, idx) => {
const v = vectorMap.get(event.id);
const sim = v ? cosineSimilarity(queryVector, v) : 0;
const rawSim = v ? cosineSimilarity(queryVector, v) : 0;
let bonus = 0;
@@ -635,14 +708,15 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
_id: event.id,
_idx: idx,
event,
similarity: sim,
finalScore: sim + bonus,
rawSim,
finalScore: rawSim + bonus,
vector: v,
_entityBonus: entityBonus,
_hasPresent: maxEntityWeight > 0,
};
});
const rawSimById = new Map(scored.map(s => [s._id, s.rawSim]));
const entityBonusById = new Map(scored.map(s => [s._id, s._entityBonus]));
const hasPresentById = new Map(scored.map(s => [s._id, s._hasPresent]));
@@ -665,14 +739,34 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
const vectorRanked = candidates.map(s => ({
event: s.event,
similarity: s.finalScore,
rawSim: s.rawSim,
vector: s.vector,
}));
const eventById = new Map(allEvents.map(e => [e.id, e]));
const fused = fuseEventsByRRF(vectorRanked, textRanked, eventById);
const hasVector = vectorRanked.length > 0;
const filtered = hasVector ? fused.filter(x => x.type !== 'TEXT') : fused;
const textOnlyStats = {
total: 0,
passedSoftCheck: 0,
filtered: 0,
finalIncluded: 0,
truncatedByLimit: 0,
};
const filtered = fused.filter(x => {
if (x.type !== 'TEXT') return true;
textOnlyStats.total++;
const sim = x.rawSim || rawSimById.get(x.id) || 0;
if (sim >= CONFIG.TEXT_SOFT_MIN_SIM) {
textOnlyStats.passedSoftCheck++;
return true;
}
textOnlyStats.filtered++;
return false;
});
const mmrInput = filtered.slice(0, CONFIG.CANDIDATE_EVENTS).map(x => ({
...x,
@@ -686,14 +780,27 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
c => c.vector || null,
c => c.rrf
);
// 构造结果
const results = mmrOutput.map(x => ({
let textOnlyCount = 0;
const finalResults = mmrOutput.filter(x => {
if (x.type !== 'TEXT') return true;
if (textOnlyCount < CONFIG.TEXT_TOTAL_MAX) {
textOnlyCount++;
return true;
}
textOnlyStats.truncatedByLimit++;
return false;
});
textOnlyStats.finalIncluded = textOnlyCount;
const results = finalResults.map(x => ({
event: x.event,
similarity: x.rrf,
_recallType: hasPresentById.get(x.event?.id) ? 'DIRECT' : 'SIMILAR',
_recallReason: x.type,
_rrfDetail: { vRank: x.vRank, tRank: x.tRank, rrf: x.rrf },
_entityBonus: entityBonusById.get(x.event?.id) || 0,
_rawSim: rawSimById.get(x.event?.id) || 0,
}));
// 统计信息附加到第一条结果
@@ -704,8 +811,9 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
textCount: textRanked.length,
hybridCount: fused.filter(x => x.type === 'HYBRID').length,
vectorOnlyCount: fused.filter(x => x.type === 'VECTOR').length,
textOnlyFiltered: fused.filter(x => x.type === 'TEXT').length,
textOnlyTotal: textOnlyStats.total,
};
results[0]._textOnlyStats = textOnlyStats;
results[0]._textGapInfo = textGapInfo;
}
@@ -729,6 +837,7 @@ function formatRecallLog({
l0Results = [],
textGapInfo = null,
expandedTerms = [],
textQueryBreakdown = null,
}) {
const lines = [
'\u2554' + '\u2550'.repeat(62) + '\u2557',
@@ -775,6 +884,40 @@ function formatRecallLog({
lines.push(` 扩散: ${expandedTerms.join('、')}`);
}
lines.push('');
lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510');
lines.push('\u2502 【文本路 Query 构成】 \u2502');
lines.push('\u2514' + '\u2500'.repeat(61) + '\u2518');
if (textQueryBreakdown) {
const bd = textQueryBreakdown;
if (bd.entities?.length) {
lines.push(` 强信号-实体 (${bd.entities.length}): ${bd.entities.slice(0, 8).join(' | ')}${bd.entities.length > 8 ? ' ...' : ''}`);
} else {
lines.push(' 强信号-实体: (无)');
}
if (bd.rareTerms?.length) {
lines.push(` 强信号-稀有词 (${bd.rareTerms.length}): ${bd.rareTerms.slice(0, 10).join(' | ')}${bd.rareTerms.length > 10 ? ' ...' : ''}`);
} else {
lines.push(' 强信号-稀有词: (无)');
}
if (bd.factsO?.length) {
lines.push(` 中信号-facts O (${bd.factsO.length}): ${bd.factsO.join(' | ')}`);
} else {
lines.push(' 中信号-facts O: (无)');
}
if (bd.expanded?.length) {
lines.push(` 背景扩展 (${bd.expanded.length}): ${bd.expanded.join(' | ')}`);
} else {
lines.push(' 背景扩展: (无)');
}
} else {
lines.push(' (降级模式，无分层信息)');
}
lines.push('');
lines.push(' 实体归一化（用于加分）:');
if (normalizedEntityWeights?.size) {
@@ -816,13 +959,37 @@ function formatRecallLog({
// L2
const rrfStats = eventResults[0]?._rrfStats || {};
const textOnlyStats = eventResults[0]?._textOnlyStats || {};
lines.push('');
lines.push(' L2 事件记忆 (RRF 混合检索):');
lines.push(` 总事件: ${allEvents.length} 条 | 最终: ${eventResults.length} 条`);
lines.push(` 向量路: ${rrfStats.vectorCount || 0} 条 | 文本路: ${rrfStats.textCount || 0} 条`);
lines.push(` HYBRID: ${rrfStats.hybridCount || 0} 条 | 纯 VECTOR: ${rrfStats.vectorOnlyCount || 0} 条 | 纯 TEXT (已过滤): ${rrfStats.textOnlyFiltered || 0} 条`);
lines.push(` HYBRID: ${rrfStats.hybridCount || 0} 条 | 纯 VECTOR: ${rrfStats.vectorOnlyCount || 0} 条`);
lines.push('');
lines.push(' TEXT-only 质量控制:');
lines.push(` 候选: ${textOnlyStats.total || 0} 条`);
lines.push(` 通过软校验 (sim>=${CONFIG.TEXT_SOFT_MIN_SIM}): ${textOnlyStats.passedSoftCheck || 0} 条`);
lines.push(` 语义过滤: ${textOnlyStats.filtered || 0} 条`);
lines.push(` 限额截断 (max=${CONFIG.TEXT_TOTAL_MAX}): ${textOnlyStats.truncatedByLimit || 0} 条`);
lines.push(` 最终入选: ${textOnlyStats.finalIncluded || 0} 条`);
const textOnlyEvents = eventResults.filter(e => e._recallReason === 'TEXT');
if (textOnlyEvents.length > 0) {
lines.push('');
lines.push(' TEXT-only 入选事件:');
textOnlyEvents.forEach((e, i) => {
const ev = e.event || {};
const id = ev.id || '?';
const title = (ev.title || '').slice(0, 25) || '(无标题)';
const sim = (e._rawSim || 0).toFixed(2);
const tRank = e._rrfDetail?.tRank ?? '?';
lines.push(` ${i + 1}. [${id}] ${title.padEnd(25)} sim=${sim} tRank=${tRank}`);
});
}
const entityBoostedEvents = eventResults.filter(e => e._entityBonus > 0).length;
lines.push('');
lines.push(` 实体加分事件: ${entityBoostedEvents} 条`);
if (textGapInfo) {
@@ -886,14 +1053,17 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
const expandedTerms = expandByFacts(queryEntities, facts, 2);
const normalizedEntityWeights = normalizeEntityWeights(queryEntityWeights);
// 构建文本查询串：最后一条消息 + 实体 + 关键词
const lastSeg = segments[segments.length - 1] || '';
const queryTextForSearch = [
lastSeg,
...queryEntities,
...expandedTerms,
...(store?.json?.keywords || []).slice(0, 5).map(k => k.text),
].join(' ');
let queryTextForSearch = '';
let textQueryBreakdown = null;
try {
const result = await buildTextSearchQuery(segments, queryEntities, facts, expandedTerms);
queryTextForSearch = result.query;
textQueryBreakdown = result.breakdown;
} catch (e) {
xbLog.warn(MODULE_ID, '文本路 Query 构建失败，降级到简单拼接', e);
const lastSeg = segments[segments.length - 1] || '';
queryTextForSearch = [lastSeg, ...queryEntities, ...expandedTerms].join(' ');
}
// L0 召回
let l0Results = [];
@@ -954,6 +1124,7 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
l0Results,
textGapInfo,
expandedTerms,
textQueryBreakdown,
});
console.group('%c[Recall]', 'color: #7c3aed; font-weight: bold');