Fix lint warnings and update retrieval modules
This commit is contained in:
341
modules/story-summary/vector/retrieval/query-builder.js
Normal file
341
modules/story-summary/vector/retrieval/query-builder.js
Normal file
@@ -0,0 +1,341 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// query-builder.js - 确定性查询构建器（无 LLM）
|
||||
//
|
||||
// 职责：
|
||||
// 1. 从最近消息 + 实体词典构建 QueryBundle_v0
|
||||
// 2. 用第一轮召回结果增强为 QueryBundle_v1
|
||||
//
|
||||
// 不负责：向量化、检索、rerank
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { getContext } from '../../../../../../../extensions.js';
|
||||
import { buildEntityLexicon, buildDisplayNameMap, extractEntitiesFromText } from './entity-lexicon.js';
|
||||
import { getSummaryStore } from '../../data/store.js';
|
||||
import { filterText } from '../utils/text-filter.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 常量
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const DIALOGUE_MAX_CHARS = 400;
|
||||
const PENDING_MAX_CHARS = 400;
|
||||
const MEMORY_HINT_MAX_CHARS = 100;
|
||||
const MEMORY_HINT_ATOMS_MAX = 5;
|
||||
const MEMORY_HINT_EVENTS_MAX = 3;
|
||||
const RERANK_QUERY_MAX_CHARS = 500;
|
||||
const RERANK_SNIPPET_CHARS = 150;
|
||||
const LEXICAL_TERMS_MAX = 10;
|
||||
const LEXICAL_TERM_MIN_LEN = 2;
|
||||
const LEXICAL_TERM_MAX_LEN = 6;
|
||||
|
||||
// 中文停用词（高频无意义词）
|
||||
const STOP_WORDS = new Set([
|
||||
'的', '了', '在', '是', '我', '有', '和', '就', '不', '人',
|
||||
'都', '一', '一个', '上', '也', '很', '到', '说', '要', '去',
|
||||
'你', '会', '着', '没有', '看', '好', '自己', '这', '他', '她',
|
||||
'它', '吗', '什么', '那', '里', '来', '吧', '呢', '啊', '哦',
|
||||
'嗯', '呀', '哈', '嘿', '喂', '哎', '唉', '哇', '呃', '嘛',
|
||||
'把', '被', '让', '给', '从', '向', '对', '跟', '比', '但',
|
||||
'而', '或', '如果', '因为', '所以', '虽然', '但是', '然后',
|
||||
'可以', '这样', '那样', '怎么', '为什么', '什么样', '哪里',
|
||||
'时候', '现在', '已经', '还是', '只是', '可能', '应该', '知道',
|
||||
'觉得', '开始', '一下', '一些', '这个', '那个', '他们', '我们',
|
||||
'你们', '自己', '起来', '出来', '进去', '回来', '过来', '下去',
|
||||
]);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 工具函数
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 清洗消息文本（与 chunk-builder / recall 保持一致）
|
||||
* @param {string} text
|
||||
* @returns {string}
|
||||
*/
|
||||
function cleanMessageText(text) {
|
||||
return filterText(text)
|
||||
.replace(/\[tts:[^\]]*\]/gi, '')
|
||||
.replace(/<state>[\s\S]*?<\/state>/gi, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 截断文本到指定长度
|
||||
* @param {string} text
|
||||
* @param {number} maxLen
|
||||
* @returns {string}
|
||||
*/
|
||||
function truncate(text, maxLen) {
|
||||
if (!text || text.length <= maxLen) return text || '';
|
||||
return text.slice(0, maxLen) + '…';
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理事件摘要（移除楼层标记）
|
||||
* @param {string} summary
|
||||
* @returns {string}
|
||||
*/
|
||||
function cleanSummary(summary) {
|
||||
return String(summary || '')
|
||||
.replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文本中提取高频实词（用于词法检索）
|
||||
*
|
||||
* 策略：按中文字符边界 + 空格/标点分词，取长度 2-6 的片段
|
||||
* 过滤停用词，按频率排序
|
||||
*
|
||||
* @param {string} text - 清洗后的文本
|
||||
* @param {number} maxTerms - 最大词数
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) {
|
||||
if (!text) return [];
|
||||
|
||||
// 提取连续中文片段 + 英文单词
|
||||
const segments = text.match(/[\u4e00-\u9fff]{2,6}|[a-zA-Z]{3,}/g) || [];
|
||||
|
||||
const freq = new Map();
|
||||
for (const seg of segments) {
|
||||
const s = seg.toLowerCase();
|
||||
if (s.length < LEXICAL_TERM_MIN_LEN || s.length > LEXICAL_TERM_MAX_LEN) continue;
|
||||
if (STOP_WORDS.has(s)) continue;
|
||||
freq.set(s, (freq.get(s) || 0) + 1);
|
||||
}
|
||||
|
||||
return Array.from(freq.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, maxTerms)
|
||||
.map(([term]) => term);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// QueryBundle 类型定义（JSDoc）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* @typedef {object} QueryBundle
|
||||
* @property {string[]} focusEntities - 焦点实体（原词形，已排除 name1）
|
||||
* @property {string} queryText_v0 - 第一轮查询文本
|
||||
* @property {string|null} queryText_v1 - 第二轮查询文本（refinement 后填充）
|
||||
* @property {string} rerankQuery - rerank 用的短查询
|
||||
* @property {string[]} lexicalTerms - MiniSearch 查询词
|
||||
* @property {Set<string>} _lexicon - 实体词典（内部使用）
|
||||
* @property {Map<string, string>} _displayMap - 标准化→原词形映射（内部使用）
|
||||
*/
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 阶段 1：构建 QueryBundle_v0
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 构建初始查询包
|
||||
*
|
||||
* @param {object[]} lastMessages - 最近 K=2 条消息
|
||||
* @param {string|null} pendingUserMessage - 用户刚输入但未进 chat 的消息
|
||||
* @param {object|null} store - getSummaryStore() 返回值（可选，内部会自动获取）
|
||||
* @param {object|null} context - { name1, name2 }（可选，内部会自动获取）
|
||||
* @returns {QueryBundle}
|
||||
*/
|
||||
export function buildQueryBundle(lastMessages, pendingUserMessage, store = null, context = null) {
|
||||
// 自动获取 store 和 context
|
||||
if (!store) store = getSummaryStore();
|
||||
if (!context) {
|
||||
const ctx = getContext();
|
||||
context = { name1: ctx.name1, name2: ctx.name2 };
|
||||
}
|
||||
|
||||
// 1. 构建实体词典
|
||||
const lexicon = buildEntityLexicon(store, context);
|
||||
const displayMap = buildDisplayNameMap(store, context);
|
||||
|
||||
// 2. 清洗消息文本
|
||||
const dialogueLines = [];
|
||||
const allCleanText = [];
|
||||
|
||||
for (const m of (lastMessages || [])) {
|
||||
const speaker = m.is_user ? (context.name1 || '用户') : (m.name || context.name2 || '角色');
|
||||
const clean = cleanMessageText(m.mes || '');
|
||||
|
||||
if (clean) {
|
||||
// ★ 修复 A：不使用楼层号，embedding 模型不需要
|
||||
dialogueLines.push(`${speaker}: ${truncate(clean, DIALOGUE_MAX_CHARS)}`);
|
||||
allCleanText.push(clean);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 处理 pendingUserMessage
|
||||
let pendingClean = '';
|
||||
if (pendingUserMessage) {
|
||||
pendingClean = cleanMessageText(pendingUserMessage);
|
||||
if (pendingClean) {
|
||||
allCleanText.push(pendingClean);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 提取焦点实体
|
||||
const combinedText = allCleanText.join(' ');
|
||||
const focusEntities = extractEntitiesFromText(combinedText, lexicon, displayMap);
|
||||
|
||||
// 5. 构建 queryText_v0
|
||||
const queryParts = [];
|
||||
|
||||
if (focusEntities.length > 0) {
|
||||
queryParts.push(`[ENTITIES]\n${focusEntities.join('\n')}`);
|
||||
}
|
||||
|
||||
if (dialogueLines.length > 0) {
|
||||
queryParts.push(`[DIALOGUE]\n${dialogueLines.join('\n')}`);
|
||||
}
|
||||
|
||||
if (pendingClean) {
|
||||
queryParts.push(`[PENDING_USER]\n${truncate(pendingClean, PENDING_MAX_CHARS)}`);
|
||||
}
|
||||
|
||||
const queryText_v0 = queryParts.join('\n\n');
|
||||
|
||||
// 6. 构建 rerankQuery（短版）
|
||||
const rerankParts = [];
|
||||
|
||||
if (focusEntities.length > 0) {
|
||||
rerankParts.push(focusEntities.join(' '));
|
||||
}
|
||||
|
||||
for (const m of (lastMessages || [])) {
|
||||
const clean = cleanMessageText(m.mes || '');
|
||||
if (clean) {
|
||||
rerankParts.push(truncate(clean, RERANK_SNIPPET_CHARS));
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingClean) {
|
||||
rerankParts.push(truncate(pendingClean, RERANK_SNIPPET_CHARS));
|
||||
}
|
||||
|
||||
const rerankQuery = truncate(rerankParts.join('\n'), RERANK_QUERY_MAX_CHARS);
|
||||
|
||||
// 7. 构建 lexicalTerms
|
||||
const entityTerms = focusEntities.map(e => e.toLowerCase());
|
||||
const textTerms = extractKeyTerms(combinedText);
|
||||
|
||||
// 合并去重：实体优先
|
||||
const termSet = new Set(entityTerms);
|
||||
for (const t of textTerms) {
|
||||
if (termSet.size >= LEXICAL_TERMS_MAX) break;
|
||||
termSet.add(t);
|
||||
}
|
||||
|
||||
const lexicalTerms = Array.from(termSet);
|
||||
|
||||
return {
|
||||
focusEntities,
|
||||
queryText_v0,
|
||||
queryText_v1: null,
|
||||
rerankQuery,
|
||||
lexicalTerms,
|
||||
_lexicon: lexicon,
|
||||
_displayMap: displayMap,
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 阶段 3：Query Refinement（用第一轮召回结果增强）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 用第一轮召回结果增强 QueryBundle
|
||||
*
|
||||
* 原地修改 bundle：
|
||||
* - queryText_v1 = queryText_v0 + [MEMORY_HINTS]
|
||||
* - focusEntities 可能扩展（从 anchorHits 的 subject/object 中补充）
|
||||
* - rerankQuery 追加 memory hints 关键词
|
||||
* - lexicalTerms 追加 memory hints 关键词
|
||||
*
|
||||
* @param {QueryBundle} bundle - 原始查询包
|
||||
* @param {object[]} anchorHits - 第一轮 L0 命中（按相似度降序）
|
||||
* @param {object[]} eventHits - 第一轮 L2 命中（按相似度降序）
|
||||
*/
|
||||
export function refineQueryBundle(bundle, anchorHits, eventHits) {
|
||||
const hints = [];
|
||||
|
||||
// 1. 从 top anchorHits 提取 memory hints
|
||||
const topAnchors = (anchorHits || []).slice(0, MEMORY_HINT_ATOMS_MAX);
|
||||
for (const hit of topAnchors) {
|
||||
const semantic = hit.atom?.semantic || '';
|
||||
if (semantic) {
|
||||
hints.push(truncate(semantic, MEMORY_HINT_MAX_CHARS));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 从 top eventHits 提取 memory hints
|
||||
const topEvents = (eventHits || []).slice(0, MEMORY_HINT_EVENTS_MAX);
|
||||
for (const hit of topEvents) {
|
||||
const ev = hit.event || {};
|
||||
const title = String(ev.title || '').trim();
|
||||
const summary = cleanSummary(ev.summary);
|
||||
const line = title && summary
|
||||
? `${title}: ${summary}`
|
||||
: title || summary;
|
||||
if (line) {
|
||||
hints.push(truncate(line, MEMORY_HINT_MAX_CHARS));
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 构建 queryText_v1
|
||||
if (hints.length > 0) {
|
||||
bundle.queryText_v1 = bundle.queryText_v0 + `\n\n[MEMORY_HINTS]\n${hints.join('\n')}`;
|
||||
} else {
|
||||
bundle.queryText_v1 = bundle.queryText_v0;
|
||||
}
|
||||
|
||||
// 4. 从 anchorHits 补充 focusEntities
|
||||
const lexicon = bundle._lexicon;
|
||||
const displayMap = bundle._displayMap;
|
||||
|
||||
if (lexicon && topAnchors.length > 0) {
|
||||
const existingSet = new Set(bundle.focusEntities.map(e => e.toLowerCase()));
|
||||
|
||||
for (const hit of topAnchors) {
|
||||
const atom = hit.atom;
|
||||
if (!atom) continue;
|
||||
|
||||
// 检查 subject 和 object
|
||||
for (const field of [atom.subject, atom.object]) {
|
||||
if (!field) continue;
|
||||
const norm = String(field).normalize('NFKC').replace(/[\u200B-\u200D\uFEFF]/g, '').trim().toLowerCase();
|
||||
if (norm.length >= 2 && lexicon.has(norm) && !existingSet.has(norm)) {
|
||||
existingSet.add(norm);
|
||||
const display = displayMap?.get(norm) || field;
|
||||
bundle.focusEntities.push(display);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 增强 rerankQuery
|
||||
if (hints.length > 0) {
|
||||
const hintKeywords = extractKeyTerms(hints.join(' '), 5);
|
||||
if (hintKeywords.length > 0) {
|
||||
const addition = hintKeywords.join(' ');
|
||||
bundle.rerankQuery = truncate(
|
||||
bundle.rerankQuery + '\n' + addition,
|
||||
RERANK_QUERY_MAX_CHARS
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. 增强 lexicalTerms
|
||||
if (hints.length > 0) {
|
||||
const hintTerms = extractKeyTerms(hints.join(' '), 5);
|
||||
const termSet = new Set(bundle.lexicalTerms);
|
||||
for (const t of hintTerms) {
|
||||
if (termSet.size >= LEXICAL_TERMS_MAX) break;
|
||||
if (!termSet.has(t)) {
|
||||
termSet.add(t);
|
||||
bundle.lexicalTerms.push(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user