// ═══════════════════════════════════════════════════════════════════════════ // query-builder.js - 确定性查询构建器(无 LLM) // // 职责: // 1. 从最近消息 + 实体词典构建 QueryBundle_v0 // 2. 用第一轮召回结果增强为 QueryBundle_v1 // // 不负责:向量化、检索、rerank // ═══════════════════════════════════════════════════════════════════════════ import { getContext } from '../../../../../../../extensions.js'; import { buildEntityLexicon, buildDisplayNameMap, extractEntitiesFromText } from './entity-lexicon.js'; import { getSummaryStore } from '../../data/store.js'; import { filterText } from '../utils/text-filter.js'; // ───────────────────────────────────────────────────────────────────────── // 常量 // ───────────────────────────────────────────────────────────────────────── const DIALOGUE_MAX_CHARS = 400; const PENDING_MAX_CHARS = 400; const MEMORY_HINT_MAX_CHARS = 100; const MEMORY_HINT_ATOMS_MAX = 5; const MEMORY_HINT_EVENTS_MAX = 3; const RERANK_QUERY_MAX_CHARS = 500; const RERANK_SNIPPET_CHARS = 150; const LEXICAL_TERMS_MAX = 10; const LEXICAL_TERM_MIN_LEN = 2; const LEXICAL_TERM_MAX_LEN = 6; // 中文停用词(高频无意义词) const STOP_WORDS = new Set([ '的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这', '他', '她', '它', '吗', '什么', '那', '里', '来', '吧', '呢', '啊', '哦', '嗯', '呀', '哈', '嘿', '喂', '哎', '唉', '哇', '呃', '嘛', '把', '被', '让', '给', '从', '向', '对', '跟', '比', '但', '而', '或', '如果', '因为', '所以', '虽然', '但是', '然后', '可以', '这样', '那样', '怎么', '为什么', '什么样', '哪里', '时候', '现在', '已经', '还是', '只是', '可能', '应该', '知道', '觉得', '开始', '一下', '一些', '这个', '那个', '他们', '我们', '你们', '自己', '起来', '出来', '进去', '回来', '过来', '下去', ]); // ───────────────────────────────────────────────────────────────────────── // 工具函数 // ───────────────────────────────────────────────────────────────────────── /** * 清洗消息文本(与 chunk-builder / recall 保持一致) * @param {string} text * @returns {string} */ function cleanMessageText(text) { return filterText(text) .replace(/\[tts:[^\]]*\]/gi, '') .replace(/[\s\S]*?<\/state>/gi, '') .trim(); } /** * 截断文本到指定长度 * @param {string} text * @param {number} maxLen * @returns {string} */ function truncate(text, maxLen) { if (!text || text.length <= maxLen) return text || ''; return text.slice(0, maxLen) + '…'; } /** * 清理事件摘要(移除楼层标记) * @param {string} summary * @returns {string} */ function cleanSummary(summary) { return String(summary || '') .replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '') .trim(); } /** * 从文本中提取高频实词(用于词法检索) * * 策略:按中文字符边界 + 空格/标点分词,取长度 2-6 的片段 * 过滤停用词,按频率排序 * * @param {string} text - 清洗后的文本 * @param {number} maxTerms - 最大词数 * @returns {string[]} */ function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) { if (!text) return []; // 提取连续中文片段 + 英文单词 const segments = text.match(/[\u4e00-\u9fff]{2,6}|[a-zA-Z]{3,}/g) || []; const freq = new Map(); for (const seg of segments) { const s = seg.toLowerCase(); if (s.length < LEXICAL_TERM_MIN_LEN || s.length > LEXICAL_TERM_MAX_LEN) continue; if (STOP_WORDS.has(s)) continue; freq.set(s, (freq.get(s) || 0) + 1); } return Array.from(freq.entries()) .sort((a, b) => b[1] - a[1]) .slice(0, maxTerms) .map(([term]) => term); } // ───────────────────────────────────────────────────────────────────────── // QueryBundle 类型定义(JSDoc) // ───────────────────────────────────────────────────────────────────────── /** * @typedef {object} QueryBundle * @property {string[]} focusEntities - 焦点实体(原词形,已排除 name1) * @property {string} queryText_v0 - 第一轮查询文本 * @property {string|null} queryText_v1 - 第二轮查询文本(refinement 后填充) * @property {string} rerankQuery - rerank 用的短查询 * @property {string[]} lexicalTerms - MiniSearch 查询词 * @property {Set} _lexicon - 实体词典(内部使用) * @property {Map} _displayMap - 标准化→原词形映射(内部使用) */ // ───────────────────────────────────────────────────────────────────────── // 阶段 1:构建 QueryBundle_v0 // ───────────────────────────────────────────────────────────────────────── /** * 构建初始查询包 * * @param {object[]} lastMessages - 最近 K=2 条消息 * @param {string|null} pendingUserMessage - 用户刚输入但未进 chat 的消息 * @param {object|null} store - getSummaryStore() 返回值(可选,内部会自动获取) * @param {object|null} context - { name1, name2 }(可选,内部会自动获取) * @returns {QueryBundle} */ export function buildQueryBundle(lastMessages, pendingUserMessage, store = null, context = null) { // 自动获取 store 和 context if (!store) store = getSummaryStore(); if (!context) { const ctx = getContext(); context = { name1: ctx.name1, name2: ctx.name2 }; } // 1. 构建实体词典 const lexicon = buildEntityLexicon(store, context); const displayMap = buildDisplayNameMap(store, context); // 2. 清洗消息文本 const dialogueLines = []; const allCleanText = []; for (const m of (lastMessages || [])) { const speaker = m.is_user ? (context.name1 || '用户') : (m.name || context.name2 || '角色'); const clean = cleanMessageText(m.mes || ''); if (clean) { // ★ 修复 A:不使用楼层号,embedding 模型不需要 dialogueLines.push(`${speaker}: ${truncate(clean, DIALOGUE_MAX_CHARS)}`); allCleanText.push(clean); } } // 3. 处理 pendingUserMessage let pendingClean = ''; if (pendingUserMessage) { pendingClean = cleanMessageText(pendingUserMessage); if (pendingClean) { allCleanText.push(pendingClean); } } // 4. 提取焦点实体 const combinedText = allCleanText.join(' '); const focusEntities = extractEntitiesFromText(combinedText, lexicon, displayMap); // 5. 构建 queryText_v0 const queryParts = []; if (focusEntities.length > 0) { queryParts.push(`[ENTITIES]\n${focusEntities.join('\n')}`); } if (dialogueLines.length > 0) { queryParts.push(`[DIALOGUE]\n${dialogueLines.join('\n')}`); } if (pendingClean) { queryParts.push(`[PENDING_USER]\n${truncate(pendingClean, PENDING_MAX_CHARS)}`); } const queryText_v0 = queryParts.join('\n\n'); // 6. 构建 rerankQuery(短版) const rerankParts = []; if (focusEntities.length > 0) { rerankParts.push(focusEntities.join(' ')); } for (const m of (lastMessages || [])) { const clean = cleanMessageText(m.mes || ''); if (clean) { rerankParts.push(truncate(clean, RERANK_SNIPPET_CHARS)); } } if (pendingClean) { rerankParts.push(truncate(pendingClean, RERANK_SNIPPET_CHARS)); } const rerankQuery = truncate(rerankParts.join('\n'), RERANK_QUERY_MAX_CHARS); // 7. 构建 lexicalTerms const entityTerms = focusEntities.map(e => e.toLowerCase()); const textTerms = extractKeyTerms(combinedText); // 合并去重:实体优先 const termSet = new Set(entityTerms); for (const t of textTerms) { if (termSet.size >= LEXICAL_TERMS_MAX) break; termSet.add(t); } const lexicalTerms = Array.from(termSet); return { focusEntities, queryText_v0, queryText_v1: null, rerankQuery, lexicalTerms, _lexicon: lexicon, _displayMap: displayMap, }; } // ───────────────────────────────────────────────────────────────────────── // 阶段 3:Query Refinement(用第一轮召回结果增强) // ───────────────────────────────────────────────────────────────────────── /** * 用第一轮召回结果增强 QueryBundle * * 原地修改 bundle: * - queryText_v1 = queryText_v0 + [MEMORY_HINTS] * - focusEntities 可能扩展(从 anchorHits 的 subject/object 中补充) * - rerankQuery 追加 memory hints 关键词 * - lexicalTerms 追加 memory hints 关键词 * * @param {QueryBundle} bundle - 原始查询包 * @param {object[]} anchorHits - 第一轮 L0 命中(按相似度降序) * @param {object[]} eventHits - 第一轮 L2 命中(按相似度降序) */ export function refineQueryBundle(bundle, anchorHits, eventHits) { const hints = []; // 1. 从 top anchorHits 提取 memory hints const topAnchors = (anchorHits || []).slice(0, MEMORY_HINT_ATOMS_MAX); for (const hit of topAnchors) { const semantic = hit.atom?.semantic || ''; if (semantic) { hints.push(truncate(semantic, MEMORY_HINT_MAX_CHARS)); } } // 2. 从 top eventHits 提取 memory hints const topEvents = (eventHits || []).slice(0, MEMORY_HINT_EVENTS_MAX); for (const hit of topEvents) { const ev = hit.event || {}; const title = String(ev.title || '').trim(); const summary = cleanSummary(ev.summary); const line = title && summary ? `${title}: ${summary}` : title || summary; if (line) { hints.push(truncate(line, MEMORY_HINT_MAX_CHARS)); } } // 3. 构建 queryText_v1 if (hints.length > 0) { bundle.queryText_v1 = bundle.queryText_v0 + `\n\n[MEMORY_HINTS]\n${hints.join('\n')}`; } else { bundle.queryText_v1 = bundle.queryText_v0; } // 4. 从 anchorHits 补充 focusEntities const lexicon = bundle._lexicon; const displayMap = bundle._displayMap; if (lexicon && topAnchors.length > 0) { const existingSet = new Set(bundle.focusEntities.map(e => e.toLowerCase())); for (const hit of topAnchors) { const atom = hit.atom; if (!atom) continue; // 检查 subject 和 object for (const field of [atom.subject, atom.object]) { if (!field) continue; const norm = String(field).normalize('NFKC').replace(/[\u200B-\u200D\uFEFF]/g, '').trim().toLowerCase(); if (norm.length >= 2 && lexicon.has(norm) && !existingSet.has(norm)) { existingSet.add(norm); const display = displayMap?.get(norm) || field; bundle.focusEntities.push(display); } } } } // 5. 增强 rerankQuery if (hints.length > 0) { const hintKeywords = extractKeyTerms(hints.join(' '), 5); if (hintKeywords.length > 0) { const addition = hintKeywords.join(' '); bundle.rerankQuery = truncate( bundle.rerankQuery + '\n' + addition, RERANK_QUERY_MAX_CHARS ); } } // 6. 增强 lexicalTerms if (hints.length > 0) { const hintTerms = extractKeyTerms(hints.join(' '), 5); const termSet = new Set(bundle.lexicalTerms); for (const t of hintTerms) { if (termSet.size >= LEXICAL_TERMS_MAX) break; if (!termSet.has(t)) { termSet.add(t); bundle.lexicalTerms.push(t); } } } }