X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update story summary and lint fixes

Browse Source
This commit is contained in:
bielie
2026-02-08 12:22:45 +08:00
parent 56e30bfe02
commit d3d818da6a
15 changed files with 2479 additions and 852 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
modules/story-summary/vector/llm/atom-extraction.js
View File

@@ -1,5 +1,5 @@
// ============================================================================
// atom-extraction.js - 30并发 + 首批错开 + 取消支持 + 进度回调
// ============================================================================
// atom-extraction.js - L0 叙事锚点提取（三层 themes 版）
// ============================================================================
import { callLLM, parseJson } from './llm-service.js';
@@ -12,7 +12,7 @@ const CONCURRENCY = 10;
const RETRY_COUNT = 2;
const RETRY_DELAY = 500;
const DEFAULT_TIMEOUT = 20000;
const STAGGER_DELAY = 80; // 首批错开延迟（ms）
const STAGGER_DELAY = 80;
let batchCancelled = false;
@@ -24,49 +24,150 @@ export function isBatchCancelled() {
return batchCancelled;
}
const SYSTEM_PROMPT = `你是叙事锚点提取器。从一轮对话（用户发言+角色回复）中提取4-8个关键锚点。
// ============================================================================
// L0 提取 Prompt（三层 themes）
// ============================================================================
const SYSTEM_PROMPT = `你是叙事锚点提取器。从一轮对话中提取4-8个关键锚点，用于后续语义检索。
输入格式：
<round>
<user>...</user>
<assistant>...</assistant>
<user name="用户名">...</user>
<assistant name="角色名">...</assistant>
</round>
只输出严格JSON（不要解释，不要前后多余文字）：
{"atoms":[{"t":"类型","s":"主体","v":"值","f":"来源"}]}
只输出严格JSON：
{"atoms":[{"t":"类型","s":"主体","o":"客体","v":"谓词","l":"地点","f":"来源","th":{"fn":[],"pt":[],"kw":[]}}]}
类型（t）：
- emo: 情绪状态（需要s主体）
- loc: 地点/场景
- act: 关键动作（需要s主体）
- rev: 揭示/发现
- ten: 冲突/张力
- dec: 决定/承诺
## 类型（t）
- emo: 情绪状态变化
- act: 关键动作/行为
- rev: 揭示/发现/真相
- dec: 决定/承诺/宣言
- ten: 冲突/张力/对立
- loc: 场景/地点变化
## 字段说明
- s: 主体（必填）
- o: 客体（可空）
- v: 谓词，15字内（必填）
- l: 地点（可空）
- f: "u"=用户 / "a"=角色（必填）
- th: 主题标签（必填，结构化对象）
## th 三层结构
fn（叙事功能）1-2个，枚举：
establish=建立设定 | escalate=升级加剧 | reveal=揭示发现 | challenge=挑战试探
commit=承诺锁定 | conflict=冲突对抗 | resolve=解决收束 | transform=转变逆转
bond=连接羁绊 | break=断裂破坏
pt（互动模式）1-3个，枚举：
power_down=上对下 | power_up=下对上 | power_equal=对等 | power_contest=争夺
asymmetric=信息不对称 | witnessed=有观众 | secluded=隔绝私密
ritual=仪式正式 | routine=日常惯例 | triangular=三方介入
kw（具体关键词）1-3个，自由格式
## 示例输出
{"atoms":[
{"t":"act","s":"艾拉","o":"古龙","v":"用圣剑刺穿心脏","l":"火山口","f":"a",
"th":{"fn":["commit"],"pt":["power_down","ritual"],"kw":["战斗","牺牲"]}},
{"t":"emo","s":"林夏","o":"陆远","v":"意识到自己喜欢他","l":"","f":"a",
"th":{"fn":["reveal","escalate"],"pt":["asymmetric","secluded"],"kw":["心动","暗恋"]}},
{"t":"dec","s":"凯尔","o":"王国","v":"放弃王位继承权","l":"王座厅","f":"a",
"th":{"fn":["commit","break"],"pt":["ritual","witnessed"],"kw":["抉择","自由"]}},
{"t":"rev","s":"","o":"","v":"管家其实是间谍","l":"","f":"a",
"th":{"fn":["reveal"],"pt":["asymmetric"],"kw":["背叛","真相"]}},
{"t":"ten","s":"兄弟二人","o":"","v":"为遗产反目","l":"","f":"a",
"th":{"fn":["conflict","break"],"pt":["power_contest"],"kw":["冲突","亲情破裂"]}}
]}
规则：
- s: 主体（谁）
- v: 简洁值，10字内
- f: "u"=用户发言中, "a"=角色回复中
- 只提取对未来检索有价值的锚点
- 无明显锚点返回空数组`;
- fn 回答"这在故事里推动了什么"
- pt 回答"这是什么结构的互动"
- kw 用于细粒度检索
- 无明显锚点时返回 {"atoms":[]}`;
const JSON_PREFILL = '{"atoms":[';
// ============================================================================
// Semantic 构建
// ============================================================================
function buildSemantic(atom, userName, aiName) {
const speaker = atom.f === 'u' ? userName : aiName;
const s = atom.s || speaker;
const type = atom.t || 'act';
const subject = atom.s || (atom.f === 'u' ? userName : aiName);
const object = atom.o || '';
const verb = atom.v || '';
const location = atom.l || '';
// 三层 themes 合并
const th = atom.th || {};
const tags = [
...(Array.isArray(th.fn) ? th.fn : []),
...(Array.isArray(th.pt) ? th.pt : []),
...(Array.isArray(th.kw) ? th.kw : []),
].filter(Boolean);
switch (atom.t) {
case 'emo': return `${s}感到${atom.v}`;
case 'loc': return `场景：${atom.v}`;
case 'act': return `${s}${atom.v}`;
case 'rev': return `揭示：${atom.v}`;
case 'ten': return `冲突：${atom.v}`;
case 'dec': return `${s}决定${atom.v}`;
default: return `${s} ${atom.v}`;
const typePart = `<${type}>`;
const themePart = tags.length > 0 ? ` [${tags.join('/')}]` : '';
const locPart = location ? ` 在${location}` : '';
const objPart = object ? ` -> ${object}` : '';
let semantic = '';
switch (type) {
case 'emo':
semantic = object
? `${typePart} ${subject} -> ${verb} (对${object})${locPart}`
: `${typePart} ${subject} -> ${verb}${locPart}`;
break;
case 'act':
semantic = `${typePart} ${subject} -> ${verb}${objPart}${locPart}`;
break;
case 'rev':
semantic = object
? `${typePart} 揭示: ${verb} (关于${object})${locPart}`
: `${typePart} 揭示: ${verb}${locPart}`;
break;
case 'dec':
semantic = object
? `${typePart} ${subject} -> ${verb} (对${object})${locPart}`
: `${typePart} ${subject} -> ${verb}${locPart}`;
break;
case 'ten':
semantic = object
? `${typePart} ${subject} <-> ${object}: ${verb}${locPart}`
: `${typePart} ${subject}: ${verb}${locPart}`;
break;
case 'loc':
semantic = location
? `${typePart} 场景: ${location} - ${verb}`
: `${typePart} 场景: ${verb}`;
break;
default:
semantic = `${typePart} ${subject} -> ${verb}${objPart}${locPart}`;
}
return semantic + themePart;
}
// ============================================================================
// 睡眠工具
// ============================================================================
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
// ============================================================================
// 单轮提取（带重试）
// ============================================================================
async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options = {}) {
const { timeout = DEFAULT_TIMEOUT } = options;
@@ -86,8 +187,6 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
const input = `<round>\n${parts.join('\n')}\n</round>`;
xbLog.info(MODULE_ID, `floor ${aiFloor} 发送输入 len=${input.length}`);
for (let attempt = 0; attempt <= RETRY_COUNT; attempt++) {
if (batchCancelled) return [];
@@ -95,16 +194,15 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
const response = await callLLM([
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: input },
{ role: 'assistant', content: '收到，开始提取并仅输出 JSON。' },
{ role: 'assistant', content: JSON_PREFILL },
], {
temperature: 0.2,
max_tokens: 500,
max_tokens: 1000,
timeout,
});
const rawText = String(response || '');
if (!rawText.trim()) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：响应为空`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
@@ -112,11 +210,13 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
return null;
}
const fullJson = JSON_PREFILL + rawText;
let parsed;
try {
parsed = parseJson(rawText);
parsed = parseJson(fullJson);
} catch (e) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：JSON 异常`);
xbLog.warn(MODULE_ID, `floor ${aiFloor} JSON解析失败`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
@@ -125,8 +225,6 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
}
if (!parsed?.atoms || !Array.isArray(parsed.atoms)) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} atoms 缺失，raw="${rawText.slice(0, 300)}"`);
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：atoms 缺失`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
@@ -141,20 +239,20 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
floor: aiFloor,
type: a.t,
subject: a.s || null,
value: String(a.v).slice(0, 30),
object: a.o || null,
value: String(a.v).slice(0, 50),
location: a.l || null,
source: a.f === 'u' ? 'user' : 'ai',
themes: a.th || { fn: [], pt: [], kw: [] },
semantic: buildSemantic(a, userName, aiName),
}));
if (!filtered.length) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} atoms 为空，raw="${rawText.slice(0, 300)}"`);
}
return filtered;
} catch (e) {
if (batchCancelled) return null;
if (attempt < RETRY_COUNT) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 第${attempt + 1}次失败，重试...`, e?.message);
await sleep(RETRY_DELAY * (attempt + 1));
continue;
}
@@ -166,18 +264,14 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
return null;
}
/**
* 单轮配对提取（增量时使用）
*/
export async function extractAtomsForRound(userMessage, aiMessage, aiFloor, options = {}) {
return extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options);
}
/**
* 批量提取（首批 staggered 启动）
* @param {Array} chat
* @param {Function} onProgress - (current, total, failed) => void
*/
// ============================================================================
// 批量提取
// ============================================================================
export async function batchExtractAtoms(chat, onProgress) {
if (!chat?.length) return [];
@@ -198,14 +292,10 @@ export async function batchExtractAtoms(chat, onProgress) {
let failed = 0;
for (let i = 0; i < pairs.length; i += CONCURRENCY) {
if (batchCancelled) {
xbLog.info(MODULE_ID, `批量提取已取消 (${completed}/${pairs.length})`);
break;
}
if (batchCancelled) break;
const batch = pairs.slice(i, i + CONCURRENCY);
// ★ 首批 staggered 启动：错开 80ms 发送
if (i === 0) {
const promises = batch.map((pair, idx) => (async () => {
await sleep(idx * STAGGER_DELAY);
@@ -213,10 +303,15 @@ export async function batchExtractAtoms(chat, onProgress) {
if (batchCancelled) return;
try {
const atoms = await extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT });
const atoms = await extractAtomsForRoundWithRetry(
pair.userMsg,
pair.aiMsg,
pair.aiFloor,
{ timeout: DEFAULT_TIMEOUT }
);
if (atoms?.length) {
allAtoms.push(...atoms);
} else {
} else if (atoms === null) {
failed++;
}
} catch {
@@ -227,14 +322,18 @@ export async function batchExtractAtoms(chat, onProgress) {
})());
await Promise.all(promises);
} else {
// 后续批次正常并行
const promises = batch.map(pair =>
extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT })
extractAtomsForRoundWithRetry(
pair.userMsg,
pair.aiMsg,
pair.aiFloor,
{ timeout: DEFAULT_TIMEOUT }
)
.then(atoms => {
if (batchCancelled) return;
if (atoms?.length) {
allAtoms.push(...atoms);
} else {
} else if (atoms === null) {
failed++;
}
completed++;
@@ -251,14 +350,12 @@ export async function batchExtractAtoms(chat, onProgress) {
await Promise.all(promises);
}
// 批次间隔
if (i + CONCURRENCY < pairs.length && !batchCancelled) {
await sleep(30);
}
}
const status = batchCancelled ? '已取消' : '完成';
xbLog.info(MODULE_ID, `批量提取${status}: ${allAtoms.length} atoms, ${completed}/${pairs.length}, ${failed} 失败`);
xbLog.info(MODULE_ID, `批量提取完成: ${allAtoms.length} atoms, ${failed} 失败`);
return allAtoms;
}

25
modules/story-summary/vector/llm/llm-service.js
View File

@@ -1,14 +1,13 @@
// ═══════════════════════════════════════════════════════════════════════════
// vector/llm/llm-service.js
// ═══════════════════════════════════════════════════════════════════════════
// vector/llm/llm-service.js - 修复 prefill 传递方式
// ═══════════════════════════════════════════════════════════════════════════
import { xbLog } from '../../../../core/debug-core.js';
import { getVectorConfig } from '../../data/config.js';
const MODULE_ID = 'vector-llm-service';
const SILICONFLOW_API_URL = 'https://api.siliconflow.cn';
const SILICONFLOW_API_URL = 'https://api.siliconflow.cn/v1';
const DEFAULT_L0_MODEL = 'Qwen/Qwen3-8B';
// 唯一 ID 计数器
let callCounter = 0;
function getStreamingModule() {
@@ -30,6 +29,7 @@ function b64UrlEncode(str) {
/**
* 统一LLM调用 - 走酒馆后端（非流式）
* 修复：assistant prefill 用 bottomassistant 参数传递
*/
export async function callLLM(messages, options = {}) {
const {
@@ -46,9 +46,16 @@ export async function callLLM(messages, options = {}) {
throw new Error('L0 requires siliconflow API key');
}
const top64 = b64UrlEncode(JSON.stringify(messages));
// ★ 关键修复：分离 assistant prefill
let topMessages = [...messages];
let assistantPrefill = '';
if (topMessages.length > 0 && topMessages[topMessages.length - 1]?.role === 'assistant') {
const lastMsg = topMessages.pop();
assistantPrefill = lastMsg.content || '';
}
// 每次调用用唯一 ID，避免 session 冲突
const top64 = b64UrlEncode(JSON.stringify(topMessages));
const uniqueId = generateUniqueId('l0');
const args = {
@@ -64,8 +71,12 @@ export async function callLLM(messages, options = {}) {
model: DEFAULT_L0_MODEL,
};
// ★ 用 bottomassistant 参数传递 prefill
if (assistantPrefill) {
args.bottomassistant = assistantPrefill;
}
try {
// 非流式直接返回结果
const result = await mod.xbgenrawCommand(args, '');
return String(result ?? '');
} catch (e) {

313
modules/story-summary/vector/llm/query-expansion.js
View File

@@ -1,52 +1,228 @@
// ═══════════════════════════════════════════════════════════════════════════
// query-expansion.js - 完整输入，不截断
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
// query-expansion.js - 检索查询生成器（三层 themes 版）
// ============================================================================
import { callLLM, parseJson } from './llm-service.js';
import { xbLog } from '../../../../core/debug-core.js';
import { filterText } from '../utils/text-filter.js';
import { getContext } from '../../../../../../../extensions.js';
import { getSummaryStore } from '../../data/store.js';
const MODULE_ID = 'query-expansion';
const SESSION_ID = 'xb6';
const SYSTEM_PROMPT = `你是检索词生成器。根据最近对话，输出用于检索历史剧情的关键词。
// ============================================================================
// 系统提示词
// ============================================================================
只输出JSON：
{"e":["显式人物/地名"],"i":["隐含人物/情绪/话题"],"q":["检索短句"]}
const SYSTEM_PROMPT = `你是检索查询生成器。根据当前对话上下文，生成用于检索历史剧情的查询语句。
规则：
- e: 对话中明确提到的人名/地名，1-4个
- i: 推断出的相关人物/情绪/话题，1-5个
- q: 用于向量检索的短句，2-3个，每个15字内
- 关注：正在讨论什么、涉及谁、情绪氛围`;
## 输出格式（严格JSON）
{
"focus": ["焦点人物"],
"fn": ["叙事功能"],
"pt": ["互动模式"],
"kw": ["关键词"],
"queries": ["DSL查询语句"]
}
/**
* Query Expansion
* @param {Array} messages - 完整消息数组（最后2-3轮）
*/
export async function expandQuery(messages, options = {}) {
const { timeout = 6000 } = options;
## fn（叙事功能）枚举
establish=建立设定 | escalate=升级加剧 | reveal=揭示发现 | challenge=挑战试探
commit=承诺锁定 | conflict=冲突对抗 | resolve=解决收束 | transform=转变逆转
bond=连接羁绊 | break=断裂破坏
if (!messages?.length) {
return { entities: [], implicit: [], queries: [] };
## pt（互动模式）枚举
power_down=上对下 | power_up=下对上 | power_equal=对等 | power_contest=争夺
asymmetric=信息不对称 | witnessed=有观众 | secluded=隔绝私密
ritual=仪式正式 | routine=日常惯例 | triangular=三方介入
## DSL 查询格式
- <act> 主体 -> 动作 (-> 客体)? (在地点)?
- <emo> 主体 -> 情绪 (对客体)?
- <dec> 主体 -> 决定/承诺 (对客体)?
- <rev> 揭示: 内容 (关于客体)?
- <ten> 主体A <-> 主体B: 冲突内容
- <loc> 场景: 地点/状态
## 规则
- focus: 核心人物，1-4个
- fn: 当前对话涉及的叙事功能，1-3个
- pt: 当前对话涉及的互动模式，1-3个
- kw: 具体关键词，1-4个
- queries: 2-4条 DSL 查询
## 示例
输入：艾拉说"那把剑...我记得它的重量，在火山口的时候"
输出：
{
"focus": ["艾拉", "古龙"],
"fn": ["commit", "bond"],
"pt": ["power_down", "ritual"],
"kw": ["圣剑", "战斗", "火山口"],
"queries": [
"<act> 艾拉 -> 战斗/使用圣剑 -> 古龙 [commit/power_down]",
"<loc> 场景: 火山口 [ritual]",
"<emo> 艾拉 -> 牺牲/决绝 [commit]"
]
}`;
// ============================================================================
// 上下文构建
// ============================================================================
function getCharacterContext() {
const context = getContext();
const char = context.characters?.[context.characterId];
if (!char) {
return { name: '', description: '', personality: '' };
}
// 完整格式化，不截断
const input = messages.map(m => {
const speaker = m.is_user ? '用户' : (m.name || '角色');
const text = filterText(m.mes || '').trim();
return `【${speaker}】\n${text}`;
}).join('\n\n');
return {
name: char.name || '',
description: (char.description || '').slice(0, 500),
personality: (char.personality || '').slice(0, 300),
};
}
function getPersonaContext() {
const context = getContext();
if (typeof window !== 'undefined' && window.power_user?.persona_description) {
return String(window.power_user.persona_description).slice(0, 500);
}
if (context.persona_description) {
return String(context.persona_description).slice(0, 500);
}
return '';
}
function getRecentEvents(count = 8) {
const store = getSummaryStore();
const events = store?.json?.events || [];
return events
.slice(-count)
.map(e => {
const time = e.timeLabel || '';
const title = e.title || '';
const participants = (e.participants || []).join('/');
const summary = (e.summary || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').slice(0, 80);
return time
? `[${time}] ${title || participants}: ${summary}`
: `${title || participants}: ${summary}`;
});
}
function getRelevantArcs(focusHint = []) {
const store = getSummaryStore();
const arcs = store?.json?.arcs || [];
if (!arcs.length) return [];
const hintSet = new Set(focusHint.map(s => String(s).toLowerCase()));
const sorted = [...arcs].sort((a, b) => {
const aHit = hintSet.has(String(a.name || '').toLowerCase()) ? 1 : 0;
const bHit = hintSet.has(String(b.name || '').toLowerCase()) ? 1 : 0;
return bHit - aHit;
});
return sorted.slice(0, 4).map(a => {
const progress = Math.round((a.progress || 0) * 100);
return `${a.name}: ${a.trajectory || '未知状态'} (${progress}%)`;
});
}
function extractNamesFromMessages(messages) {
const names = new Set();
for (const m of messages) {
if (m.name) names.add(m.name);
}
const text = messages.map(m => m.mes || '').join(' ');
const namePattern = /[\u4e00-\u9fff]{2,4}/g;
const matches = text.match(namePattern) || [];
const freq = {};
for (const name of matches) {
freq[name] = (freq[name] || 0) + 1;
}
Object.entries(freq)
.filter(([, count]) => count >= 2)
.forEach(([name]) => names.add(name));
return Array.from(names).slice(0, 6);
}
// ============================================================================
// 主函数
// ============================================================================
export async function expandQuery(messages, options = {}) {
const { pendingUserMessage = null, timeout = 6000 } = options;
if (!messages?.length && !pendingUserMessage) {
return { focus: [], fn: [], pt: [], kw: [], queries: [] };
}
const T0 = performance.now();
const character = getCharacterContext();
const persona = getPersonaContext();
const nameHints = extractNamesFromMessages(messages || []);
const recentEvents = getRecentEvents(8);
const arcs = getRelevantArcs(nameHints);
const dialogueParts = [];
for (const m of (messages || [])) {
const speaker = m.is_user ? '用户' : (m.name || '角色');
const text = filterText(m.mes || '').trim();
if (text) {
dialogueParts.push(`【${speaker}】\n${text.slice(0, 400)}`);
}
}
if (pendingUserMessage) {
dialogueParts.push(`【用户（刚输入）】\n${filterText(pendingUserMessage).slice(0, 400)}`);
}
const inputParts = [];
if (character.name) {
inputParts.push(`## 当前角色\n${character.name}: ${character.description || character.personality || '无描述'}`);
}
if (persona) {
inputParts.push(`## 用户人设\n${persona}`);
}
if (recentEvents.length) {
inputParts.push(`## 近期剧情\n${recentEvents.map((e, i) => `${i + 1}. ${e}`).join('\n')}`);
}
if (arcs.length) {
inputParts.push(`## 角色状态\n${arcs.join('\n')}`);
}
inputParts.push(`## 最近对话\n${dialogueParts.join('\n\n')}`);
const input = inputParts.join('\n\n');
try {
const response = await callLLM([
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: input },
], {
temperature: 0.15,
max_tokens: 250,
max_tokens: 500,
timeout,
sessionId: SESSION_ID,
});
@@ -54,49 +230,104 @@ export async function expandQuery(messages, options = {}) {
const parsed = parseJson(response);
if (!parsed) {
xbLog.warn(MODULE_ID, 'JSON解析失败', response?.slice(0, 200));
return { entities: [], implicit: [], queries: [] };
return { focus: [], fn: [], pt: [], kw: [], queries: [] };
}
const result = {
entities: Array.isArray(parsed.e) ? parsed.e.slice(0, 5) : [],
implicit: Array.isArray(parsed.i) ? parsed.i.slice(0, 6) : [],
queries: Array.isArray(parsed.q) ? parsed.q.slice(0, 4) : [],
focus: Array.isArray(parsed.focus) ? parsed.focus.slice(0, 5) : [],
fn: Array.isArray(parsed.fn) ? parsed.fn.slice(0, 4) : [],
pt: Array.isArray(parsed.pt) ? parsed.pt.slice(0, 4) : [],
kw: Array.isArray(parsed.kw) ? parsed.kw.slice(0, 5) : [],
queries: Array.isArray(parsed.queries) ? parsed.queries.slice(0, 5) : [],
};
xbLog.info(MODULE_ID, `完成 (${Math.round(performance.now() - T0)}ms) e=${result.entities.length} i=${result.implicit.length} q=${result.queries.length}`);
xbLog.info(MODULE_ID, `完成 (${Math.round(performance.now() - T0)}ms) focus=[${result.focus.join(',')}] fn=[${result.fn.join(',')}]`);
return result;
} catch (e) {
xbLog.error(MODULE_ID, '调用失败', e);
return { entities: [], implicit: [], queries: [] };
return { focus: [], fn: [], pt: [], kw: [], queries: [] };
}
}
// ============================================================================
// 缓存
// ============================================================================
const cache = new Map();
const CACHE_TTL = 300000;
function hashMessages(messages) {
const text = messages.slice(-2).map(m => (m.mes || '').slice(0, 100)).join('|');
function hashMessages(messages, pending = '') {
const text = (messages || [])
.slice(-3)
.map(m => (m.mes || '').slice(0, 100))
.join('|') + '|' + (pending || '').slice(0, 100);
let h = 0;
for (let i = 0; i < text.length; i++) h = ((h << 5) - h + text.charCodeAt(i)) | 0;
for (let i = 0; i < text.length; i++) {
h = ((h << 5) - h + text.charCodeAt(i)) | 0;
}
return h.toString(36);
}
export async function expandQueryCached(messages, options = {}) {
const key = hashMessages(messages);
const key = hashMessages(messages, options.pendingUserMessage);
const cached = cache.get(key);
if (cached && Date.now() - cached.time < CACHE_TTL) return cached.result;
if (cached && Date.now() - cached.time < CACHE_TTL) {
return cached.result;
}
const result = await expandQuery(messages, options);
if (result.entities.length || result.queries.length) {
if (cache.size > 50) cache.delete(cache.keys().next().value);
if (result.focus.length || result.queries.length) {
if (cache.size > 50) {
cache.delete(cache.keys().next().value);
}
cache.set(key, { result, time: Date.now() });
}
return result;
}
// ============================================================================
// 辅助函数：构建检索文本
// ============================================================================
/**
* 将 expansion 结果转换为检索文本
* 三层 themes 自然拼入，让向量自动编码
*/
export function buildSearchText(expansion) {
return [...(expansion.entities || []), ...(expansion.implicit || []), ...(expansion.queries || [])]
.filter(Boolean).join(' ');
const parts = [];
// focus 人物
if (expansion.focus?.length) {
parts.push(expansion.focus.join(' '));
}
// fn + pt + kw 合并为标签
const tags = [
...(expansion.fn || []),
...(expansion.pt || []),
...(expansion.kw || []),
].filter(Boolean);
if (tags.length) {
parts.push(`[${tags.join('/')}]`);
}
// queries
if (expansion.queries?.length) {
parts.push(...expansion.queries);
}
return parts.filter(Boolean).join(' ').slice(0, 1500);
}
/**
* 提取实体列表（兼容旧接口）
*/
export function getEntitiesFromExpansion(expansion) {
return expansion?.focus || [];
}

184
modules/story-summary/vector/llm/reranker.js Normal file
View File

@@ -0,0 +1,184 @@
// ═══════════════════════════════════════════════════════════════════════════
// Reranker - 硅基 bge-reranker-v2-m3
// 对候选文档进行精排，过滤与 query 不相关的内容
// ═══════════════════════════════════════════════════════════════════════════
import { xbLog } from '../../../../core/debug-core.js';
import { getApiKey } from './siliconflow.js';
const MODULE_ID = 'reranker';
const RERANK_URL = 'https://api.siliconflow.cn/v1/rerank';
const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3';
const DEFAULT_TIMEOUT = 15000;
const MAX_DOCUMENTS = 100; // API 限制
/**
* 对文档列表进行 Rerank 精排
*
* @param {string} query - 查询文本
* @param {Array<string>} documents - 文档文本列表
* @param {object} options - 选项
* @param {number} options.topN - 返回前 N 个结果，默认 40
* @param {number} options.timeout - 超时时间，默认 15000ms
* @param {AbortSignal} options.signal - 取消信号
* @returns {Promise<Array<{index: number, relevance_score: number}>>} 排序后的结果
*/
export async function rerank(query, documents, options = {}) {
const { topN = 40, timeout = DEFAULT_TIMEOUT, signal } = options;
if (!query?.trim()) {
xbLog.warn(MODULE_ID, 'query 为空，跳过 rerank');
return documents.map((_, i) => ({ index: i, relevance_score: 0.5 }));
}
if (!documents?.length) {
return [];
}
const key = getApiKey();
if (!key) {
xbLog.warn(MODULE_ID, '未配置 API Key，跳过 rerank');
return documents.map((_, i) => ({ index: i, relevance_score: 0.5 }));
}
// 截断超长文档列表
const truncatedDocs = documents.slice(0, MAX_DOCUMENTS);
if (documents.length > MAX_DOCUMENTS) {
xbLog.warn(MODULE_ID, `文档数 ${documents.length} 超过限制 ${MAX_DOCUMENTS}，已截断`);
}
// 过滤空文档，记录原始索引
const validDocs = [];
const indexMap = []; // validDocs index → original index
for (let i = 0; i < truncatedDocs.length; i++) {
const text = String(truncatedDocs[i] || '').trim();
if (text) {
validDocs.push(text);
indexMap.push(i);
}
}
if (!validDocs.length) {
xbLog.warn(MODULE_ID, '无有效文档，跳过 rerank');
return [];
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const T0 = performance.now();
const response = await fetch(RERANK_URL, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: RERANK_MODEL,
query: query.slice(0, 1000), // 限制 query 长度
documents: validDocs,
top_n: Math.min(topN, validDocs.length),
return_documents: false,
}),
signal: signal || controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
const errorText = await response.text().catch(() => '');
throw new Error(`Rerank API ${response.status}: ${errorText.slice(0, 200)}`);
}
const data = await response.json();
const results = data.results || [];
// 映射回原始索引
const mapped = results.map(r => ({
index: indexMap[r.index],
relevance_score: r.relevance_score ?? 0,
}));
const elapsed = Math.round(performance.now() - T0);
xbLog.info(MODULE_ID, `Rerank 完成: ${validDocs.length} docs → ${results.length} selected (${elapsed}ms)`);
return mapped;
} catch (e) {
clearTimeout(timeoutId);
if (e?.name === 'AbortError') {
xbLog.warn(MODULE_ID, 'Rerank 超时或取消');
} else {
xbLog.error(MODULE_ID, 'Rerank 失败', e);
}
// 降级：返回原顺序，分数均匀分布
return documents.slice(0, topN).map((_, i) => ({
index: i,
relevance_score: 1 - (i / documents.length) * 0.5,
}));
}
}
/**
* 对 chunk 对象列表进行 Rerank
*
* @param {string} query - 查询文本
* @param {Array<object>} chunks - chunk 对象列表，需要有 text 字段
* @param {object} options - 选项
* @returns {Promise<Array<object>>} 排序后的 chunk 列表，带 _rerankScore 字段
*/
export async function rerankChunks(query, chunks, options = {}) {
const { topN = 40, minScore = 0.1 } = options;
if (!chunks?.length) return [];
if (chunks.length <= topN) {
// 数量不超限，仍然 rerank 以获取分数，但不过滤
const texts = chunks.map(c => c.text || c.semantic || '');
const results = await rerank(query, texts, { topN: chunks.length, ...options });
const scoreMap = new Map(results.map(r => [r.index, r.relevance_score]));
return chunks.map((c, i) => ({
...c,
_rerankScore: scoreMap.get(i) ?? 0.5,
})).sort((a, b) => b._rerankScore - a._rerankScore);
}
const texts = chunks.map(c => c.text || c.semantic || '');
const results = await rerank(query, texts, { topN, ...options });
// 过滤低分 + 排序
const selected = results
.filter(r => r.relevance_score >= minScore)
.sort((a, b) => b.relevance_score - a.relevance_score)
.map(r => ({
...chunks[r.index],
_rerankScore: r.relevance_score,
}));
return selected;
}
/**
* 测试 Rerank 服务连接
*/
export async function testRerankService() {
const key = getApiKey();
if (!key) {
throw new Error('请配置硅基 API Key');
}
try {
const results = await rerank('测试查询', ['测试文档1', '测试文档2'], { topN: 2 });
return {
success: true,
message: `连接成功，返回 ${results.length} 个结果`,
};
} catch (e) {
throw new Error(`连接失败: ${e.message}`);
}
}