chore: update story summary and lint fixes

This commit is contained in:
2026-02-08 12:22:45 +08:00
parent 56e30bfe02
commit d3d818da6a
15 changed files with 2479 additions and 852 deletions

View File

@@ -11,6 +11,8 @@ const PROVIDER_MAP = {
custom: "custom",
};
const JSON_PREFILL = '{"mindful_prelude": {';
const LLM_PROMPT_CONFIG = {
topSystem: `Story Analyst: This task involves narrative comprehension and structured incremental summarization, representing creative story analysis at the intersection of plot tracking and character development. As a story analyst, you will conduct systematic evaluation of provided dialogue content to generate structured incremental summary data.
[Read the settings for this task]
@@ -161,6 +163,16 @@ Before generating, observe the USER and analyze carefully:
- What arc PROGRESS was made?
- What facts changed? (status/position/ownership/relationships)
## factUpdates 规则
- 目的: 纠错 & 世界一致性约束,只记录硬性事实
- s+p 为键,相同键会覆盖旧值
- isState: true=核心约束(位置/身份/生死/关系)false=有容量上限会被清理
- 关系类: p="对X的看法"trend 必填(破裂|厌恶|反感|陌生|投缘|亲密|交融)
- 删除: {s, p, retracted: true},不需要 o 字段
- 更新: {s, p, o, isState, trend?}
- 谓词规范化: 复用已有谓词,不要发明同义词
- 只输出有变化的条目,确保少、硬、稳定
## Output Format
\`\`\`json
{
@@ -170,7 +182,7 @@ Before generating, observe the USER and analyze carefully:
"fact_changes": "识别到的事实变化概述"
},
"keywords": [
{"text": "综合已有+新内容的全关键词(5-10个)", "weight": "核心|重要|一般"}
{"text": "综合历史+新内容的全剧情关键词(5-10个)", "weight": "核心|重要|一般"}
],
"events": [
{
@@ -178,7 +190,7 @@ Before generating, observe the USER and analyze carefully:
"title": "地点·事件标题",
"timeLabel": "时间线标签(如:开场、第二天晚上)",
"summary": "1-2句话描述涵盖丰富信息素末尾标注楼层(#X-Y)",
"participants": ["参与角色名"],
"participants": ["参与角色名,不要使用人称代词或别名,只用正式人名"],
"type": "相遇|冲突|揭示|抉择|羁绊|转变|收束|日常",
"weight": "核心|主线|转折|点睛|氛围",
"causedBy": ["evt-12", "evt-14"]
@@ -186,30 +198,15 @@ Before generating, observe the USER and analyze carefully:
],
"newCharacters": ["仅本次首次出现的角色名"],
"arcUpdates": [
{"name": "角色名", "trajectory": "当前阶段描述(15字内)", "progress": 0.0-1.0, "newMoment": "本次新增的关键时刻"}
{"name": "角色名,不要使用人称代词或别名,只用正式人名", "trajectory": "当前阶段描述(15字内)", "progress": 0.0-1.0, "newMoment": "本次新增的关键时刻"}
],
"factUpdates": [
{
"s": "主体",
"p": "谓词(复用已有谓词,避免同义词)",
"o": "当前值",
"isState": true/false,
"trend": "仅关系类:破裂|厌恶|反感|陌生|投缘|亲密|交融"
}
{"s": "主体", "p": "谓词", "o": "当前值", "isState": true, "trend": "仅关系类填"},
{"s": "要删除的主体", "p": "要删除的谓词", "retracted": true}
]
}
\`\`\`
## factUpdates 规则
- 目的: 纠错 & 世界一致性约束,只记录硬性事实
- s+p 为键,相同键会覆盖旧值
- isState: true=核心约束(位置/身份/生死/关系)false=有容量上限会被清理
- 关系类: p="对X的看法"trend 必填
- 删除: 设置 retracted: true
- 谓词规范化: 复用已有谓词,不要发明同义词
- 只输出有变化的条目,确保少、硬、稳定
## CRITICAL NOTES
- events.id 从 evt-{nextEventId} 开始编号
- 仅输出【增量】内容,已有事件绝不重复
@@ -242,7 +239,7 @@ All checks passed. Beginning incremental extraction...
userConfirm: `怎么截断了重新完整生成只输出JSON不要任何其他内容
</Chat_History>`,
assistantPrefill: `非常抱歉现在重新完整生成JSON。`
assistantPrefill: JSON_PREFILL
};
// ═══════════════════════════════════════════════════════════════════════════
@@ -437,5 +434,5 @@ export async function generateSummary(options) {
console.log(rawOutput);
console.groupEnd();
return rawOutput;
return JSON_PREFILL + rawOutput;
}

File diff suppressed because it is too large Load Diff

View File

@@ -257,7 +257,7 @@
}
function updateVectorStats(stats) {
$('vector-atom-count').textContent = stats.stateAtoms || 0;
$('vector-atom-count').textContent = stats.stateVectors || 0;
$('vector-chunk-count').textContent = stats.chunkCount || 0;
$('vector-event-count').textContent = stats.eventVectors || 0;
}
@@ -276,19 +276,36 @@
const pending = stats.pending || 0;
const empty = stats.empty || 0;
const fail = stats.fail || 0;
const atomsCount = stats.atomsCount || 0;
$('anchor-extracted').textContent = extracted;
$('anchor-total').textContent = total;
$('anchor-pending').textContent = pending;
const extra = document.getElementById('anchor-extra');
if (extra) extra.textContent = `${empty} · 失败 ${fail}`;
$('anchor-atoms-count').textContent = atomsCount;
const pendingWrap = $('anchor-pending-wrap');
if (pendingWrap) {
pendingWrap.classList.toggle('hidden', pending === 0);
}
// 显示 empty/fail 信息
const extraWrap = $('anchor-extra-wrap');
const extraSep = $('anchor-extra-sep');
const extra = $('anchor-extra');
if (extraWrap && extra) {
if (empty > 0 || fail > 0) {
const parts = [];
if (empty > 0) parts.push(`${empty}`);
if (fail > 0) parts.push(`失败 ${fail}`);
extra.textContent = parts.join(' · ');
extraWrap.style.display = '';
if (extraSep) extraSep.style.display = '';
} else {
extraWrap.style.display = 'none';
if (extraSep) extraSep.style.display = 'none';
}
}
const emptyWarning = $('vector-empty-l0-warning');
if (emptyWarning) {
emptyWarning.classList.toggle('hidden', extracted > 0);
@@ -966,6 +983,7 @@ function initVectorUI() {
};
}
function setRecallLog(text) {
lastRecallLogText = text || '';
updateRecallLogDisplay();
@@ -974,14 +992,27 @@ function initVectorUI() {
function updateRecallLogDisplay() {
const content = $('recall-log-content');
if (!content) return;
if (lastRecallLogText) {
content.textContent = lastRecallLogText;
content.classList.remove('recall-empty');
} else {
setHtml(content, '<div class="recall-empty">暂无召回日志<br><br>当 AI 生成回复时,系统会自动进行记忆召回。<br>召回日志将显示:<br>• 查询文本<br>• L1 片段匹配结果<br>• L2 事件召回详情<br>• 耗时统计</div>');
setHtml(content, `<div class="recall-empty">
暂无召回日志<br><br>
当 AI 生成回复时,系统会自动进行记忆召回。<br><br>
召回日志将显示:<br>
• [L0] Query Understanding - 意图识别<br>
• [L1] Constraints - 硬约束注入<br>
• [L2] Narrative Retrieval - 事件召回<br>
• [L3] Evidence Assembly - 证据装配<br>
• [L4] Prompt Formatting - 格式化<br>
• [Budget] Token 预算使用情况<br>
• [Quality] 质量指标与潜在问题
</div>`);
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Editor
// ═══════════════════════════════════════════════════════════════════════════

View File

@@ -2873,3 +2873,28 @@ h1 span {
padding: 6px 10px;
}
}
/* ═══════════════════════════════════════════════════════════════════════════
Metrics Log Styling
═══════════════════════════════════════════════════════════════════════════ */
#recall-log-content {
font-family: 'SF Mono', Monaco, Consolas, 'Courier New', monospace;
font-size: 11px;
line-height: 1.5;
white-space: pre;
overflow-x: auto;
tab-size: 4;
}
#recall-log-content .metric-warn {
color: #f59e0b;
}
#recall-log-content .metric-error {
color: #ef4444;
}
#recall-log-content .metric-good {
color: #22c55e;
}

View File

@@ -116,7 +116,6 @@
<div class="sel-trigger" id="char-sel-trigger">
<span id="sel-char-text">选择角色</span>
</div>
<div class="settings-hint" id="anchor-extra" style="margin-top:-6px"></div>
<div class="sel-opts" id="char-sel-opts">
<div class="sel-opt" data-value="">暂无角色</div>
</div>
@@ -425,6 +424,16 @@
<div class="anchor-stat-pending" id="anchor-pending-wrap">
<span>(待提取 <strong id="anchor-pending">0</strong> 楼)</span>
</div>
<span class="anchor-stat-sep">·</span>
<div class="anchor-stat-item">
<span class="anchor-stat-label">L0 Atoms:</span>
<span class="anchor-stat-value"><strong id="anchor-atoms-count">0</strong>
</span>
</div>
<span class="anchor-stat-sep" id="anchor-extra-sep" style="display:none">·</span>
<div class="anchor-stat-item" id="anchor-extra-wrap" style="display:none">
<span id="anchor-extra"></span>
</div>
</div>
<!-- 进度条 -->
@@ -452,7 +461,7 @@
<label>当前聊天向量</label>
<div class="vector-stats" id="vector-stats">
<div class="vector-stat-col">
<span class="vector-stat-label">L0 Atoms:</span>
<span class="vector-stat-label">L0 Vectors:</span>
<span class="vector-stat-value"><strong
id="vector-atom-count">0</strong></span>
</div>

View File

@@ -90,7 +90,7 @@ import { exportVectors, importVectors } from "./vector/storage/vector-io.js";
const MODULE_ID = "storySummary";
const SUMMARY_CONFIG_KEY = "storySummaryPanelConfig";
const iframePath = `${extensionFolderPath}/modules/story-summary/story-summary.html`;
const VALID_SECTIONS = ["keywords", "events", "characters", "arcs", "world"];
const VALID_SECTIONS = ["keywords", "events", "characters", "arcs", "facts"];
const MESSAGE_EVENT = "message";
// ═══════════════════════════════════════════════════════════════════════════
@@ -236,7 +236,6 @@ async function sendVectorStatsToFrame() {
const stats = await getStorageStats(chatId);
const chunkStatus = await getChunkBuildStatus();
const totalMessages = chat?.length || 0;
const stateAtomsCount = getStateAtomsCount();
const stateVectorsCount = await getStateVectorsCount(chatId);
const cfg = getVectorConfig();
@@ -256,7 +255,6 @@ async function sendVectorStatsToFrame() {
builtFloors: chunkStatus.builtFloors,
totalFloors: chunkStatus.totalFloors,
totalMessages,
stateAtoms: stateAtomsCount,
stateVectors: stateVectorsCount,
},
mismatch,
@@ -265,7 +263,8 @@ async function sendVectorStatsToFrame() {
async function sendAnchorStatsToFrame() {
const stats = await getAnchorStats();
postToFrame({ type: "ANCHOR_STATS", stats });
const atomsCount = getStateAtomsCount();
postToFrame({ type: "ANCHOR_STATS", stats: { ...stats, atomsCount } });
}
async function handleAnchorGenerate() {
@@ -290,10 +289,15 @@ async function handleAnchorGenerate() {
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "分析中..." });
try {
// Phase 1: L0 提取 + Phase 2: L0 向量化(在 incrementalExtractAtoms 内部完成)
await incrementalExtractAtoms(chatId, chat, (message, current, total) => {
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current, total, message });
});
// Phase 3: 处理 pending L1 Chunks
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "向量化 L1..." });
await buildIncrementalChunks({ vectorConfig: vectorCfg });
await sendAnchorStatsToFrame();
await sendVectorStatsToFrame();
@@ -1212,9 +1216,11 @@ async function handleChatChanged() {
if (frameReady) {
await sendFrameBaseData(store, newLength);
sendFrameFullData(store, newLength);
sendAnchorStatsToFrame();
sendVectorStatsToFrame();
}
// 检测向量完整性并提醒(仅提醒,不自动操作)
setTimeout(() => checkVectorIntegrityAndWarn(), 2000);
}

View File

@@ -1,5 +1,5 @@
// ============================================================================
// atom-extraction.js - 30并发 + 首批错开 + 取消支持 + 进度回调
// ============================================================================
// atom-extraction.js - L0 叙事锚点提取(三层 themes 版)
// ============================================================================
import { callLLM, parseJson } from './llm-service.js';
@@ -12,7 +12,7 @@ const CONCURRENCY = 10;
const RETRY_COUNT = 2;
const RETRY_DELAY = 500;
const DEFAULT_TIMEOUT = 20000;
const STAGGER_DELAY = 80; // 首批错开延迟ms
const STAGGER_DELAY = 80;
let batchCancelled = false;
@@ -24,49 +24,150 @@ export function isBatchCancelled() {
return batchCancelled;
}
const SYSTEM_PROMPT = `你是叙事锚点提取器。从一轮对话(用户发言+角色回复中提取4-8个关键锚点。
// ============================================================================
// L0 提取 Prompt三层 themes
// ============================================================================
const SYSTEM_PROMPT = `你是叙事锚点提取器。从一轮对话中提取4-8个关键锚点用于后续语义检索。
输入格式:
<round>
<user>...</user>
<assistant>...</assistant>
<user name="用户名">...</user>
<assistant name="角色名">...</assistant>
</round>
只输出严格JSON(不要解释,不要前后多余文字)
{"atoms":[{"t":"类型","s":"主体","v":"值","f":"来源"}]}
只输出严格JSON
{"atoms":[{"t":"类型","s":"主体","o":"客体","v":"谓词","l":"地点","f":"来源","th":{"fn":[],"pt":[],"kw":[]}}]}
类型t
- emo: 情绪状态需要s主体
- loc: 地点/场景
- act: 关键动作需要s主体
- rev: 揭示/发现
- ten: 冲突/张力
- dec: 决定/承诺
## 类型t
- emo: 情绪状态变化
- act: 关键动作/行为
- rev: 揭示/发现/真相
- dec: 决定/承诺/宣言
- ten: 冲突/张力/对立
- loc: 场景/地点变化
## 字段说明
- s: 主体(必填)
- o: 客体(可空)
- v: 谓词15字内必填
- l: 地点(可空)
- f: "u"=用户 / "a"=角色(必填)
- th: 主题标签(必填,结构化对象)
## th 三层结构
fn叙事功能1-2个枚举
establish=建立设定 | escalate=升级加剧 | reveal=揭示发现 | challenge=挑战试探
commit=承诺锁定 | conflict=冲突对抗 | resolve=解决收束 | transform=转变逆转
bond=连接羁绊 | break=断裂破坏
pt互动模式1-3个枚举
power_down=上对下 | power_up=下对上 | power_equal=对等 | power_contest=争夺
asymmetric=信息不对称 | witnessed=有观众 | secluded=隔绝私密
ritual=仪式正式 | routine=日常惯例 | triangular=三方介入
kw具体关键词1-3个自由格式
## 示例输出
{"atoms":[
{"t":"act","s":"艾拉","o":"古龙","v":"用圣剑刺穿心脏","l":"火山口","f":"a",
"th":{"fn":["commit"],"pt":["power_down","ritual"],"kw":["战斗","牺牲"]}},
{"t":"emo","s":"林夏","o":"陆远","v":"意识到自己喜欢他","l":"","f":"a",
"th":{"fn":["reveal","escalate"],"pt":["asymmetric","secluded"],"kw":["心动","暗恋"]}},
{"t":"dec","s":"凯尔","o":"王国","v":"放弃王位继承权","l":"王座厅","f":"a",
"th":{"fn":["commit","break"],"pt":["ritual","witnessed"],"kw":["抉择","自由"]}},
{"t":"rev","s":"","o":"","v":"管家其实是间谍","l":"","f":"a",
"th":{"fn":["reveal"],"pt":["asymmetric"],"kw":["背叛","真相"]}},
{"t":"ten","s":"兄弟二人","o":"","v":"为遗产反目","l":"","f":"a",
"th":{"fn":["conflict","break"],"pt":["power_contest"],"kw":["冲突","亲情破裂"]}}
]}
规则:
- s: 主体(谁)
- v: 简洁值10字内
- f: "u"=用户发言中, "a"=角色回复中
- 只提取对未来检索有价值的锚点
- 无明显锚点返回空数组`;
- fn 回答"这在故事里推动了什么"
- pt 回答"这是什么结构的互动"
- kw 用于细粒度检索
- 无明显锚点时返回 {"atoms":[]}`;
const JSON_PREFILL = '{"atoms":[';
// ============================================================================
// Semantic 构建
// ============================================================================
function buildSemantic(atom, userName, aiName) {
const speaker = atom.f === 'u' ? userName : aiName;
const s = atom.s || speaker;
const type = atom.t || 'act';
const subject = atom.s || (atom.f === 'u' ? userName : aiName);
const object = atom.o || '';
const verb = atom.v || '';
const location = atom.l || '';
switch (atom.t) {
case 'emo': return `${s}感到${atom.v}`;
case 'loc': return `场景:${atom.v}`;
case 'act': return `${s}${atom.v}`;
case 'rev': return `揭示:${atom.v}`;
case 'ten': return `冲突:${atom.v}`;
case 'dec': return `${s}决定${atom.v}`;
default: return `${s} ${atom.v}`;
// 三层 themes 合并
const th = atom.th || {};
const tags = [
...(Array.isArray(th.fn) ? th.fn : []),
...(Array.isArray(th.pt) ? th.pt : []),
...(Array.isArray(th.kw) ? th.kw : []),
].filter(Boolean);
const typePart = `<${type}>`;
const themePart = tags.length > 0 ? ` [${tags.join('/')}]` : '';
const locPart = location ? `${location}` : '';
const objPart = object ? ` -> ${object}` : '';
let semantic = '';
switch (type) {
case 'emo':
semantic = object
? `${typePart} ${subject} -> ${verb} (对${object})${locPart}`
: `${typePart} ${subject} -> ${verb}${locPart}`;
break;
case 'act':
semantic = `${typePart} ${subject} -> ${verb}${objPart}${locPart}`;
break;
case 'rev':
semantic = object
? `${typePart} 揭示: ${verb} (关于${object})${locPart}`
: `${typePart} 揭示: ${verb}${locPart}`;
break;
case 'dec':
semantic = object
? `${typePart} ${subject} -> ${verb} (对${object})${locPart}`
: `${typePart} ${subject} -> ${verb}${locPart}`;
break;
case 'ten':
semantic = object
? `${typePart} ${subject} <-> ${object}: ${verb}${locPart}`
: `${typePart} ${subject}: ${verb}${locPart}`;
break;
case 'loc':
semantic = location
? `${typePart} 场景: ${location} - ${verb}`
: `${typePart} 场景: ${verb}`;
break;
default:
semantic = `${typePart} ${subject} -> ${verb}${objPart}${locPart}`;
}
return semantic + themePart;
}
// ============================================================================
// 睡眠工具
// ============================================================================
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
// ============================================================================
// 单轮提取(带重试)
// ============================================================================
async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options = {}) {
const { timeout = DEFAULT_TIMEOUT } = options;
@@ -86,8 +187,6 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
const input = `<round>\n${parts.join('\n')}\n</round>`;
xbLog.info(MODULE_ID, `floor ${aiFloor} 发送输入 len=${input.length}`);
for (let attempt = 0; attempt <= RETRY_COUNT; attempt++) {
if (batchCancelled) return [];
@@ -95,16 +194,15 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
const response = await callLLM([
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: input },
{ role: 'assistant', content: '收到,开始提取并仅输出 JSON。' },
{ role: 'assistant', content: JSON_PREFILL },
], {
temperature: 0.2,
max_tokens: 500,
max_tokens: 1000,
timeout,
});
const rawText = String(response || '');
if (!rawText.trim()) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败:响应为空`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
@@ -112,11 +210,13 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
return null;
}
const fullJson = JSON_PREFILL + rawText;
let parsed;
try {
parsed = parseJson(rawText);
parsed = parseJson(fullJson);
} catch (e) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败JSON 异常`);
xbLog.warn(MODULE_ID, `floor ${aiFloor} JSON解析失败`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
@@ -125,8 +225,6 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
}
if (!parsed?.atoms || !Array.isArray(parsed.atoms)) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} atoms 缺失raw="${rawText.slice(0, 300)}"`);
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败atoms 缺失`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
@@ -141,20 +239,20 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
floor: aiFloor,
type: a.t,
subject: a.s || null,
value: String(a.v).slice(0, 30),
object: a.o || null,
value: String(a.v).slice(0, 50),
location: a.l || null,
source: a.f === 'u' ? 'user' : 'ai',
themes: a.th || { fn: [], pt: [], kw: [] },
semantic: buildSemantic(a, userName, aiName),
}));
if (!filtered.length) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} atoms 为空raw="${rawText.slice(0, 300)}"`);
}
return filtered;
} catch (e) {
if (batchCancelled) return null;
if (attempt < RETRY_COUNT) {
xbLog.warn(MODULE_ID, `floor ${aiFloor}${attempt + 1}次失败,重试...`, e?.message);
await sleep(RETRY_DELAY * (attempt + 1));
continue;
}
@@ -166,18 +264,14 @@ async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, op
return null;
}
/**
* 单轮配对提取(增量时使用)
*/
export async function extractAtomsForRound(userMessage, aiMessage, aiFloor, options = {}) {
return extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options);
}
/**
* 批量提取(首批 staggered 启动)
* @param {Array} chat
* @param {Function} onProgress - (current, total, failed) => void
*/
// ============================================================================
// 批量提取
// ============================================================================
export async function batchExtractAtoms(chat, onProgress) {
if (!chat?.length) return [];
@@ -198,14 +292,10 @@ export async function batchExtractAtoms(chat, onProgress) {
let failed = 0;
for (let i = 0; i < pairs.length; i += CONCURRENCY) {
if (batchCancelled) {
xbLog.info(MODULE_ID, `批量提取已取消 (${completed}/${pairs.length})`);
break;
}
if (batchCancelled) break;
const batch = pairs.slice(i, i + CONCURRENCY);
// ★ 首批 staggered 启动:错开 80ms 发送
if (i === 0) {
const promises = batch.map((pair, idx) => (async () => {
await sleep(idx * STAGGER_DELAY);
@@ -213,10 +303,15 @@ export async function batchExtractAtoms(chat, onProgress) {
if (batchCancelled) return;
try {
const atoms = await extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT });
const atoms = await extractAtomsForRoundWithRetry(
pair.userMsg,
pair.aiMsg,
pair.aiFloor,
{ timeout: DEFAULT_TIMEOUT }
);
if (atoms?.length) {
allAtoms.push(...atoms);
} else {
} else if (atoms === null) {
failed++;
}
} catch {
@@ -227,14 +322,18 @@ export async function batchExtractAtoms(chat, onProgress) {
})());
await Promise.all(promises);
} else {
// 后续批次正常并行
const promises = batch.map(pair =>
extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT })
extractAtomsForRoundWithRetry(
pair.userMsg,
pair.aiMsg,
pair.aiFloor,
{ timeout: DEFAULT_TIMEOUT }
)
.then(atoms => {
if (batchCancelled) return;
if (atoms?.length) {
allAtoms.push(...atoms);
} else {
} else if (atoms === null) {
failed++;
}
completed++;
@@ -251,14 +350,12 @@ export async function batchExtractAtoms(chat, onProgress) {
await Promise.all(promises);
}
// 批次间隔
if (i + CONCURRENCY < pairs.length && !batchCancelled) {
await sleep(30);
}
}
const status = batchCancelled ? '已取消' : '完成';
xbLog.info(MODULE_ID, `批量提取${status}: ${allAtoms.length} atoms, ${completed}/${pairs.length}, ${failed} 失败`);
xbLog.info(MODULE_ID, `批量提取完成: ${allAtoms.length} atoms, ${failed} 失败`);
return allAtoms;
}

View File

@@ -1,14 +1,13 @@
// ═══════════════════════════════════════════════════════════════════════════
// vector/llm/llm-service.js
// ═══════════════════════════════════════════════════════════════════════════
// vector/llm/llm-service.js - 修复 prefill 传递方式
// ═══════════════════════════════════════════════════════════════════════════
import { xbLog } from '../../../../core/debug-core.js';
import { getVectorConfig } from '../../data/config.js';
const MODULE_ID = 'vector-llm-service';
const SILICONFLOW_API_URL = 'https://api.siliconflow.cn';
const SILICONFLOW_API_URL = 'https://api.siliconflow.cn/v1';
const DEFAULT_L0_MODEL = 'Qwen/Qwen3-8B';
// 唯一 ID 计数器
let callCounter = 0;
function getStreamingModule() {
@@ -30,6 +29,7 @@ function b64UrlEncode(str) {
/**
* 统一LLM调用 - 走酒馆后端(非流式)
* 修复assistant prefill 用 bottomassistant 参数传递
*/
export async function callLLM(messages, options = {}) {
const {
@@ -46,9 +46,16 @@ export async function callLLM(messages, options = {}) {
throw new Error('L0 requires siliconflow API key');
}
const top64 = b64UrlEncode(JSON.stringify(messages));
// ★ 关键修复:分离 assistant prefill
let topMessages = [...messages];
let assistantPrefill = '';
// 每次调用用唯一 ID避免 session 冲突
if (topMessages.length > 0 && topMessages[topMessages.length - 1]?.role === 'assistant') {
const lastMsg = topMessages.pop();
assistantPrefill = lastMsg.content || '';
}
const top64 = b64UrlEncode(JSON.stringify(topMessages));
const uniqueId = generateUniqueId('l0');
const args = {
@@ -64,8 +71,12 @@ export async function callLLM(messages, options = {}) {
model: DEFAULT_L0_MODEL,
};
// ★ 用 bottomassistant 参数传递 prefill
if (assistantPrefill) {
args.bottomassistant = assistantPrefill;
}
try {
// 非流式直接返回结果
const result = await mod.xbgenrawCommand(args, '');
return String(result ?? '');
} catch (e) {

View File

@@ -1,52 +1,228 @@
// ═══════════════════════════════════════════════════════════════════════════
// query-expansion.js - 完整输入,不截断
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
// query-expansion.js - 检索查询生成器(三层 themes 版)
// ============================================================================
import { callLLM, parseJson } from './llm-service.js';
import { xbLog } from '../../../../core/debug-core.js';
import { filterText } from '../utils/text-filter.js';
import { getContext } from '../../../../../../../extensions.js';
import { getSummaryStore } from '../../data/store.js';
const MODULE_ID = 'query-expansion';
const SESSION_ID = 'xb6';
const SYSTEM_PROMPT = `你是检索词生成器。根据最近对话,输出用于检索历史剧情的关键词。
// ============================================================================
// 系统提示词
// ============================================================================
只输出JSON
{"e":["显式人物/地名"],"i":["隐含人物/情绪/话题"],"q":["检索短句"]}
const SYSTEM_PROMPT = `你是检索查询生成器。根据当前对话上下文,生成用于检索历史剧情的查询语句。
规则:
- e: 对话中明确提到的人名/地名1-4个
- i: 推断出的相关人物/情绪/话题1-5个
- q: 用于向量检索的短句2-3个每个15字内
- 关注:正在讨论什么、涉及谁、情绪氛围`;
/**
* Query Expansion
* @param {Array} messages - 完整消息数组最后2-3轮
*/
export async function expandQuery(messages, options = {}) {
const { timeout = 6000 } = options;
if (!messages?.length) {
return { entities: [], implicit: [], queries: [] };
## 输出格式严格JSON
{
"focus": ["焦点人物"],
"fn": ["叙事功能"],
"pt": ["互动模式"],
"kw": ["关键词"],
"queries": ["DSL查询语句"]
}
// 完整格式化,不截断
const input = messages.map(m => {
const speaker = m.is_user ? '用户' : (m.name || '角色');
const text = filterText(m.mes || '').trim();
return `${speaker}\n${text}`;
}).join('\n\n');
## fn叙事功能枚举
establish=建立设定 | escalate=升级加剧 | reveal=揭示发现 | challenge=挑战试探
commit=承诺锁定 | conflict=冲突对抗 | resolve=解决收束 | transform=转变逆转
bond=连接羁绊 | break=断裂破坏
## pt互动模式枚举
power_down=上对下 | power_up=下对上 | power_equal=对等 | power_contest=争夺
asymmetric=信息不对称 | witnessed=有观众 | secluded=隔绝私密
ritual=仪式正式 | routine=日常惯例 | triangular=三方介入
## DSL 查询格式
- <act> 主体 -> 动作 (-> 客体)? (在地点)?
- <emo> 主体 -> 情绪 (对客体)?
- <dec> 主体 -> 决定/承诺 (对客体)?
- <rev> 揭示: 内容 (关于客体)?
- <ten> 主体A <-> 主体B: 冲突内容
- <loc> 场景: 地点/状态
## 规则
- focus: 核心人物1-4个
- fn: 当前对话涉及的叙事功能1-3个
- pt: 当前对话涉及的互动模式1-3个
- kw: 具体关键词1-4个
- queries: 2-4条 DSL 查询
## 示例
输入:艾拉说"那把剑...我记得它的重量,在火山口的时候"
输出:
{
"focus": ["艾拉", "古龙"],
"fn": ["commit", "bond"],
"pt": ["power_down", "ritual"],
"kw": ["圣剑", "战斗", "火山口"],
"queries": [
"<act> 艾拉 -> 战斗/使用圣剑 -> 古龙 [commit/power_down]",
"<loc> 场景: 火山口 [ritual]",
"<emo> 艾拉 -> 牺牲/决绝 [commit]"
]
}`;
// ============================================================================
// 上下文构建
// ============================================================================
function getCharacterContext() {
const context = getContext();
const char = context.characters?.[context.characterId];
if (!char) {
return { name: '', description: '', personality: '' };
}
return {
name: char.name || '',
description: (char.description || '').slice(0, 500),
personality: (char.personality || '').slice(0, 300),
};
}
function getPersonaContext() {
const context = getContext();
if (typeof window !== 'undefined' && window.power_user?.persona_description) {
return String(window.power_user.persona_description).slice(0, 500);
}
if (context.persona_description) {
return String(context.persona_description).slice(0, 500);
}
return '';
}
function getRecentEvents(count = 8) {
const store = getSummaryStore();
const events = store?.json?.events || [];
return events
.slice(-count)
.map(e => {
const time = e.timeLabel || '';
const title = e.title || '';
const participants = (e.participants || []).join('/');
const summary = (e.summary || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').slice(0, 80);
return time
? `[${time}] ${title || participants}: ${summary}`
: `${title || participants}: ${summary}`;
});
}
function getRelevantArcs(focusHint = []) {
const store = getSummaryStore();
const arcs = store?.json?.arcs || [];
if (!arcs.length) return [];
const hintSet = new Set(focusHint.map(s => String(s).toLowerCase()));
const sorted = [...arcs].sort((a, b) => {
const aHit = hintSet.has(String(a.name || '').toLowerCase()) ? 1 : 0;
const bHit = hintSet.has(String(b.name || '').toLowerCase()) ? 1 : 0;
return bHit - aHit;
});
return sorted.slice(0, 4).map(a => {
const progress = Math.round((a.progress || 0) * 100);
return `${a.name}: ${a.trajectory || '未知状态'} (${progress}%)`;
});
}
function extractNamesFromMessages(messages) {
const names = new Set();
for (const m of messages) {
if (m.name) names.add(m.name);
}
const text = messages.map(m => m.mes || '').join(' ');
const namePattern = /[\u4e00-\u9fff]{2,4}/g;
const matches = text.match(namePattern) || [];
const freq = {};
for (const name of matches) {
freq[name] = (freq[name] || 0) + 1;
}
Object.entries(freq)
.filter(([, count]) => count >= 2)
.forEach(([name]) => names.add(name));
return Array.from(names).slice(0, 6);
}
// ============================================================================
// 主函数
// ============================================================================
export async function expandQuery(messages, options = {}) {
const { pendingUserMessage = null, timeout = 6000 } = options;
if (!messages?.length && !pendingUserMessage) {
return { focus: [], fn: [], pt: [], kw: [], queries: [] };
}
const T0 = performance.now();
const character = getCharacterContext();
const persona = getPersonaContext();
const nameHints = extractNamesFromMessages(messages || []);
const recentEvents = getRecentEvents(8);
const arcs = getRelevantArcs(nameHints);
const dialogueParts = [];
for (const m of (messages || [])) {
const speaker = m.is_user ? '用户' : (m.name || '角色');
const text = filterText(m.mes || '').trim();
if (text) {
dialogueParts.push(`${speaker}\n${text.slice(0, 400)}`);
}
}
if (pendingUserMessage) {
dialogueParts.push(`【用户(刚输入)】\n${filterText(pendingUserMessage).slice(0, 400)}`);
}
const inputParts = [];
if (character.name) {
inputParts.push(`## 当前角色\n${character.name}: ${character.description || character.personality || '无描述'}`);
}
if (persona) {
inputParts.push(`## 用户人设\n${persona}`);
}
if (recentEvents.length) {
inputParts.push(`## 近期剧情\n${recentEvents.map((e, i) => `${i + 1}. ${e}`).join('\n')}`);
}
if (arcs.length) {
inputParts.push(`## 角色状态\n${arcs.join('\n')}`);
}
inputParts.push(`## 最近对话\n${dialogueParts.join('\n\n')}`);
const input = inputParts.join('\n\n');
try {
const response = await callLLM([
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: input },
], {
temperature: 0.15,
max_tokens: 250,
max_tokens: 500,
timeout,
sessionId: SESSION_ID,
});
@@ -54,49 +230,104 @@ export async function expandQuery(messages, options = {}) {
const parsed = parseJson(response);
if (!parsed) {
xbLog.warn(MODULE_ID, 'JSON解析失败', response?.slice(0, 200));
return { entities: [], implicit: [], queries: [] };
return { focus: [], fn: [], pt: [], kw: [], queries: [] };
}
const result = {
entities: Array.isArray(parsed.e) ? parsed.e.slice(0, 5) : [],
implicit: Array.isArray(parsed.i) ? parsed.i.slice(0, 6) : [],
queries: Array.isArray(parsed.q) ? parsed.q.slice(0, 4) : [],
focus: Array.isArray(parsed.focus) ? parsed.focus.slice(0, 5) : [],
fn: Array.isArray(parsed.fn) ? parsed.fn.slice(0, 4) : [],
pt: Array.isArray(parsed.pt) ? parsed.pt.slice(0, 4) : [],
kw: Array.isArray(parsed.kw) ? parsed.kw.slice(0, 5) : [],
queries: Array.isArray(parsed.queries) ? parsed.queries.slice(0, 5) : [],
};
xbLog.info(MODULE_ID, `完成 (${Math.round(performance.now() - T0)}ms) e=${result.entities.length} i=${result.implicit.length} q=${result.queries.length}`);
xbLog.info(MODULE_ID, `完成 (${Math.round(performance.now() - T0)}ms) focus=[${result.focus.join(',')}] fn=[${result.fn.join(',')}]`);
return result;
} catch (e) {
xbLog.error(MODULE_ID, '调用失败', e);
return { entities: [], implicit: [], queries: [] };
return { focus: [], fn: [], pt: [], kw: [], queries: [] };
}
}
// ============================================================================
// 缓存
// ============================================================================
const cache = new Map();
const CACHE_TTL = 300000;
function hashMessages(messages) {
const text = messages.slice(-2).map(m => (m.mes || '').slice(0, 100)).join('|');
function hashMessages(messages, pending = '') {
const text = (messages || [])
.slice(-3)
.map(m => (m.mes || '').slice(0, 100))
.join('|') + '|' + (pending || '').slice(0, 100);
let h = 0;
for (let i = 0; i < text.length; i++) h = ((h << 5) - h + text.charCodeAt(i)) | 0;
for (let i = 0; i < text.length; i++) {
h = ((h << 5) - h + text.charCodeAt(i)) | 0;
}
return h.toString(36);
}
export async function expandQueryCached(messages, options = {}) {
const key = hashMessages(messages);
const key = hashMessages(messages, options.pendingUserMessage);
const cached = cache.get(key);
if (cached && Date.now() - cached.time < CACHE_TTL) return cached.result;
if (cached && Date.now() - cached.time < CACHE_TTL) {
return cached.result;
}
const result = await expandQuery(messages, options);
if (result.entities.length || result.queries.length) {
if (cache.size > 50) cache.delete(cache.keys().next().value);
if (result.focus.length || result.queries.length) {
if (cache.size > 50) {
cache.delete(cache.keys().next().value);
}
cache.set(key, { result, time: Date.now() });
}
return result;
}
// ============================================================================
// 辅助函数:构建检索文本
// ============================================================================
/**
* 将 expansion 结果转换为检索文本
* 三层 themes 自然拼入,让向量自动编码
*/
export function buildSearchText(expansion) {
return [...(expansion.entities || []), ...(expansion.implicit || []), ...(expansion.queries || [])]
.filter(Boolean).join(' ');
const parts = [];
// focus 人物
if (expansion.focus?.length) {
parts.push(expansion.focus.join(' '));
}
// fn + pt + kw 合并为标签
const tags = [
...(expansion.fn || []),
...(expansion.pt || []),
...(expansion.kw || []),
].filter(Boolean);
if (tags.length) {
parts.push(`[${tags.join('/')}]`);
}
// queries
if (expansion.queries?.length) {
parts.push(...expansion.queries);
}
return parts.filter(Boolean).join(' ').slice(0, 1500);
}
/**
* 提取实体列表(兼容旧接口)
*/
export function getEntitiesFromExpansion(expansion) {
return expansion?.focus || [];
}

View File

@@ -0,0 +1,184 @@
// ═══════════════════════════════════════════════════════════════════════════
// Reranker - 硅基 bge-reranker-v2-m3
// 对候选文档进行精排,过滤与 query 不相关的内容
// ═══════════════════════════════════════════════════════════════════════════
import { xbLog } from '../../../../core/debug-core.js';
import { getApiKey } from './siliconflow.js';
const MODULE_ID = 'reranker';
const RERANK_URL = 'https://api.siliconflow.cn/v1/rerank';
const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3';
const DEFAULT_TIMEOUT = 15000;
const MAX_DOCUMENTS = 100; // API 限制
/**
* 对文档列表进行 Rerank 精排
*
* @param {string} query - 查询文本
* @param {Array<string>} documents - 文档文本列表
* @param {object} options - 选项
* @param {number} options.topN - 返回前 N 个结果,默认 40
* @param {number} options.timeout - 超时时间,默认 15000ms
* @param {AbortSignal} options.signal - 取消信号
* @returns {Promise<Array<{index: number, relevance_score: number}>>} 排序后的结果
*/
export async function rerank(query, documents, options = {}) {
const { topN = 40, timeout = DEFAULT_TIMEOUT, signal } = options;
if (!query?.trim()) {
xbLog.warn(MODULE_ID, 'query 为空,跳过 rerank');
return documents.map((_, i) => ({ index: i, relevance_score: 0.5 }));
}
if (!documents?.length) {
return [];
}
const key = getApiKey();
if (!key) {
xbLog.warn(MODULE_ID, '未配置 API Key跳过 rerank');
return documents.map((_, i) => ({ index: i, relevance_score: 0.5 }));
}
// 截断超长文档列表
const truncatedDocs = documents.slice(0, MAX_DOCUMENTS);
if (documents.length > MAX_DOCUMENTS) {
xbLog.warn(MODULE_ID, `文档数 ${documents.length} 超过限制 ${MAX_DOCUMENTS},已截断`);
}
// 过滤空文档,记录原始索引
const validDocs = [];
const indexMap = []; // validDocs index → original index
for (let i = 0; i < truncatedDocs.length; i++) {
const text = String(truncatedDocs[i] || '').trim();
if (text) {
validDocs.push(text);
indexMap.push(i);
}
}
if (!validDocs.length) {
xbLog.warn(MODULE_ID, '无有效文档,跳过 rerank');
return [];
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const T0 = performance.now();
const response = await fetch(RERANK_URL, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: RERANK_MODEL,
query: query.slice(0, 1000), // 限制 query 长度
documents: validDocs,
top_n: Math.min(topN, validDocs.length),
return_documents: false,
}),
signal: signal || controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
const errorText = await response.text().catch(() => '');
throw new Error(`Rerank API ${response.status}: ${errorText.slice(0, 200)}`);
}
const data = await response.json();
const results = data.results || [];
// 映射回原始索引
const mapped = results.map(r => ({
index: indexMap[r.index],
relevance_score: r.relevance_score ?? 0,
}));
const elapsed = Math.round(performance.now() - T0);
xbLog.info(MODULE_ID, `Rerank 完成: ${validDocs.length} docs → ${results.length} selected (${elapsed}ms)`);
return mapped;
} catch (e) {
clearTimeout(timeoutId);
if (e?.name === 'AbortError') {
xbLog.warn(MODULE_ID, 'Rerank 超时或取消');
} else {
xbLog.error(MODULE_ID, 'Rerank 失败', e);
}
// 降级:返回原顺序,分数均匀分布
return documents.slice(0, topN).map((_, i) => ({
index: i,
relevance_score: 1 - (i / documents.length) * 0.5,
}));
}
}
/**
* 对 chunk 对象列表进行 Rerank
*
* @param {string} query - 查询文本
* @param {Array<object>} chunks - chunk 对象列表,需要有 text 字段
* @param {object} options - 选项
* @returns {Promise<Array<object>>} 排序后的 chunk 列表,带 _rerankScore 字段
*/
export async function rerankChunks(query, chunks, options = {}) {
const { topN = 40, minScore = 0.1 } = options;
if (!chunks?.length) return [];
if (chunks.length <= topN) {
// 数量不超限,仍然 rerank 以获取分数,但不过滤
const texts = chunks.map(c => c.text || c.semantic || '');
const results = await rerank(query, texts, { topN: chunks.length, ...options });
const scoreMap = new Map(results.map(r => [r.index, r.relevance_score]));
return chunks.map((c, i) => ({
...c,
_rerankScore: scoreMap.get(i) ?? 0.5,
})).sort((a, b) => b._rerankScore - a._rerankScore);
}
const texts = chunks.map(c => c.text || c.semantic || '');
const results = await rerank(query, texts, { topN, ...options });
// 过滤低分 + 排序
const selected = results
.filter(r => r.relevance_score >= minScore)
.sort((a, b) => b.relevance_score - a.relevance_score)
.map(r => ({
...chunks[r.index],
_rerankScore: r.relevance_score,
}));
return selected;
}
/**
* 测试 Rerank 服务连接
*/
export async function testRerankService() {
const key = getApiKey();
if (!key) {
throw new Error('请配置硅基 API Key');
}
try {
const results = await rerank('测试查询', ['测试文档1', '测试文档2'], { topN: 2 });
return {
success: true,
message: `连接成功,返回 ${results.length} 个结果`,
};
} catch (e) {
throw new Error(`连接失败: ${e.message}`);
}
}

View File

@@ -1,9 +1,11 @@
// ============================================================================
// state-integration.js - L0 记忆锚点管理
// 支持增量提取、清空、取消
// ============================================================================
// state-integration.js - L0 状态层集成
// Phase 1: 批量 LLM 提取(只存文本)
// Phase 2: 统一向量化(提取完成后)
// ============================================================================
import { getContext } from '../../../../../../../extensions.js';
import { saveMetadataDebounced } from '../../../../../../../extensions.js';
import { xbLog } from '../../../../core/debug-core.js';
import {
saveStateAtoms,
@@ -26,9 +28,15 @@ import { filterText } from '../utils/text-filter.js';
const MODULE_ID = 'state-integration';
// ★ 并发配置
const CONCURRENCY = 30;
const STAGGER_DELAY = 30;
let initialized = false;
let extractionCancelled = false;
export function cancelL0Extraction() {
extractionCancelled = true;
cancelBatchExtraction();
}
@@ -53,6 +61,7 @@ export async function getAnchorStats() {
return { extracted: 0, total: 0, pending: 0, empty: 0, fail: 0 };
}
// 统计 AI 楼层
const aiFloors = [];
for (let i = 0; i < chat.length; i++) {
if (!chat[i]?.is_user) aiFloors.push(i);
@@ -71,14 +80,20 @@ export async function getAnchorStats() {
}
const total = aiFloors.length;
const completed = ok + empty;
const pending = Math.max(0, total - completed);
const processed = ok + empty + fail;
const pending = Math.max(0, total - processed);
return { extracted: completed, total, pending, empty, fail };
return {
extracted: ok + empty,
total,
pending,
empty,
fail
};
}
// ============================================================================
// 增量提取
// 增量提取 - Phase 1 提取文本Phase 2 统一向量化
// ============================================================================
function buildL0InputText(userMessage, aiMessage) {
@@ -102,6 +117,9 @@ export async function incrementalExtractAtoms(chatId, chat, onProgress) {
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return { built: 0 };
// ★ 重置取消标志
extractionCancelled = false;
const pendingPairs = [];
for (let i = 0; i < chat.length; i++) {
@@ -109,6 +127,7 @@ export async function incrementalExtractAtoms(chatId, chat, onProgress) {
if (!msg || msg.is_user) continue;
const st = getL0FloorStatus(i);
// ★ 只跳过 ok 和 emptyfail 的可以重试
if (st?.status === 'ok' || st?.status === 'empty') {
continue;
}
@@ -125,23 +144,47 @@ export async function incrementalExtractAtoms(chatId, chat, onProgress) {
}
if (!pendingPairs.length) {
onProgress?.(0, 0, '已全部提取');
onProgress?.('已全部提取', 0, 0);
return { built: 0 };
}
xbLog.info(MODULE_ID, `增量 L0 提取pending=${pendingPairs.length}`);
xbLog.info(MODULE_ID, `增量 L0 提取pending=${pendingPairs.length}, concurrency=${CONCURRENCY}`);
let completed = 0;
let failed = 0;
const total = pendingPairs.length;
let builtAtoms = 0;
for (const pair of pendingPairs) {
// ★ Phase 1: 收集所有新提取的 atoms不向量化
const allNewAtoms = [];
// ★ 30 并发批次处理
for (let i = 0; i < pendingPairs.length; i += CONCURRENCY) {
// ★ 检查取消
if (extractionCancelled) {
xbLog.info(MODULE_ID, `用户取消,已完成 ${completed}/${total}`);
break;
}
const batch = pendingPairs.slice(i, i + CONCURRENCY);
const promises = batch.map((pair, idx) => (async () => {
// 首批错开启动,避免瞬间打满
if (i === 0) {
await new Promise(r => setTimeout(r, idx * STAGGER_DELAY));
}
// 再次检查取消
if (extractionCancelled) return;
const floor = pair.aiFloor;
const prev = getL0FloorStatus(floor);
try {
const atoms = await extractAtomsForRound(pair.userMsg, pair.aiMsg, floor, { timeout: 20000 });
if (extractionCancelled) return;
if (atoms == null) {
throw new Error('llm_failed');
}
@@ -151,28 +194,59 @@ export async function incrementalExtractAtoms(chatId, chat, onProgress) {
} else {
atoms.forEach(a => a.chatId = chatId);
saveStateAtoms(atoms);
await vectorizeAtoms(chatId, atoms);
// ★ Phase 1: 只收集,不向量化
allNewAtoms.push(...atoms);
setL0FloorStatus(floor, { status: 'ok', atoms: atoms.length });
builtAtoms += atoms.length;
}
} catch (e) {
if (extractionCancelled) return;
setL0FloorStatus(floor, {
status: 'fail',
attempts: (prev?.attempts || 0) + 1,
reason: String(e?.message || e).replace(/\s+/g, ' ').slice(0, 120),
});
failed++;
} finally {
if (!extractionCancelled) {
completed++;
onProgress?.(`L0: ${completed}/${total}`, completed, total);
onProgress?.(`提取: ${completed}/${total}`, completed, total);
}
}
})());
await Promise.all(promises);
// 批次间短暂间隔
if (i + CONCURRENCY < pendingPairs.length && !extractionCancelled) {
await new Promise(r => setTimeout(r, 30));
}
}
xbLog.info(MODULE_ID, `增量 L0 完成atoms=${builtAtoms}, floors=${pendingPairs.length}`);
// ★ 立即保存文本,不要等防抖
try {
saveMetadataDebounced?.();
} catch { }
// ★ Phase 2: 统一向量化所有新提取的 atoms
if (allNewAtoms.length > 0 && !extractionCancelled) {
onProgress?.(`向量化 L0: 0/${allNewAtoms.length}`, 0, allNewAtoms.length);
await vectorizeAtoms(chatId, allNewAtoms, (current, total) => {
onProgress?.(`向量化 L0: ${current}/${total}`, current, total);
});
}
xbLog.info(MODULE_ID, `L0 ${extractionCancelled ? '已取消' : '完成'}atoms=${builtAtoms}, completed=${completed}/${total}, failed=${failed}`);
return { built: builtAtoms };
}
async function vectorizeAtoms(chatId, atoms) {
// ============================================================================
// 向量化(支持进度回调)
// ============================================================================
async function vectorizeAtoms(chatId, atoms, onProgress) {
if (!atoms?.length) return;
const vectorCfg = getVectorConfig();
@@ -180,14 +254,27 @@ async function vectorizeAtoms(chatId, atoms) {
const texts = atoms.map(a => a.semantic);
const fingerprint = getEngineFingerprint(vectorCfg);
const batchSize = 20;
try {
const vectors = await embed(texts, { timeout: 30000 });
const allVectors = [];
const items = atoms.map((a, i) => ({
for (let i = 0; i < texts.length; i += batchSize) {
if (extractionCancelled) break;
const batch = texts.slice(i, i + batchSize);
const vectors = await embed(batch, { timeout: 30000 });
allVectors.push(...vectors);
onProgress?.(allVectors.length, texts.length);
}
if (extractionCancelled) return;
const items = atoms.slice(0, allVectors.length).map((a, i) => ({
atomId: a.atomId,
floor: a.floor,
vector: vectors[i],
vector: allVectors[i],
}));
await saveStateVectors(chatId, items, fingerprint);
@@ -207,11 +294,17 @@ export async function clearAllAtomsAndVectors(chatId) {
if (chatId) {
await clearStateVectors(chatId);
}
// ★ 立即保存
try {
saveMetadataDebounced?.();
} catch { }
xbLog.info(MODULE_ID, '已清空所有记忆锚点');
}
// ============================================================================
// 实时增量AI 消息后触发)- 保留原有逻辑
// 实时增量AI 消息后触发)- 保持不变
// ============================================================================
let extractionQueue = [];
@@ -245,7 +338,9 @@ async function processQueue() {
atoms.forEach(a => a.chatId = chatId);
saveStateAtoms(atoms);
await vectorizeAtoms(chatId, atoms);
// 单楼实时处理:立即向量化
await vectorizeAtomsSimple(chatId, atoms);
xbLog.info(MODULE_ID, `floor ${aiFloor}: ${atoms.length} atoms 已存储`);
} catch (e) {
@@ -256,6 +351,31 @@ async function processQueue() {
isProcessing = false;
}
// 简单向量化(无进度回调,用于单楼实时处理)
async function vectorizeAtomsSimple(chatId, atoms) {
if (!atoms?.length) return;
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return;
const texts = atoms.map(a => a.semantic);
const fingerprint = getEngineFingerprint(vectorCfg);
try {
const vectors = await embed(texts, { timeout: 30000 });
const items = atoms.map((a, i) => ({
atomId: a.atomId,
floor: a.floor,
vector: vectors[i],
}));
await saveStateVectors(chatId, items, fingerprint);
} catch (e) {
xbLog.error(MODULE_ID, 'L0 向量化失败', e);
}
}
// ============================================================================
// 回滚钩子
// ============================================================================
@@ -301,7 +421,7 @@ export async function rebuildStateVectors(chatId, vectorCfg) {
xbLog.info(MODULE_ID, `重建 L0 向量: ${atoms.length} 条 atom`);
await clearStateVectors(chatId);
await vectorizeAtoms(chatId, atoms);
await vectorizeAtomsSimple(chatId, atoms);
return { built: atoms.length };
}

View File

@@ -131,16 +131,44 @@ export function stateToVirtualChunks(l0Results) {
// ═══════════════════════════════════════════════════════════════════════════
/**
* 合并 L0 和 L1 chunks,每楼层最多保留 limit 条
* @param {Array} l0Chunks - 虚拟 chunks已按相似度排序
* @param {Array} l1Chunks - 真实 chunks已按相似度排序
* 合并 L0 和 L1 chunks
* @param {Array} l0Chunks - L0 虚拟 chunks带 similarity
* @param {Array} l1Chunks - L1 真实 chunks无 similarity
* @param {number} limit - 每楼层上限
* @returns {Array} 合并后的 chunks
*/
export function mergeAndSparsify(l0Chunks, l1Chunks, limit = 2) {
// 构建 L0 楼层 → 最高 similarity 映射
const floorSimilarity = new Map();
for (const c of (l0Chunks || [])) {
const existing = floorSimilarity.get(c.floor) || 0;
if ((c.similarity || 0) > existing) {
floorSimilarity.set(c.floor, c.similarity || 0);
}
}
// L1 继承所属楼层的 L0 similarity
const l1WithScore = (l1Chunks || []).map(c => ({
...c,
similarity: floorSimilarity.get(c.floor) || 0.5,
}));
// 合并并按相似度排序
const all = [...(l0Chunks || []), ...(l1Chunks || [])]
.sort((a, b) => b.similarity - a.similarity);
const all = [...(l0Chunks || []), ...l1WithScore]
.sort((a, b) => {
// 相似度优先
const simDiff = (b.similarity || 0) - (a.similarity || 0);
if (Math.abs(simDiff) > 0.01) return simDiff;
// 同楼层L0 优先于 L1
if (a.floor === b.floor) {
if (a.isL0 && !b.isL0) return -1;
if (!a.isL0 && b.isL0) return 1;
}
// 按楼层升序
return a.floor - b.floor;
});
// 每楼层稀疏去重
const byFloor = new Map();
@@ -153,8 +181,9 @@ export function mergeAndSparsify(l0Chunks, l1Chunks, limit = 2) {
}
}
// 扁平化并保持相似度排序
// 扁平化并保持排序
return Array.from(byFloor.values())
.flat()
.sort((a, b) => b.similarity - a.similarity);
.sort((a, b) => (b.similarity || 0) - (a.similarity || 0));
}

View File

@@ -0,0 +1,388 @@
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Metrics Collector
// 召回质量指标收集与格式化
// ═══════════════════════════════════════════════════════════════════════════
/**
* 创建空的指标对象
*/
export function createMetrics() {
return {
// L0 Query Understanding
l0: {
needRecall: false,
intent: '',
focusEntities: [],
queries: [],
implicitTopics: [],
queryExpansionTime: 0,
atomsMatched: 0,
floorsHit: 0,
topAtoms: [],
},
// L1 Constraints (Facts)
l1: {
factsTotal: 0,
factsInjected: 0,
factsFiltered: 0,
tokens: 0,
samples: [],
},
// L2 Narrative Retrieval
l2: {
eventsInStore: 0,
eventsConsidered: 0,
eventsSelected: 0,
byRecallType: { direct: 0, causal: 0, context: 0 },
similarityDistribution: { min: 0, max: 0, mean: 0, median: 0 },
entityFilterStats: null,
causalChainDepth: 0,
causalEventsCount: 0,
entitiesLoaded: 0,
entityNames: [],
retrievalTime: 0,
},
// L3 Evidence Assembly
l3: {
floorsFromL0: 0,
// 候选规模rerank 前)
chunksInRange: 0,
chunksInRangeByType: { l0Virtual: 0, l1Real: 0 },
// 最终注入rerank + sparse 后)
chunksSelected: 0,
chunksSelectedByType: { l0Virtual: 0, l1Real: 0 },
// 上下文配对
contextPairsAdded: 0,
tokens: 0,
assemblyTime: 0,
// Rerank 相关
rerankApplied: false,
beforeRerank: 0,
afterRerank: 0,
rerankTime: 0,
rerankScoreDistribution: null,
},
// L4 Formatting
l4: {
sectionsIncluded: [],
formattingTime: 0,
},
// Budget Summary
budget: {
total: 0,
limit: 0,
utilization: 0,
breakdown: {
constraints: 0,
events: 0,
entities: 0,
chunks: 0,
recentOrphans: 0,
arcs: 0,
},
},
// Total Timing
timing: {
queryExpansion: 0,
l0Search: 0,
l1Constraints: 0,
l2Retrieval: 0,
l3Retrieval: 0,
l3Rerank: 0,
l3Assembly: 0,
l4Formatting: 0,
total: 0,
},
// Quality Indicators
quality: {
constraintCoverage: 100,
eventPrecisionProxy: 0,
evidenceDensity: 0,
potentialIssues: [],
},
};
}
/**
* 计算相似度分布统计
*/
export function calcSimilarityStats(similarities) {
if (!similarities?.length) {
return { min: 0, max: 0, mean: 0, median: 0 };
}
const sorted = [...similarities].sort((a, b) => a - b);
const sum = sorted.reduce((a, b) => a + b, 0);
return {
min: Number(sorted[0].toFixed(3)),
max: Number(sorted[sorted.length - 1].toFixed(3)),
mean: Number((sum / sorted.length).toFixed(3)),
median: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)),
};
}
/**
* 格式化指标为可读日志
*/
export function formatMetricsLog(metrics) {
const m = metrics;
const lines = [];
lines.push('');
lines.push('═══════════════════════════════════════════════════════════════════');
lines.push(' Recall Metrics Report ');
lines.push('═══════════════════════════════════════════════════════════════════');
lines.push('');
// L0 Query Understanding
lines.push('[L0] Query Understanding');
lines.push(`├─ need_recall: ${m.l0.needRecall}`);
if (m.l0.needRecall) {
lines.push(`├─ intent: ${m.l0.intent || 'mixed'}`);
lines.push(`├─ focus_entities: [${(m.l0.focusEntities || []).join(', ')}]`);
lines.push(`├─ queries: [${(m.l0.queries || []).slice(0, 3).join(', ')}]`);
lines.push(`├─ query_expansion_time: ${m.l0.queryExpansionTime}ms`);
lines.push(`├─ atoms_matched: ${m.l0.atomsMatched || 0}`);
lines.push(`└─ floors_hit: ${m.l0.floorsHit || 0}`);
}
lines.push('');
// L1 Constraints
lines.push('[L1] Constraints (Facts)');
lines.push(`├─ facts_total: ${m.l1.factsTotal}`);
lines.push(`├─ facts_filtered: ${m.l1.factsFiltered || 0}`);
lines.push(`├─ facts_injected: ${m.l1.factsInjected}`);
lines.push(`├─ tokens: ${m.l1.tokens}`);
if (m.l1.samples && m.l1.samples.length > 0) {
lines.push(`└─ samples: "${m.l1.samples.slice(0, 2).join('", "')}"`);
}
lines.push('');
// L2 Narrative Retrieval
lines.push('[L2] Narrative Retrieval');
lines.push(`├─ events_in_store: ${m.l2.eventsInStore}`);
lines.push(`├─ events_considered: ${m.l2.eventsConsidered}`);
if (m.l2.entityFilterStats) {
const ef = m.l2.entityFilterStats;
lines.push(`├─ entity_filter:`);
lines.push(`│ ├─ focus_entities: [${(ef.focusEntities || []).join(', ')}]`);
lines.push(`│ ├─ before_filter: ${ef.before}`);
lines.push(`│ ├─ after_filter: ${ef.after}`);
lines.push(`│ └─ filtered_out: ${ef.filtered}`);
}
lines.push(`├─ events_selected: ${m.l2.eventsSelected}`);
lines.push(`├─ by_recall_type:`);
lines.push(`│ ├─ direct: ${m.l2.byRecallType.direct}`);
lines.push(`│ ├─ causal: ${m.l2.byRecallType.causal}`);
lines.push(`│ └─ context: ${m.l2.byRecallType.context}`);
const sim = m.l2.similarityDistribution;
if (sim && sim.max > 0) {
lines.push(`├─ similarity_distribution:`);
lines.push(`│ ├─ min: ${sim.min}`);
lines.push(`│ ├─ max: ${sim.max}`);
lines.push(`│ ├─ mean: ${sim.mean}`);
lines.push(`│ └─ median: ${sim.median}`);
}
lines.push(`├─ causal_chain: depth=${m.l2.causalChainDepth}, events=${m.l2.causalEventsCount}`);
lines.push(`├─ entities_loaded: ${m.l2.entitiesLoaded} [${(m.l2.entityNames || []).join(', ')}]`);
lines.push(`└─ retrieval_time: ${m.l2.retrievalTime}ms`);
lines.push('');
// L3 Evidence Assembly
lines.push('[L3] Evidence Assembly');
lines.push(`├─ floors_from_l0: ${m.l3.floorsFromL0}`);
// 候选规模
lines.push(`├─ chunks_in_range: ${m.l3.chunksInRange}`);
if (m.l3.chunksInRangeByType) {
const cir = m.l3.chunksInRangeByType;
lines.push(`│ ├─ l0_virtual: ${cir.l0Virtual || 0}`);
lines.push(`│ └─ l1_real: ${cir.l1Real || 0}`);
}
// Rerank 信息
if (m.l3.rerankApplied) {
lines.push(`├─ rerank_applied: true`);
lines.push(`│ ├─ before: ${m.l3.beforeRerank}`);
lines.push(`│ ├─ after: ${m.l3.afterRerank}`);
lines.push(`│ └─ time: ${m.l3.rerankTime}ms`);
if (m.l3.rerankScoreDistribution) {
const rd = m.l3.rerankScoreDistribution;
lines.push(`├─ rerank_scores: min=${rd.min}, max=${rd.max}, mean=${rd.mean}`);
}
} else {
lines.push(`├─ rerank_applied: false`);
}
// 最终注入规模
lines.push(`├─ chunks_selected: ${m.l3.chunksSelected}`);
if (m.l3.chunksSelectedByType) {
const cs = m.l3.chunksSelectedByType;
lines.push(`│ ├─ l0_virtual: ${cs.l0Virtual || 0}`);
lines.push(`│ └─ l1_real: ${cs.l1Real || 0}`);
}
lines.push(`├─ context_pairs_added: ${m.l3.contextPairsAdded}`);
lines.push(`├─ tokens: ${m.l3.tokens}`);
lines.push(`└─ assembly_time: ${m.l3.assemblyTime}ms`);
lines.push('');
// L4 Formatting
lines.push('[L4] Prompt Formatting');
lines.push(`├─ sections: [${(m.l4.sectionsIncluded || []).join(', ')}]`);
lines.push(`└─ formatting_time: ${m.l4.formattingTime}ms`);
lines.push('');
// Budget Summary
lines.push('[Budget Summary]');
lines.push(`├─ total_tokens: ${m.budget.total}`);
lines.push(`├─ budget_limit: ${m.budget.limit}`);
lines.push(`├─ utilization: ${m.budget.utilization}%`);
lines.push(`└─ breakdown:`);
const bd = m.budget.breakdown || {};
lines.push(` ├─ constraints (L1): ${bd.constraints || 0}`);
lines.push(` ├─ events (L2): ${bd.events || 0}`);
lines.push(` ├─ chunks (L3): ${bd.chunks || 0}`);
lines.push(` ├─ recent_orphans: ${bd.recentOrphans || 0}`);
lines.push(` └─ arcs: ${bd.arcs || 0}`);
lines.push('');
// Timing
lines.push('[Timing]');
lines.push(`├─ query_expansion: ${m.timing.queryExpansion}ms`);
lines.push(`├─ l0_search: ${m.timing.l0Search}ms`);
lines.push(`├─ l1_constraints: ${m.timing.l1Constraints}ms`);
lines.push(`├─ l2_retrieval: ${m.timing.l2Retrieval}ms`);
lines.push(`├─ l3_retrieval: ${m.timing.l3Retrieval}ms`);
if (m.timing.l3Rerank > 0) {
lines.push(`├─ l3_rerank: ${m.timing.l3Rerank}ms`);
}
lines.push(`├─ l3_assembly: ${m.timing.l3Assembly}ms`);
lines.push(`├─ l4_formatting: ${m.timing.l4Formatting}ms`);
lines.push(`└─ total: ${m.timing.total}ms`);
lines.push('');
// Quality Indicators
lines.push('[Quality Indicators]');
lines.push(`├─ constraint_coverage: ${m.quality.constraintCoverage}%`);
lines.push(`├─ event_precision_proxy: ${m.quality.eventPrecisionProxy}`);
lines.push(`├─ evidence_density: ${m.quality.evidenceDensity}%`);
if (m.quality.potentialIssues && m.quality.potentialIssues.length > 0) {
lines.push(`└─ potential_issues:`);
m.quality.potentialIssues.forEach((issue, i) => {
const prefix = i === m.quality.potentialIssues.length - 1 ? ' └─' : ' ├─';
lines.push(`${prefix}${issue}`);
});
} else {
lines.push(`└─ potential_issues: none`);
}
lines.push('');
lines.push('═══════════════════════════════════════════════════════════════════');
lines.push('');
return lines.join('\n');
}
/**
* 检测潜在问题
*/
export function detectIssues(metrics) {
const issues = [];
const m = metrics;
// 召回比例问题
if (m.l2.eventsConsidered > 0) {
const selectRatio = m.l2.eventsSelected / m.l2.eventsConsidered;
if (selectRatio < 0.1) {
issues.push(`Event selection ratio too low (${(selectRatio * 100).toFixed(1)}%) - threshold may be too high`);
}
if (selectRatio > 0.6 && m.l2.eventsConsidered > 10) {
issues.push(`Event selection ratio high (${(selectRatio * 100).toFixed(1)}%) - may include noise`);
}
}
// 实体过滤问题
if (m.l2.entityFilterStats) {
const ef = m.l2.entityFilterStats;
if (ef.filtered === 0 && ef.before > 10) {
issues.push(`No events filtered by entity - focus entities may be too broad or missing`);
}
if (ef.before > 0 && ef.filtered > ef.before * 0.8) {
issues.push(`Too many events filtered (${ef.filtered}/${ef.before}) - focus may be too narrow`);
}
}
// 相似度问题
if (m.l2.similarityDistribution && m.l2.similarityDistribution.min > 0 && m.l2.similarityDistribution.min < 0.5) {
issues.push(`Low similarity events included (min=${m.l2.similarityDistribution.min})`);
}
// 因果链问题
if (m.l2.eventsSelected > 0 && m.l2.causalEventsCount === 0 && m.l2.byRecallType.direct === 0) {
issues.push('No direct or causal events - query expansion may be inaccurate');
}
// L0 atoms 问题
if ((m.l0.atomsMatched || 0) === 0) {
issues.push('L0 atoms not matched - may need to generate anchors');
}
// Rerank 相关问题
if (m.l3.rerankApplied) {
if (m.l3.beforeRerank > 0 && m.l3.afterRerank > 0) {
const filterRatio = 1 - (m.l3.afterRerank / m.l3.beforeRerank);
if (filterRatio > 0.7) {
issues.push(`High rerank filter ratio (${(filterRatio * 100).toFixed(0)}%) - many irrelevant chunks removed`);
}
}
if (m.l3.rerankScoreDistribution) {
const rd = m.l3.rerankScoreDistribution;
if (rd.max < 0.5) {
issues.push(`Low rerank scores (max=${rd.max}) - query may be poorly matched`);
}
if (rd.mean < 0.3) {
issues.push(`Very low average rerank score (mean=${rd.mean}) - context may be weak`);
}
}
if (m.l3.rerankTime > 2000) {
issues.push(`Slow rerank (${m.l3.rerankTime}ms) - may affect response time`);
}
}
// 证据密度问题(基于 selected 的构成)
if (m.l3.chunksSelected > 0 && m.l3.chunksSelectedByType) {
const l1Real = m.l3.chunksSelectedByType.l1Real || 0;
const density = l1Real / m.l3.chunksSelected;
if (density < 0.3 && m.l3.chunksSelected > 10) {
issues.push(`Low L1 chunk ratio in selected (${(density * 100).toFixed(0)}%) - may lack concrete evidence`);
}
}
// 预算问题
if (m.budget.utilization > 90) {
issues.push(`High budget utilization (${m.budget.utilization}%) - may be truncating content`);
}
// 性能问题
if (m.timing.total > 5000) {
issues.push(`Slow recall (${m.timing.total}ms) - consider optimization`);
}
return issues;
}

File diff suppressed because it is too large Load Diff

View File

@@ -166,6 +166,7 @@ class StreamingGeneration {
if (!source) throw new Error(`不支持的 api: ${opts.api}`);
const model = String(opts.model || '').trim();
const msgCount = Array.isArray(messages) ? messages.length : null;
if (!model) {
try { xbLog.error('streamingGeneration', 'missing model', null); } catch {}
@@ -175,7 +176,6 @@ class StreamingGeneration {
try {
try {
if (xbLog.isEnabled?.()) {
const msgCount = Array.isArray(messages) ? messages.length : null;
xbLog.info('streamingGeneration', `callAPI stream=${!!stream} api=${String(opts.api || '')} model=${model} messages=${msgCount ?? '-'}`);
}
} catch {}
@@ -286,10 +286,34 @@ class StreamingGeneration {
}
const logSendRequestError = (err, streamMode) => {
if (err?.name !== 'AbortError') {
const safeApiUrl = String(cmdApiUrl || reverseProxy || oai_settings?.custom_url || '').trim();
try {
xbLog.error('streamingGeneration', 'sendRequest failed', {
message: err?.message || String(err),
name: err?.name,
stream: !!streamMode,
api: String(opts.api || ''),
model,
msgCount,
apiurl: safeApiUrl,
});
} catch {}
console.error('[xbgen:callAPI] sendRequest failed:', err);
}
};
if (stream) {
const payload = ChatCompletionService.createRequestData(body);
const streamFactory = await ChatCompletionService.sendRequest(payload, false, abortSignal);
let streamFactory;
try {
streamFactory = await ChatCompletionService.sendRequest(payload, false, abortSignal);
} catch (err) {
logSendRequestError(err, true);
throw err;
}
const generator = (typeof streamFactory === 'function') ? streamFactory() : streamFactory;
@@ -350,7 +374,13 @@ class StreamingGeneration {
})();
} else {
const payload = ChatCompletionService.createRequestData(body);
const extracted = await ChatCompletionService.sendRequest(payload, false, abortSignal);
let extracted;
try {
extracted = await ChatCompletionService.sendRequest(payload, false, abortSignal);
} catch (err) {
logSendRequestError(err, false);
throw err;
}
let result = '';
if (extracted && typeof extracted === 'object') {