X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add L0 index and anchor UI updates

Browse Source
This commit is contained in:
bielie
2026-02-06 11:22:02 +08:00
parent c36efe6805
commit 44ca06f9b9
23 changed files with 1749 additions and 3898 deletions
Download Patch File Download Diff File Expand all files Collapse all files

251
modules/story-summary/vector/llm/atom-extraction.js Normal file
View File

@@ -0,0 +1,251 @@
// ============================================================================
// atom-extraction.js - 30并发 + 首批错开 + 取消支持 + 进度回调
// ============================================================================
import { callLLM, parseJson } from './llm-service.js';
import { xbLog } from '../../../../core/debug-core.js';
import { filterText } from '../utils/text-filter.js';
const MODULE_ID = 'atom-extraction';
const CONCURRENCY = 10;
const RETRY_COUNT = 2;
const RETRY_DELAY = 500;
const DEFAULT_TIMEOUT = 20000;
const STAGGER_DELAY = 80; // 首批错开延迟（ms）
let batchCancelled = false;
export function cancelBatchExtraction() {
batchCancelled = true;
}
export function isBatchCancelled() {
return batchCancelled;
}
const SYSTEM_PROMPT = `你是叙事锚点提取器。从一轮对话（用户发言+角色回复）中提取4-8个关键锚点。
只输出JSON：
{"atoms":[{"t":"类型","s":"主体","v":"值","f":"来源"}]}
类型（t）：
- emo: 情绪状态（需要s主体）
- loc: 地点/场景
- act: 关键动作（需要s主体）
- rev: 揭示/发现
- ten: 冲突/张力
- dec: 决定/承诺
规则：
- s: 主体（谁）
- v: 简洁值，10字内
- f: "u"=用户发言中, "a"=角色回复中
- 只提取对未来检索有价值的锚点
- 无明显锚点返回空数组`;
function buildSemantic(atom, userName, aiName) {
const speaker = atom.f === 'u' ? userName : aiName;
const s = atom.s || speaker;
switch (atom.t) {
case 'emo': return `${s}感到${atom.v}`;
case 'loc': return `场景：${atom.v}`;
case 'act': return `${s}${atom.v}`;
case 'rev': return `揭示：${atom.v}`;
case 'ten': return `冲突：${atom.v}`;
case 'dec': return `${s}决定${atom.v}`;
default: return `${s} ${atom.v}`;
}
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options = {}) {
const { timeout = DEFAULT_TIMEOUT } = options;
if (!aiMessage?.mes?.trim()) return [];
const parts = [];
const userName = userMessage?.name || '用户';
const aiName = aiMessage.name || '角色';
if (userMessage?.mes?.trim()) {
const userText = filterText(userMessage.mes);
parts.push(`【用户：${userName}】\n${userText}`);
}
const aiText = filterText(aiMessage.mes);
parts.push(`【角色：${aiName}】\n${aiText}`);
const input = parts.join('\n\n---\n\n');
xbLog.info(MODULE_ID, `floor ${aiFloor} 发送输入 len=${input.length}`);
for (let attempt = 0; attempt <= RETRY_COUNT; attempt++) {
if (batchCancelled) return [];
try {
const response = await callLLM([
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: input },
], {
temperature: 0.2,
max_tokens: 500,
timeout,
});
if (!response || !String(response).trim()) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：响应为空`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
}
return [];
}
let parsed;
try {
parsed = parseJson(response);
} catch (e) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：JSON 异常`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
}
return [];
}
if (!parsed?.atoms || !Array.isArray(parsed.atoms)) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：atoms 缺失`);
if (attempt < RETRY_COUNT) {
await sleep(RETRY_DELAY);
continue;
}
return [];
}
return parsed.atoms
.filter(a => a?.t && a?.v)
.map((a, idx) => ({
atomId: `atom-${aiFloor}-${idx}`,
floor: aiFloor,
type: a.t,
subject: a.s || null,
value: String(a.v).slice(0, 30),
source: a.f === 'u' ? 'user' : 'ai',
semantic: buildSemantic(a, userName, aiName),
}));
} catch (e) {
if (batchCancelled) return [];
if (attempt < RETRY_COUNT) {
xbLog.warn(MODULE_ID, `floor ${aiFloor} 第${attempt + 1}次失败，重试...`, e?.message);
await sleep(RETRY_DELAY * (attempt + 1));
continue;
}
xbLog.error(MODULE_ID, `floor ${aiFloor} 失败`, e);
return [];
}
}
return [];
}
/**
* 单轮配对提取（增量时使用）
*/
export async function extractAtomsForRound(userMessage, aiMessage, aiFloor, options = {}) {
return extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options);
}
/**
* 批量提取（首批 staggered 启动）
* @param {Array} chat
* @param {Function} onProgress - (current, total, failed) => void
*/
export async function batchExtractAtoms(chat, onProgress) {
if (!chat?.length) return [];
batchCancelled = false;
const pairs = [];
for (let i = 0; i < chat.length; i++) {
if (!chat[i].is_user) {
const userMsg = (i > 0 && chat[i - 1]?.is_user) ? chat[i - 1] : null;
pairs.push({ userMsg, aiMsg: chat[i], aiFloor: i });
}
}
if (!pairs.length) return [];
const allAtoms = [];
let completed = 0;
let failed = 0;
for (let i = 0; i < pairs.length; i += CONCURRENCY) {
if (batchCancelled) {
xbLog.info(MODULE_ID, `批量提取已取消 (${completed}/${pairs.length})`);
break;
}
const batch = pairs.slice(i, i + CONCURRENCY);
// ★ 首批 staggered 启动：错开 80ms 发送
if (i === 0) {
const promises = batch.map((pair, idx) => (async () => {
await sleep(idx * STAGGER_DELAY);
if (batchCancelled) return;
try {
const atoms = await extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT });
if (atoms?.length) {
allAtoms.push(...atoms);
} else {
failed++;
}
} catch {
failed++;
}
completed++;
onProgress?.(completed, pairs.length, failed);
})());
await Promise.all(promises);
} else {
// 后续批次正常并行
const promises = batch.map(pair =>
extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT })
.then(atoms => {
if (batchCancelled) return;
if (atoms?.length) {
allAtoms.push(...atoms);
} else {
failed++;
}
completed++;
onProgress?.(completed, pairs.length, failed);
})
.catch(() => {
if (batchCancelled) return;
failed++;
completed++;
onProgress?.(completed, pairs.length, failed);
})
);
await Promise.all(promises);
}
// 批次间隔
if (i + CONCURRENCY < pairs.length && !batchCancelled) {
await sleep(30);
}
}
const status = batchCancelled ? '已取消' : '完成';
xbLog.info(MODULE_ID, `批量提取${status}: ${allAtoms.length} atoms, ${completed}/${pairs.length}, ${failed} 失败`);
return allAtoms;
}

72
modules/story-summary/vector/llm/llm-service.js Normal file
View File

@@ -0,0 +1,72 @@
// ═══════════════════════════════════════════════════════════════════════════
// vector/llm/llm-service.js
// ═══════════════════════════════════════════════════════════════════════════
import { xbLog } from '../../../../core/debug-core.js';
const MODULE_ID = 'vector-llm-service';
// 唯一 ID 计数器
let callCounter = 0;
function getStreamingModule() {
const mod = window.xiaobaixStreamingGeneration;
return mod?.xbgenrawCommand ? mod : null;
}
function generateUniqueId(prefix = 'llm') {
callCounter = (callCounter + 1) % 100000;
return `${prefix}-${callCounter}-${Date.now().toString(36)}`;
}
function b64UrlEncode(str) {
const utf8 = new TextEncoder().encode(String(str));
let bin = '';
utf8.forEach(b => bin += String.fromCharCode(b));
return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
}
/**
* 统一LLM调用 - 走酒馆后端（非流式）
*/
export async function callLLM(messages, options = {}) {
const {
temperature = 0.2,
max_tokens = 500,
} = options;
const mod = getStreamingModule();
if (!mod) throw new Error('生成模块未加载');
const top64 = b64UrlEncode(JSON.stringify(messages));
// ★ 每次调用用唯一 ID，避免 session 冲突
const uniqueId = generateUniqueId('l0');
const args = {
as: 'user',
nonstream: 'true',
top64,
id: uniqueId,
temperature: String(temperature),
max_tokens: String(max_tokens),
};
try {
// 非流式直接返回结果
const result = await mod.xbgenrawCommand(args, '');
return String(result ?? '');
} catch (e) {
xbLog.error(MODULE_ID, 'LLM调用失败', e);
throw e;
}
}
export function parseJson(text) {
if (!text) return null;
let s = text.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/i, '').trim();
try { return JSON.parse(s); } catch { }
const i = s.indexOf('{'), j = s.lastIndexOf('}');
if (i !== -1 && j > i) try { return JSON.parse(s.slice(i, j + 1)); } catch { }
return null;
}

102
modules/story-summary/vector/llm/query-expansion.js Normal file
View File

@@ -0,0 +1,102 @@
// ═══════════════════════════════════════════════════════════════════════════
// query-expansion.js - 完整输入，不截断
// ═══════════════════════════════════════════════════════════════════════════
import { callLLM, parseJson } from './llm-service.js';
import { xbLog } from '../../../../core/debug-core.js';
import { filterText } from '../utils/text-filter.js';
const MODULE_ID = 'query-expansion';
const SESSION_ID = 'xb6';
const SYSTEM_PROMPT = `你是检索词生成器。根据最近对话，输出用于检索历史剧情的关键词。
只输出JSON：
{"e":["显式人物/地名"],"i":["隐含人物/情绪/话题"],"q":["检索短句"]}
规则：
- e: 对话中明确提到的人名/地名，1-4个
- i: 推断出的相关人物/情绪/话题，1-5个
- q: 用于向量检索的短句，2-3个，每个15字内
- 关注：正在讨论什么、涉及谁、情绪氛围`;
/**
* Query Expansion
* @param {Array} messages - 完整消息数组（最后2-3轮）
*/
export async function expandQuery(messages, options = {}) {
const { timeout = 6000 } = options;
if (!messages?.length) {
return { entities: [], implicit: [], queries: [] };
}
// 完整格式化，不截断
const input = messages.map(m => {
const speaker = m.is_user ? '用户' : (m.name || '角色');
const text = filterText(m.mes || '').trim();
return `【${speaker}】\n${text}`;
}).join('\n\n');
const T0 = performance.now();
try {
const response = await callLLM([
{ role: 'system', content: SYSTEM_PROMPT },
{ role: 'user', content: input },
], {
temperature: 0.15,
max_tokens: 250,
timeout,
sessionId: SESSION_ID,
});
const parsed = parseJson(response);
if (!parsed) {
xbLog.warn(MODULE_ID, 'JSON解析失败', response?.slice(0, 200));
return { entities: [], implicit: [], queries: [] };
}
const result = {
entities: Array.isArray(parsed.e) ? parsed.e.slice(0, 5) : [],
implicit: Array.isArray(parsed.i) ? parsed.i.slice(0, 6) : [],
queries: Array.isArray(parsed.q) ? parsed.q.slice(0, 4) : [],
};
xbLog.info(MODULE_ID, `完成 (${Math.round(performance.now() - T0)}ms) e=${result.entities.length} i=${result.implicit.length} q=${result.queries.length}`);
return result;
} catch (e) {
xbLog.error(MODULE_ID, '调用失败', e);
return { entities: [], implicit: [], queries: [] };
}
}
// 缓存
const cache = new Map();
const CACHE_TTL = 300000;
function hashMessages(messages) {
const text = messages.slice(-2).map(m => (m.mes || '').slice(0, 100)).join('|');
let h = 0;
for (let i = 0; i < text.length; i++) h = ((h << 5) - h + text.charCodeAt(i)) | 0;
return h.toString(36);
}
export async function expandQueryCached(messages, options = {}) {
const key = hashMessages(messages);
const cached = cache.get(key);
if (cached && Date.now() - cached.time < CACHE_TTL) return cached.result;
const result = await expandQuery(messages, options);
if (result.entities.length || result.queries.length) {
if (cache.size > 50) cache.delete(cache.keys().next().value);
cache.set(key, { result, time: Date.now() });
}
return result;
}
export function buildSearchText(expansion) {
return [...(expansion.entities || []), ...(expansion.implicit || []), ...(expansion.queries || [])]
.filter(Boolean).join(' ');
}

59
modules/story-summary/vector/llm/siliconflow.js Normal file
View File

@@ -0,0 +1,59 @@
// ═══════════════════════════════════════════════════════════════════════════
// siliconflow.js - 仅保留 Embedding
// ═══════════════════════════════════════════════════════════════════════════
const BASE_URL = 'https://api.siliconflow.cn';
const EMBEDDING_MODEL = 'BAAI/bge-m3';
export function getApiKey() {
try {
const raw = localStorage.getItem('summary_panel_config');
if (raw) {
const parsed = JSON.parse(raw);
return parsed.vector?.online?.key || null;
}
} catch { }
return null;
}
export async function embed(texts, options = {}) {
if (!texts?.length) return [];
const key = getApiKey();
if (!key) throw new Error('未配置硅基 API Key');
const { timeout = 30000, signal } = options;
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const response = await fetch(`${BASE_URL}/v1/embeddings`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: EMBEDDING_MODEL,
input: texts,
}),
signal: signal || controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
const errorText = await response.text().catch(() => '');
throw new Error(`Embedding ${response.status}: ${errorText.slice(0, 200)}`);
}
const data = await response.json();
return (data.data || [])
.sort((a, b) => a.index - b.index)
.map(item => Array.isArray(item.embedding) ? item.embedding : Array.from(item.embedding));
} finally {
clearTimeout(timeoutId);
}
}
export { EMBEDDING_MODEL as MODELS };

28
modules/story-summary/vector/pipeline/chunk-builder.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Chunk Builder
// 标准 RAG chunking: ~200 tokens per chunk
// ═══════════════════════════════════════════════════════════════════════════
@@ -19,6 +19,7 @@ import {
import { embed, getEngineFingerprint } from '../utils/embedder.js';
import { xbLog } from '../../../../core/debug-core.js';
import { filterText } from '../utils/text-filter.js';
import { extractAndStoreAtomsForRound } from './state-integration.js';
const MODULE_ID = 'chunk-builder';
@@ -201,8 +202,7 @@ export async function buildAllChunks(options = {}) {
await saveChunks(chatId, allChunks);
const texts = allChunks.map(c => c.text);
const isLocal = vectorConfig.engine === 'local';
const batchSize = isLocal ? 5 : 20;
const batchSize = 20;
let completed = 0;
let errors = 0;
@@ -302,6 +302,7 @@ export async function buildIncrementalChunks(options = {}) {
}
}
// ═══════════════════════════════════════════════════════════════════════════
// L1 同步（消息变化时调用）
// ═══════════════════════════════════════════════════════════════════════════
@@ -337,13 +338,6 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
if (!chatId || lastFloor < 0 || !message) return;
if (!vectorConfig?.enabled) return;
// 本地模型未加载时跳过（避免意外触发下载或报错）
if (vectorConfig.engine === "local") {
const { isLocalModelLoaded, DEFAULT_LOCAL_MODEL } = await import("../utils/embedder.js");
const modelId = vectorConfig.local?.modelId || DEFAULT_LOCAL_MODEL;
if (!isLocalModelLoaded(modelId)) return;
}
// 删除该楼层旧的
await deleteChunksAtFloor(chatId, lastFloor);
@@ -367,4 +361,18 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
} catch (e) {
xbLog.error(MODULE_ID, `消息同步失败：floor ${lastFloor}`, e);
}
// L0 配对提取（仅 AI 消息触发）
if (!message.is_user) {
const { chat } = getContext();
const userFloor = lastFloor - 1;
const userMessage = (userFloor >= 0 && chat[userFloor]?.is_user) ? chat[userFloor] : null;
try {
await extractAndStoreAtomsForRound(lastFloor, message, userMessage);
} catch (e) {
xbLog.warn(MODULE_ID, `Atom 提取失败: floor ${lastFloor}`, e);
}
}
}

306
modules/story-summary/vector/pipeline/state-integration.js
View File

@@ -1,7 +1,7 @@
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - State Integration (L0)
// 事件监听 + 回滚钩子注册
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
// state-integration.js - L0 记忆锚点管理
// 支持增量提取、清空、取消
// ============================================================================
import { getContext } from '../../../../../../../extensions.js';
import { xbLog } from '../../../../core/debug-core.js';
@@ -11,70 +11,174 @@ import {
deleteStateAtomsFromFloor,
deleteStateVectorsFromFloor,
getStateAtoms,
clearStateAtoms,
clearStateVectors,
getL0FloorStatus,
setL0FloorStatus,
clearL0Index,
deleteL0IndexFromFloor,
} from '../storage/state-store.js';
import { embed, getEngineFingerprint } from '../utils/embedder.js';
import { embed } from '../llm/siliconflow.js';
import { extractAtomsForRound, cancelBatchExtraction } from '../llm/atom-extraction.js';
import { getVectorConfig } from '../../data/config.js';
import { getEngineFingerprint } from '../utils/embedder.js';
import { filterText } from '../utils/text-filter.js';
const MODULE_ID = 'state-integration';
let initialized = false;
// ═══════════════════════════════════════════════════════════════════════════
export function cancelL0Extraction() {
cancelBatchExtraction();
}
// ============================================================================
// 初始化
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
export function initStateIntegration() {
if (initialized) return;
initialized = true;
// 监听变量团队的事件
$(document).on('xiaobaix:variables:stateAtomsGenerated', handleStateAtomsGenerated);
// 注册回滚钩子
globalThis.LWB_StateRollbackHook = handleStateRollback;
xbLog.info(MODULE_ID, 'L0 状态层集成已初始化');
}
// ═══════════════════════════════════════════════════════════════════════════
// 事件处理
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
// 统计
// ============================================================================
async function handleStateAtomsGenerated(e, data) {
const { atoms } = data || {};
if (!atoms?.length) return;
const { chatId } = getContext();
if (!chatId) return;
const validAtoms = atoms.filter(a => a?.chatId === chatId);
if (!validAtoms.length) {
xbLog.warn(MODULE_ID, `atoms.chatId 不匹配，期望 ${chatId}，跳过`);
return;
export async function getAnchorStats() {
const { chat } = getContext();
if (!chat?.length) {
return { extracted: 0, total: 0, pending: 0, empty: 0, fail: 0 };
}
xbLog.info(MODULE_ID, `收到 ${validAtoms.length} 个 StateAtom`);
// 1. 存入 chat_metadata（持久化）
saveStateAtoms(validAtoms);
// 2. 向量化并存入 IndexedDB
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) {
xbLog.info(MODULE_ID, '向量未启用，跳过 L0 向量化');
return;
const aiFloors = [];
for (let i = 0; i < chat.length; i++) {
if (!chat[i]?.is_user) aiFloors.push(i);
}
await vectorizeAtoms(chatId, validAtoms, vectorCfg);
let ok = 0;
let empty = 0;
let fail = 0;
for (const f of aiFloors) {
const s = getL0FloorStatus(f);
if (!s) continue;
if (s.status === 'ok') ok++;
else if (s.status === 'empty') empty++;
else if (s.status === 'fail') fail++;
}
const total = aiFloors.length;
const completed = ok + empty;
const pending = Math.max(0, total - completed);
return { extracted: completed, total, pending, empty, fail };
}
async function vectorizeAtoms(chatId, atoms, vectorCfg) {
// ============================================================================
// 增量提取
// ============================================================================
function buildL0InputText(userMessage, aiMessage) {
const parts = [];
const userName = userMessage?.name || '用户';
const aiName = aiMessage?.name || '角色';
if (userMessage?.mes?.trim()) {
parts.push(`【用户：${userName}】\n${filterText(userMessage.mes).trim()}`);
}
if (aiMessage?.mes?.trim()) {
parts.push(`【角色：${aiName}】\n${filterText(aiMessage.mes).trim()}`);
}
return parts.join('\n\n---\n\n').trim();
}
export async function incrementalExtractAtoms(chatId, chat, onProgress) {
if (!chatId || !chat?.length) return { built: 0 };
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return { built: 0 };
const pendingPairs = [];
for (let i = 0; i < chat.length; i++) {
const msg = chat[i];
if (!msg || msg.is_user) continue;
const st = getL0FloorStatus(i);
if (st?.status === 'ok' || st?.status === 'empty') {
continue;
}
const userMsg = (i > 0 && chat[i - 1]?.is_user) ? chat[i - 1] : null;
const inputText = buildL0InputText(userMsg, msg);
if (!inputText) {
setL0FloorStatus(i, { status: 'empty', reason: 'filtered_empty', atoms: 0 });
continue;
}
pendingPairs.push({ userMsg, aiMsg: msg, aiFloor: i });
}
if (!pendingPairs.length) {
onProgress?.(0, 0, '已全部提取');
return { built: 0 };
}
xbLog.info(MODULE_ID, `增量 L0 提取：pending=${pendingPairs.length}`);
let completed = 0;
const total = pendingPairs.length;
let builtAtoms = 0;
for (const pair of pendingPairs) {
const floor = pair.aiFloor;
const prev = getL0FloorStatus(floor);
try {
const atoms = await extractAtomsForRound(pair.userMsg, pair.aiMsg, floor, { timeout: 20000 });
if (!atoms?.length) {
setL0FloorStatus(floor, { status: 'empty', reason: 'llm_empty', atoms: 0 });
} else {
atoms.forEach(a => a.chatId = chatId);
saveStateAtoms(atoms);
await vectorizeAtoms(chatId, atoms);
setL0FloorStatus(floor, { status: 'ok', atoms: atoms.length });
builtAtoms += atoms.length;
}
} catch (e) {
setL0FloorStatus(floor, {
status: 'fail',
attempts: (prev?.attempts || 0) + 1,
reason: String(e?.message || e).replace(/\s+/g, ' ').slice(0, 120),
});
} finally {
completed++;
onProgress?.(`L0: ${completed}/${total}`, completed, total);
}
}
xbLog.info(MODULE_ID, `增量 L0 完成：atoms=${builtAtoms}, floors=${pendingPairs.length}`);
return { built: builtAtoms };
}
async function vectorizeAtoms(chatId, atoms) {
if (!atoms?.length) return;
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return;
const texts = atoms.map(a => a.semantic);
const fingerprint = getEngineFingerprint(vectorCfg);
try {
const vectors = await embed(texts, vectorCfg);
const vectors = await embed(texts, { timeout: 30000 });
const items = atoms.map((a, i) => ({
atomId: a.atomId,
@@ -83,34 +187,106 @@ async function vectorizeAtoms(chatId, atoms, vectorCfg) {
}));
await saveStateVectors(chatId, items, fingerprint);
xbLog.info(MODULE_ID, `L0 向量化完成: ${items.length} 个`);
xbLog.info(MODULE_ID, `L0 向量化完成: ${items.length} 条`);
} catch (e) {
xbLog.error(MODULE_ID, 'L0 向量化失败', e);
// 不阻塞，向量可后续通过"生成向量"重建
}
}
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
// 清空
// ============================================================================
export async function clearAllAtomsAndVectors(chatId) {
clearStateAtoms();
clearL0Index();
if (chatId) {
await clearStateVectors(chatId);
}
xbLog.info(MODULE_ID, '已清空所有记忆锚点');
}
// ============================================================================
// 实时增量（AI 消息后触发）- 保留原有逻辑
// ============================================================================
let extractionQueue = [];
let isProcessing = false;
export async function extractAndStoreAtomsForRound(aiFloor, aiMessage, userMessage) {
const { chatId } = getContext();
if (!chatId) return;
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return;
extractionQueue.push({ aiFloor, aiMessage, userMessage, chatId });
processQueue();
}
async function processQueue() {
if (isProcessing || extractionQueue.length === 0) return;
isProcessing = true;
while (extractionQueue.length > 0) {
const { aiFloor, aiMessage, userMessage, chatId } = extractionQueue.shift();
try {
const atoms = await extractAtomsForRound(userMessage, aiMessage, aiFloor, { timeout: 12000 });
if (!atoms?.length) {
xbLog.info(MODULE_ID, `floor ${aiFloor}: 无有效 atoms`);
continue;
}
atoms.forEach(a => a.chatId = chatId);
saveStateAtoms(atoms);
await vectorizeAtoms(chatId, atoms);
xbLog.info(MODULE_ID, `floor ${aiFloor}: ${atoms.length} atoms 已存储`);
} catch (e) {
xbLog.error(MODULE_ID, `floor ${aiFloor} 处理失败`, e);
}
}
isProcessing = false;
}
// ============================================================================
// 回滚钩子
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
async function handleStateRollback(floor) {
xbLog.info(MODULE_ID, `收到回滚请求: floor >= ${floor}`);
const { chatId } = getContext();
// 1. 删除 chat_metadata 中的 atoms
deleteStateAtomsFromFloor(floor);
deleteL0IndexFromFloor(floor);
// 2. 删除 IndexedDB 中的 vectors
if (chatId) {
await deleteStateVectorsFromFloor(chatId, floor);
}
}
// ═══════════════════════════════════════════════════════════════════════════
// 重建向量（供"生成向量"按钮调用）
// ═══════════════════════════════════════════════════════════════════════════
// ============================================================================
// 兼容旧接口
// ============================================================================
export async function batchExtractAndStoreAtoms(chatId, chat, onProgress) {
if (!chatId || !chat?.length) return { built: 0 };
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return { built: 0 };
xbLog.info(MODULE_ID, `开始批量 L0 提取: ${chat.length} 条消息`);
clearStateAtoms();
clearL0Index();
await clearStateVectors(chatId);
return await incrementalExtractAtoms(chatId, chat, onProgress);
}
export async function rebuildStateVectors(chatId, vectorCfg) {
if (!chatId || !vectorCfg?.enabled) return { built: 0 };
@@ -118,36 +294,10 @@ export async function rebuildStateVectors(chatId, vectorCfg) {
const atoms = getStateAtoms();
if (!atoms.length) return { built: 0 };
xbLog.info(MODULE_ID, `开始重建 L0 向量: ${atoms.length} 个 atom`);
xbLog.info(MODULE_ID, `重建 L0 向量: ${atoms.length} 条 atom`);
// 清空旧向量
await clearStateVectors(chatId);
await vectorizeAtoms(chatId, atoms);
// 重新向量化
const fingerprint = getEngineFingerprint(vectorCfg);
const batchSize = vectorCfg.engine === 'local' ? 5 : 25;
let built = 0;
for (let i = 0; i < atoms.length; i += batchSize) {
const batch = atoms.slice(i, i + batchSize);
const texts = batch.map(a => a.semantic);
try {
const vectors = await embed(texts, vectorCfg);
const items = batch.map((a, j) => ({
atomId: a.atomId,
floor: a.floor,
vector: vectors[j],
}));
await saveStateVectors(chatId, items, fingerprint);
built += items.length;
} catch (e) {
xbLog.error(MODULE_ID, `L0 向量化批次失败: ${i}-${i + batchSize}`, e);
}
}
xbLog.info(MODULE_ID, `L0 向量重建完成: ${built}/${atoms.length}`);
return { built };
return { built: atoms.length };
}

129
modules/story-summary/vector/retrieval/entity.js
View File

@@ -1,129 +0,0 @@
// ═══════════════════════════════════════════════════════════════════════════
// Entity Recognition & Relation Graph
// 实体识别与关系扩散
// ═══════════════════════════════════════════════════════════════════════════
/**
* 从文本中匹配已知实体
* @param {string} text - 待匹配文本
* @param {Set<string>} knownEntities - 已知实体集合
* @returns {string[]} - 匹配到的实体
*/
export function matchEntities(text, knownEntities) {
if (!text || !knownEntities?.size) return [];
const matched = new Set();
for (const entity of knownEntities) {
// 精确包含
if (text.includes(entity)) {
matched.add(entity);
continue;
}
// 处理简称：如果实体是"林黛玉"，文本包含"黛玉"
if (entity.length >= 3) {
const shortName = entity.slice(-2); // 取后两字
if (text.includes(shortName)) {
matched.add(entity);
}
}
}
return Array.from(matched);
}
/**
* 从角色数据和事件中收集所有已知实体
*/
export function collectKnownEntities(characters, events) {
const entities = new Set();
// 从主要角色
(characters?.main || []).forEach(m => {
const name = typeof m === 'string' ? m : m.name;
if (name) entities.add(name);
});
// 从关系
(characters?.relationships || []).forEach(r => {
if (r.from) entities.add(r.from);
if (r.to) entities.add(r.to);
});
// 从事件参与者
(events || []).forEach(e => {
(e.participants || []).forEach(p => {
if (p) entities.add(p);
});
});
return entities;
}
/**
* 构建关系邻接表
* @param {Array} relationships - 关系数组
* @returns {Map<string, Array<{target: string, weight: number}>>}
*/
export function buildRelationGraph(relationships) {
const graph = new Map();
const trendWeight = {
'交融': 1.0,
'亲密': 0.9,
'投缘': 0.7,
'陌生': 0.3,
'反感': 0.5,
'厌恶': 0.6,
'破裂': 0.7,
};
for (const rel of relationships || []) {
if (!rel.from || !rel.to) continue;
const weight = trendWeight[rel.trend] || 0.5;
// 双向
if (!graph.has(rel.from)) graph.set(rel.from, []);
if (!graph.has(rel.to)) graph.set(rel.to, []);
graph.get(rel.from).push({ target: rel.to, weight });
graph.get(rel.to).push({ target: rel.from, weight });
}
return graph;
}
/**
* 关系扩散（1跳）
* @param {string[]} focusEntities - 焦点实体
* @param {Map} graph - 关系图
* @param {number} decayFactor - 衰减因子
* @returns {Map<string, number>} - 实体 -> 激活分数
*/
export function spreadActivation(focusEntities, graph, decayFactor = 0.5) {
const activation = new Map();
// 焦点实体初始分数 1.0
for (const entity of focusEntities) {
activation.set(entity, 1.0);
}
// 1跳扩散
for (const entity of focusEntities) {
const neighbors = graph.get(entity) || [];
for (const { target, weight } of neighbors) {
const spreadScore = weight * decayFactor;
const existing = activation.get(target) || 0;
// 取最大值，不累加
if (spreadScore > existing) {
activation.set(target, spreadScore);
}
}
}
return activation;
}

1088
modules/story-summary/vector/retrieval/recall.js
View File

File diff suppressed because it is too large Load Diff

237
modules/story-summary/vector/retrieval/text-search.js
View File

@@ -1,237 +0,0 @@
// text-search.js - 最终版
import MiniSearch from '../../../../libs/minisearch.mjs';
const STOP_WORDS = new Set([
'的', '了', '是', '在', '和', '与', '或', '但', '而', '却',
'这', '那', '他', '她', '它', '我', '你', '们', '着', '过',
'把', '被', '给', '让', '向', '就', '都', '也', '还', '又',
'很', '太', '更', '最', '只', '才', '已', '正', '会', '能',
'要', '可', '得', '地', '之', '所', '以', '为', '于', '有',
'不', '去', '来', '上', '下', '里', '说', '看', '吧', '呢',
'啊', '吗', '呀', '哦', '嗯', '么',
'の', 'に', 'は', 'を', 'が', 'と', 'で', 'へ', 'や', 'か',
'も', 'な', 'よ', 'ね', 'わ', 'です', 'ます', 'した', 'ない',
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been',
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from',
'and', 'or', 'but', 'if', 'that', 'this', 'it', 'its',
'i', 'you', 'he', 'she', 'we', 'they', 'my', 'your', 'his',
]);
function tokenize(text) {
const s = String(text || '').toLowerCase().trim();
if (!s) return [];
const tokens = new Set();
// CJK Bigram + Trigram
const cjk = s.match(/[\u4e00-\u9fff\u3400-\u4dbf]+/g) || [];
for (const seg of cjk) {
const chars = [...seg].filter(c => !STOP_WORDS.has(c));
for (let i = 0; i < chars.length - 1; i++) {
tokens.add(chars[i] + chars[i + 1]);
}
for (let i = 0; i < chars.length - 2; i++) {
tokens.add(chars[i] + chars[i + 1] + chars[i + 2]);
}
}
// 日语假名
const kana = s.match(/[\u3040-\u309f\u30a0-\u30ff]{2,}/g) || [];
for (const k of kana) {
if (!STOP_WORDS.has(k)) tokens.add(k);
}
// 英文
const en = s.match(/[a-z]{2,}/g) || [];
for (const w of en) {
if (!STOP_WORDS.has(w)) tokens.add(w);
}
return [...tokens];
}
let idx = null;
let lastRevision = null;
function stripFloorTag(s) {
return String(s || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').trim();
}
export function ensureEventTextIndex(events, revision) {
if (!events?.length) {
idx = null;
lastRevision = null;
return;
}
if (idx && revision === lastRevision) return;
try {
idx = new MiniSearch({
fields: ['title', 'summary', 'participants'],
storeFields: ['id'],
tokenize,
searchOptions: { tokenize },
});
idx.addAll(events.map(e => ({
id: e.id,
title: e.title || '',
summary: stripFloorTag(e.summary),
participants: (e.participants || []).join(' '),
})));
lastRevision = revision;
} catch (e) {
console.error('[text-search] Index build failed:', e);
idx = null;
}
}
/**
* BM25 检索，返回 top-K 候选给 RRF
*
* 设计原则：
* - 不做分数过滤（BM25 分数跨查询不可比）
* - 不做匹配数过滤（bigram 让一个词产生多个 token）
* - 只做 top-K（BM25 排序本身有区分度）
* - 质量过滤交给 RRF 后的 hasVector 过滤
*/
/**
* 动态 top-K：累积分数占比法
*
* 原理：BM25 分数服从幂律分布，少数高分条目贡献大部分总分
* 取累积分数达到阈值的最小 K
*
* 参考：帕累托法则（80/20 法则）在信息检索中的应用
*/
export function dynamicTopK(scores, coverage = 0.90, minK = 15, maxK = 80) {
if (!scores.length) return 0;
const total = scores.reduce((a, b) => a + b, 0);
if (total <= 0) return Math.min(minK, scores.length);
let cumulative = 0;
for (let i = 0; i < scores.length; i++) {
cumulative += scores[i];
if (cumulative / total >= coverage) {
return Math.max(minK, Math.min(maxK, i + 1));
}
}
return Math.min(maxK, scores.length);
}
export function searchEventsByText(queryText, limit = 80) {
if (!idx || !queryText?.trim()) return [];
try {
const results = idx.search(queryText, {
boost: { title: 4, participants: 2, summary: 1 },
fuzzy: false,
prefix: false,
});
if (!results.length) return [];
const scores = results.map(r => r.score);
const k = dynamicTopK(scores, 0.90, 15, limit);
const output = results.slice(0, k).map((r, i) => ({
id: r.id,
textRank: i + 1,
score: r.score,
}));
const total = scores.reduce((a, b) => a + b, 0);
const kCumulative = scores.slice(0, k).reduce((a, b) => a + b, 0);
output._gapInfo = {
total: results.length,
returned: k,
coverage: ((kCumulative / total) * 100).toFixed(1) + '%',
scoreRange: {
top: scores[0]?.toFixed(1),
cutoff: scores[k - 1]?.toFixed(1),
p50: scores[Math.floor(scores.length / 2)]?.toFixed(1),
last: scores[scores.length - 1]?.toFixed(1),
},
};
return output;
} catch (e) {
console.error('[text-search] Search failed:', e);
return [];
}
}
export function clearEventTextIndex() {
idx = null;
lastRevision = null;
}
// ---------------------------------------------------------------------------
// Chunk 文本索引（待整理区 L1 补充）
// ---------------------------------------------------------------------------
let chunkIdx = null;
let chunkIdxRevision = null;
export function ensureChunkTextIndex(chunks, revision) {
if (chunkIdx && revision === chunkIdxRevision) return;
try {
chunkIdx = new MiniSearch({
fields: ['text'],
storeFields: ['chunkId', 'floor'],
tokenize,
searchOptions: { tokenize },
});
chunkIdx.addAll(chunks.map(c => ({
id: c.chunkId,
chunkId: c.chunkId,
floor: c.floor,
text: c.text || '',
})));
chunkIdxRevision = revision;
} catch (e) {
console.error('[text-search] Chunk index build failed:', e);
chunkIdx = null;
}
}
export function searchChunksByText(query, floorMin, floorMax, limit = 20) {
if (!chunkIdx || !query?.trim()) return [];
try {
const results = chunkIdx.search(query, {
fuzzy: false,
prefix: false,
});
const filtered = results.filter(r => r.floor >= floorMin && r.floor <= floorMax);
if (!filtered.length) return [];
const scores = filtered.map(r => r.score);
const k = dynamicTopK(scores, 0.85, 5, limit);
return filtered.slice(0, k).map((r, i) => ({
chunkId: r.chunkId,
floor: r.floor,
textRank: i + 1,
score: r.score,
}));
} catch (e) {
console.error('[text-search] Chunk search failed:', e);
return [];
}
}
export function clearChunkTextIndex() {
chunkIdx = null;
chunkIdxRevision = null;
}

287
modules/story-summary/vector/retrieval/tokenizer.js
View File

@@ -1,287 +0,0 @@
import { xbLog } from '../../../../core/debug-core.js';
import { extensionFolderPath } from '../../../../core/constants.js';
const MODULE_ID = 'tokenizer';
// ═══════════════════════════════════════════════════════════════════════════
// 词性过滤
// ═══════════════════════════════════════════════════════════════════════════
// 保留的词性（名词类 + 英文）
const KEEP_POS_PREFIXES = ['n', 'eng'];
function shouldKeepByPos(pos) {
return KEEP_POS_PREFIXES.some(prefix => pos.startsWith(prefix));
}
// ═══════════════════════════════════════════════════════════════════════════
// 语言检测
// ═══════════════════════════════════════════════════════════════════════════
function shouldUseJieba(text) {
const zh = (text.match(/[\u4e00-\u9fff]/g) || []).length;
return zh >= 5;
}
function detectMainLanguage(text) {
const zh = (text.match(/[\u4e00-\u9fff]/g) || []).length;
const jp = (text.match(/[\u3040-\u309f\u30a0-\u30ff]/g) || []).length;
const en = (text.match(/[a-zA-Z]/g) || []).length;
const total = zh + jp + en || 1;
if (jp / total > 0.2) return 'jp';
if (en / total > 0.5) return 'en';
return 'zh';
}
// 替换原有的大停用词表
const STOP_WORDS = new Set([
// 系统词
'用户', '角色', '玩家', '旁白', 'user', 'assistant', 'system',
// 时间泛词
'时候', '现在', '今天', '明天', '昨天', '早上', '晚上',
// 方位泛词
'这里', '那里', '上面', '下面', '里面', '外面',
// 泛化名词
'东西', '事情', '事儿', '地方', '样子', '意思', '感觉',
'一下', '一些', '一点', '一会', '一次',
]);
// 英文停用词（fallback 用）
const EN_STOP_WORDS = new Set([
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been',
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
'could', 'should', 'may', 'might', 'must', 'can',
'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from',
'and', 'or', 'but', 'if', 'that', 'this', 'it', 'its',
'i', 'you', 'he', 'she', 'we', 'they',
'my', 'your', 'his', 'her', 'our', 'their',
'what', 'which', 'who', 'whom', 'where', 'when', 'why', 'how',
]);
let jiebaModule = null;
let jiebaReady = false;
let jiebaLoading = false;
async function ensureJieba() {
if (jiebaReady) return true;
if (jiebaLoading) {
for (let i = 0; i < 50; i++) {
await new Promise(r => setTimeout(r, 100));
if (jiebaReady) return true;
}
return false;
}
jiebaLoading = true;
try {
const jiebaPath = `/${extensionFolderPath}/libs/jieba-wasm/jieba_rs_wasm.js`;
// eslint-disable-next-line no-unsanitized/method
jiebaModule = await import(jiebaPath);
if (jiebaModule.default) {
await jiebaModule.default();
}
jiebaReady = true;
xbLog.info(MODULE_ID, 'jieba-wasm 加载成功');
const keys = Object.getOwnPropertyNames(jiebaModule || {});
const dkeys = Object.getOwnPropertyNames(jiebaModule?.default || {});
xbLog.info(MODULE_ID, `jieba keys: ${keys.join(',')}`);
xbLog.info(MODULE_ID, `jieba default keys: ${dkeys.join(',')}`);
xbLog.info(MODULE_ID, `jieba.tag: ${typeof jiebaModule?.tag}`);
return true;
} catch (e) {
xbLog.error(MODULE_ID, 'jieba-wasm 加载失败', e);
jiebaLoading = false;
return false;
}
}
function fallbackTokenize(text) {
const tokens = [];
const lang = detectMainLanguage(text);
// 英文
const enMatches = text.match(/[a-zA-Z]{2,20}/gi) || [];
tokens.push(...enMatches.filter(w => !EN_STOP_WORDS.has(w.toLowerCase())));
// 日语假名
if (lang === 'jp') {
const kanaMatches = text.match(/[\u3040-\u309f\u30a0-\u30ff]{2,10}/g) || [];
tokens.push(...kanaMatches);
}
// 中文/日语汉字
const zhMatches = text.match(/[\u4e00-\u9fff]{2,6}/g) || [];
tokens.push(...zhMatches);
// 数字+汉字组合
const numZhMatches = text.match(/\d+[\u4e00-\u9fff]{1,4}/g) || [];
tokens.push(...numZhMatches);
return tokens;
}
export async function extractNouns(text, options = {}) {
const { minLen = 2, maxCount = 0 } = options;
if (!text?.trim()) return [];
// 中文为主 → 用 jieba
if (shouldUseJieba(text)) {
const hasJieba = await ensureJieba();
if (hasJieba && jiebaModule?.tag) {
try {
const tagged = jiebaModule.tag(text, true);
const result = [];
const seen = new Set();
const list = Array.isArray(tagged) ? tagged : [];
for (const item of list) {
let word = '';
let pos = '';
if (Array.isArray(item)) {
[word, pos] = item;
} else if (item && typeof item === 'object') {
word = item.word || item.w || item.text || item.term || '';
pos = item.tag || item.pos || item.p || '';
}
if (!word || !pos) continue;
if (word.length < minLen) continue;
if (!shouldKeepByPos(pos)) continue;
if (STOP_WORDS.has(word)) continue;
if (seen.has(word)) continue;
seen.add(word);
result.push(word);
if (maxCount > 0 && result.length >= maxCount) break;
}
return result;
} catch (e) {
xbLog.warn(MODULE_ID, 'jieba tag 失败：' + (e && e.message ? e.message : String(e)));
}
}
}
// 非中文 / jieba 失败 → fallback
const tokens = fallbackTokenize(text);
const result = [];
const seen = new Set();
for (const t of tokens) {
if (t.length < minLen) continue;
if (STOP_WORDS.has(t)) continue;
if (seen.has(t)) continue;
seen.add(t);
result.push(t);
if (maxCount > 0 && result.length >= maxCount) break;
}
return result;
}
export async function extractRareTerms(text, maxCount = 15) {
if (!text?.trim()) return [];
// 中文为主 → 用 jieba
if (shouldUseJieba(text)) {
const hasJieba = await ensureJieba();
if (hasJieba && jiebaModule?.tag) {
try {
const tagged = jiebaModule.tag(text, true);
const candidates = [];
const seen = new Set();
const list = Array.isArray(tagged) ? tagged : [];
for (const item of list) {
let word = '';
let pos = '';
if (Array.isArray(item)) {
[word, pos] = item;
} else if (item && typeof item === 'object') {
word = item.word || item.w || item.text || item.term || '';
pos = item.tag || item.pos || item.p || '';
}
if (!word || !pos) continue;
if (word.length < 2) continue;
if (!shouldKeepByPos(pos)) continue;
if (STOP_WORDS.has(word)) continue;
if (seen.has(word)) continue;
seen.add(word);
// 稀有度评分
let score = 0;
if (word.length >= 4) score += 3;
else if (word.length >= 3) score += 1;
if (/[a-zA-Z]/.test(word)) score += 2;
if (/\d/.test(word)) score += 1;
// 专名词性加分
if (['nr', 'ns', 'nt', 'nz'].some(p => pos.startsWith(p))) score += 2;
candidates.push({ term: word, score });
}
candidates.sort((a, b) => b.score - a.score);
return candidates.slice(0, maxCount).map(x => x.term);
} catch (e) {
xbLog.warn(MODULE_ID, 'jieba tag 失败：' + (e && e.message ? e.message : String(e)));
}
}
}
// 非中文 / jieba 失败 → fallback
const allNouns = await extractNouns(text, { minLen: 2, maxCount: 0 });
const scored = allNouns.map(t => {
let score = 0;
if (t.length >= 4) score += 3;
else if (t.length >= 3) score += 1;
if (/[a-zA-Z]/.test(t)) score += 2;
if (/\d/.test(t)) score += 1;
return { term: t, score };
});
scored.sort((a, b) => b.score - a.score);
return scored.slice(0, maxCount).map(x => x.term);
}
export async function extractNounsFromFactsO(facts, relevantSubjects, maxCount = 5) {
if (!facts?.length || !relevantSubjects?.size) return [];
const oTexts = [];
for (const f of facts) {
if (f.retracted) continue;
// 只取相关主体的 facts
const s = String(f.s || '').trim();
if (!relevantSubjects.has(s)) continue;
const o = String(f.o || '').trim();
if (!o) continue;
// 跳过太长的 O（可能是完整句子）
if (o.length > 30) continue;
oTexts.push(o);
}
if (!oTexts.length) return [];
const combined = oTexts.join(' ');
return await extractNouns(combined, { minLen: 2, maxCount });
}
export { ensureJieba };

76
modules/story-summary/vector/storage/state-store.js
View File

@@ -35,6 +35,58 @@ function ensureStateAtomsArray() {
return chat_metadata.extensions[EXT_ID].stateAtoms;
}
// L0Index: per-floor status (ok | empty | fail)
function ensureL0Index() {
chat_metadata.extensions ||= {};
chat_metadata.extensions[EXT_ID] ||= {};
chat_metadata.extensions[EXT_ID].l0Index ||= { version: 1, byFloor: {} };
chat_metadata.extensions[EXT_ID].l0Index.byFloor ||= {};
return chat_metadata.extensions[EXT_ID].l0Index;
}
export function getL0Index() {
return ensureL0Index();
}
export function getL0FloorStatus(floor) {
const idx = ensureL0Index();
return idx.byFloor?.[String(floor)] || null;
}
export function setL0FloorStatus(floor, record) {
const idx = ensureL0Index();
idx.byFloor[String(floor)] = {
...record,
floor,
updatedAt: Date.now(),
};
saveMetadataDebounced();
}
export function clearL0Index() {
const idx = ensureL0Index();
idx.byFloor = {};
saveMetadataDebounced();
}
export function deleteL0IndexFromFloor(fromFloor) {
const idx = ensureL0Index();
const keys = Object.keys(idx.byFloor || {});
let deleted = 0;
for (const k of keys) {
const f = Number(k);
if (Number.isFinite(f) && f >= fromFloor) {
delete idx.byFloor[k];
deleted++;
}
}
if (deleted > 0) {
saveMetadataDebounced();
xbLog.info(MODULE_ID, `删除 ${deleted} 条 L0Index (floor >= ${fromFloor})`);
}
return deleted;
}
/**
* 获取当前聊天的所有 StateAtoms
*/
@@ -113,6 +165,30 @@ export function getStateAtomsCount() {
return ensureStateAtomsArray().length;
}
/**
* Return floors that already have extracted atoms.
*/
export function getExtractedFloors() {
const floors = new Set();
const arr = ensureStateAtomsArray();
for (const atom of arr) {
if (typeof atom?.floor === 'number' && atom.floor >= 0) {
floors.add(atom.floor);
}
}
return floors;
}
/**
* Replace all stored StateAtoms.
*/
export function replaceStateAtoms(atoms) {
const next = Array.isArray(atoms) ? atoms : [];
chat_metadata.extensions[EXT_ID].stateAtoms = next;
saveMetadataDebounced();
xbLog.info(MODULE_ID, `替换 StateAtoms: ${next.length} 条`);
}
// ═══════════════════════════════════════════════════════════════════════════
// StateVector 操作（IndexedDB）
// ═══════════════════════════════════════════════════════════════════════════

697
modules/story-summary/vector/utils/embedder.js
View File

@@ -1,648 +1,83 @@
// ═══════════════════════════════════════════════════════════════════════════
// Story Summary - Embedding Service
// 统一的向量生成接口（本地模型 / 在线服务）
// Story Summary - Embedder (v2 - 统一硅基)
// 所有 embedding 请求转发到 siliconflow.js
// ═══════════════════════════════════════════════════════════════════════════
import { xbLog } from '../../../../core/debug-core.js';
const MODULE_ID = 'embedding';
import { embed as sfEmbed, getApiKey } from '../llm/siliconflow.js';
// ═══════════════════════════════════════════════════════════════════════════
// 本地模型配置
// 统一 embed 接口
// ═══════════════════════════════════════════════════════════════════════════
export const LOCAL_MODELS = {
'bge-small-zh': {
id: 'bge-small-zh',
name: '中文轻量 (51MB)',
hfId: 'Xenova/bge-small-zh-v1.5',
dims: 512,
desc: '手机/低配适用',
},
'bge-base-zh': {
id: 'bge-base-zh',
name: '中文标准 (102MB)',
hfId: 'Xenova/bge-base-zh-v1.5',
dims: 768,
desc: 'PC 推荐，效果更好',
},
'e5-small': {
id: 'e5-small',
name: '多语言 (118MB)',
hfId: 'Xenova/multilingual-e5-small',
dims: 384,
desc: '非中文用户',
},
};
export const DEFAULT_LOCAL_MODEL = 'bge-small-zh';
export async function embed(texts, config, options = {}) {
// 忽略旧的 config 参数，统一走硅基
return await sfEmbed(texts, options);
}
// ═══════════════════════════════════════════════════════════════════════════
// 在线服务配置
// 指纹（简化版）
// ═══════════════════════════════════════════════════════════════════════════
export function getEngineFingerprint(config) {
// 统一使用硅基 bge-m3
return 'siliconflow:bge-m3:1024';
}
// ═══════════════════════════════════════════════════════════════════════════
// 状态检查（简化版）
// ═══════════════════════════════════════════════════════════════════════════
export async function checkLocalModelStatus() {
// 不再支持本地模型
return { status: 'not_supported', message: '请使用在线服务' };
}
export function isLocalModelLoaded() {
return false;
}
export async function downloadLocalModel() {
throw new Error('本地模型已移除，请使用在线服务');
}
export function cancelDownload() { }
export async function deleteLocalModelCache() { }
// ═══════════════════════════════════════════════════════════════════════════
// 在线服务测试
// ═══════════════════════════════════════════════════════════════════════════
export async function testOnlineService() {
const key = getApiKey();
if (!key) {
throw new Error('请配置硅基 API Key');
}
try {
const [vec] = await sfEmbed(['测试连接']);
return { success: true, dims: vec?.length || 0 };
} catch (e) {
throw new Error(`连接失败: ${e.message}`);
}
}
export async function fetchOnlineModels() {
// 硅基模型固定
return ['BAAI/bge-m3'];
}
// ═══════════════════════════════════════════════════════════════════════════
// 兼容旧接口
// ═══════════════════════════════════════════════════════════════════════════
export const DEFAULT_LOCAL_MODEL = 'bge-m3';
export const LOCAL_MODELS = {};
export const ONLINE_PROVIDERS = {
siliconflow: {
id: 'siliconflow',
name: '硅基流动',
baseUrl: 'https://api.siliconflow.cn',
canFetchModels: false,
defaultModels: [
'BAAI/bge-m3',
'BAAI/bge-large-zh-v1.5',
'BAAI/bge-small-zh-v1.5',
],
},
cohere: {
id: 'cohere',
name: 'Cohere',
baseUrl: 'https://api.cohere.ai',
canFetchModels: false,
defaultModels: [
'embed-multilingual-v3.0',
'embed-english-v3.0',
],
// Cohere 使用不同的 API 格式
customEmbed: true,
},
openai: {
id: 'openai',
name: 'OpenAI 兼容',
baseUrl: '',
canFetchModels: true,
defaultModels: [],
},
};
// ═══════════════════════════════════════════════════════════════════════════
// 本地模型状态管理
// ═══════════════════════════════════════════════════════════════════════════
// 已加载的模型实例：{ modelId: pipeline }
const loadedPipelines = {};
// 当前正在下载的模型
let downloadingModelId = null;
let downloadAbortController = null;
// Worker for local embedding
let embeddingWorker = null;
let workerRequestId = 0;
const workerCallbacks = new Map();
function getWorker() {
if (!embeddingWorker) {
const workerPath = new URL('./embedder.worker.js', import.meta.url).href;
embeddingWorker = new Worker(workerPath, { type: 'module' });
embeddingWorker.onmessage = (e) => {
const { requestId, ...data } = e.data || {};
const callback = workerCallbacks.get(requestId);
if (callback) {
callback(data);
if (data.type === 'result' || data.type === 'error' || data.type === 'loaded') {
workerCallbacks.delete(requestId);
}
}
};
}
return embeddingWorker;
}
function workerRequest(message) {
return new Promise((resolve, reject) => {
const requestId = ++workerRequestId;
const worker = getWorker();
workerCallbacks.set(requestId, (data) => {
if (data.type === 'error') {
reject(new Error(data.error));
} else if (data.type === 'result') {
resolve(data.vectors);
} else if (data.type === 'loaded') {
resolve(true);
}
});
worker.postMessage({ ...message, requestId });
});
}
// ═══════════════════════════════════════════════════════════════════════════
// 本地模型操作
// ═══════════════════════════════════════════════════════════════════════════
/**
* 检查指定本地模型的状态
* 只读取缓存，绝不触发下载
*/
export async function checkLocalModelStatus(modelId = DEFAULT_LOCAL_MODEL) {
const modelConfig = LOCAL_MODELS[modelId];
if (!modelConfig) {
return { status: 'error', message: '未知模型' };
}
// 已加载到内存
if (loadedPipelines[modelId]) {
return { status: 'ready', message: '已就绪' };
}
// 正在下载
if (downloadingModelId === modelId) {
return { status: 'downloading', message: '下载中' };
}
// 检查 IndexedDB 缓存
const hasCache = await checkModelCache(modelConfig.hfId);
if (hasCache) {
return { status: 'cached', message: '已缓存，可加载' };
}
return { status: 'not_downloaded', message: '未下载' };
}
/**
* 检查 IndexedDB 中是否有模型缓存
*/
async function checkModelCache(hfId) {
return new Promise((resolve) => {
try {
const request = indexedDB.open('transformers-cache', 1);
request.onerror = () => resolve(false);
request.onsuccess = (event) => {
const db = event.target.result;
const storeNames = Array.from(db.objectStoreNames);
db.close();
// 检查是否有该模型的缓存
const modelKey = hfId.replace('/', '_');
const hasModel = storeNames.some(name =>
name.includes(modelKey) || name.includes('onnx')
);
resolve(hasModel);
};
request.onupgradeneeded = () => resolve(false);
} catch {
resolve(false);
}
});
}
/**
* 下载/加载本地模型
* @param {string} modelId - 模型ID
* @param {Function} onProgress - 进度回调 (0-100)
* @returns {Promise<boolean>}
*/
export async function downloadLocalModel(modelId = DEFAULT_LOCAL_MODEL, onProgress) {
const modelConfig = LOCAL_MODELS[modelId];
if (!modelConfig) {
throw new Error(`未知模型: ${modelId}`);
}
// 已加载
if (loadedPipelines[modelId]) {
onProgress?.(100);
return true;
}
// 正在下载其他模型
if (downloadingModelId && downloadingModelId !== modelId) {
throw new Error(`正在下载其他模型: ${downloadingModelId}`);
}
// 正在下载同一模型，等待完成
if (downloadingModelId === modelId) {
xbLog.info(MODULE_ID, `模型 ${modelId} 正在加载中...`);
return new Promise((resolve, reject) => {
const check = () => {
if (loadedPipelines[modelId]) {
resolve(true);
} else if (downloadingModelId !== modelId) {
reject(new Error('下载已取消'));
} else {
setTimeout(check, 200);
}
};
check();
});
}
downloadingModelId = modelId;
downloadAbortController = new AbortController();
try {
xbLog.info(MODULE_ID, `开始下载模型: ${modelId}`);
return await new Promise((resolve, reject) => {
const requestId = ++workerRequestId;
const worker = getWorker();
workerCallbacks.set(requestId, (data) => {
if (data.type === 'progress') {
onProgress?.(data.percent);
} else if (data.type === 'loaded') {
loadedPipelines[modelId] = true;
workerCallbacks.delete(requestId);
resolve(true);
} else if (data.type === 'error') {
workerCallbacks.delete(requestId);
reject(new Error(data.error));
}
});
worker.postMessage({
type: 'load',
modelId,
hfId: modelConfig.hfId,
requestId
});
});
} finally {
downloadingModelId = null;
downloadAbortController = null;
}
}
export function cancelDownload() {
if (downloadAbortController) {
downloadAbortController.abort();
xbLog.info(MODULE_ID, '下载已取消');
}
downloadingModelId = null;
downloadAbortController = null;
}
/**
* 删除指定模型的缓存
*/
export async function deleteLocalModelCache(modelId = null) {
try {
// 删除 IndexedDB
await new Promise((resolve, reject) => {
const request = indexedDB.deleteDatabase('transformers-cache');
request.onsuccess = () => resolve();
request.onerror = () => reject(request.error);
request.onblocked = () => {
xbLog.warn(MODULE_ID, 'IndexedDB 删除被阻塞');
resolve();
};
});
// 删除 CacheStorage
if (window.caches) {
const cacheNames = await window.caches.keys();
for (const name of cacheNames) {
if (name.includes('transformers') || name.includes('huggingface') || name.includes('xenova')) {
await window.caches.delete(name);
}
}
}
// 清除内存中的 pipeline
if (modelId && loadedPipelines[modelId]) {
delete loadedPipelines[modelId];
} else {
Object.keys(loadedPipelines).forEach(key => delete loadedPipelines[key]);
}
xbLog.info(MODULE_ID, '模型缓存已清除');
return true;
} catch (e) {
xbLog.error(MODULE_ID, '清除缓存失败', e);
throw e;
}
}
/**
* 使用本地模型生成向量
*/
async function embedLocal(texts, modelId = DEFAULT_LOCAL_MODEL) {
if (!loadedPipelines[modelId]) {
await downloadLocalModel(modelId);
}
return await workerRequest({ type: 'embed', texts });
}
export function isLocalModelLoaded(modelId = DEFAULT_LOCAL_MODEL) {
return !!loadedPipelines[modelId];
}
/**
* 获取本地模型信息
*/
export function getLocalModelInfo(modelId = DEFAULT_LOCAL_MODEL) {
return LOCAL_MODELS[modelId] || null;
}
// ═══════════════════════════════════════════════════════════════════════════
// 在线服务操作
// ═══════════════════════════════════════════════════════════════════════════
/**
* 测试在线服务连接
*/
export async function testOnlineService(provider, config) {
const { url, key, model } = config;
if (!key) {
throw new Error('请填写 API Key');
}
if (!model) {
throw new Error('请选择模型');
}
const providerConfig = ONLINE_PROVIDERS[provider];
const baseUrl = (providerConfig?.baseUrl || url || '').replace(/\/+$/, '');
if (!baseUrl) {
throw new Error('请填写 API URL');
}
try {
if (provider === 'cohere') {
// Cohere 使用不同的 API 格式
const response = await fetch(`${baseUrl}/v1/embed`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: model,
texts: ['测试连接'],
input_type: 'search_document',
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`API 返回 ${response.status}: ${error}`);
}
const data = await response.json();
const dims = data.embeddings?.[0]?.length || 0;
if (dims === 0) {
throw new Error('API 返回的向量维度为 0');
}
return { success: true, dims };
} else {
// OpenAI 兼容格式
const response = await fetch(`${baseUrl}/v1/embeddings`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: model,
input: ['测试连接'],
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`API 返回 ${response.status}: ${error}`);
}
const data = await response.json();
const dims = data.data?.[0]?.embedding?.length || 0;
if (dims === 0) {
throw new Error('API 返回的向量维度为 0');
}
return { success: true, dims };
}
} catch (e) {
if (e.name === 'TypeError' && e.message.includes('fetch')) {
throw new Error('网络错误，请检查 URL 是否正确');
}
throw e;
}
}
/**
* 拉取在线模型列表（仅 OpenAI 兼容）
*/
export async function fetchOnlineModels(config) {
const { url, key } = config;
if (!url || !key) {
throw new Error('请填写 URL 和 Key');
}
const baseUrl = url.replace(/\/+$/, '').replace(/\/v1$/, '');
const response = await fetch(`${baseUrl}/v1/models`, {
headers: {
'Authorization': `Bearer ${key}`,
'Accept': 'application/json',
},
});
if (!response.ok) {
throw new Error(`获取模型列表失败: ${response.status}`);
}
const data = await response.json();
const models = data.data?.map(m => m.id).filter(Boolean) || [];
// 过滤出 embedding 相关的模型
const embeddingModels = models.filter(m => {
const lower = m.toLowerCase();
return lower.includes('embed') ||
lower.includes('bge') ||
lower.includes('e5') ||
lower.includes('gte');
});
return embeddingModels.length > 0 ? embeddingModels : models.slice(0, 20);
}
/**
* 使用在线服务生成向量
*/
async function embedOnline(texts, provider, config, options = {}) {
const { url, key, model } = config;
const signal = options?.signal;
const providerConfig = ONLINE_PROVIDERS[provider];
const baseUrl = (providerConfig?.baseUrl || url || '').replace(/\/+$/, '');
// 永远重试：指数退避 + 上限 + 抖动
const BASE_WAIT_MS = 1200;
const MAX_WAIT_MS = 15000;
const sleepAbortable = (ms) => new Promise((resolve, reject) => {
if (signal?.aborted) return reject(new DOMException('Aborted', 'AbortError'));
const t = setTimeout(resolve, ms);
if (signal) {
signal.addEventListener('abort', () => {
clearTimeout(t);
reject(new DOMException('Aborted', 'AbortError'));
}, { once: true });
}
});
let attempt = 0;
while (true) {
attempt++;
try {
let response;
if (provider === 'cohere') {
response = await fetch(`${baseUrl}/v1/embed`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: model,
texts: texts,
input_type: 'search_document',
}),
signal,
});
} else {
response = await fetch(`${baseUrl}/v1/embeddings`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: model,
input: texts,
}),
signal,
});
}
// 需要“永远重试”的典型状态：
// - 429：限流
// - 403：配额/风控/未实名等（你提到的硅基未认证）
// - 5xx：服务端错误
const retryableStatus = (s) => s === 429 || s === 403 || (s >= 500 && s <= 599);
if (!response.ok) {
const errorText = await response.text().catch(() => '');
if (retryableStatus(response.status)) {
const exp = Math.min(MAX_WAIT_MS, BASE_WAIT_MS * Math.pow(2, Math.min(attempt, 6) - 1));
const jitter = Math.floor(Math.random() * 350);
const waitMs = exp + jitter;
await sleepAbortable(waitMs);
continue;
}
// 非可恢复错误：直接抛出（比如 400 参数错、401 key 错等）
const err = new Error(`API 返回 ${response.status}: ${errorText}`);
err.status = response.status;
throw err;
}
const data = await response.json();
if (provider === 'cohere') {
return (data.embeddings || []).map(e => Array.isArray(e) ? e : Array.from(e));
}
return (data.data || []).map(item => {
const embedding = item.embedding;
return Array.isArray(embedding) ? embedding : Array.from(embedding);
});
} catch (e) {
// 取消：必须立刻退出
if (e?.name === 'AbortError') throw e;
// 网络错误：永远重试
const exp = Math.min(MAX_WAIT_MS, BASE_WAIT_MS * Math.pow(2, Math.min(attempt, 6) - 1));
const jitter = Math.floor(Math.random() * 350);
const waitMs = exp + jitter;
await sleepAbortable(waitMs);
}
}
}
// ═══════════════════════════════════════════════════════════════════════════
// 统一接口
// ═══════════════════════════════════════════════════════════════════════════
/**
* 生成向量（统一接口）
* @param {string[]} texts - 要向量化的文本数组
* @param {Object} config - 配置
* @returns {Promise<number[][]>}
*/
export async function embed(texts, config, options = {}) {
if (!texts?.length) return [];
const { engine, local, online } = config;
if (engine === 'local') {
const modelId = local?.modelId || DEFAULT_LOCAL_MODEL;
return await embedLocal(texts, modelId);
} else if (engine === 'online') {
const provider = online?.provider || 'siliconflow';
if (!online?.key || !online?.model) {
throw new Error('在线服务配置不完整');
}
return await embedOnline(texts, provider, online, options);
} else {
throw new Error(`未知的引擎类型: ${engine}`);
}
}
/**
* 获取当前引擎的唯一标识（用于检查向量是否匹配）
*/
// Concurrent embed for online services (local falls back to sequential)
export async function embedBatchesConcurrent(textBatches, config, concurrency = 3) {
if (config.engine === 'local' || textBatches.length <= 1) {
const results = [];
for (const batch of textBatches) {
results.push(await embed(batch, config));
}
return results;
}
const results = new Array(textBatches.length);
let index = 0;
async function worker() {
while (index < textBatches.length) {
const i = index++;
results[i] = await embed(textBatches[i], config);
}
}
await Promise.all(
Array(Math.min(concurrency, textBatches.length))
.fill(null)
.map(() => worker())
);
return results;
}
export function getEngineFingerprint(config) {
if (config.engine === 'local') {
const modelId = config.local?.modelId || DEFAULT_LOCAL_MODEL;
const modelConfig = LOCAL_MODELS[modelId];
return `local:${modelId}:${modelConfig?.dims || 512}`;
} else if (config.engine === 'online') {
const provider = config.online?.provider || 'unknown';
const model = config.online?.model || 'unknown';
return `online:${provider}:${model}`;
} else {
return 'unknown';
}
}